From 1b64959e2c3ac82b97eff6f5b49653ea3f5e7dcc Mon Sep 17 00:00:00 2001 From: pengzhisheng Date: Thu, 6 Mar 2025 21:16:31 +0800 Subject: [PATCH 01/37] fix manager-api bug --- .../src/main/java/xiaozhi/common/exception/ErrorCode.java | 5 +++++ .../java/xiaozhi/modules/sys/service/SysDictTypeService.java | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java index 22fd6100..ffa78cf5 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java +++ b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java @@ -35,4 +35,9 @@ public interface ErrorCode { int REDIS_ERROR = 10027; int JOB_ERROR = 10028; int INVALID_SYMBOL = 10029; + + + int PASSWORD_LENGTH_ERROR = 10030; + int PASSWORD_WEAK_ERROR = 10031; + int DEL_MYSELF_ERROR = 10032; } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/SysDictTypeService.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/SysDictTypeService.java index 0b4b5150..29906afb 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/SysDictTypeService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/SysDictTypeService.java @@ -23,4 +23,8 @@ public interface SysDictTypeService extends BaseService { void update(SysDictTypeDTO dto); void delete(Long[] ids); + + List getAllList(); + + List getDictTypeList(); } \ No newline at end of file From 8f76db9e411474eaa968cad26554d09d6b7e642a Mon Sep 17 00:00:00 2001 From: pengzhisheng Date: Wed, 12 Mar 2025 16:53:49 +0800 Subject: [PATCH 02/37] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=99=BB=E5=BD=95?= =?UTF-8?q?=E6=B3=A8=E5=86=8C=E6=B5=81=E7=A8=8B=EF=BC=8C=E8=B7=91=E9=80=9A?= =?UTF-8?q?=E6=B3=A8=E5=86=8C=E7=99=BB=E5=BD=95=E3=80=81=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../security/controller/LoginController.java | 7 ++++--- .../modules/security/dto/LoginDTO.java | 2 +- main/manager-web/package-lock.json | 7 ++++++- main/manager-web/src/apis/api.js | 4 ++-- main/manager-web/src/views/login.vue | 4 +++- main/manager-web/src/views/register.vue | 6 +++--- main/manager-web/vue.config.js | 19 ++++++++++++++----- 7 files changed, 33 insertions(+), 16 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java b/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java index 3f549ffb..85c5952b 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java @@ -51,7 +51,7 @@ public class LoginController { throw new RenException("验证码错误，请重新获取"); } // 按照用户名获取用户 - SysUserDTO userDTO = sysUserService.getByUsername(login.getMobile()); + SysUserDTO userDTO = sysUserService.getByUsername(login.getUsername()); // 判断用户是否存在 if (userDTO == null) { throw new RenException("请检测用户和密码是否输入错误"); @@ -72,15 +72,16 @@ public class LoginController { throw new RenException("验证码错误，请重新获取"); } // 按照用户名获取用户 - SysUserDTO userDTO = sysUserService.getByUsername(login.getMobile()); + SysUserDTO userDTO = sysUserService.getByUsername(login.getUsername()); if (userDTO != null){ throw new RenException("此手机号码已经注册过"); } userDTO = new SysUserDTO(); - userDTO.setUsername(login.getMobile()); + userDTO.setUsername(login.getUsername()); userDTO.setPassword(login.getPassword()); sysUserService.save(userDTO); return new Result(); + } } \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/dto/LoginDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/security/dto/LoginDTO.java index 076324ef..2fc0d3b2 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/dto/LoginDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/dto/LoginDTO.java @@ -15,7 +15,7 @@ public class LoginDTO implements Serializable { @Schema(description = "手机号码") @NotBlank(message = "{sysuser.username.require}") - private String mobile; + private String username; @Schema(description = "密码") @NotBlank(message = "{sysuser.password.require}") diff --git a/main/manager-web/package-lock.json b/main/manager-web/package-lock.json index cbfc40df..5af654f0 100644 --- a/main/manager-web/package-lock.json +++ b/main/manager-web/package-lock.json @@ -14,7 +14,8 @@ "vue": "^2.6.14", "vue-axios": "^3.5.2", "vue-router": "^3.6.5", - "vuex": "^3.6.2" + "vuex": "^3.6.2", + "xiaozhi": "file:" }, "devDependencies": { "@vue/cli-plugin-router": "~5.0.0", @@ -9157,6 +9158,10 @@ } } }, + "node_modules/xiaozhi": { + "resolved": "", + "link": true + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmmirror.com/y18n/-/y18n-5.0.8.tgz", diff --git a/main/manager-web/src/apis/api.js b/main/manager-web/src/apis/api.js index ebe36f49..e11b60d6 100755 --- a/main/manager-web/src/apis/api.js +++ b/main/manager-web/src/apis/api.js @@ -7,9 +7,9 @@ import user from './module/user.js' * 如果你想调用8002端口，就用'/xiaozhi-esp32-api/api/v1'，请与vue.config.js的devServer配置相结合，方便跨域请求 * */ -const DEV_API_SERVICE = 'https://apifoxmock.com/m1/5931378-5618560-default' +// const DEV_API_SERVICE = 'https://apifoxmock.com/m1/5931378-5618560-default' // 8002开发完成完成后使用这个 -// const DEV_API_SERVICE = '/xiaozhi-esp32-api/api/v1' +const DEV_API_SERVICE = '/xiaozhi-esp32-api' /** * 根据开发环境返回接口url diff --git a/main/manager-web/src/views/login.vue b/main/manager-web/src/views/login.vue index 12c74ca4..6c73f82f 100644 --- a/main/manager-web/src/views/login.vue +++ b/main/manager-web/src/views/login.vue @@ -75,7 +75,8 @@ export default { form: { username: '', password: '', - captcha: '' + captcha: '', + captchaId: '' }, captchaUuid: '', captchaUrl: '' @@ -114,6 +115,7 @@ export default { return } + this.form.captchaId = this.captchaUuid Api.user.login(this.form, ({data}) => { showSuccess('登陆成功！') goToPage('/home') diff --git a/main/manager-web/src/views/register.vue b/main/manager-web/src/views/register.vue index 43de61e4..3e7839a8 100644 --- a/main/manager-web/src/views/register.vue +++ b/main/manager-web/src/views/register.vue @@ -95,7 +95,7 @@ export default { password: '', confirmPassword: '', captcha: '', - uuid: '' + captchaId: '' }, captchaUrl: '' } @@ -106,8 +106,8 @@ export default { methods: { // 复用验证码获取方法 fetchCaptcha() { - this.form.uuid = Date.now().toString() - Api.user.getCaptcha(this.form.uuid, (res) => { + this.form.captchaId = Date.now().toString() + Api.user.getCaptcha(this.form.captchaId, (res) => { if (res.status === 200) { const blob = new Blob([res.data], {type: res.data.type}); this.captchaUrl = URL.createObjectURL(blob); diff --git a/main/manager-web/vue.config.js b/main/manager-web/vue.config.js index 01e210fa..9e72e068 100644 --- a/main/manager-web/vue.config.js +++ b/main/manager-web/vue.config.js @@ -1,14 +1,23 @@ const { defineConfig } = require('@vue/cli-service'); const dotenv = require('dotenv'); +// 确保加载 .env 文件 dotenv.config(); module.exports = defineConfig({ devServer: { // Bug 修复：将代理配置为环境变量中定义的 API 基础 URL - proxy: process.env.VUE_APP_API_BASE_URL, - client: { - overlay: false, + proxy: { + '/xiaozhi-esp32-api': { + target: process.env.VUE_APP_API_BASE_URL || 'http://localhost:8002', // 后端 API 的基础 URL + changeOrigin: true, // 允许跨域 + // pathRewrite: { + // '^/api': '', // 路径重写 + // }, + }, + }, + client: { + overlay: false, + }, }, - } -}) +}); From 0f41039b99e341164ff1d2124655318bd8078253 Mon Sep 17 00:00:00 2001 From: pengzhisheng Date: Wed, 12 Mar 2025 17:14:48 +0800 Subject: [PATCH 03/37] update package --- main/manager-web/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main/manager-web/package.json b/main/manager-web/package.json index e1423d6a..fddf19aa 100644 --- a/main/manager-web/package.json +++ b/main/manager-web/package.json @@ -13,7 +13,8 @@ "vue": "^2.6.14", "vue-axios": "^3.5.2", "vue-router": "^3.6.5", - "vuex": "^3.6.2" + "vuex": "^3.6.2", + "xiaozhi": "file:" }, "devDependencies": { "@vue/cli-plugin-router": "~5.0.0", From 5b2957e7eb5e7d648db3b99cea588b6df6b36c27 Mon Sep 17 00:00:00 2001 From: pengzhisheng Date: Fri, 14 Mar 2025 09:54:54 +0800 Subject: [PATCH 04/37] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=99=BB=E5=BD=95=20to?= =?UTF-8?q?ken=20=E6=A0=A1=E9=AA=8C=EF=BC=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、后端新增 /api/v1/user/info 接口，优化 Oauth2Filter.getRequestToken 逻辑； 2、前端 /api/v1/user/login 登录成功后，保存 token 至浏览器本地； 3、前端请求添加本地 token。 --- .../xiaozhi/common/constant/Constant.java | 2 ++ .../modules/security/config/ShiroConfig.java | 6 ++++ .../security/controller/LoginController.java | 25 ++++++++++++++++- .../modules/security/oauth2/Oauth2Filter.java | 15 ++++++++-- .../modules/security/oauth2/Oauth2Realm.java | 10 +++++++ .../security/service/SysUserTokenService.java | 6 +++- .../service/impl/SysUserTokenServiceImpl.java | 24 +++++++++++++++- .../modules/sys/service/SysUserService.java | 2 ++ .../sys/service/impl/SysUserServiceImpl.java | 7 +++++ main/manager-web/src/apis/httpRequest.js | 6 ++-- main/manager-web/src/apis/module/user.js | 2 +- main/manager-web/src/store/index.js | 12 ++++++++ main/manager-web/src/views/home.vue | 8 +++--- main/manager-web/src/views/login.vue | 28 ++++++++++++------- 14 files changed, 131 insertions(+), 22 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index ffe7ff3c..10135cef 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -78,6 +78,8 @@ public interface Constant { */ String TOKEN_HEADER = "token"; + String AUTHORIZATION = "Authorization"; + /** * 路径分割符 */ diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java b/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java index f770c79f..35cde8aa 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java @@ -58,6 +58,12 @@ public class ShiroConfig { filters.put("oauth2", new Oauth2Filter()); shiroFilter.setFilters(filters); + //添加Shiro的内置过滤器 + /*anon：无需认证就可以访问 + authc：必须认证了才能让问 + user：必须拥有，记住我功能，才能访问 + perms：拥有对某个资源的权限才能访问 + role：拥有某个角色权限才能访问*/ Map filterMap = new LinkedHashMap<>(); filterMap.put("/webjars/**", "anon"); filterMap.put("/druid/**", "anon"); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java b/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java index 85c5952b..af3f451c 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java @@ -4,11 +4,16 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletResponse; import lombok.AllArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.*; import xiaozhi.common.exception.ErrorCode; import xiaozhi.common.exception.RenException; +import xiaozhi.common.page.TokenDTO; import xiaozhi.common.utils.Result; import xiaozhi.common.validator.AssertUtils; +import xiaozhi.modules.security.dao.SysUserTokenDao; import xiaozhi.modules.security.dto.LoginDTO; import xiaozhi.modules.security.password.PasswordUtils; import xiaozhi.modules.security.service.CaptchaService; @@ -31,6 +36,7 @@ public class LoginController { private final SysUserTokenService sysUserTokenService; private final CaptchaService captchaService; + private static final Logger logger = LoggerFactory.getLogger(LoginController.class); @GetMapping("/captcha") @Operation(summary = "验证码") @@ -44,7 +50,7 @@ public class LoginController { @PostMapping("/login") @Operation(summary = "登录") - public Result login( @RequestBody LoginDTO login) { + public Result login(@RequestBody LoginDTO login) { // 验证是否正确输入验证码 boolean validate = captchaService.validate(login.getCaptchaId(), login.getCaptcha()); if (!validate) { @@ -84,4 +90,21 @@ public class LoginController { } + @GetMapping("/info") + @Operation(summary = "用户信息获取") + public Result info(@RequestHeader("Authorization")String authorization) { + logger.info("the authorization:{}", authorization); + + String token; + if (StringUtils.isBlank(authorization) && authorization.contains("Bearer ")) { + throw new RenException(ErrorCode.UNAUTHORIZED); + } + token = authorization.replace("Bearer ", ""); + if (StringUtils.isBlank(token)) { + throw new RenException(ErrorCode.UNAUTHORIZED); + } + SysUserDTO sysUserDTO = sysUserTokenService.getUserByToken(token); + Result result = new Result(); + return result.ok(sysUserDTO); + } } \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Filter.java b/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Filter.java index 3d8cf0cb..26486b7b 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Filter.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Filter.java @@ -11,6 +11,7 @@ import org.apache.shiro.web.filter.authc.AuthenticatingFilter; import org.springframework.web.bind.annotation.RequestMethod; import xiaozhi.common.constant.Constant; import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; import xiaozhi.common.utils.HttpContextUtils; import xiaozhi.common.utils.JsonUtils; import xiaozhi.common.utils.Result; @@ -89,12 +90,22 @@ public class Oauth2Filter extends AuthenticatingFilter { * 获取请求的token */ private String getRequestToken(HttpServletRequest httpRequest) { + String token; //从header中获取token - String token = httpRequest.getHeader(Constant.TOKEN_HEADER); + String authorization = httpRequest.getHeader(Constant.AUTHORIZATION); + if (StringUtils.isBlank(authorization) && authorization.contains("Bearer ")) { + throw new RenException(ErrorCode.UNAUTHORIZED); + } + token = authorization.replace("Bearer ", ""); //如果header中不存在token，则从参数中获取token if (StringUtils.isBlank(token)) { - token = httpRequest.getParameter(Constant.TOKEN_HEADER); + authorization = httpRequest.getParameter(Constant.AUTHORIZATION); + + if (StringUtils.isBlank(authorization) && authorization.contains("Bearer ")) { + throw new RenException(ErrorCode.UNAUTHORIZED); + } + token = authorization.replace("Bearer ", ""); } return token; } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Realm.java b/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Realm.java index 3f8a3eeb..8c297e17 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Realm.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Realm.java @@ -6,12 +6,15 @@ import org.apache.shiro.authz.AuthorizationInfo; import org.apache.shiro.authz.SimpleAuthorizationInfo; import org.apache.shiro.realm.AuthorizingRealm; import org.apache.shiro.subject.PrincipalCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import xiaozhi.common.exception.ErrorCode; import xiaozhi.common.user.UserDetail; import xiaozhi.common.utils.ConvertUtils; import xiaozhi.common.utils.MessageUtils; +import xiaozhi.modules.security.controller.LoginController; import xiaozhi.modules.security.entity.SysUserTokenEntity; import xiaozhi.modules.security.service.ShiroService; import xiaozhi.modules.sys.entity.SysUserEntity; @@ -30,6 +33,8 @@ public class Oauth2Realm extends AuthorizingRealm { @Resource private ShiroService shiroService; + private static final Logger logger = LoggerFactory.getLogger(Oauth2Realm.class); + @Override public boolean supports(AuthenticationToken token) { return token instanceof Oauth2Token; @@ -82,6 +87,11 @@ public class Oauth2Realm extends AuthorizingRealm { userDetail.setToken(accessToken); //账号锁定 + if (userDetail.getStatus() == null) { + logger.error("账号状态异常，status 不能为空"); + throw new DisabledAccountException(MessageUtils.getMessage(ErrorCode.ACCOUNT_DISABLE)); + } + if (userDetail.getStatus() == 0) { throw new LockedAccountException(MessageUtils.getMessage(ErrorCode.ACCOUNT_LOCK)); } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java b/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java index 220b18b5..74882dea 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java @@ -1,9 +1,11 @@ package xiaozhi.modules.security.service; import xiaozhi.common.page.PageData; +import xiaozhi.common.page.TokenDTO; import xiaozhi.common.service.BaseService; import xiaozhi.common.utils.Result; import xiaozhi.modules.security.entity.SysUserTokenEntity; +import xiaozhi.modules.sys.dto.SysUserDTO; import java.util.Map; @@ -19,7 +21,9 @@ public interface SysUserTokenService extends BaseService { * * @param userId 用户ID */ - Result createToken(Long userId); + Result createToken(Long userId); + + SysUserDTO getUserByToken(String token); /** * 退出 diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java index 903ca242..caa24801 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java @@ -1,6 +1,9 @@ package xiaozhi.modules.security.service.impl; import cn.hutool.core.date.DateUtil; +import lombok.AllArgsConstructor; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; import xiaozhi.common.page.TokenDTO; import xiaozhi.common.service.impl.BaseServiceImpl; import xiaozhi.common.utils.HttpContextUtils; @@ -10,18 +13,23 @@ import xiaozhi.modules.security.entity.SysUserTokenEntity; import xiaozhi.modules.security.oauth2.TokenGenerator; import xiaozhi.modules.security.service.SysUserTokenService; import org.springframework.stereotype.Service; +import xiaozhi.modules.sys.dto.SysUserDTO; +import xiaozhi.modules.sys.service.SysUserService; import java.util.Date; +@AllArgsConstructor @Service public class SysUserTokenServiceImpl extends BaseServiceImpl implements SysUserTokenService { + + private final SysUserService sysUserService; /** * 12小时后过期 */ private final static int EXPIRE = 3600 * 12; @Override - public Result createToken(Long userId) { + public Result createToken(Long userId) { //用户token String token; @@ -70,6 +78,20 @@ public class SysUserTokenServiceImpl extends BaseServiceImpl { SysUserDTO getByUsername(String username); + SysUserDTO getByUserId(Long userId); + void save(SysUserDTO dto); void delete(Long[] ids); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java index 5c3a10c5..8f576137 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java @@ -41,6 +41,13 @@ public class SysUserServiceImpl extends BaseServiceImpl { // 打印失败响应 - console.log(res) + console.log('catch', res) httpHandlerError(res, this._failCallback) }) return this @@ -97,6 +98,7 @@ function sendRequest() { */ // 在错误处理函数中添加日志 function httpHandlerError(info, callBack) { + console.log('httpHandlerError', info) /** 请求成功，退出该函数 可以根据项目需求来判断是否请求成功。这里判断的是status为200的时候是成功 */ let networkError = false diff --git a/main/manager-web/src/apis/module/user.js b/main/manager-web/src/apis/module/user.js index 94de7b3f..6f09c28f 100755 --- a/main/manager-web/src/apis/module/user.js +++ b/main/manager-web/src/apis/module/user.js @@ -32,7 +32,7 @@ export default { }, // 获取设备信息 getHomeList(callback) { - RequestService.sendRequest().url(`${getServiceUrl()}/api/v1/user/device/bind`).method('GET') + RequestService.sendRequest().url(`${getServiceUrl()}/api/v1/device/bind`).method('GET') .success((res) => { RequestService.clearRequestTime() callback(res) diff --git a/main/manager-web/src/store/index.js b/main/manager-web/src/store/index.js index ceffa8e3..0f810894 100644 --- a/main/manager-web/src/store/index.js +++ b/main/manager-web/src/store/index.js @@ -1,14 +1,26 @@ import Vue from 'vue' import Vuex from 'vuex' +import Constant from '../utils/constant' Vue.use(Vuex) export default new Vuex.Store({ state: { + token: '' }, getters: { + getToken(state) { + if (!state.token) { + state.token = localStorage.getItem('token') + } + return state.token + } }, mutations: { + setToken(state, token) { + state.token = token + localStorage.token = token + } }, actions: { }, diff --git a/main/manager-web/src/views/home.vue b/main/manager-web/src/views/home.vue index 4d0181f5..80420e13 100644 --- a/main/manager-web/src/views/home.vue +++ b/main/manager-web/src/views/home.vue @@ -27,7 +27,7 @@ @@ -251,7 +251,7 @@ export default { label: '双皮奶' }], userInfo: { - mobile: '' // 初始化用户信息 + username: '' // 初始化用户信息 } }; }, @@ -331,8 +331,8 @@ export default { } }, mounted() { - this.fetchUserInfo(); // 组件加载时获取用户信息 - this.getList(); // 初始化设备列表 + // this.fetchUserInfo(); // 组件加载时获取用户信息 + // this.getList(); // 初始化设备列表 } } diff --git a/main/manager-web/src/views/login.vue b/main/manager-web/src/views/login.vue index d52d2c9b..d6083635 100644 --- a/main/manager-web/src/views/login.vue +++ b/main/manager-web/src/views/login.vue @@ -87,18 +87,22 @@ export default { }, methods: { fetchCaptcha() { - this.captchaUuid = getUUID(); + if (this.$store.getters.getToken) { + goToPage('/home') + } else { + this.captchaUuid = getUUID(); - Api.user.getCaptcha(this.captchaUuid, (res) => { - if (res.status === 200) { - const blob = new Blob([res.data], {type: res.data.type}); - this.captchaUrl = URL.createObjectURL(blob); + Api.user.getCaptcha(this.captchaUuid, (res) => { + if (res.status === 200) { + const blob = new Blob([res.data], {type: res.data.type}); + this.captchaUrl = URL.createObjectURL(blob); - } else { - console.error('验证码加载异常:', error); - showDanger('验证码加载失败，点击刷新') - } - }); + } else { + console.error('验证码加载异常:', error); + showDanger('验证码加载失败，点击刷新') + } + }); + } }, async login() { @@ -119,8 +123,12 @@ export default { Api.user.login(this.form, ({data}) => { console.log(data) showSuccess('登陆成功！') + + this.$store.commit('setToken', JSON.stringify(data.data)) + goToPage('/home') }) + setTimeout(() => { this.fetchCaptcha() }, 1000) From 08e57936fcff8b26e5e3d4722c23c4a4f840addd Mon Sep 17 00:00:00 2001 From: Jad Date: Mon, 17 Mar 2025 13:45:49 +0800 Subject: [PATCH 05/37] =?UTF-8?q?feat:=20ASR=E5=A2=9E=E5=8A=A0sherpa-onnx?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=20#315=20(#379)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 +- README.md | 1 + README_en.md | 1 + main/xiaozhi-server/config.yaml | 4 + .../core/providers/asr/sherpa_onnx_local.py | 164 ++++++++++++++++++ main/xiaozhi-server/requirements.txt | 4 +- 6 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py diff --git a/.gitignore b/.gitignore index 698d638c..a25b3401 100644 --- a/.gitignore +++ b/.gitignore @@ -141,8 +141,8 @@ music/ # Cython debug symbols cython_debug/ *.iml -model.pt tmp +.history .DS_Store main/xiaozhi-server/data main/manager-web/node_modules @@ -151,3 +151,6 @@ main/manager-web/node_modules .private_config.yaml .env.development +# model files +main/xiaozhi-server/models/SenseVoiceSmall/model.pt +main/xiaozhi-server/models/sherpa-onnx* diff --git a/README.md b/README.md index 121dfbe6..efc45101 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,7 @@ server: | 类型 | 平台名称 | 使用方式 | 收费模式 | 备注 | |:---:|:---------:|:----:|:----:|:--:| | ASR | FunASR | 本地使用 | 免费 | | +| ASR | SherpaASR | 本地使用 | 免费 | | | ASR | DoubaoASR | 接口调用 | 收费 | | --- diff --git a/README_en.md b/README_en.md index 8c251a44..88306675 100644 --- a/README_en.md +++ b/README_en.md @@ -165,6 +165,7 @@ In fact, any LLM that supports OpenAI API calls can be integrated. | Type | Platform Name | Usage Method | Pricing Model | Remarks | |:----:|:-------------------:|:------------:|:-------------:|:-------:| | ASR | FunASR | Local | Free | | +| ASR | SherpaASR | Local | Free | | | ASR | DoubaoASR | API call | Paid | | --- diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 7fa8a24a..31210e9e 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -120,6 +120,10 @@ ASR: type: fun_local model_dir: models/SenseVoiceSmall output_dir: tmp/ + SherpaASR: + type: sherpa_onnx_local + model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 + output_dir: tmp/ DoubaoASR: type: doubao appid: 你的火山引擎语音合成服务appid diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py new file mode 100644 index 00000000..3dfa3162 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -0,0 +1,164 @@ +import time +import wave +import os +import sys +import io +from config.logger import setup_logging +from typing import Optional, Tuple, List +import uuid +import opuslib_next +from core.providers.asr.base import ASRProviderBase + +import numpy as np +import sherpa_onnx + +from modelscope.hub.file_download import model_file_download + +TAG = __name__ +logger = setup_logging() + + +# 捕获标准输出 +class CaptureOutput: + def __enter__(self): + self._output = io.StringIO() + self._original_stdout = sys.stdout + sys.stdout = self._output + + def __exit__(self, exc_type, exc_value, traceback): + sys.stdout = self._original_stdout + self.output = self._output.getvalue() + self._output.close() + + # 将捕获到的内容通过 logger 输出 + if self.output: + logger.bind(tag=TAG).info(self.output.strip()) + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + self.model_dir = config.get("model_dir") + self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + # 初始化模型文件路径 + model_files = { + "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"), + "tokens.txt": os.path.join(self.model_dir, "tokens.txt") + } + + # 下载并检查模型文件 + try: + for file_name, file_path in model_files.items(): + if not os.path.isfile(file_path): + logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}") + model_file_download( + model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue", + file_path=file_name, + local_dir=self.model_dir + ) + + if not os.path.isfile(file_path): + raise FileNotFoundError(f"模型文件下载失败: {file_path}") + + self.model_path = model_files["model.int8.onnx"] + self.tokens_path = model_files["tokens.txt"] + + except Exception as e: + logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}") + raise + + with CaptureOutput(): + self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice( + model=self.model_path, + tokens=self.tokens_path, + num_threads=2, + sample_rate=16000, + feature_dim=80, + decoding_method="greedy_search", + debug=False, + use_itn=True, + ) + + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """将Opus音频数据解码并保存为WAV文件""" + file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + file_path = os.path.join(self.output_dir, file_name) + + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path + + def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]: + """ + Args: + wave_filename: + Path to a wave file. It should be single channel and each sample should + be 16-bit. Its sample rate does not need to be 16kHz. + Returns: + Return a tuple containing: + - A 1-D array of dtype np.float32 containing the samples, which are + normalized to the range [-1, 1]. + - sample rate of the wave file + """ + + with wave.open(wave_filename) as f: + assert f.getnchannels() == 1, f.getnchannels() + assert f.getsampwidth() == 2, f.getsampwidth() # it is in bytes + num_samples = f.getnframes() + samples = f.readframes(num_samples) + samples_int16 = np.frombuffer(samples, dtype=np.int16) + samples_float32 = samples_int16.astype(np.float32) + + samples_float32 = samples_float32 / 32768 + return samples_float32, f.getframerate() + + async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + """语音转文本主处理逻辑""" + file_path = None + try: + # 保存音频文件 + start_time = time.time() + file_path = self.save_audio_to_file(opus_data, session_id) + logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + + # 语音识别 + start_time = time.time() + s = self.model.create_stream() + samples, sample_rate = self.read_wave(file_path) + s.accept_waveform(sample_rate, samples) + self.model.decode_stream(s) + text = s.result.text + logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + + return text, file_path + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) + return "", None + + finally: + # 文件清理逻辑 + if self.delete_audio_file and file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") + except Exception as e: + logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index bcd6b4ee..58362e02 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -19,4 +19,6 @@ loguru==0.7.3 requests==2.32.3 cozepy==0.12.0 mem0ai==0.1.62 -bs4==0.0.2 \ No newline at end of file +bs4==0.0.2 +modelscope==1.23.2 +sherpa_onnx==1.11.0 \ No newline at end of file From c7dec8045e1e3da3e9d5d1189a9cf636ab2a04d0 Mon Sep 17 00:00:00 2001 From: pengzhisheng Date: Mon, 17 Mar 2025 23:25:24 +0800 Subject: [PATCH 06/37] =?UTF-8?q?=E5=90=8E=E7=AB=AF=E6=96=B0=E5=A2=9E=20/a?= =?UTF-8?q?pi/v1/user/change-password=20=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-api/pom.xml | 4 +++ .../common/utils/HttpContextUtils.java | 13 +++++++++ .../java/xiaozhi/common/utils/Result.java | 5 ++++ .../device/controller/DeviceController.java | 3 +- .../security/controller/LoginController.java | 29 ++++++++++--------- .../security/service/SysUserTokenService.java | 3 ++ .../service/impl/SysUserTokenServiceImpl.java | 23 +++++++++++++++ .../modules/sys/service/SysUserService.java | 2 ++ .../sys/service/impl/SysUserServiceImpl.java | 26 +++++++++++++++++ 9 files changed, 93 insertions(+), 15 deletions(-) diff --git a/main/manager-api/pom.xml b/main/manager-api/pom.xml index eacbdff9..a1c249a7 100644 --- a/main/manager-api/pom.xml +++ b/main/manager-api/pom.xml @@ -168,6 +168,10 @@ lombok true + + org.springframework.boot + spring-boot-starter-actuator + diff --git a/main/manager-api/src/main/java/xiaozhi/common/utils/HttpContextUtils.java b/main/manager-api/src/main/java/xiaozhi/common/utils/HttpContextUtils.java index 3629d553..efa320a8 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/utils/HttpContextUtils.java +++ b/main/manager-api/src/main/java/xiaozhi/common/utils/HttpContextUtils.java @@ -8,6 +8,8 @@ import org.springframework.util.DigestUtils; import org.springframework.web.context.request.RequestAttributes; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; import java.util.Date; import java.util.Enumeration; @@ -41,6 +43,17 @@ public class HttpContextUtils { return token; } + public static String getToken(String authorization) { + String token; + if (StringUtils.isBlank(authorization) && authorization.contains("Bearer ")) { + throw new RenException(ErrorCode.UNAUTHORIZED); + } + token = authorization.replace("Bearer ", ""); + if (StringUtils.isBlank(token)) { + throw new RenException(ErrorCode.TOKEN_NOT_EMPTY); + } + return token; + } public static Map getParameterMap(HttpServletRequest request) { Enumeration parameters = request.getParameterNames(); diff --git a/main/manager-api/src/main/java/xiaozhi/common/utils/Result.java b/main/manager-api/src/main/java/xiaozhi/common/utils/Result.java index c21bdbf9..89999420 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/utils/Result.java +++ b/main/manager-api/src/main/java/xiaozhi/common/utils/Result.java @@ -37,6 +37,11 @@ public class Result implements Serializable { @Schema(description = "响应数据") private T data; + public Result ok() { + this.setCode(0); + return this; + } + public Result ok(T data) { this.setData(data); return this; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/controller/DeviceController.java b/main/manager-api/src/main/java/xiaozhi/modules/device/controller/DeviceController.java index 1974689c..d458e9f9 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/controller/DeviceController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/controller/DeviceController.java @@ -25,11 +25,10 @@ import xiaozhi.modules.security.user.SecurityUser; import java.util.List; import java.util.Map; +@Tag(name = "设备管理") @AllArgsConstructor @RestController @RequestMapping("/device") -@Tag(name = "设备管理") - public class DeviceController { private final DeviceService deviceService; private final RedisUtils redisUtils; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java b/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java index af3f451c..130ef27f 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java @@ -4,20 +4,20 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletResponse; import lombok.AllArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.*; import xiaozhi.common.exception.ErrorCode; import xiaozhi.common.exception.RenException; import xiaozhi.common.page.TokenDTO; +import xiaozhi.common.utils.HttpContextUtils; import xiaozhi.common.utils.Result; import xiaozhi.common.validator.AssertUtils; -import xiaozhi.modules.security.dao.SysUserTokenDao; import xiaozhi.modules.security.dto.LoginDTO; import xiaozhi.modules.security.password.PasswordUtils; import xiaozhi.modules.security.service.CaptchaService; import xiaozhi.modules.security.service.SysUserTokenService; +import xiaozhi.modules.sys.dto.PasswordDTO; import xiaozhi.modules.sys.dto.SysUserDTO; import xiaozhi.modules.sys.service.SysUserService; @@ -26,7 +26,6 @@ import java.io.IOException; /** * 登录控制层 */ -@Tag(name = "登录管理") @AllArgsConstructor @RestController @RequestMapping("/user") @@ -92,19 +91,23 @@ public class LoginController { @GetMapping("/info") @Operation(summary = "用户信息获取") - public Result info(@RequestHeader("Authorization")String authorization) { + public Result info(@RequestHeader("Authorization")String authorization) { logger.info("the authorization:{}", authorization); - String token; - if (StringUtils.isBlank(authorization) && authorization.contains("Bearer ")) { - throw new RenException(ErrorCode.UNAUTHORIZED); - } - token = authorization.replace("Bearer ", ""); - if (StringUtils.isBlank(token)) { - throw new RenException(ErrorCode.UNAUTHORIZED); - } + String token = HttpContextUtils.getToken(authorization); SysUserDTO sysUserDTO = sysUserTokenService.getUserByToken(token); - Result result = new Result(); + Result result = new Result<>(); return result.ok(sysUserDTO); } + + @PutMapping("/change-password") + @Operation(summary = "修改用户密码") + public Result changePassword(@RequestHeader("Authorization")String authorization, + @RequestBody PasswordDTO passwordDTO) { + + String token = HttpContextUtils.getToken(authorization); + sysUserTokenService.changePassword(token, passwordDTO); + + return new Result<>().ok(); + } } \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java b/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java index 74882dea..6cfef3ec 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/service/SysUserTokenService.java @@ -5,6 +5,7 @@ import xiaozhi.common.page.TokenDTO; import xiaozhi.common.service.BaseService; import xiaozhi.common.utils.Result; import xiaozhi.modules.security.entity.SysUserTokenEntity; +import xiaozhi.modules.sys.dto.PasswordDTO; import xiaozhi.modules.sys.dto.SysUserDTO; import java.util.Map; @@ -32,4 +33,6 @@ public interface SysUserTokenService extends BaseService { */ void logout(Long userId); + void changePassword(String token, PasswordDTO passwordDTO); + } \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java index caa24801..c2aaeecc 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/security/service/impl/SysUserTokenServiceImpl.java @@ -13,6 +13,7 @@ import xiaozhi.modules.security.entity.SysUserTokenEntity; import xiaozhi.modules.security.oauth2.TokenGenerator; import xiaozhi.modules.security.service.SysUserTokenService; import org.springframework.stereotype.Service; +import xiaozhi.modules.sys.dto.PasswordDTO; import xiaozhi.modules.sys.dto.SysUserDTO; import xiaozhi.modules.sys.service.SysUserService; @@ -81,6 +82,9 @@ public class SysUserTokenServiceImpl extends BaseServiceImpl { void delete(Long[] ids); + void changePassword(Long userId, PasswordDTO passwordDTO); } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java index 8f576137..17d1939c 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysUserServiceImpl.java @@ -10,6 +10,7 @@ import xiaozhi.common.service.impl.BaseServiceImpl; import xiaozhi.common.utils.ConvertUtils; import xiaozhi.modules.security.password.PasswordUtils; import xiaozhi.modules.sys.dao.SysUserDao; +import xiaozhi.modules.sys.dto.PasswordDTO; import xiaozhi.modules.sys.dto.SysUserDTO; import xiaozhi.modules.sys.entity.SysUserEntity; import xiaozhi.modules.sys.enums.SuperAdminEnum; @@ -81,6 +82,31 @@ public class SysUserServiceImpl extends BaseServiceImpl queryWrapper = new QueryWrapper<>(); return baseDao.selectCount(queryWrapper); From 2c1fc30bfb2b562fea63b77ae152b99dc43ffc08 Mon Sep 17 00:00:00 2001 From: aileenfun Date: Tue, 18 Mar 2025 00:53:03 +0800 Subject: [PATCH 07/37] =?UTF-8?q?gemini=E5=8F=AF=E4=BB=A5=E5=8D=95?= =?UTF-8?q?=E7=8B=AC=E8=B5=B0=E4=BB=A3=E7=90=86=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E5=85=A8=E5=B1=80=E4=BB=A3=E7=90=86=E3=80=82=20=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E4=BB=A3=E7=90=86=E5=AE=B9=E6=98=93=E8=AE=A9=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E4=B8=AD=E5=85=B6=E4=BB=96=E6=A8=A1=E5=9D=97=E4=B9=9F?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E4=BB=A3=E7=90=86=EF=BC=8C=E9=80=A0=E6=88=90?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E8=AE=BF=E9=97=AE=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E3=80=82=20=E6=B5=8B=E8=AF=95=E6=9C=80=E6=96=B0=E7=9A=84=20gem?= =?UTF-8?q?ini=20flash=202.0=E5=8F=AF=E4=BB=A5=E4=BD=BF=E7=94=A8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 4 +- .../core/providers/llm/gemini/gemini.py | 81 +++++++++++++------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 21f37c5b..cc2545e3 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -193,7 +193,9 @@ LLM: # token申请地址： https://aistudio.google.com/apikey # 若部署地无法访问接口，需要开启科学上网 api_key: 你的gemini web key - model_name: "gemini-1.5-pro" # gemini-1.5-pro 是免费的 + model_name: "gemini-2.0-flash" + http_proxy: "" #"http://127.0.0.1:10808" + https_proxy: "" #http://127.0.0.1:10808" CozeLLM: # 定义LLM API类型 type: coze diff --git a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py index f753ac3e..e559bd8e 100644 --- a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py +++ b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py @@ -1,14 +1,19 @@ import google.generativeai as genai from core.utils.util import check_model_key from core.providers.llm.base import LLMProviderBase - +from config.logger import setup_logging +import requests +import json +TAG = __name__ +logger = setup_logging() class LLMProvider(LLMProviderBase): def __init__(self, config): """初始化Gemini LLM Provider""" self.model_name = config.get("model_name", "gemini-1.5-pro") self.api_key = config.get("api_key") - + self.http_proxy=config.get("http_proxy") + self.https_proxy = config.get("https_proxy") have_key = check_model_key("LLM", self.api_key) if not have_key: @@ -16,6 +21,19 @@ class LLMProvider(LLMProviderBase): try: # 初始化Gemini客户端 + # 配置代理（如果提供了代理配置） + self.proxies=None + if self.http_proxy is not "" or self.https_proxy is not "": + + self.proxies = { + "http": self.http_proxy, + "https": self.https_proxy, + } + logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}") + # 使用猴子补丁修改 google-generativeai 库的请求会话 + + # 使用 session 对象配置 genai + genai.configure(api_key=self.api_key) self.model = genai.GenerativeModel(self.model_name) @@ -46,35 +64,48 @@ class LLMProvider(LLMProviderBase): if content: chat_history.append({ "role": role, - "parts": [content] + "parts": [{"text":content}] + }) # 获取当前消息 current_msg = dialogue[-1]["content"] - # 创建新的聊天会话 - chat = self.model.start_chat(history=chat_history) + # 构建请求体 + request_body = { + "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}], + "generationConfig": self.generation_config + } - # 发送消息并获取流式响应 - response = chat.send_message( - current_msg, - stream=True, - generation_config=self.generation_config - ) + # 构建请求URL + url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}" - # 处理流式响应 - for chunk in response: - if hasattr(chunk, 'text') and chunk.text: - yield chunk.text + # 构建请求头 + headers = { + "Content-Type": "application/json", + } - except Exception as e: - error_msg = str(e) - logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}") - - # 针对不同错误返回友好提示 - if "Rate limit" in error_msg: - yield "【Gemini服务请求太频繁,请稍后再试】" - elif "Invalid API key" in error_msg: - yield "【Gemini API key无效】" + # 发送POST请求,经测试手动 request 无法使用 stream 模式，但是文本消息其实也很快，stream 与否也无所谓 + if self.proxies: + response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies) else: - yield f"【Gemini服务响应异常: {error_msg}】" + response = requests.post(url, headers=headers, json=request_body, stream=False) + response.raise_for_status() + try: + data = response.json() # 直接解析JSON + if 'candidates' in data and data['candidates']: + yield data['candidates'][0]['content']['parts'][0]['text'] + else: + yield "未找到候选回复。" + except json.JSONDecodeError as e: + yield f"JSON解码错误：{e}" + except Exception as e: + yield f"发生错误：{e}" + + + except requests.exceptions.RequestException as e: + yield f"请求失败：{e}" + except json.JSONDecodeError as e: + yield f"JSON解码错误：{e}" + except Exception as e: + yield f"发生错误：{e}" From 3d97fab16d94a3374eac3c64b0baa888f9102b5a Mon Sep 17 00:00:00 2001 From: HaoDuoYu <63943547+diaoling6665@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:05:33 +0800 Subject: [PATCH 08/37] Add files via upload --- .../core/providers/llm/openai/openai.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/main/xiaozhi-server/core/providers/llm/openai/openai.py b/main/xiaozhi-server/core/providers/llm/openai/openai.py index 95cd85f9..2feb4847 100644 --- a/main/xiaozhi-server/core/providers/llm/openai/openai.py +++ b/main/xiaozhi-server/core/providers/llm/openai/openai.py @@ -7,19 +7,27 @@ class LLMProvider(LLMProviderBase): def __init__(self, config): self.model_name = config.get("model_name") self.api_key = config.get("api_key") - if 'base_url' in config: - self.base_url = config.get("base_url") - else: - self.base_url = config.get("url") + self.url = config.get("url") + self.top_p = config.get("top_p") + self.top_k = config.get("top_k") + self.temperature = config.get("temperature") + self.max_tokens = config.get("max_tokens") + self.frequency_penalty = config.get("frequency_penalty") + check_model_key("LLM", self.api_key) - self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url) + self.client = openai.OpenAI(api_key=self.api_key, base_url=self.url) def response(self, session_id, dialogue): try: responses = self.client.chat.completions.create( model=self.model_name, messages=dialogue, - stream=True + stream=True, + temperature=self.temperature, + max_tokens=self.max_tokens, + top_p=self.top_p, + top_k=self.top_k, + frequency_penalty=self.frequency_penalty ) is_active = True @@ -51,6 +59,11 @@ class LLMProvider(LLMProviderBase): messages=dialogue, stream=True, tools=functions, + temperature=self.temperature, + max_tokens=self.max_tokens, + top_p=self.top_p, + top_k=self.top_k, + frequency_penalty=self.frequency_penalty ) for chunk in stream: From f5d9b478d57c4c4b07e428e052a3bc2d40ffb0cd Mon Sep 17 00:00:00 2001 From: HaoDuoYu <63943547+diaoling6665@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:07:53 +0800 Subject: [PATCH 09/37] Update config.yaml --- main/xiaozhi-server/config.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 21f37c5b..fcae7485 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -141,6 +141,8 @@ VAD: min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些 LLM: + #所有openai类型均可以修改超参，以AliLLM为例 + # 当前支持的type为openai、dify、ollama，可自行适配 AliLLM: # 定义LLM API类型 @@ -149,6 +151,11 @@ LLM: base_url: https://dashscope.aliyuncs.com/compatible-mode/v1 model_name: qwen-turbo api_key: 你的deepseek web key + temperature: 0.7 # 温度值 + max_tokens: 500 # 最大生成token数 + top_p: 1 + top_k: 50 + frequency_penalty: 0 # 频率惩罚 DoubaoLLM: # 定义LLM API类型 type: openai @@ -469,4 +476,4 @@ manager: enabled: false ip: 0.0.0.0 port: 8002 -use_private_config: false \ No newline at end of file +use_private_config: false From 878809ecb4c00545eb8112ec295ddaee1a2f6fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AC=A3=E5=8D=97=E7=A7=91=E6=8A=80?= Date: Tue, 18 Mar 2025 13:25:34 +0800 Subject: [PATCH 10/37] =?UTF-8?q?update:=E4=BF=AE=E5=A4=8Diot=E7=9A=84bug?= =?UTF-8?q?=20(#407)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 修复manual模式无法识别 (#404) * feat(docs): 新增Issues模板 * fix: 补全core依赖 * fix: 修复manual模式无法识别 * 去除重复依赖.txt 已经有torch和torchaudio --------- Co-authored-by: 欣南科技 * 修复iot功能中表达式问题 (#400) * Custom paths asr tts (#388) * #164 自定义asr、tts缓存目录,项目启动自动创建目录 * #164 自定义asr、tts缓存目录,项目启动自动创建目录 * fix:修复语音无法找到新配置项output_file的bug * fix：电脑不支持iot音量控制bug --------- Co-authored-by: Junsen <66542771+Huang-junsen@users.noreply.github.com> Co-authored-by: tang Co-authored-by: shudongW <178200623@qq.com> Co-authored-by: hrz <1710360675@qq.com> --- main/xiaozhi-server/config.yaml | 32 ++++++------ main/xiaozhi-server/config/settings.py | 49 +++++++++++++++++-- main/xiaozhi-server/core/handle/iotHandle.py | 8 +-- main/xiaozhi-server/core/handle/textHandle.py | 6 ++- .../xiaozhi-server/core/providers/tts/base.py | 2 +- .../core/providers/tts/custom.py | 2 +- .../core/providers/tts/openai.py | 2 +- .../core/providers/tts/ttson.py | 2 +- main/xiaozhi-server/docker-compose.yml | 1 + .../functions/raise_and_lower_the_volume.py | 2 + main/xiaozhi-server/requirements.txt | 2 +- 11 files changed, 78 insertions(+), 30 deletions(-) mode change 100755 => 100644 main/xiaozhi-server/docker-compose.yml diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 689b93fb..f3ca5706 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -114,10 +114,10 @@ Memory: # 每月有1000次免费调用 api_key: 你的mem0ai api key nomem: - # 不想使用记忆功能，可以使用nomem + # 不想使用记忆功能，可以使用nomem type: nomem mem_local_short: - # 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器 + # 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器 type: mem_local_short ASR: @@ -225,7 +225,7 @@ TTS: # 定义TTS API类型 type: edge voice: zh-CN-XiaoxiaoNeural - output_file: tmp/ + output_dir: tmp/ DoubaoTTS: # 定义TTS API类型 type: doubao @@ -235,7 +235,7 @@ TTS: # 地址：https://console.volcengine.com/speech/service/8 api_url: https://openspeech.bytedance.com/api/v1/tts voice: BV001_streaming - output_file: tmp/ + output_dir: tmp/ authorization: "Bearer;" appid: 你的火山引擎语音合成服务appid access_token: 你的火山引擎语音合成服务access_token @@ -246,7 +246,7 @@ TTS: # token申请地址 https://cloud.siliconflow.cn/account/ak model: FunAudioLLM/CosyVoice2-0.5B voice: FunAudioLLM/CosyVoice2-0.5B:alex - output_file: tmp/ + output_dir: tmp/ access_token: 你的硅基流动API密钥 response_format: wav CozeCnTTS: @@ -254,7 +254,7 @@ TTS: # COZECN TTS # token申请地址 https://www.coze.cn/open/oauth/pats voice: 7426720361733046281 - output_file: tmp/ + output_dir: tmp/ access_token: 你的coze web key response_format: wav FishSpeech: @@ -267,7 +267,7 @@ TTS: #--decoder-config-name firefly_gan_vq #--compile type: fishspeech - output_file: tmp/ + output_dir: tmp/ response_format: wav reference_id: null reference_audio: ["/tmp/test.wav",] @@ -291,7 +291,7 @@ TTS: #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml type: gpt_sovits_v2 url: "http://127.0.0.1:9880/tts" - output_file: tmp/ + output_dir: tmp/ text_lang: "auto" ref_audio_path: "caixukun.wav" prompt_text: "" @@ -316,7 +316,7 @@ TTS: #python api.py type: gpt_sovits_v3 url: "http://127.0.0.1:9880" - output_file: tmp/ + output_dir: tmp/ text_language: "auto" refer_wav_path: "caixukun.wav" prompt_language: "zh" @@ -337,7 +337,7 @@ TTS: # api_key地址：https://platform.minimaxi.com/user-center/basic-information/interface-key # 定义TTS API类型 type: minimax - output_file: tmp/ + output_dir: tmp/ group_id: 你的minimax平台groupID api_key: 你的minimax平台接口密钥 model: "speech-01-turbo" @@ -374,7 +374,7 @@ TTS: # token地址：https://nls-portal.console.aliyun.com/overview # 定义TTS API类型 type: aliyun - output_file: tmp/ + output_dir: tmp/ appkey: 你的阿里云智能语音交互服务项目Appkey token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret voice: xiaoyun @@ -397,7 +397,7 @@ TTS: api_url: https://api.302ai.cn/doubao/tts_hd authorization: "Bearer " voice: "zh_female_wanwanxiaohe_moon_bigtts" - output_file: tmp/ + output_dir: tmp/ access_token: "你的302API密钥" ACGNTTS: #在线网址：https://acgn.ttson.cn/ @@ -414,7 +414,7 @@ TTS: to_lang: ZH url: https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token= format: mp3 - output_file: tmp/ + output_dir: tmp/ emotion: 1 OpenAITTS: # openai官方文本转语音服务，可支持全球大多数语种 @@ -428,7 +428,7 @@ TTS: voice: onyx # 语速范围0.25-4.0 speed: 1 - output_file: tmp/ + output_dir: tmp/ CustomTTS: # 自定义的TTS接口服务，请求参数可自定义 # 要求接口使用GET方式请求，并返回音频文件 @@ -443,7 +443,7 @@ TTS: headers: # 自定义请求头 # Authorization: Bearer xxxx format: wav # 接口返回的音频格式 - output_file: tmp/ + output_dir: tmp/ # 模块测试配置 module_test: test_sentences: # 自定义测试语句 @@ -466,4 +466,4 @@ manager: enabled: false ip: 0.0.0.0 port: 8002 -use_private_config: false \ No newline at end of file +use_private_config: false diff --git a/main/xiaozhi-server/config/settings.py b/main/xiaozhi-server/config/settings.py index 0582bf76..bb81cfe9 100644 --- a/main/xiaozhi-server/config/settings.py +++ b/main/xiaozhi-server/config/settings.py @@ -7,9 +7,48 @@ from core.utils.util import read_config, get_project_dir default_config_file = "config.yaml" +def ensure_directories(config): + """确保所有配置路径存在""" + dirs_to_create = set() + project_dir = get_project_dir() # 获取项目根目录 + # 日志文件目录 + log_dir = config.get('log', {}).get('log_dir', 'tmp') + dirs_to_create.add(os.path.join(project_dir, log_dir)) + + # ASR/TTS模块输出目录 + for module in ['ASR', 'TTS']: + for provider in config.get(module, {}).values(): + output_dir = provider.get('output_dir', '') + if output_dir: + dirs_to_create.add(output_dir) + + # 根据selected_module创建模型目录 + selected_modules = config.get('selected_module', {}) + for module_type in ['ASR', 'LLM', 'TTS']: + selected_provider = selected_modules.get(module_type) + if not selected_provider: + continue + provider_config = config.get(module_type, {}).get(selected_provider, {}) + output_dir = provider_config.get('output_dir') + if output_dir: + full_model_dir = os.path.join(project_dir, output_dir) + dirs_to_create.add(full_model_dir) + + # 统一创建目录（保留原data目录创建） + for dir_path in dirs_to_create: + try: + os.makedirs(dir_path, exist_ok=True) + except PermissionError: + print(f"警告：无法创建目录 {dir_path}，请检查写入权限") + + def get_config_file(): global default_config_file - # 判断是否存在私有的配置文件 + """获取配置文件路径，优先使用私有配置文件（若存在）。 + + Returns: + str: 配置文件路径（相对路径或默认路径） + """ config_file = default_config_file if os.path.exists(get_project_dir() + "data/." + default_config_file): config_file = "data/." + default_config_file @@ -20,9 +59,13 @@ def load_config(): """加载配置文件""" parser = argparse.ArgumentParser(description="Server configuration") config_file = get_config_file() + parser.add_argument("--config_path", type=str, default=config_file) args = parser.parse_args() - return read_config(args.config_path) + config = read_config(args.config_path) + # 初始化目录 + ensure_directories(config) + return config def update_config(config): @@ -67,7 +110,7 @@ def find_missing_keys(new_config, old_config, parent_key=''): def check_config_file(): old_config_file = get_config_file() global default_config_file - if not old_config_file.startswith('data'): + if not 'data' in old_config_file: return old_config = read_config(get_project_dir() + old_config_file) new_config = read_config(get_project_dir() + default_config_file) diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index 9def78cd..1cb875cb 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -197,12 +197,12 @@ def register_device_type(descriptor): # 为每个属性创建查询函数 for prop_name, prop_info in descriptor["properties"].items(): - func_name = f"get_{device_name.lower()}_{prop_name.lower()}" + func_name = f"get_{{device_name.lower()}}_{{prop_name.lower()}}" func_desc = { "type": "function", "function": { "name": func_name, - "description": f"查询{descriptor['description']}的{prop_info['description']}", + "description": f"查询{{descriptor['description']}}的{{prop_info['description']}}", "parameters": { "type": "object", "properties": { @@ -212,7 +212,7 @@ def register_device_type(descriptor): }, "response_failure": { "type": "string", - "description": "查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'" + "description": f"查询失败时的友好回复，例如：'无法获取{{device_name}}的{{prop_info['description']}}'" } }, "required": ["response_success", "response_failure"] @@ -256,7 +256,7 @@ def register_device_type(descriptor): "type": "function", "function": { "name": func_name, - "description": f"{descriptor['description']} - {method_info['description']}", + "description": f"{{descriptor['description']}} - {{method_info['description']}}", "parameters": { "type": "object", "properties": parameters, diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index cc28dbb4..514de127 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -2,7 +2,7 @@ from config.logger import setup_logging import json from core.handle.abortHandle import handleAbortMessage from core.handle.helloHandle import handleHelloMessage -from core.handle.receiveAudioHandle import startToChat +from core.handle.receiveAudioHandle import startToChat, handleAudioMessage from core.handle.iotHandle import handleIotDescriptors, handleIotStatus TAG = __name__ @@ -24,13 +24,15 @@ async def handleTextMessage(conn, message): elif msg_json["type"] == "listen": if "mode" in msg_json: conn.client_listen_mode = msg_json["mode"] - logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}") + logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn. client_listen_mode}") if msg_json["state"] == "start": conn.client_have_voice = True conn.client_voice_stop = False elif msg_json["state"] == "stop": conn.client_have_voice = True conn.client_voice_stop = True + if len(conn.asr_audio) > 0: + await handleAudioMessage(conn, b'') elif msg_json["state"] == "detect": conn.asr_server_receive = False conn.client_have_voice = False diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index e6f013ca..d4a059f8 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -13,7 +13,7 @@ logger = setup_logging() class TTSProviderBase(ABC): def __init__(self, config, delete_audio_file): self.delete_audio_file = delete_audio_file - self.output_file = config.get("output_file") + self.output_file = config.get("output_dir") @abstractmethod def generate_filename(self): diff --git a/main/xiaozhi-server/core/providers/tts/custom.py b/main/xiaozhi-server/core/providers/tts/custom.py index d5447878..3417790f 100644 --- a/main/xiaozhi-server/core/providers/tts/custom.py +++ b/main/xiaozhi-server/core/providers/tts/custom.py @@ -15,7 +15,7 @@ class TTSProvider(TTSProviderBase): self.headers = config.get("headers", {}) self.params = config.get("params") self.format = config.get("format", "wav") - self.output_file = config.get("output_file", "tmp/") + self.output_file = config.get("output_dir", "tmp/") def generate_filename(self): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}") diff --git a/main/xiaozhi-server/core/providers/tts/openai.py b/main/xiaozhi-server/core/providers/tts/openai.py index dbdf311f..1849c128 100644 --- a/main/xiaozhi-server/core/providers/tts/openai.py +++ b/main/xiaozhi-server/core/providers/tts/openai.py @@ -14,7 +14,7 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("voice", "alloy") self.response_format = "wav" self.speed = config.get("speed", 1.0) - self.output_file = config.get("output_file", "tmp/") + self.output_file = config.get("output_dir", "tmp/") check_model_key("TTS", self.api_key) def generate_filename(self, extension=".wav"): diff --git a/main/xiaozhi-server/core/providers/tts/ttson.py b/main/xiaozhi-server/core/providers/tts/ttson.py index e56fd02d..e9fee109 100644 --- a/main/xiaozhi-server/core/providers/tts/ttson.py +++ b/main/xiaozhi-server/core/providers/tts/ttson.py @@ -17,7 +17,7 @@ class TTSProvider(TTSProviderBase): self.volume_change_dB = config.get("volume_change_dB", 0) self.speed_factor = config.get("speed_factor", 1) self.stream = config.get("stream", False) - self.output_file = config.get("output_file") + self.output_file = config.get("output_dir") self.pitch_factor = config.get("pitch_factor", 0) self.format = config.get("format", "mp3") self.emotion = config.get("emotion", 1) diff --git a/main/xiaozhi-server/docker-compose.yml b/main/xiaozhi-server/docker-compose.yml old mode 100755 new mode 100644 index d74d77f2..c22e7545 --- a/main/xiaozhi-server/docker-compose.yml +++ b/main/xiaozhi-server/docker-compose.yml @@ -21,6 +21,7 @@ services: - ./data:/opt/xiaozhi-esp32-server/data # 模型文件挂接，很重要 - ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt + # #智控台还没开发好，还不能完全使用，会报很多错误，如果是非技术人员，请不要启用智控台服务 # xiaozhi-esp32-server-web: # image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest diff --git a/main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py b/main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py index 2a5aa965..c77cb78b 100644 --- a/main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py +++ b/main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py @@ -49,6 +49,8 @@ def raise_and_lower_the_volume(conn, action: str): async def _raise_and_lower_the_volume(conn, action): volume = await get_iot_status(conn, "Speaker", "volume") + if volume is None: + raise Exception("你的设备不支持音量控制") if action == 'raise': volume += 10 elif action == 'lower': diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index bcd6b4ee..81d0cdfd 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -19,4 +19,4 @@ loguru==0.7.3 requests==2.32.3 cozepy==0.12.0 mem0ai==0.1.62 -bs4==0.0.2 \ No newline at end of file +bs4==0.0.2 From bd556c33a0aa2d7d23e73e70633bf3ed9d78aa3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AC=A3=E5=8D=97=E7=A7=91=E6=8A=80?= Date: Tue, 18 Mar 2025 14:26:27 +0800 Subject: [PATCH 11/37] =?UTF-8?q?fix:=E6=89=BE=E4=B8=8D=E5=88=B0function?= =?UTF-8?q?=E5=8D=A1=E4=BD=8Fbug=20(#408)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: hrz <1710360675@qq.com> --- main/xiaozhi-server/core/connection.py | 53 ++++++++++++-------- main/xiaozhi-server/core/handle/iotHandle.py | 8 +-- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 2ea7ac06..718398c3 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -197,7 +197,7 @@ class ConnectionHandler: self.dialogue.put(Message(role="system", content=self.prompt)) self.func_handler = FunctionHandler(self.config) - + def change_system_prompt(self, prompt): self.prompt = prompt # 找到原来的role==system，替换原来的系统提示 @@ -303,7 +303,7 @@ class ConnectionHandler: self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)) return True - def chat_with_function_calling(self, query, tool_call = False): + def chat_with_function_calling(self, query, tool_call=False): self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}") """Chat with function calling for intent detection using streaming""" if self.isNeedAuth(): @@ -311,7 +311,7 @@ class ConnectionHandler: future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop) future.result() return True - + if not tool_call: self.dialogue.put(Message(role="user", content=query)) @@ -320,7 +320,7 @@ class ConnectionHandler: response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 - + try: start_time = time.time() @@ -328,7 +328,7 @@ class ConnectionHandler: future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop) memory_str = future.result() - #self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") + # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") # 使用支持functions的streaming接口 llm_responses = self.llm.response_with_functions( @@ -351,8 +351,8 @@ class ConnectionHandler: content_arguments = "" for response in llm_responses: content, tools_call = response - if content is not None and len(content)>0: - if len(response_message)<=0 and (content=="```" or "" in content): + if content is not None and len(content) > 0: + if len(response_message) <= 0 and (content == "```" or "" in content): tool_call_flag = True if tools_call is not None: @@ -366,7 +366,7 @@ class ConnectionHandler: if content is not None and len(content) > 0: if tool_call_flag: - content_arguments+=content + content_arguments += content else: response_message.append(content) @@ -422,16 +422,17 @@ class ConnectionHandler: else: function_arguments = json.loads(function_arguments) if not bHasError: - self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}") + self.logger.bind(tag=TAG).info( + f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}") function_call_data = { "name": function_name, "id": function_id, "arguments": function_arguments } result = self.func_handler.handle_llm_function_call(self, function_call_data) - self._handle_function_result(result, function_call_data, text_index+1) + self._handle_function_result(result, function_call_data, text_index + 1) - # 处理最后剩余的文本 + # 处理最后剩余的文本 full_text = "".join(response_message) remaining_text = full_text[processed_chars:] if remaining_text: @@ -443,7 +444,7 @@ class ConnectionHandler: self.tts_queue.put(future) # 存储对话内容 - if len(response_message)>0: + if len(response_message) > 0: self.dialogue.put(Message(role="assistant", content="".join(response_message))) self.llm_finish_task = True @@ -452,32 +453,40 @@ class ConnectionHandler: return True def _handle_function_result(self, result, function_call_data, text_index): - if result.action == Action.RESPONSE: # 直接回复前端 + if result.action == Action.RESPONSE: # 直接回复前端 text = result.response self.recode_first_last_text(text, text_index) future = self.executor.submit(self.speak_and_play, text, text_index) self.tts_queue.put(future) self.dialogue.put(Message(role="assistant", content=text)) - elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 - + elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 + text = result.result if text is not None and len(text) > 0: function_id = function_call_data["id"] function_name = function_call_data["name"] function_arguments = function_call_data["arguments"] self.dialogue.put(Message(role='assistant', - tool_calls=[{"id": function_id, - "function": {"arguments": function_arguments,"name": function_name}, - "type": 'function', - "index": 0}])) + tool_calls=[{"id": function_id, + "function": {"arguments": function_arguments, + "name": function_name}, + "type": 'function', + "index": 0}])) self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text)) self.chat_with_function_calling(text, tool_call=True) elif result.action == Action.NOTFOUND: - text = result.response + text = result.result + self.recode_first_last_text(text, text_index) + future = self.executor.submit(self.speak_and_play, text, text_index) + self.tts_queue.put(future) + self.dialogue.put(Message(role="assistant", content=text)) else: - text = result.response - + text = result.result + self.recode_first_last_text(text, text_index) + future = self.executor.submit(self.speak_and_play, text, text_index) + self.tts_queue.put(future) + self.dialogue.put(Message(role="assistant", content=text)) def _tts_priority_thread(self): while not self.stop_event.is_set(): diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index 1cb875cb..7de2ef57 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -197,12 +197,12 @@ def register_device_type(descriptor): # 为每个属性创建查询函数 for prop_name, prop_info in descriptor["properties"].items(): - func_name = f"get_{{device_name.lower()}}_{{prop_name.lower()}}" + func_name = f"get_{device_name.lower()}_{prop_name.lower()}" func_desc = { "type": "function", "function": { "name": func_name, - "description": f"查询{{descriptor['description']}}的{{prop_info['description']}}", + "description": f"查询{descriptor['description']}的{prop_info['description']}", "parameters": { "type": "object", "properties": { @@ -212,7 +212,7 @@ def register_device_type(descriptor): }, "response_failure": { "type": "string", - "description": f"查询失败时的友好回复，例如：'无法获取{{device_name}}的{{prop_info['description']}}'" + "description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'" } }, "required": ["response_success", "response_failure"] @@ -256,7 +256,7 @@ def register_device_type(descriptor): "type": "function", "function": { "name": func_name, - "description": f"{{descriptor['description']}} - {{method_info['description']}}", + "description": f"{descriptor['description']} - {method_info['description']}", "parameters": { "type": "object", "properties": parameters, From e58da80f8e097664c989a0fa44584d023fedb552 Mon Sep 17 00:00:00 2001 From: Xvsenfeng <1458612070@qq.com> Date: Tue, 18 Mar 2025 15:58:31 +0800 Subject: [PATCH 12/37] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E5=92=8C=E6=96=87=E6=9C=AC=E7=94=9F=E6=88=90=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 4 ++ .../core/providers/llm/dify/dify.py | 61 ++++++++++++++----- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 21f37c5b..89a6f7f9 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -186,6 +186,10 @@ LLM: # 如果使用DifyLLM，配置文件里prompt(提示词)是无效的，需要在dify控制台设置提示词 base_url: https://api.dify.cn/v1 api_key: 你的DifyLLM web key + # 使用的对话模式 可以选择工作流 workflows/run 对话模式 chat-messages 文本生成 completion-messages + # 使用workflows进行返回的时候输入参数为 query 返回参数的名字要设置为 answer + # 文本生成的默认输入参数也是query + mode: workflows/run GeminiLLM: type: gemini # 谷歌Gemini API，需要先在Google Cloud控制台创建API密钥并获取api_key diff --git a/main/xiaozhi-server/core/providers/llm/dify/dify.py b/main/xiaozhi-server/core/providers/llm/dify/dify.py index f3c62639..02ef5588 100644 --- a/main/xiaozhi-server/core/providers/llm/dify/dify.py +++ b/main/xiaozhi-server/core/providers/llm/dify/dify.py @@ -9,6 +9,7 @@ logger = setup_logging() class LLMProvider(LLMProviderBase): def __init__(self, config): self.api_key = config["api_key"] + self.mode = config.get("mode", "workflows/run") self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/') self.session_conversation_map = {} # 存储session_id和conversation_id的映射 @@ -19,27 +20,59 @@ class LLMProvider(LLMProviderBase): conversation_id = self.session_conversation_map.get(session_id) # 发起流式请求 - with requests.post( - f"{self.base_url}/chat-messages", - headers={"Authorization": f"Bearer {self.api_key}"}, - json={ + if self.mode == "chat-messages": + request_json = { "query": last_msg["content"], "response_mode": "streaming", "user": session_id, "inputs": {}, "conversation_id": conversation_id - }, + } + elif self.mode == "workflows/run": + request_json = { + "inputs": {"query": last_msg["content"]}, + "response_mode": "streaming", + "user": session_id + } + elif self.mode == "completion-messages": + request_json = { + "inputs": {"query": last_msg["content"]}, + "response_mode": "streaming", + "user": session_id + } + + with requests.post( + f"{self.base_url}/{self.mode}", + headers={"Authorization": f"Bearer {self.api_key}"}, + json=request_json, stream=True ) as r: - for line in r.iter_lines(): - if line.startswith(b'data: '): - event = json.loads(line[6:]) - # 如果没有找到conversation_id，则获取此次conversation_id - if not conversation_id: - conversation_id = event.get('conversation_id') - self.session_conversation_map[session_id] = conversation_id # 更新映射 - if event.get('answer'): - yield event['answer'] + if self.mode == "chat-messages": + for line in r.iter_lines(): + if line.startswith(b'data: '): + event = json.loads(line[6:]) + # 如果没有找到conversation_id，则获取此次conversation_id + if not conversation_id: + conversation_id = event.get('conversation_id') + self.session_conversation_map[session_id] = conversation_id # 更新映射 + if event.get('answer'): + yield event['answer'] + elif self.mode == "workflows/run": + for line in r.iter_lines(): + # logger.bind(tag=TAG).info(f"chat message response: {line}") + if line.startswith(b'data: '): + event = json.loads(line[6:]) + if event.get('event') == "workflow_finished": + if event['data']['status'] == "succeeded": + yield event['data']['outputs']['answer'] + else: + yield "【服务响应异常】" + elif self.mode == "completion-messages": + for line in r.iter_lines(): + if line.startswith(b'data: '): + event = json.loads(line[6:]) + if event.get('answer'): + yield event['answer'] except Exception as e: logger.bind(tag=TAG).error(f"Error in response generation: {e}") From c796af7d0775031a6cf2969d00aa66745e843306 Mon Sep 17 00:00:00 2001 From: Huang <32005838+openrz@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:42:17 +0800 Subject: [PATCH 13/37] Hot fix (#415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix:找不到function卡住bug * fix:客户端拾音模式日志bug --- main/xiaozhi-server/core/handle/textHandle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index 514de127..c63b9a33 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -24,7 +24,7 @@ async def handleTextMessage(conn, message): elif msg_json["type"] == "listen": if "mode" in msg_json: conn.client_listen_mode = msg_json["mode"] - logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn. client_listen_mode}") + logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}") if msg_json["state"] == "start": conn.client_have_voice = True conn.client_voice_stop = False @@ -43,6 +43,6 @@ async def handleTextMessage(conn, message): if "descriptors" in msg_json: await handleIotDescriptors(conn, msg_json["descriptors"]) if "states" in msg_json: - await handleIotStatus(conn, msg_json["states"]) + await handleIotStatus(conn, msg_json["states"]) except json.JSONDecodeError: await conn.websocket.send(message) From 34fa65132d8b13e651b668093faf6d210a74477a Mon Sep 17 00:00:00 2001 From: aileenfun Date: Tue, 18 Mar 2025 21:04:21 +0800 Subject: [PATCH 14/37] =?UTF-8?q?gemini=E7=8B=AC=E7=AB=8B=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E6=A8=A1=E5=BC=8F=20=E5=9C=A8=E4=BB=A3=E7=90=86?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=B8=8B=E6=89=8B=E5=8A=A8=E7=BB=84=E6=88=90?= =?UTF-8?q?request=EF=BC=8C=E6=97=A0=E8=AE=BA=20Steam=20=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E4=B8=BA=20True=EF=BC=8C=E8=BF=94=E5=9B=9E=E7=9A=84=E5=9D=87?= =?UTF-8?q?=E6=98=AF=E9=9D=9Esteam=E5=80=BC=E3=80=82=20=E4=B8=8D=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E4=BB=A3=E7=90=86=E6=97=B6=E8=87=AA=E5=8A=A8=E5=88=87?= =?UTF-8?q?=E6=8D=A2=E4=B8=BAgoogle=E7=9A=84=E5=AE=98=E6=96=B9api=EF=BC=8C?= =?UTF-8?q?=E8=B5=B0=20stream=E6=A8=A1=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 全局代理容易让系统中其他模块也使用代理，造成无法访问的问题。 测试最新的 gemini flash 2.0可以使用。 --- .../core/providers/llm/gemini/gemini.py | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py index e559bd8e..298bf71e 100644 --- a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py +++ b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py @@ -85,22 +85,48 @@ class LLMProvider(LLMProviderBase): "Content-Type": "application/json", } - # 发送POST请求,经测试手动 request 无法使用 stream 模式，但是文本消息其实也很快，stream 与否也无所谓 + # 发送POST请求,经测试手动 request 无法使用 stream 模式 if self.proxies: + logger.bind(tag=TAG).info(f"Gemini response mode ") response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies) + try: + data = response.json() # 直接解析JSON + if 'candidates' in data and data['candidates']: + yield data['candidates'][0]['content']['parts'][0]['text'] + else: + yield "未找到候选回复。" + except json.JSONDecodeError as e: + yield f"JSON解码错误：{e}" + except Exception as e: + yield f"发生错误：{e}" else: - response = requests.post(url, headers=headers, json=request_body, stream=False) - response.raise_for_status() - try: - data = response.json() # 直接解析JSON - if 'candidates' in data and data['candidates']: - yield data['candidates'][0]['content']['parts'][0]['text'] - else: - yield "未找到候选回复。" - except json.JSONDecodeError as e: - yield f"JSON解码错误：{e}" - except Exception as e: - yield f"发生错误：{e}" + logger.bind(tag=TAG).info(f"Gemini stream mode ") + chat = self.model.start_chat(history=chat_history) + + # 发送消息并获取流式响应 + response = chat.send_message( + current_msg, + stream=True, + generation_config=self.generation_config + ) + # 处理流式响应 + for chunk in response: + if hasattr(chunk, 'text') and chunk.text: + yield chunk.text + + except Exception as e: + error_msg = str(e) + logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}") + + # 针对不同错误返回友好提示 + if "Rate limit" in error_msg: + yield "【Gemini服务请求太频繁,请稍后再试】" + elif "Invalid API key" in error_msg: + yield "【Gemini API key无效】" + else: + yield f"【Gemini服务响应异常: {error_msg}】" + + except requests.exceptions.RequestException as e: From f3738a1630f0994c0a81c21a1b93894f8e6173f6 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Tue, 18 Mar 2025 21:49:57 +0800 Subject: [PATCH 15/37] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/AddDeviceDialog.vue | 39 ++-- .../src/components/AddWisdomBodyDialog.vue | 87 +++++++++ .../manager-web/src/components/DeviceItem.vue | 34 ++-- main/manager-web/src/components/HeaderBar.vue | 83 ++++---- main/manager-web/src/router/index.js | 9 + .../src/views/DeviceManagement.vue | 169 ++++++++++++++++ main/manager-web/src/views/home.vue | 65 ++++--- main/manager-web/src/views/roleConfig.vue | 180 +++++++++++++----- 8 files changed, 522 insertions(+), 144 deletions(-) create mode 100644 main/manager-web/src/components/AddWisdomBodyDialog.vue create mode 100644 main/manager-web/src/views/DeviceManagement.vue diff --git a/main/manager-web/src/components/AddDeviceDialog.vue b/main/manager-web/src/components/AddDeviceDialog.vue index a9eec57c..e7ba492e 100644 --- a/main/manager-web/src/components/AddDeviceDialog.vue +++ b/main/manager-web/src/components/AddDeviceDialog.vue @@ -1,21 +1,22 @@ @@ -46,28 +49,28 @@ export default { \ No newline at end of file diff --git a/main/manager-web/src/components/HeaderBar.vue b/main/manager-web/src/components/HeaderBar.vue index ac87a339..758959b3 100644 --- a/main/manager-web/src/components/HeaderBar.vue +++ b/main/manager-web/src/components/HeaderBar.vue @@ -1,29 +1,29 @@ diff --git a/main/manager-web/src/components/HeaderBar.vue b/main/manager-web/src/components/HeaderBar.vue index 758959b3..78b18970 100644 --- a/main/manager-web/src/components/HeaderBar.vue +++ b/main/manager-web/src/components/HeaderBar.vue @@ -19,13 +19,13 @@
- + + style="width: 14px;height: 14px;margin-right: 11px;cursor: pointer;" @click="handleSearch" />
@@ -33,17 +33,57 @@ diff --git a/main/manager-web/src/views/home.vue b/main/manager-web/src/views/home.vue index 4b9c6eb5..15a215b9 100644 --- a/main/manager-web/src/views/home.vue +++ b/main/manager-web/src/views/home.vue @@ -1,7 +1,7 @@ diff --git a/main/manager-web/src/views/home.vue b/main/manager-web/src/views/home.vue index 4b9c6eb5..0c914e77 100644 --- a/main/manager-web/src/views/home.vue +++ b/main/manager-web/src/views/home.vue @@ -1,7 +1,7 @@