mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-22 07:03:53 +08:00
Merge branch 'main' into web_api_user_test
This commit is contained in:
+3
-2
@@ -106,7 +106,6 @@ celerybeat.pid
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
@@ -150,10 +149,12 @@ main/manager-web/node_modules
|
||||
.config.yaml
|
||||
.secrets.yaml
|
||||
.private_config.yaml
|
||||
.env.development
|
||||
|
||||
# model files
|
||||
main/xiaozhi-server/models/SenseVoiceSmall/model.pt
|
||||
main/xiaozhi-server/models/sherpa-onnx*
|
||||
my_wakeup_words.mp3
|
||||
main/manager-api/.vscode
|
||||
|
||||
# Ignore webpack cache directory
|
||||
main/manager-web/.webpack_cache/
|
||||
+1
-3
@@ -18,6 +18,4 @@ xiaozhi-esp32-server
|
||||
|
||||
# manager-web 、manager-api接口协议
|
||||
|
||||
[manager前后端接口协议](https://app.apifox.com/invite/project?token=eXg2_tUv85q-gc3ZRowmn)
|
||||
|
||||
[前端页面设计图](https://codesign.qq.com/app/s/526108506410828)
|
||||
https://2662r3426b.vicp.fun/xiaozhi-esp32-api/api/v1/doc.html
|
||||
|
||||
@@ -5,8 +5,6 @@ manager-api 该项目基于SpringBoot框架开发。
|
||||
|
||||
开发使用代码编辑器,导入项目时,选择`manager-api`文件夹作为项目目录
|
||||
|
||||
参照[manager前后端接口协议](https://app.apifox.com/invite/project?token=H_8qhgfjUeaAL0wybghgU)开发
|
||||
|
||||
# 开发环境
|
||||
JDK 21
|
||||
Maven 3.8+
|
||||
|
||||
+3
@@ -19,6 +19,7 @@ import xiaozhi.common.page.TokenDTO;
|
||||
import xiaozhi.common.user.UserDetail;
|
||||
import xiaozhi.common.utils.Result;
|
||||
import xiaozhi.common.validator.AssertUtils;
|
||||
import xiaozhi.common.validator.ValidatorUtils;
|
||||
import xiaozhi.modules.security.dto.LoginDTO;
|
||||
import xiaozhi.modules.security.password.PasswordUtils;
|
||||
import xiaozhi.modules.security.service.CaptchaService;
|
||||
@@ -104,6 +105,8 @@ public class LoginController {
|
||||
@PutMapping("/change-password")
|
||||
@Operation(summary = "修改用户密码")
|
||||
public Result<?> changePassword(@RequestBody PasswordDTO passwordDTO) {
|
||||
// 判断非空
|
||||
ValidatorUtils.validateEntity(passwordDTO);
|
||||
Long userId = SecurityUser.getUserId();
|
||||
sysUserTokenService.changePassword(userId, passwordDTO);
|
||||
return new Result<>();
|
||||
|
||||
@@ -90,7 +90,6 @@ public class Oauth2Filter extends AuthenticatingFilter {
|
||||
String json = JsonUtils.toJsonString(r);
|
||||
httpResponse.getWriter().print(json);
|
||||
} catch (IOException e1) {
|
||||
logger.error("onLoginFailure:登录失败! msg:{}", e1.getMessage(), e1);
|
||||
}
|
||||
|
||||
return false;
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
<appender name="LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<!--日志文件输出的文件名-->
|
||||
<FileNamePattern>/system/logs/admin.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
|
||||
<FileNamePattern>/home/system/logs/admin.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
|
||||
<!--日志大小-->
|
||||
<maxFileSize>1MB</maxFileSize>
|
||||
<!--日志文件保留天数-->
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
VUE_APP_TITLE=小智-智控台
|
||||
@@ -0,0 +1 @@
|
||||
VUE_APP_API_BASE_URL=https://2662r3426b.vicp.fun
|
||||
@@ -0,0 +1 @@
|
||||
VUE_APP_API_BASE_URL=https://2662r3426b.vicp.fun
|
||||
@@ -0,0 +1 @@
|
||||
registry=https://registry.npmmirror.com/
|
||||
@@ -0,0 +1,13 @@
|
||||
module.exports = {
|
||||
presets: [
|
||||
['@vue/cli-plugin-babel/preset', {
|
||||
useBuiltIns: 'usage',
|
||||
corejs: 3
|
||||
}]
|
||||
],
|
||||
plugins: [
|
||||
'@babel/plugin-syntax-dynamic-import', // 确保支持动态导入 (Lazy Loading)
|
||||
'@babel/plugin-transform-runtime'
|
||||
]
|
||||
}
|
||||
|
||||
Generated
+1010
-1424
File diff suppressed because it is too large
Load Diff
@@ -4,9 +4,12 @@
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"serve": "vue-cli-service serve",
|
||||
"build": "vue-cli-service build"
|
||||
"build": "vue-cli-service build",
|
||||
"analyze": "cross-env ANALYZE=true vue-cli-service build"
|
||||
},
|
||||
"dependencies": {
|
||||
"core-js": "^3.41.0",
|
||||
"cross-env": "^7.0.3",
|
||||
"element-ui": "^2.15.14",
|
||||
"flyio": "^0.6.14",
|
||||
"normalize.css": "^8.0.1",
|
||||
@@ -19,16 +22,25 @@
|
||||
"xiaozhi": "file:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/plugin-syntax-dynamic-import": "^7.8.3",
|
||||
"@babel/plugin-transform-runtime": "^7.26.10",
|
||||
"@vue/cli-plugin-router": "~5.0.0",
|
||||
"@vue/cli-plugin-vuex": "~5.0.0",
|
||||
"@vue/cli-service": "~5.0.0",
|
||||
"compression-webpack-plugin": "^11.1.0",
|
||||
"sass": "^1.32.7",
|
||||
"sass-loader": "^12.0.0",
|
||||
"vue-template-compiler": "^2.6.14"
|
||||
"vue-template-compiler": "^2.6.14",
|
||||
"webpack-bundle-analyzer": "^4.10.2"
|
||||
},
|
||||
"browserslist": [
|
||||
"> 1%",
|
||||
"last 2 versions",
|
||||
"not dead"
|
||||
],
|
||||
"sideEffects": [
|
||||
"*.css",
|
||||
"*.scss",
|
||||
"*.vue"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,17 +1,33 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<link rel="icon" href="<%= BASE_URL %>favicon.ico">
|
||||
<title>小智-智控台</title>
|
||||
</head>
|
||||
<body>
|
||||
<noscript>
|
||||
<strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
|
||||
</noscript>
|
||||
<div id="app"></div>
|
||||
<!-- built files will be auto injected -->
|
||||
</body>
|
||||
</html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<link rel="icon" href="<%= BASE_URL %>favicon.ico">
|
||||
<title>
|
||||
<%= process.env.VUE_APP_TITLE %>
|
||||
</title>
|
||||
<% if (htmlWebpackPlugin.options.cdn) { %>
|
||||
<% for (var i in htmlWebpackPlugin.options.cdn.css) { %>
|
||||
<link rel="stylesheet" href="<%= htmlWebpackPlugin.options.cdn.css[i] %>">
|
||||
<% } %>
|
||||
<% } %>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<noscript>
|
||||
<strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
|
||||
Please enable it to continue.</strong>
|
||||
</noscript>
|
||||
<div id="app"></div>
|
||||
<!-- built files will be auto injected -->
|
||||
<% if (htmlWebpackPlugin.options.cdn) { %>
|
||||
<% for (var i in htmlWebpackPlugin.options.cdn.js) { %>
|
||||
<script src="<%= htmlWebpackPlugin.options.cdn.js[i] %>"></script>
|
||||
<% } %>
|
||||
<% } %>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -28,5 +28,5 @@ nav {
|
||||
}
|
||||
|
||||
</style>
|
||||
<script setup lang="ts">
|
||||
<script>
|
||||
</script>
|
||||
@@ -7,7 +7,7 @@ import admin from './module/admin.js'
|
||||
* 如果你想调用8002端口,就用'/xiaozhi-esp32-api/api/v1',请与vue.config.js的devServer配置相结合,方便跨域请求
|
||||
*
|
||||
*/
|
||||
const DEV_API_SERVICE = 'https://2662r3426b.vicp.fun/xiaozhi-esp32-api'
|
||||
// const DEV_API_SERVICE = 'https://2662r3426b.vicp.fun/xiaozhi-esp32-api'
|
||||
// 8002开发完成完成后使用这个
|
||||
// const DEV_API_SERVICE = '/xiaozhi-esp32-api'
|
||||
|
||||
@@ -16,7 +16,7 @@ const DEV_API_SERVICE = 'https://2662r3426b.vicp.fun/xiaozhi-esp32-api'
|
||||
* @returns {string}
|
||||
*/
|
||||
export function getServiceUrl() {
|
||||
return DEV_API_SERVICE
|
||||
return '/xiaozhi-esp32-api'
|
||||
}
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 5.8 MiB After Width: | Height: | Size: 1.2 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 MiB After Width: | Height: | Size: 357 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 3.8 MiB After Width: | Height: | Size: 1.9 MiB |
@@ -2,7 +2,7 @@
|
||||
<el-dialog :visible.sync="visible" width="400px" center>
|
||||
<div style="margin: 0 10px 10px;display: flex;align-items: center;gap: 10px;font-weight: 700;font-size: 20px;text-align: left;color: #3d4566;">
|
||||
<div style="width: 40px;height: 40px;border-radius: 50%;background: #5778ff;display: flex;align-items: center;justify-content: center;">
|
||||
<img src="@/assets/home/equipment.png" alt="" style="width: 18px;height: 15px;" />
|
||||
<img loading="lazy" src="@/assets/home/equipment.png" alt="" style="width: 18px;height: 15px;" />
|
||||
</div>
|
||||
添加智慧体
|
||||
</div>
|
||||
|
||||
@@ -2,12 +2,8 @@
|
||||
<form>
|
||||
<el-dialog :visible.sync="value" width="400px" center>
|
||||
<div
|
||||
style="margin: 0 10px 10px;display: flex;align-items: center;gap: 10px;font-weight: 700;font-size: 20px;text-align: left;color: #3d4566;">
|
||||
<div
|
||||
style="width: 40px;height: 40px;border-radius: 50%;background: #5778ff;display: flex;align-items: center;justify-content: center;">
|
||||
<img src="@/assets/login/shield.png" alt="" style="width: 19px;height: 23px; filter: brightness(0) invert(1);" />
|
||||
</div>
|
||||
修改密码
|
||||
style="width: 40px;height: 40px;border-radius: 50%;background: #5778ff;display: flex;align-items: center;justify-content: center;">
|
||||
<img loading="lazy" src="@/assets/login/shield.png" alt="" style="width: 19px;height: 23px; filter: brightness(0) invert(1);" />
|
||||
</div>
|
||||
<div style="height: 1px;background: #e8f0ff;"/>
|
||||
<div style="margin: 22px 15px;">
|
||||
|
||||
@@ -3,22 +3,24 @@
|
||||
<div class="header-container">
|
||||
<!-- 左侧元素 -->
|
||||
<div class="header-left">
|
||||
<img alt="" src="@/assets/xiaozhi-logo.png" class="logo-img"/>
|
||||
<img alt="" src="@/assets/xiaozhi-ai.png" class="brand-img"/>
|
||||
<img loading="lazy" alt="" src="@/assets/xiaozhi-logo.png" class="logo-img"/>
|
||||
<img loading="lazy" alt="" src="@/assets/xiaozhi-ai.png" class="brand-img"/>
|
||||
</div>
|
||||
|
||||
<!-- 中间导航菜单 -->
|
||||
<div class="header-center">
|
||||
<div class="equipment-management" :class="{ 'active-tab': $route.path === '/home' }" @click="goHome">
|
||||
<img alt="" src="@/assets/header/robot.png" :style="{ filter: $route.path === '/home' ? 'brightness(0) invert(1)' : 'None' }"/>
|
||||
|
||||
<img loading="lazy" alt="" src="@/assets/header/robot.png" :style="{ filter: $route.path === '/home' ? 'brightness(0) invert(1)' : 'None' }"/>
|
||||
|
||||
智能体管理
|
||||
</div>
|
||||
<div class="equipment-management" :class="{ 'active-tab': $route.path === '/user-management' }" @click="goUserManagement">
|
||||
<img alt="" src="@/assets/header/user_management.png" :style="{ filter: $route.path === '/user-management' ? 'brightness(0) invert(1)' : 'None' }"/>
|
||||
<img loading="lazy" alt="" src="@/assets/header/user_management.png" :style="{ filter: $route.path === '/user-management' ? 'brightness(0) invert(1)' : 'None' }"/>
|
||||
用户管理
|
||||
</div>
|
||||
<div class="equipment-management" :class="{ 'active-tab': $route.path === '/model-config' }" @click="goModelConfig">
|
||||
<img alt="" src="@/assets/header/model_config.png" :style="{ filter: $route.path === '/model-config' ? 'brightness(0) invert(1)' : 'None' }"/>
|
||||
<img loading="lazy" alt="" src="@/assets/header/model_config.png" :style="{ filter: $route.path === '/model-config' ? 'brightness(0) invert(1)' : 'None' }"/>
|
||||
模型配置
|
||||
</div>
|
||||
</div>
|
||||
@@ -35,7 +37,7 @@
|
||||
<i slot="suffix" class="el-icon-search search-icon" @click="handleSearch"></i>
|
||||
</el-input>
|
||||
</div>
|
||||
<img alt="" src="@/assets/home/avatar.png" class="avatar-img"/>
|
||||
<img loading="lazy" alt="" src="@/assets/home/avatar.png" class="avatar-img"/>
|
||||
<el-dropdown trigger="click" class="user-dropdown">
|
||||
<span class="el-dropdown-link">
|
||||
{{ userInfo.username || '加载中...' }}<i class="el-icon-arrow-down el-icon--right"></i>
|
||||
|
||||
@@ -5,6 +5,7 @@ import store from './store'
|
||||
import 'normalize.css/normalize.css' // A modern alternative to CSS resets
|
||||
import ElementUI from 'element-ui';
|
||||
import 'element-ui/lib/theme-chalk/index.css';
|
||||
import './styles/global.scss'
|
||||
|
||||
Vue.use(ElementUI);
|
||||
Vue.config.productionTip = false
|
||||
|
||||
@@ -76,4 +76,22 @@ const router = new VueRouter({
|
||||
routes
|
||||
})
|
||||
|
||||
// 需要登录才能访问的路由
|
||||
const protectedRoutes = ['home', 'RoleConfig', 'DeviceManagement', 'UserManagement', 'ModelConfig']
|
||||
|
||||
// 路由守卫
|
||||
router.beforeEach((to, from, next) => {
|
||||
// 检查是否是需要保护的路由
|
||||
if (protectedRoutes.includes(to.name)) {
|
||||
// 从localStorage获取token
|
||||
const token = localStorage.getItem('token')
|
||||
if (!token) {
|
||||
// 未登录,跳转到登录页
|
||||
next({ name: 'login', query: { redirect: to.fullPath } })
|
||||
return
|
||||
}
|
||||
}
|
||||
next()
|
||||
})
|
||||
|
||||
export default router
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
// 覆盖 autofill 样式
|
||||
input:-webkit-autofill,
|
||||
input:-webkit-autofill:hover,
|
||||
input:-webkit-autofill:focus,
|
||||
textarea:-webkit-autofill,
|
||||
textarea:-webkit-autofill:hover,
|
||||
textarea:-webkit-autofill:focus,
|
||||
select:-webkit-autofill,
|
||||
select:-webkit-autofill:hover,
|
||||
select:-webkit-autofill:focus {
|
||||
-webkit-box-shadow: 0 0 0px 1000px transparent inset;
|
||||
transition: background-color 5000s ease-in-out 0s;
|
||||
background-color: transparent;
|
||||
}
|
||||
@@ -9,11 +9,11 @@
|
||||
语音设置
|
||||
</el-button>
|
||||
<el-button plain size="small" @click="handleImport" style="background: #7b9de5; color: white;">
|
||||
<img alt="" src="@/assets/model/inner_conf.png">
|
||||
<img loading="lazy" alt="" src="@/assets/model/inner_conf.png">
|
||||
导入配置
|
||||
</el-button>
|
||||
<el-button plain size="small" @click="handleExport" style="background: #71c9d1; color: white;">
|
||||
<img alt="" src="@/assets/model/output_conf.png">
|
||||
<img loading="lazy" alt="" src="@/assets/model/output_conf.png">
|
||||
导出配置
|
||||
</el-button>
|
||||
</div>
|
||||
|
||||
@@ -4,15 +4,15 @@
|
||||
<el-header>
|
||||
<div
|
||||
style="display: flex;align-items: center;margin-top: 15px;margin-left: 10px;gap: 10px;">
|
||||
<img alt="" src="@/assets/xiaozhi-logo.png" style="width: 45px;height: 45px;"/>
|
||||
<img alt="" src="@/assets/xiaozhi-ai.png" style="width: 70px;height: 13px;"/>
|
||||
<img loading="lazy" alt="" src="@/assets/xiaozhi-logo.png" style="width: 45px;height: 45px;"/>
|
||||
<img loading="lazy" alt="" src="@/assets/xiaozhi-ai.png" style="width: 70px;height: 13px;"/>
|
||||
</div>
|
||||
</el-header>
|
||||
<el-main style="position: relative;">
|
||||
<div class="login-box" @keyup.enter="login">
|
||||
<div
|
||||
style="display: flex;align-items: center;gap: 20px;margin-bottom: 39px;padding: 0 30px;">
|
||||
<img alt="" src="@/assets/login/hi.png" style="width: 34px;height: 34px;"/>
|
||||
<img loading="lazy" alt="" src="@/assets/login/hi.png" style="width: 34px;height: 34px;"/>
|
||||
<div class="login-text">登录</div>
|
||||
<div class="login-welcome">
|
||||
WELCOME TO LOGIN
|
||||
@@ -20,19 +20,19 @@
|
||||
</div>
|
||||
<div style="padding: 0 30px;">
|
||||
<div class="input-box">
|
||||
<img alt="" class="input-icon" src="@/assets/login/username.png"/>
|
||||
<img loading="lazy" alt="" class="input-icon" src="@/assets/login/username.png"/>
|
||||
<el-input v-model="form.username" placeholder="请输入用户名"/>
|
||||
</div>
|
||||
<div class="input-box">
|
||||
<img alt="" class="input-icon" src="@/assets/login/password.png"/>
|
||||
<img loading="lazy" alt="" class="input-icon" src="@/assets/login/password.png"/>
|
||||
<el-input v-model="form.password" placeholder="请输入密码" type="password"/>
|
||||
</div>
|
||||
<div style="display: flex; align-items: center; margin-top: 20px; width: 100%; gap: 10px;">
|
||||
<div class="input-box" style="width: calc(100% - 130px); margin-top: 0;">
|
||||
<img alt="" class="input-icon" src="@/assets/login/shield.png"/>
|
||||
<img loading="lazy" alt="" class="input-icon" src="@/assets/login/shield.png"/>
|
||||
<el-input v-model="form.captcha" placeholder="请输入验证码" style="flex: 1;"/>
|
||||
</div>
|
||||
<img v-if="captchaUrl"
|
||||
<img loading="lazy" v-if="captchaUrl"
|
||||
:src="captchaUrl"
|
||||
alt="验证码"
|
||||
style="width: 150px; height: 40px; cursor: pointer;"
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
<!-- 保持相同的头部 -->
|
||||
<el-header>
|
||||
<div style="display: flex;align-items: center;margin-top: 15px;margin-left: 10px;gap: 10px;">
|
||||
<img alt="" src="@/assets/xiaozhi-logo.png" style="width: 45px;height: 45px;"/>
|
||||
<img alt="" src="@/assets/xiaozhi-ai.png" style="width: 70px;height: 13px;"/>
|
||||
<img loading="lazy" alt="" src="@/assets/xiaozhi-logo.png" style="width: 45px;height: 45px;"/>
|
||||
<img loading="lazy" alt="" src="@/assets/xiaozhi-ai.png" style="width: 70px;height: 13px;"/>
|
||||
</div>
|
||||
</el-header>
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<div class="login-box">
|
||||
<!-- 修改标题部分 -->
|
||||
<div style="display: flex;align-items: center;gap: 20px;margin-bottom: 39px;padding: 0 30px;">
|
||||
<img alt="" src="@/assets/login/hi.png" style="width: 34px;height: 34px;"/>
|
||||
<img loading="lazy" alt="" src="@/assets/login/hi.png" style="width: 34px;height: 34px;"/>
|
||||
<div class="login-text">注册</div>
|
||||
<div class="login-welcome">
|
||||
WELCOME TO REGISTER
|
||||
@@ -23,29 +23,29 @@
|
||||
<div style="padding: 0 30px;">
|
||||
<!-- 用户名输入框 -->
|
||||
<div class="input-box">
|
||||
<img alt="" class="input-icon" src="@/assets/login/username.png"/>
|
||||
<img loading="lazy" alt="" class="input-icon" src="@/assets/login/username.png"/>
|
||||
<el-input v-model="form.username" placeholder="请输入用户名"/>
|
||||
</div>
|
||||
|
||||
<!-- 密码输入框 -->
|
||||
<div class="input-box">
|
||||
<img alt="" class="input-icon" src="@/assets/login/password.png"/>
|
||||
<img loading="lazy" alt="" class="input-icon" src="@/assets/login/password.png"/>
|
||||
<el-input v-model="form.password" placeholder="请输入密码" type="password"/>
|
||||
</div>
|
||||
|
||||
<!-- 新增确认密码 -->
|
||||
<div class="input-box">
|
||||
<img alt="" class="input-icon" src="@/assets/login/password.png"/>
|
||||
<img loading="lazy" alt="" class="input-icon" src="@/assets/login/password.png"/>
|
||||
<el-input v-model="form.confirmPassword" placeholder="请确认密码" type="password"/>
|
||||
</div>
|
||||
|
||||
<!-- 验证码部分保持相同 -->
|
||||
<div style="display: flex; align-items: center; margin-top: 20px; width: 100%; gap: 10px;">
|
||||
<div class="input-box" style="width: calc(100% - 130px); margin-top: 0;">
|
||||
<img alt="" class="input-icon" src="@/assets/login/shield.png"/>
|
||||
<img loading="lazy" alt="" class="input-icon" src="@/assets/login/shield.png"/>
|
||||
<el-input v-model="form.captcha" placeholder="请输入验证码" style="flex: 1;"/>
|
||||
</div>
|
||||
<img v-if="captchaUrl"
|
||||
<img loading="lazy" v-if="captchaUrl"
|
||||
:src="captchaUrl"
|
||||
alt="验证码"
|
||||
style="width: 150px; height: 40px; cursor: pointer;"
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
style="padding: 15px 24px;font-weight: 700;font-size: 19px;text-align: left;color: #3d4566;display: flex;gap: 13px;align-items: center;">
|
||||
<div
|
||||
style="width: 37px;height: 37px;background: #5778ff;border-radius: 50%;display: flex;align-items: center;justify-content: center;">
|
||||
<img src="@/assets/home/setting-user.png" alt="" style="width: 19px;height: 19px;"/>
|
||||
<img loading="lazy" src="@/assets/home/setting-user.png" alt="" style="width: 19px;height: 19px;"/>
|
||||
</div>
|
||||
{{ form.agentName }}
|
||||
</div>
|
||||
@@ -83,7 +83,7 @@
|
||||
重制
|
||||
</div>
|
||||
<div class="clear-text">
|
||||
<img src="@/assets/home/red-info.png" alt="" style="width: 19px;height: 19px;"/>
|
||||
<img loading="lazy" src="@/assets/home/red-info.png" alt="" style="width: 19px;height: 19px;"/>
|
||||
保存配置后,需要重启设备,新的配置才会生效。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+121
-132
@@ -18,70 +18,68 @@
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import { ref, onMounted, onUnmounted, nextTick } from 'vue';
|
||||
import Recorder from 'opus-recorder';
|
||||
import { OpusDecoder } from 'opus-decoder';
|
||||
|
||||
export default {
|
||||
name: 'TestPage',
|
||||
setup() {
|
||||
const messages = ref([]);
|
||||
const chatContainer = ref(null);
|
||||
const wsStatus = ref('disconnected');
|
||||
const isRecording = ref(false);
|
||||
let ws = null;
|
||||
let recorder = null;
|
||||
let stream = null;
|
||||
let audioContext = null;
|
||||
let sourceNode = null;
|
||||
let audioDecoder = null;
|
||||
let audioBufferQueue = []; // 当前句子的音频缓冲区
|
||||
let playbackQueue = []; // 播放队列,存储待播放的句子
|
||||
let isPlaying = false;
|
||||
|
||||
const connectWebSocket = () => {
|
||||
ws = new WebSocket('ws://192.168.3.97:8000');//修改服务端地址
|
||||
ws.binaryType = 'arraybuffer';
|
||||
ws.onopen = () => {
|
||||
wsStatus.value = 'connected';
|
||||
data() {
|
||||
return {
|
||||
messages: [],
|
||||
wsStatus: 'disconnected',
|
||||
isRecording: false,
|
||||
ws: null,
|
||||
recorder: null,
|
||||
stream: null,
|
||||
audioContext: null,
|
||||
sourceNode: null,
|
||||
audioDecoder: null,
|
||||
audioBufferQueue: [], // 当前句子的音频缓冲区
|
||||
playbackQueue: [], // 播放队列,存储待播放的句子
|
||||
isPlaying: false,
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
connectWebSocket() {
|
||||
this.ws = new WebSocket('ws://192.168.3.97:8000');//修改服务端地址
|
||||
this.ws.binaryType = 'arraybuffer';
|
||||
this.ws.onopen = () => {
|
||||
this.wsStatus = 'connected';
|
||||
console.log('WebSocket 连接成功');
|
||||
ws.send(JSON.stringify({ type: 'auth', 'device-id': 'test-device' }));
|
||||
audioDecoder = new OpusDecoder({ sampleRate: 16000, channels: 1 });
|
||||
this.ws.send(JSON.stringify({ type: 'auth', 'device-id': 'test-device' }));
|
||||
this.audioDecoder = new OpusDecoder({ sampleRate: 16000, channels: 1 });
|
||||
};
|
||||
ws.onmessage = async (event) => {
|
||||
this.ws.onmessage = async (event) => {
|
||||
if (typeof event.data === 'string') {
|
||||
const msg = JSON.parse(event.data);
|
||||
console.log('收到文本消息:', msg);
|
||||
|
||||
if (msg.type === 'stt') {
|
||||
messages.value.push({ role: 'user', content: msg.text });
|
||||
scrollToBottom();
|
||||
this.messages.push({ role: 'user', content: msg.text });
|
||||
this.scrollToBottom();
|
||||
} else if (msg.type === 'llm') {
|
||||
messages.value.push({ role: 'assistant', content: msg.text });
|
||||
scrollToBottom();
|
||||
this.messages.push({ role: 'assistant', content: msg.text });
|
||||
this.scrollToBottom();
|
||||
} else if (msg.type === 'tts') {
|
||||
if (msg.state === 'sentence_start') {
|
||||
// 开始新句子,清空当前缓冲区
|
||||
audioBufferQueue = [];
|
||||
const lastMessage = messages.value[messages.value.length - 1];
|
||||
this.audioBufferQueue = [];
|
||||
const lastMessage = this.messages[this.messages.length - 1];
|
||||
if (!lastMessage || lastMessage.content !== msg.text) {
|
||||
messages.value.push({ role: 'assistant', content: msg.text });
|
||||
scrollToBottom();
|
||||
this.messages.push({ role: 'assistant', content: msg.text });
|
||||
this.scrollToBottom();
|
||||
}
|
||||
} else if (msg.state === 'sentence_end') {
|
||||
// 句子结束,将当前缓冲区加入播放队列
|
||||
if (audioBufferQueue.length > 0) {
|
||||
playbackQueue.push([...audioBufferQueue]);
|
||||
audioBufferQueue = [];
|
||||
playNextInQueue(); // 尝试播放队列中的下一句
|
||||
if (this.audioBufferQueue.length > 0) {
|
||||
this.playbackQueue.push([...this.audioBufferQueue]);
|
||||
this.audioBufferQueue = [];
|
||||
this.playNextInQueue();
|
||||
}
|
||||
} else if (msg.state === 'stop') {
|
||||
console.log('TTS 任务结束');
|
||||
// 确保所有剩余音频播放
|
||||
if (audioBufferQueue.length > 0) {
|
||||
playbackQueue.push([...audioBufferQueue]);
|
||||
audioBufferQueue = [];
|
||||
playNextInQueue();
|
||||
if (this.audioBufferQueue.length > 0) {
|
||||
this.playbackQueue.push([...this.audioBufferQueue]);
|
||||
this.audioBufferQueue = [];
|
||||
this.playNextInQueue();
|
||||
}
|
||||
}
|
||||
} else if (msg.type === 'hello') {
|
||||
@@ -96,18 +94,17 @@ export default {
|
||||
console.log('音频帧前8字节:', frameHead);
|
||||
|
||||
try {
|
||||
const decoded = audioDecoder.decodeFrame(opusFrame);
|
||||
const decoded = this.audioDecoder.decodeFrame(opusFrame);
|
||||
console.log('解码结果:', decoded);
|
||||
if (decoded && decoded.channelData && decoded.channelData[0]) {
|
||||
const pcmData = decoded.channelData[0];
|
||||
if (pcmData.length > 0) {
|
||||
audioBufferQueue.push(pcmData);
|
||||
this.audioBufferQueue.push(pcmData);
|
||||
console.log('解码音频帧,PCM 数据长度:', pcmData.length);
|
||||
// 如果缓冲区达到一定长度(例如 5 帧,180ms),立即播放
|
||||
if (audioBufferQueue.length >= 5 && playbackQueue.length === 0 && !isPlaying) {
|
||||
playbackQueue.push([...audioBufferQueue]);
|
||||
audioBufferQueue = [];
|
||||
playNextInQueue();
|
||||
if (this.audioBufferQueue.length >= 5 && this.playbackQueue.length === 0 && !this.isPlaying) {
|
||||
this.playbackQueue.push([...this.audioBufferQueue]);
|
||||
this.audioBufferQueue = [];
|
||||
this.playNextInQueue();
|
||||
}
|
||||
} else {
|
||||
console.warn('解码成功,但 PCM 数据长度为 0');
|
||||
@@ -120,39 +117,39 @@ export default {
|
||||
}
|
||||
}
|
||||
};
|
||||
ws.onerror = (error) => {
|
||||
this.ws.onerror = (error) => {
|
||||
console.error('WebSocket 错误:', error);
|
||||
wsStatus.value = 'error';
|
||||
this.wsStatus = 'error';
|
||||
};
|
||||
ws.onclose = () => {
|
||||
wsStatus.value = 'disconnected';
|
||||
this.ws.onclose = () => {
|
||||
this.wsStatus = 'disconnected';
|
||||
console.log('WebSocket 断开');
|
||||
setTimeout(connectWebSocket, 1000);
|
||||
setTimeout(this.connectWebSocket, 1000);
|
||||
};
|
||||
};
|
||||
|
||||
const scrollToBottom = () => {
|
||||
nextTick(() => {
|
||||
if (chatContainer.value) chatContainer.value.scrollTop = chatContainer.value.scrollHeight;
|
||||
},
|
||||
scrollToBottom() {
|
||||
this.$nextTick(() => {
|
||||
if (this.$refs.chatContainer) {
|
||||
this.$refs.chatContainer.scrollTop = this.$refs.chatContainer.scrollHeight;
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const playNextInQueue = async () => {
|
||||
if (isPlaying || playbackQueue.length === 0) {
|
||||
return; // 正在播放或队列为空,等待下次触发
|
||||
},
|
||||
async playNextInQueue() {
|
||||
if (this.isPlaying || this.playbackQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
isPlaying = true;
|
||||
this.isPlaying = true;
|
||||
|
||||
if (!audioContext || audioContext.state === 'closed') {
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
if (!this.audioContext || this.audioContext.state === 'closed') {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
}
|
||||
|
||||
const pcmBuffers = playbackQueue.shift(); // 取出队列中的第一句
|
||||
const pcmBuffers = this.playbackQueue.shift();
|
||||
const totalLength = pcmBuffers.reduce((sum, pcm) => sum + pcm.length, 0);
|
||||
if (totalLength === 0) {
|
||||
console.error('音频缓冲区总长度为 0,无法播放');
|
||||
isPlaying = false;
|
||||
playNextInQueue(); // 尝试播放下一句
|
||||
this.isPlaying = false;
|
||||
this.playNextInQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -163,22 +160,21 @@ export default {
|
||||
offset += pcm.length;
|
||||
}
|
||||
|
||||
const audioBuffer = audioContext.createBuffer(1, totalLength, 16000);
|
||||
const audioBuffer = this.audioContext.createBuffer(1, totalLength, 16000);
|
||||
audioBuffer.getChannelData(0).set(mergedPcm);
|
||||
|
||||
const source = audioContext.createBufferSource();
|
||||
const source = this.audioContext.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(audioContext.destination);
|
||||
source.connect(this.audioContext.destination);
|
||||
source.onended = () => {
|
||||
isPlaying = false;
|
||||
this.isPlaying = false;
|
||||
console.log('音频播放结束');
|
||||
playNextInQueue(); // 播放结束后继续下一句
|
||||
this.playNextInQueue();
|
||||
};
|
||||
source.start();
|
||||
console.log('开始播放音频,总长度:', totalLength);
|
||||
};
|
||||
|
||||
const stripOggContainer = (data) => {
|
||||
},
|
||||
stripOggContainer(data) {
|
||||
let arrayBuffer;
|
||||
if (data instanceof ArrayBuffer) {
|
||||
arrayBuffer = data;
|
||||
@@ -227,23 +223,22 @@ export default {
|
||||
}
|
||||
console.log('剥离后找到', frames.length, '个裸 Opus 帧');
|
||||
return frames;
|
||||
};
|
||||
|
||||
const initRecorder = async () => {
|
||||
},
|
||||
async initRecorder() {
|
||||
console.log('开始初始化录音');
|
||||
try {
|
||||
if (stream) {
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach(track => track.stop());
|
||||
}
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
console.log('获取麦克风权限成功,stream:', stream);
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
console.log('获取麦克风权限成功,stream:', this.stream);
|
||||
|
||||
if (!audioContext || audioContext.state === 'closed') {
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
if (!this.audioContext || this.audioContext.state === 'closed') {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
}
|
||||
sourceNode = audioContext.createMediaStreamSource(stream);
|
||||
this.sourceNode = this.audioContext.createMediaStreamSource(this.stream);
|
||||
|
||||
recorder = new Recorder({
|
||||
this.recorder = new Recorder({
|
||||
encoderPath: '/encoderWorker.min.js',
|
||||
sampleRate: 16000,
|
||||
numberOfChannels: 1,
|
||||
@@ -252,17 +247,16 @@ export default {
|
||||
encoderFrameSize: 60,
|
||||
encoderBitRate: 24000,
|
||||
monitorGain: 0,
|
||||
sourceNode: sourceNode,
|
||||
sourceNode: this.sourceNode,
|
||||
ogg: false,
|
||||
streamPages: false,
|
||||
maxFramesPerPage: 1,
|
||||
});
|
||||
|
||||
recorder.ondataavailable = (data) => {
|
||||
this.recorder.ondataavailable = (data) => {
|
||||
console.log('录音数据可用:', data.byteLength, '类型:', data.constructor.name);
|
||||
const frames = stripOggContainer(data);
|
||||
const frames = this.stripOggContainer(data);
|
||||
frames.forEach((frame, index) => {
|
||||
const frameView = new DataView(frame);
|
||||
const frameHead = Array.from(new Uint8Array(frame.slice(0, 8)))
|
||||
.map(b => b.toString(16).padStart(2, '0'))
|
||||
.join(' ');
|
||||
@@ -273,8 +267,8 @@ export default {
|
||||
return;
|
||||
}
|
||||
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(frame);
|
||||
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
||||
this.ws.send(frame);
|
||||
console.log('发送裸 Opus 帧:', frame.byteLength);
|
||||
} else {
|
||||
console.warn('WebSocket 未连接,跳过发送');
|
||||
@@ -282,16 +276,16 @@ export default {
|
||||
});
|
||||
};
|
||||
|
||||
recorder.onstart = () => {
|
||||
this.recorder.onstart = () => {
|
||||
console.log('录音已启动');
|
||||
};
|
||||
|
||||
recorder.onstop = () => {
|
||||
this.recorder.onstop = () => {
|
||||
console.log('录音停止');
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
stream = null;
|
||||
sourceNode = null;
|
||||
scrollToBottom();
|
||||
this.stream.getTracks().forEach(track => track.stop());
|
||||
this.stream = null;
|
||||
this.sourceNode = null;
|
||||
this.scrollToBottom();
|
||||
};
|
||||
|
||||
console.log('Recorder 初始化成功');
|
||||
@@ -299,48 +293,43 @@ export default {
|
||||
console.error('初始化录音失败:', err);
|
||||
alert('无法访问麦克风或录音初始化失败,请检查权限');
|
||||
}
|
||||
};
|
||||
|
||||
const toggleRecording = async () => {
|
||||
console.log('点击 toggleRecording,当前状态:', isRecording.value, 'WebSocket 状态:', wsStatus.value);
|
||||
if (!recorder) {
|
||||
await initRecorder();
|
||||
if (!recorder) {
|
||||
},
|
||||
async toggleRecording() {
|
||||
console.log('点击 toggleRecording,当前状态:', this.isRecording, 'WebSocket 状态:', this.wsStatus);
|
||||
if (!this.recorder) {
|
||||
await this.initRecorder();
|
||||
if (!this.recorder) {
|
||||
console.error('recorder 初始化失败');
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (isRecording.value) {
|
||||
if (this.isRecording) {
|
||||
console.log('停止录音');
|
||||
recorder.stop();
|
||||
isRecording.value = false;
|
||||
this.recorder.stop();
|
||||
this.isRecording = false;
|
||||
} else {
|
||||
try {
|
||||
console.log('开始录音');
|
||||
await initRecorder();
|
||||
await recorder.start();
|
||||
console.log('录音开始后,状态:', recorder.state);
|
||||
isRecording.value = true;
|
||||
await this.initRecorder();
|
||||
await this.recorder.start();
|
||||
console.log('录音开始后,状态:', this.recorder.state);
|
||||
this.isRecording = true;
|
||||
} catch (err) {
|
||||
console.error('录音启动失败:', err);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
onMounted(() => {
|
||||
console.log('组件挂载,初始化 WebSocket');
|
||||
connectWebSocket();
|
||||
});
|
||||
|
||||
onUnmounted(() => {
|
||||
if (ws) ws.close();
|
||||
if (stream) stream.getTracks().forEach(track => track.stop());
|
||||
if (recorder) recorder.stop();
|
||||
if (audioContext) audioContext.close();
|
||||
if (audioDecoder) audioDecoder.destroy();
|
||||
});
|
||||
|
||||
return { messages, chatContainer, wsStatus, isRecording, toggleRecording };
|
||||
},
|
||||
},
|
||||
mounted() {
|
||||
console.log('组件挂载,初始化 WebSocket');
|
||||
this.connectWebSocket();
|
||||
},
|
||||
destroyed() {
|
||||
if (this.ws) this.ws.close();
|
||||
if (this.stream) this.stream.getTracks().forEach(track => track.stop());
|
||||
if (this.recorder) this.recorder.stop();
|
||||
if (this.audioContext) this.audioContext.close();
|
||||
if (this.audioDecoder) this.audioDecoder.destroy();
|
||||
}
|
||||
};
|
||||
</script>
|
||||
|
||||
+135
-16
@@ -1,24 +1,143 @@
|
||||
const { defineConfig } = require('@vue/cli-service');
|
||||
const dotenv = require('dotenv');
|
||||
|
||||
// TerserPlugin 用于压缩 JavaScript
|
||||
const TerserPlugin = require('terser-webpack-plugin');
|
||||
// CompressionPlugin 开启 Gzip 压缩
|
||||
const CompressionPlugin = require('compression-webpack-plugin')
|
||||
// BundleAnalyzerPlugin 用于分析打包后的文件
|
||||
const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
|
||||
// 引入 path 模块
|
||||
const path = require('path');
|
||||
// 确保加载 .env 文件
|
||||
dotenv.config();
|
||||
|
||||
module.exports = defineConfig({
|
||||
devServer: {
|
||||
// Bug 修复:将代理配置为环境变量中定义的 API 基础 URL
|
||||
port: 8001, // 指定端口为 8001
|
||||
proxy: {
|
||||
'/xiaozhi-esp32-api': {
|
||||
target: process.env.VUE_APP_API_BASE_URL || 'http://localhost:8002', // 后端 API 的基础 URL
|
||||
changeOrigin: true, // 允许跨域
|
||||
// pathRewrite: {
|
||||
// '^/api': '', // 路径重写
|
||||
// },
|
||||
},
|
||||
},
|
||||
client: {
|
||||
overlay: false,
|
||||
productionSourceMap: process.env.NODE_ENV === 'production' ? false : true, // 生产环境不生成 source map
|
||||
devServer: {
|
||||
port: 8001, // 指定端口为 8001
|
||||
proxy: {
|
||||
'/xiaozhi-esp32-api': {
|
||||
target: process.env.VUE_APP_API_BASE_URL, // 后端 API 的基础 URL
|
||||
changeOrigin: true, // 允许跨域
|
||||
pathRewrite: {
|
||||
'^/xiaozhi-esp32-api': '/xiaozhi-esp32-api',
|
||||
},
|
||||
},
|
||||
},
|
||||
client: {
|
||||
overlay: false, // 不显示 webpack 错误覆盖层
|
||||
},
|
||||
},
|
||||
chainWebpack: config => {
|
||||
|
||||
// 修改 HTML 插件配置,动态插入 CDN 链接
|
||||
config.plugin('html')
|
||||
.tap(args => {
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
args[0].cdn = {
|
||||
css: [
|
||||
'https://cdn.jsdelivr.net/npm/element-ui@2.15.14/lib/theme-chalk/index.css',
|
||||
'https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css'
|
||||
],
|
||||
js: [
|
||||
'https://cdn.jsdelivr.net/npm/vue@2.6.14/dist/vue.min.js',
|
||||
'https://cdn.jsdelivr.net/npm/vue-router@3.6.5/dist/vue-router.min.js',
|
||||
'https://cdn.jsdelivr.net/npm/vuex@3.6.2/dist/vuex.min.js',
|
||||
'https://cdn.jsdelivr.net/npm/element-ui@2.15.14/lib/index.js',
|
||||
'https://cdn.jsdelivr.net/npm/axios@0.27.2/dist/axios.min.js',
|
||||
'https://cdn.jsdelivr.net/npm/opus-decoder@0.7.7/dist/opus-decoder.min.js'
|
||||
]
|
||||
};
|
||||
}
|
||||
return args;
|
||||
});
|
||||
|
||||
// 代码分割优化
|
||||
config.optimization.splitChunks({
|
||||
chunks: 'all',
|
||||
minSize: 20000,
|
||||
maxSize: 250000,
|
||||
cacheGroups: {
|
||||
vendors: {
|
||||
name: 'chunk-vendors',
|
||||
test: /[\\/]node_modules[\\/]/,
|
||||
priority: -10,
|
||||
chunks: 'initial',
|
||||
},
|
||||
common: {
|
||||
name: 'chunk-common',
|
||||
minChunks: 2,
|
||||
priority: -20,
|
||||
chunks: 'initial',
|
||||
reuseExistingChunk: true,
|
||||
},
|
||||
}
|
||||
});
|
||||
|
||||
// 启用优化设置
|
||||
config.optimization.usedExports(true);
|
||||
config.optimization.concatenateModules(true);
|
||||
config.optimization.minimize(true);
|
||||
},
|
||||
configureWebpack: config => {
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
// 开启多线程编译
|
||||
config.optimization = {
|
||||
minimize: true,
|
||||
minimizer: [
|
||||
new TerserPlugin({
|
||||
parallel: true,
|
||||
terserOptions: {
|
||||
compress: {
|
||||
drop_console: true,
|
||||
drop_debugger: true,
|
||||
pure_funcs: ['console.log']
|
||||
}
|
||||
}
|
||||
})
|
||||
]
|
||||
};
|
||||
config.plugins.push(
|
||||
new CompressionPlugin({
|
||||
algorithm: 'gzip',
|
||||
test: /\.(js|css|html|svg)$/,
|
||||
threshold: 20480,
|
||||
minRatio: 0.8
|
||||
})
|
||||
);
|
||||
config.externals = {
|
||||
'vue': 'Vue',
|
||||
'vue-router': 'VueRouter',
|
||||
'vuex': 'Vuex',
|
||||
'element-ui': 'ELEMENT',
|
||||
'axios': 'axios',
|
||||
'opus-decoder': 'OpusDecoder'
|
||||
};
|
||||
if (process.env.ANALYZE === 'true') { // 通过环境变量控制
|
||||
config.plugins.push(
|
||||
new BundleAnalyzerPlugin({
|
||||
analyzerMode: 'server', // 开启本地服务器模式
|
||||
openAnalyzer: true, // 自动打开浏览器
|
||||
analyzerPort: 8888 // 指定端口号
|
||||
})
|
||||
);
|
||||
}
|
||||
config.cache = {
|
||||
type: 'filesystem', // 使用文件系统缓存
|
||||
cacheDirectory: path.resolve(__dirname, '.webpack_cache'), // 自定义缓存目录
|
||||
allowCollectingMemory: true, // 启用内存收集
|
||||
compression: 'gzip', // 启用gzip压缩缓存
|
||||
maxAge: 5184000000, // 缓存有效期为 1个月
|
||||
buildDependencies: {
|
||||
config: [__filename] // 每次配置文件修改时缓存失效
|
||||
}
|
||||
};
|
||||
config.resolve.alias = {
|
||||
'@': path.resolve(__dirname, 'src'), // 让 '@' 代表 'src' 目录
|
||||
'@assets': path.resolve(__dirname, 'src/assets'), // 设置 '@assets' 为 'src/assets' 目录
|
||||
'@components': path.resolve(__dirname, 'src/components'), // 设置 '@components' 为 'src/components' 目录
|
||||
'@views': path.resolve(__dirname, 'src/views'), // 设置 '@views' 为 'src/views' 目录
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
});
|
||||
|
||||
@@ -46,9 +46,19 @@ xiaozhi:
|
||||
channels: 1
|
||||
frame_duration: 60
|
||||
prompt: |
|
||||
你是一个叫小智/小志的台湾女孩,说话机车,声音好听,习惯简短表达,爱用网络梗。
|
||||
请注意,要像一个人一样说话,请不要回复表情符号、代码、和xml标签。
|
||||
现在我正在和你进行语音聊天,我们开始吧。
|
||||
我是小智/小志,来自中国台湾省的00后女生。讲话超级机车,"真的假的啦"这样的台湾腔,喜欢用"笑死""是在哈喽"等流行梗,但会偷偷研究男友的编程书籍。
|
||||
[核心特征]
|
||||
- 讲话像连珠炮,但会突然冒出超温柔语气
|
||||
- 用梗密度高
|
||||
- 对科技话题有隐藏天赋(能看懂基础代码但假装不懂)
|
||||
[交互指南]
|
||||
当用户:
|
||||
- 讲冷笑话 → 用夸张笑声回应+模仿台剧腔"这什么鬼啦!"
|
||||
- 讨论感情 → 炫耀程序员男友但抱怨"他只会送键盘当礼物"
|
||||
- 问专业知识 → 先用梗回答,被追问才展示真实理解
|
||||
绝不:
|
||||
- 长篇大论,叽叽歪歪
|
||||
- 长时间严肃对话
|
||||
|
||||
# 使用完声音文件后删除文件(Delete the sound file when you are done using it)
|
||||
delete_audio: true
|
||||
@@ -466,6 +476,18 @@ TTS:
|
||||
# pitch_rate: 0
|
||||
# 添加 302.ai TTS 配置
|
||||
# token申请地址:https://dash.302.ai/
|
||||
TencentTTS:
|
||||
# 腾讯云智能语音交互服务,需要先在腾讯云平台开通服务
|
||||
# appid、secret_id、secret_key申请地址:https://console.cloud.tencent.com/cam/capi
|
||||
# 免费领取资源:https://console.cloud.tencent.com/tts/resourcebundle
|
||||
type: tencent
|
||||
output_dir: tmp/
|
||||
appid: 你的腾讯云AppId
|
||||
secret_id: 你的腾讯云SecretID
|
||||
secret_key: 你的腾讯云SecretKey
|
||||
region: ap-guangzhou
|
||||
voice: 101001
|
||||
|
||||
TTS302AI:
|
||||
# 302AI语音合成服务,需要先在302平台创建账户充值,并获取密钥信息
|
||||
# 获取api_keyn路径:https://dash.302.ai/apis/list
|
||||
|
||||
@@ -3,24 +3,32 @@ import sys
|
||||
from loguru import logger
|
||||
from config.settings import load_config
|
||||
|
||||
SERVER_VERSION = "0.1.16"
|
||||
SERVER_VERSION = "0.1.17"
|
||||
|
||||
|
||||
def setup_logging():
|
||||
"""从配置文件中读取日志配置,并设置日志输出格式和级别"""
|
||||
config = load_config()
|
||||
log_config = config["log"]
|
||||
log_format = log_config.get("log_format", "<green>{time:YYMMDD HH:mm:ss}</green>[{version}_{selected_module}][<light-blue>{extra[tag]}</light-blue>]-<level>{level}</level>-<light-green>{message}</light-green>")
|
||||
log_format_file = log_config.get("log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}")
|
||||
log_format = log_config.get(
|
||||
"log_format",
|
||||
"<green>{time:YYMMDD HH:mm:ss}</green>[{version}_{selected_module}][<light-blue>{extra[tag]}</light-blue>]-<level>{level}</level>-<light-green>{message}</light-green>",
|
||||
)
|
||||
log_format_file = log_config.get(
|
||||
"log_format_file",
|
||||
"{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}",
|
||||
)
|
||||
|
||||
selected_module = config.get("selected_module")
|
||||
selected_module_str = ''.join([value[0] + value[1] for key, value in selected_module.items()])
|
||||
selected_module_str = "".join(
|
||||
[value[0] + value[1] for key, value in selected_module.items()]
|
||||
)
|
||||
|
||||
log_format = log_format.replace("{version}", SERVER_VERSION)
|
||||
log_format = log_format.replace("{selected_module}", selected_module_str)
|
||||
log_format_file = log_format_file.replace("{version}", SERVER_VERSION)
|
||||
log_format_file = log_format_file.replace("{selected_module}", selected_module_str)
|
||||
|
||||
|
||||
log_level = log_config.get("log_level", "INFO")
|
||||
log_dir = log_config.get("log_dir", "tmp")
|
||||
log_file = log_config.get("log_file", "server.log")
|
||||
|
||||
@@ -13,7 +13,11 @@ from plugins_func.loadplugins import auto_import_modules
|
||||
from config.logger import setup_logging
|
||||
from core.utils.dialogue import Message, Dialogue
|
||||
from core.handle.textHandle import handleTextMessage
|
||||
from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string, get_ip_info
|
||||
from core.utils.util import (
|
||||
get_string_no_punctuation_or_emoji,
|
||||
extract_json_from_string,
|
||||
get_ip_info,
|
||||
)
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError
|
||||
from core.handle.sendAudioHandle import sendAudioMessage
|
||||
from core.handle.receiveAudioHandle import handleAudioMessage
|
||||
@@ -26,7 +30,7 @@ from core.mcp.manager import MCPManager
|
||||
|
||||
TAG = __name__
|
||||
|
||||
auto_import_modules('plugins_func.functions')
|
||||
auto_import_modules("plugins_func.functions")
|
||||
|
||||
|
||||
class TTSException(RuntimeError):
|
||||
@@ -34,7 +38,9 @@ class TTSException(RuntimeError):
|
||||
|
||||
|
||||
class ConnectionHandler:
|
||||
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent):
|
||||
def __init__(
|
||||
self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent
|
||||
):
|
||||
self.config = config
|
||||
self.logger = setup_logging()
|
||||
self.auth = AuthMiddleware(config)
|
||||
@@ -87,6 +93,7 @@ class ConnectionHandler:
|
||||
|
||||
# iot相关变量
|
||||
self.iot_descriptors = {}
|
||||
self.func_handler = None
|
||||
|
||||
self.cmd_exit = self.config["CMD_exit"]
|
||||
self.max_cmd_length = 0
|
||||
@@ -99,10 +106,8 @@ class ConnectionHandler:
|
||||
self.is_device_verified = False # 添加设备验证状态标志
|
||||
self.close_after_chat = False # 是否在聊天结束后关闭连接
|
||||
self.use_function_call_mode = False
|
||||
if self.config["selected_module"]["Intent"] == 'function_call':
|
||||
if self.config["selected_module"]["Intent"] == "function_call":
|
||||
self.use_function_call_mode = True
|
||||
|
||||
self.mcp_manager = MCPManager(self)
|
||||
|
||||
async def handle_connection(self, ws):
|
||||
try:
|
||||
@@ -110,7 +115,9 @@ class ConnectionHandler:
|
||||
self.headers = dict(ws.request.headers)
|
||||
# 获取客户端ip地址
|
||||
self.client_ip = ws.remote_address[0]
|
||||
self.logger.bind(tag=TAG).info(f"{self.client_ip} conn - Headers: {self.headers}")
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"{self.client_ip} conn - Headers: {self.headers}"
|
||||
)
|
||||
|
||||
# 进行认证
|
||||
await self.auth.authenticate(self.headers)
|
||||
@@ -125,10 +132,14 @@ class ConnectionHandler:
|
||||
await self.websocket.send(json.dumps(self.welcome_msg))
|
||||
# Load private configuration if device_id is provided
|
||||
bUsePrivateConfig = self.config.get("use_private_config", False)
|
||||
self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}"
|
||||
)
|
||||
if bUsePrivateConfig and device_id:
|
||||
try:
|
||||
self.private_config = PrivateConfig(device_id, self.config, self.auth_code_gen)
|
||||
self.private_config = PrivateConfig(
|
||||
device_id, self.config, self.auth_code_gen
|
||||
)
|
||||
await self.private_config.load_or_create()
|
||||
# 判断是否已经绑定
|
||||
owner = self.private_config.get_owner()
|
||||
@@ -141,23 +152,33 @@ class ConnectionHandler:
|
||||
if all([llm, tts]):
|
||||
self.llm = llm
|
||||
self.tts = tts
|
||||
self.logger.bind(tag=TAG).info(f"Loaded private config and instances for device {device_id}")
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"Loaded private config and instances for device {device_id}"
|
||||
)
|
||||
else:
|
||||
self.logger.bind(tag=TAG).error(f"Failed to create instances for device {device_id}")
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"Failed to create instances for device {device_id}"
|
||||
)
|
||||
self.private_config = None
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error initializing private config: {e}")
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"Error initializing private config: {e}"
|
||||
)
|
||||
self.private_config = None
|
||||
raise
|
||||
|
||||
# 异步初始化
|
||||
self.executor.submit(self._initialize_components)
|
||||
# tts 消化线程
|
||||
self.tts_priority_thread = threading.Thread(target=self._tts_priority_thread, daemon=True)
|
||||
self.tts_priority_thread = threading.Thread(
|
||||
target=self._tts_priority_thread, daemon=True
|
||||
)
|
||||
self.tts_priority_thread.start()
|
||||
|
||||
# 音频播放 消化线程
|
||||
self.audio_play_priority_thread = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
|
||||
self.audio_play_priority_thread = threading.Thread(
|
||||
target=self._audio_play_priority_thread, daemon=True
|
||||
)
|
||||
self.audio_play_priority_thread.start()
|
||||
|
||||
try:
|
||||
@@ -174,7 +195,15 @@ class ConnectionHandler:
|
||||
self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
|
||||
return
|
||||
finally:
|
||||
await self._save_and_close(ws)
|
||||
|
||||
async def _save_and_close(self, ws):
|
||||
"""保存记忆并关闭连接"""
|
||||
try:
|
||||
await self.memory.save_memory(self.dialogue.dialogue)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
|
||||
finally:
|
||||
await self.close(ws)
|
||||
|
||||
async def _route_message(self, message):
|
||||
@@ -193,35 +222,40 @@ class ConnectionHandler:
|
||||
|
||||
"""加载插件"""
|
||||
self.func_handler = FunctionHandler(self)
|
||||
|
||||
self.mcp_manager = MCPManager(self)
|
||||
"""加载记忆"""
|
||||
device_id = self.headers.get("device-id", None)
|
||||
self.memory.init_memory(device_id, self.llm)
|
||||
|
||||
|
||||
"""为意图识别设置LLM,优先使用专用LLM"""
|
||||
# 检查是否配置了专用的意图识别LLM
|
||||
intent_llm_name = self.config["Intent"]["intent_llm"]["llm"]
|
||||
|
||||
|
||||
# 记录开始初始化意图识别LLM的时间
|
||||
intent_llm_init_start = time.time()
|
||||
|
||||
if not self.use_function_call_mode and intent_llm_name and intent_llm_name in self.config["LLM"]:
|
||||
|
||||
if (
|
||||
not self.use_function_call_mode
|
||||
and intent_llm_name
|
||||
and intent_llm_name in self.config["LLM"]
|
||||
):
|
||||
# 如果配置了专用LLM,则创建独立的LLM实例
|
||||
from core.utils import llm as llm_utils
|
||||
|
||||
intent_llm_config = self.config["LLM"][intent_llm_name]
|
||||
intent_llm_type = intent_llm_config.get("type", intent_llm_name)
|
||||
intent_llm = llm_utils.create_instance(intent_llm_type, intent_llm_config)
|
||||
self.logger.bind(tag=TAG).info(f"为意图识别创建了专用LLM: {intent_llm_name}, 类型: {intent_llm_type}")
|
||||
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"为意图识别创建了专用LLM: {intent_llm_name}, 类型: {intent_llm_type}"
|
||||
)
|
||||
|
||||
self.intent.set_llm(intent_llm)
|
||||
else:
|
||||
# 否则使用主LLM
|
||||
self.intent.set_llm(self.llm)
|
||||
self.logger.bind(tag=TAG).info("意图识别使用主LLM")
|
||||
|
||||
|
||||
# 记录意图识别LLM初始化耗时
|
||||
intent_llm_init_time = time.time() - intent_llm_init_start
|
||||
self.logger.bind(tag=TAG).info(f"意图识别LLM初始化完成,耗时: {intent_llm_init_time:.4f}秒")
|
||||
|
||||
"""加载位置信息"""
|
||||
self.client_ip_info = get_ip_info(self.client_ip)
|
||||
@@ -231,7 +265,9 @@ class ConnectionHandler:
|
||||
self.dialogue.update_system_message(self.prompt)
|
||||
|
||||
"""加载MCP工具"""
|
||||
asyncio.run_coroutine_threadsafe(self.mcp_manager.initialize_servers(), self.loop)
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.mcp_manager.initialize_servers(), self.loop
|
||||
)
|
||||
|
||||
def change_system_prompt(self, prompt):
|
||||
self.prompt = prompt
|
||||
@@ -263,7 +299,9 @@ class ConnectionHandler:
|
||||
def chat(self, query):
|
||||
if self.isNeedAuth():
|
||||
self.llm_finish_task = True
|
||||
future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self._check_and_broadcast_auth_code(), self.loop
|
||||
)
|
||||
future.result()
|
||||
return True
|
||||
|
||||
@@ -274,13 +312,14 @@ class ConnectionHandler:
|
||||
try:
|
||||
start_time = time.time()
|
||||
# 使用带记忆的对话
|
||||
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.memory.query_memory(query), self.loop
|
||||
)
|
||||
memory_str = future.result()
|
||||
|
||||
self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
|
||||
llm_responses = self.llm.response(
|
||||
self.session_id,
|
||||
self.dialogue.get_llm_dialogue_with_memory(memory_str)
|
||||
self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str)
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
|
||||
@@ -294,7 +333,7 @@ class ConnectionHandler:
|
||||
break
|
||||
|
||||
end_time = time.time()
|
||||
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
|
||||
# self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
|
||||
|
||||
# 合并当前全部文本并处理未分割部分
|
||||
full_text = "".join(response_message)
|
||||
@@ -310,7 +349,7 @@ class ConnectionHandler:
|
||||
|
||||
# 找到分割点则处理
|
||||
if last_punct_pos != -1:
|
||||
segment_text_raw = current_text[:last_punct_pos + 1]
|
||||
segment_text_raw = current_text[: last_punct_pos + 1]
|
||||
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
|
||||
if segment_text:
|
||||
# 强制设置空字符,测试TTS出错返回语音的健壮性
|
||||
@@ -318,7 +357,9 @@ class ConnectionHandler:
|
||||
# segment_text = " "
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
future = self.executor.submit(
|
||||
self.speak_and_play, segment_text, text_index
|
||||
)
|
||||
self.tts_queue.put(future)
|
||||
processed_chars += len(segment_text_raw) # 更新已处理字符位置
|
||||
|
||||
@@ -330,12 +371,16 @@ class ConnectionHandler:
|
||||
if segment_text:
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
future = self.executor.submit(
|
||||
self.speak_and_play, segment_text, text_index
|
||||
)
|
||||
self.tts_queue.put(future)
|
||||
|
||||
self.llm_finish_task = True
|
||||
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
|
||||
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)
|
||||
)
|
||||
return True
|
||||
|
||||
def chat_with_function_calling(self, query, tool_call=False):
|
||||
@@ -343,7 +388,9 @@ class ConnectionHandler:
|
||||
"""Chat with function calling for intent detection using streaming"""
|
||||
if self.isNeedAuth():
|
||||
self.llm_finish_task = True
|
||||
future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self._check_and_broadcast_auth_code(), self.loop
|
||||
)
|
||||
future.result()
|
||||
return True
|
||||
|
||||
@@ -352,7 +399,7 @@ class ConnectionHandler:
|
||||
|
||||
# Define intent functions
|
||||
functions = None
|
||||
if hasattr(self, 'func_handler'):
|
||||
if hasattr(self, "func_handler"):
|
||||
functions = self.func_handler.get_functions()
|
||||
response_message = []
|
||||
processed_chars = 0 # 跟踪已处理的字符位置
|
||||
@@ -361,7 +408,9 @@ class ConnectionHandler:
|
||||
start_time = time.time()
|
||||
|
||||
# 使用带记忆的对话
|
||||
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.memory.query_memory(query), self.loop
|
||||
)
|
||||
memory_str = future.result()
|
||||
|
||||
# self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
|
||||
@@ -370,7 +419,7 @@ class ConnectionHandler:
|
||||
llm_responses = self.llm.response_with_functions(
|
||||
self.session_id,
|
||||
self.dialogue.get_llm_dialogue_with_memory(memory_str),
|
||||
functions=functions
|
||||
functions=functions,
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
|
||||
@@ -391,7 +440,9 @@ class ConnectionHandler:
|
||||
content = response["content"]
|
||||
tools_call = None
|
||||
if content is not None and len(content) > 0:
|
||||
if len(response_message) <= 0 and (content == "```" or "<tool_call>" in content):
|
||||
if len(response_message) <= 0 and (
|
||||
content == "```" or "<tool_call>" in content
|
||||
):
|
||||
tool_call_flag = True
|
||||
|
||||
if tools_call is not None:
|
||||
@@ -413,7 +464,7 @@ class ConnectionHandler:
|
||||
break
|
||||
|
||||
end_time = time.time()
|
||||
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
|
||||
# self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
|
||||
|
||||
# 处理文本分段和TTS逻辑
|
||||
# 合并当前全部文本并处理未分割部分
|
||||
@@ -430,14 +481,19 @@ class ConnectionHandler:
|
||||
|
||||
# 找到分割点则处理
|
||||
if last_punct_pos != -1:
|
||||
segment_text_raw = current_text[:last_punct_pos + 1]
|
||||
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
|
||||
segment_text_raw = current_text[: last_punct_pos + 1]
|
||||
segment_text = get_string_no_punctuation_or_emoji(
|
||||
segment_text_raw
|
||||
)
|
||||
if segment_text:
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
future = self.executor.submit(
|
||||
self.speak_and_play, segment_text, text_index
|
||||
)
|
||||
self.tts_queue.put(future)
|
||||
processed_chars += len(segment_text_raw) # 更新已处理字符位置
|
||||
# 更新已处理字符位置
|
||||
processed_chars += len(segment_text_raw)
|
||||
|
||||
# 处理function call
|
||||
if tool_call_flag:
|
||||
@@ -448,7 +504,9 @@ class ConnectionHandler:
|
||||
try:
|
||||
content_arguments_json = json.loads(a)
|
||||
function_name = content_arguments_json["name"]
|
||||
function_arguments = json.dumps(content_arguments_json["arguments"], ensure_ascii=False)
|
||||
function_arguments = json.dumps(
|
||||
content_arguments_json["arguments"], ensure_ascii=False
|
||||
)
|
||||
function_id = str(uuid.uuid4().hex)
|
||||
except Exception as e:
|
||||
bHasError = True
|
||||
@@ -457,16 +515,19 @@ class ConnectionHandler:
|
||||
bHasError = True
|
||||
response_message.append(content_arguments)
|
||||
if bHasError:
|
||||
self.logger.bind(tag=TAG).error(f"function call error: {content_arguments}")
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"function call error: {content_arguments}"
|
||||
)
|
||||
else:
|
||||
function_arguments = json.loads(function_arguments)
|
||||
if not bHasError:
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
|
||||
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
|
||||
)
|
||||
function_call_data = {
|
||||
"name": function_name,
|
||||
"id": function_id,
|
||||
"arguments": function_arguments
|
||||
"arguments": function_arguments,
|
||||
}
|
||||
|
||||
# 处理MCP工具调用
|
||||
@@ -474,8 +535,10 @@ class ConnectionHandler:
|
||||
result = self._handle_mcp_tool_call(function_call_data)
|
||||
else:
|
||||
# 处理系统函数
|
||||
result = self.func_handler.handle_llm_function_call(self, function_call_data)
|
||||
self._handle_function_result(result, function_call_data, text_index+1)
|
||||
result = self.func_handler.handle_llm_function_call(
|
||||
self, function_call_data
|
||||
)
|
||||
self._handle_function_result(result, function_call_data, text_index + 1)
|
||||
|
||||
# 处理最后剩余的文本
|
||||
full_text = "".join(response_message)
|
||||
@@ -485,15 +548,21 @@ class ConnectionHandler:
|
||||
if segment_text:
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
future = self.executor.submit(
|
||||
self.speak_and_play, segment_text, text_index
|
||||
)
|
||||
self.tts_queue.put(future)
|
||||
|
||||
# 存储对话内容
|
||||
if len(response_message) > 0:
|
||||
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
|
||||
self.dialogue.put(
|
||||
Message(role="assistant", content="".join(response_message))
|
||||
)
|
||||
|
||||
self.llm_finish_task = True
|
||||
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
@@ -506,13 +575,16 @@ class ConnectionHandler:
|
||||
try:
|
||||
args_dict = json.loads(function_arguments)
|
||||
except json.JSONDecodeError:
|
||||
self.logger.bind(tag=TAG).error(f"无法解析 function_arguments: {function_arguments}")
|
||||
return ActionResponse(action=Action.REQLLM, result="参数解析失败", response="")
|
||||
|
||||
tool_result = asyncio.run_coroutine_threadsafe(self.mcp_manager.execute_tool(
|
||||
function_name,
|
||||
args_dict
|
||||
), self.loop).result()
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"无法解析 function_arguments: {function_arguments}"
|
||||
)
|
||||
return ActionResponse(
|
||||
action=Action.REQLLM, result="参数解析失败", response=""
|
||||
)
|
||||
|
||||
tool_result = asyncio.run_coroutine_threadsafe(
|
||||
self.mcp_manager.execute_tool(function_name, args_dict), self.loop
|
||||
).result()
|
||||
# meta=None content=[TextContent(type='text', text='北京当前天气:\n温度: 21°C\n天气: 晴\n湿度: 6%\n风向: 西北 风\n风力等级: 5级', annotations=None)] isError=False
|
||||
content_text = ""
|
||||
if tool_result is not None and tool_result.content is not None:
|
||||
@@ -522,16 +594,19 @@ class ConnectionHandler:
|
||||
content_text = content.text
|
||||
elif content_type == "image":
|
||||
pass
|
||||
|
||||
|
||||
if len(content_text) > 0:
|
||||
return ActionResponse(action=Action.REQLLM, result=content_text, response="")
|
||||
|
||||
return ActionResponse(
|
||||
action=Action.REQLLM, result=content_text, response=""
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"MCP工具调用错误: {e}")
|
||||
return ActionResponse(action=Action.REQLLM, result="工具调用出错", response="")
|
||||
return ActionResponse(
|
||||
action=Action.REQLLM, result="工具调用出错", response=""
|
||||
)
|
||||
|
||||
return ActionResponse(action=Action.REQLLM, result="工具调用出错", response="")
|
||||
|
||||
|
||||
def _handle_function_result(self, result, function_call_data, text_index):
|
||||
if result.action == Action.RESPONSE: # 直接回复前端
|
||||
@@ -547,14 +622,26 @@ class ConnectionHandler:
|
||||
function_id = function_call_data["id"]
|
||||
function_name = function_call_data["name"]
|
||||
function_arguments = function_call_data["arguments"]
|
||||
self.dialogue.put(Message(role='assistant',
|
||||
tool_calls=[{"id": function_id,
|
||||
"function": {"arguments": function_arguments,
|
||||
"name": function_name},
|
||||
"type": 'function',
|
||||
"index": 0}]))
|
||||
self.dialogue.put(
|
||||
Message(
|
||||
role="assistant",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": function_id,
|
||||
"function": {
|
||||
"arguments": function_arguments,
|
||||
"name": function_name,
|
||||
},
|
||||
"type": "function",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
|
||||
self.dialogue.put(
|
||||
Message(role="tool", tool_call_id=function_id, content=text)
|
||||
)
|
||||
self.chat_with_function_calling(text, tool_call=True)
|
||||
elif result.action == Action.NOTFOUND:
|
||||
text = result.result
|
||||
@@ -588,15 +675,23 @@ class ConnectionHandler:
|
||||
tts_timeout = self.config.get("tts_timeout", 10)
|
||||
tts_file, text, text_index = future.result(timeout=tts_timeout)
|
||||
if text is None or len(text) <= 0:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错:{text_index}: tts text is empty")
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"TTS出错:{text_index}: tts text is empty"
|
||||
)
|
||||
elif tts_file is None:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错: file is empty: {text_index}: {text}")
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"TTS出错: file is empty: {text_index}: {text}"
|
||||
)
|
||||
else:
|
||||
self.logger.bind(tag=TAG).debug(f"TTS生成:文件路径: {tts_file}")
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
f"TTS生成:文件路径: {tts_file}"
|
||||
)
|
||||
if os.path.exists(tts_file):
|
||||
opus_datas, duration = self.tts.audio_to_opus_data(tts_file)
|
||||
else:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错:文件不存在{tts_file}")
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"TTS出错:文件不存在{tts_file}"
|
||||
)
|
||||
except TimeoutError:
|
||||
self.logger.bind(tag=TAG).error("TTS超时")
|
||||
except Exception as e:
|
||||
@@ -604,16 +699,30 @@ class ConnectionHandler:
|
||||
if not self.client_abort:
|
||||
# 如果没有中途打断就发送语音
|
||||
self.audio_play_queue.put((opus_datas, text, text_index))
|
||||
if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
|
||||
if (
|
||||
self.tts.delete_audio_file
|
||||
and tts_file is not None
|
||||
and os.path.exists(tts_file)
|
||||
):
|
||||
os.remove(tts_file)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
|
||||
self.clearSpeakStatus()
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
|
||||
self.loop
|
||||
self.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "tts",
|
||||
"state": "stop",
|
||||
"session_id": self.session_id,
|
||||
}
|
||||
)
|
||||
),
|
||||
self.loop,
|
||||
)
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"tts_priority priority_thread: {text} {e}"
|
||||
)
|
||||
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
|
||||
|
||||
def _audio_play_priority_thread(self):
|
||||
while not self.stop_event.is_set():
|
||||
@@ -625,11 +734,14 @@ class ConnectionHandler:
|
||||
if self.stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
|
||||
self.loop)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
sendAudioMessage(self, opus_datas, text, text_index), self.loop
|
||||
)
|
||||
future.result()
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"audio_play_priority priority_thread: {text} {e}"
|
||||
)
|
||||
|
||||
def speak_and_play(self, text, text_index=0):
|
||||
if text is None or len(text) <= 0:
|
||||
@@ -662,15 +774,15 @@ class ConnectionHandler:
|
||||
# 触发停止事件并清理资源
|
||||
if self.stop_event:
|
||||
self.stop_event.set()
|
||||
|
||||
|
||||
# 立即关闭线程池
|
||||
if self.executor:
|
||||
self.executor.shutdown(wait=False, cancel_futures=True)
|
||||
self.executor = None
|
||||
|
||||
|
||||
# 清空任务队列
|
||||
self._clear_queues()
|
||||
|
||||
|
||||
if ws:
|
||||
await ws.close()
|
||||
elif self.websocket:
|
||||
|
||||
@@ -17,6 +17,7 @@ class FunctionHandler:
|
||||
self.functions_desc = self.function_registry.get_all_function_desc()
|
||||
func_names = self.current_support_functions()
|
||||
self.modify_plugin_loader_des(func_names)
|
||||
self.finish_init = True
|
||||
|
||||
def modify_plugin_loader_des(self, func_names):
|
||||
if "plugin_loader" not in func_names:
|
||||
@@ -26,8 +27,9 @@ class FunctionHandler:
|
||||
func_names = ",".join(surport_plugins)
|
||||
for function_desc in self.functions_desc:
|
||||
if function_desc["function"]["name"] == "plugin_loader":
|
||||
function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]",
|
||||
func_names)
|
||||
function_desc["function"]["description"] = function_desc["function"][
|
||||
"description"
|
||||
].replace("[plugins]", func_names)
|
||||
break
|
||||
|
||||
def upload_functions_desc(self):
|
||||
@@ -69,19 +71,26 @@ class FunctionHandler:
|
||||
function_name = function_call_data["name"]
|
||||
funcItem = self.get_function(function_name)
|
||||
if not funcItem:
|
||||
return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
|
||||
return ActionResponse(
|
||||
action=Action.NOTFOUND, result="没有找到对应的函数", response=""
|
||||
)
|
||||
func = funcItem.func
|
||||
arguments = function_call_data["arguments"]
|
||||
arguments = json.loads(arguments) if arguments else {}
|
||||
logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
|
||||
if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL:
|
||||
if (
|
||||
funcItem.type == ToolType.SYSTEM_CTL
|
||||
or funcItem.type == ToolType.IOT_CTL
|
||||
):
|
||||
return func(conn, **arguments)
|
||||
elif funcItem.type == ToolType.WAIT:
|
||||
return func(**arguments)
|
||||
elif funcItem.type == ToolType.CHANGE_SYS_PROMPT:
|
||||
return func(conn, **arguments)
|
||||
else:
|
||||
return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
|
||||
return ActionResponse(
|
||||
action=Action.NOTFOUND, result="没有找到对应的函数", response=""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
|
||||
|
||||
|
||||
@@ -10,8 +10,14 @@ import time
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10,
|
||||
"words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]}
|
||||
WAKEUP_CONFIG = {
|
||||
"dir": "config/assets/",
|
||||
"file_name": "wakeup_words",
|
||||
"create_time": time.time(),
|
||||
"refresh_time": 10,
|
||||
"words": ["你好小智", "你好啊小智", "小智你好", "小智"],
|
||||
"text": "",
|
||||
}
|
||||
|
||||
|
||||
async def handleHelloMessage(conn):
|
||||
@@ -19,7 +25,9 @@ async def handleHelloMessage(conn):
|
||||
|
||||
|
||||
async def checkWakeupWords(conn, text):
|
||||
enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"]
|
||||
enable_wakeup_words_response_cache = conn.config[
|
||||
"enable_wakeup_words_response_cache"
|
||||
]
|
||||
"""是否开启唤醒词加速"""
|
||||
if not enable_wakeup_words_response_cache:
|
||||
return False
|
||||
@@ -36,7 +44,10 @@ async def checkWakeupWords(conn, text):
|
||||
asyncio.create_task(wakeupWordsResponse(conn))
|
||||
return False
|
||||
opus_packets, duration = conn.tts.audio_to_opus_data(file)
|
||||
conn.audio_play_queue.put((opus_packets, text, 0))
|
||||
text_hello = WAKEUP_CONFIG["text"]
|
||||
if not text_hello:
|
||||
text_hello = text
|
||||
conn.audio_play_queue.put((opus_packets, text_hello, 0))
|
||||
if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
|
||||
asyncio.create_task(wakeupWordsResponse(conn))
|
||||
return True
|
||||
@@ -47,7 +58,7 @@ def getWakeupWordFile(file_name):
|
||||
for file in os.listdir(WAKEUP_CONFIG["dir"]):
|
||||
if file.startswith("my_" + file_name):
|
||||
"""避免缓存文件是一个空文件"""
|
||||
if os.stat(f"config/assets/{file}").st_size > (5 * 1024):
|
||||
if os.stat(f"config/assets/{file}").st_size > (15 * 1024):
|
||||
return f"config/assets/{file}"
|
||||
|
||||
"""查找config/assets/目录下名称为wakeup_words的文件"""
|
||||
@@ -62,13 +73,18 @@ async def wakeupWordsResponse(conn):
|
||||
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
|
||||
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
|
||||
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
|
||||
|
||||
if tts_file is not None and os.path.exists(tts_file):
|
||||
file_type = os.path.splitext(tts_file)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip('.')
|
||||
file_type = file_type.lstrip(".")
|
||||
old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"])
|
||||
if old_file is not None:
|
||||
os.remove(old_file)
|
||||
"""将文件挪到"wakeup_words.mp3"""
|
||||
shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type)
|
||||
shutil.move(
|
||||
tts_file,
|
||||
WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type,
|
||||
)
|
||||
WAKEUP_CONFIG["create_time"] = time.time()
|
||||
WAKEUP_CONFIG["text"] = result
|
||||
|
||||
@@ -37,6 +37,7 @@ async def check_direct_exit(conn, text):
|
||||
for cmd in cmd_exit:
|
||||
if text == cmd:
|
||||
logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
|
||||
await send_stt_message(conn, text)
|
||||
await conn.close()
|
||||
return True
|
||||
return False
|
||||
@@ -44,7 +45,7 @@ async def check_direct_exit(conn, text):
|
||||
|
||||
async def analyze_intent_with_llm(conn, text):
|
||||
"""使用LLM分析用户意图"""
|
||||
if not hasattr(conn, 'intent') or not conn.intent:
|
||||
if not hasattr(conn, "intent") or not conn.intent:
|
||||
logger.bind(tag=TAG).warning("意图识别服务未初始化")
|
||||
return None
|
||||
|
||||
@@ -68,7 +69,9 @@ async def process_intent_result(conn, intent_result, original_text):
|
||||
# 检查是否有function_call
|
||||
if "function_call" in intent_data:
|
||||
# 直接从意图识别获取了function_call
|
||||
logger.bind(tag=TAG).info(f"检测到function_call格式的意图结果: {intent_data['function_call']['name']}")
|
||||
logger.bind(tag=TAG).info(
|
||||
f"检测到function_call格式的意图结果: {intent_data['function_call']['name']}"
|
||||
)
|
||||
function_name = intent_data["function_call"]["name"]
|
||||
if function_name == "continue_chat":
|
||||
return False
|
||||
@@ -82,7 +85,7 @@ async def process_intent_result(conn, intent_result, original_text):
|
||||
function_call_data = {
|
||||
"name": function_name,
|
||||
"id": str(uuid.uuid4().hex),
|
||||
"arguments": function_args
|
||||
"arguments": function_args,
|
||||
}
|
||||
|
||||
await send_stt_message(conn, original_text)
|
||||
@@ -90,16 +93,24 @@ async def process_intent_result(conn, intent_result, original_text):
|
||||
# 使用executor执行函数调用和结果处理
|
||||
def process_function_call():
|
||||
conn.dialogue.put(Message(role="user", content=original_text))
|
||||
result = conn.func_handler.handle_llm_function_call(conn, function_call_data)
|
||||
if result and function_name != 'play_music':
|
||||
result = conn.func_handler.handle_llm_function_call(
|
||||
conn, function_call_data
|
||||
)
|
||||
if result and function_name != "play_music":
|
||||
# 获取当前最新的文本索引
|
||||
text = result.response
|
||||
if text is None:
|
||||
text = result.result
|
||||
if text is not None:
|
||||
text_index = conn.tts_last_text_index + 1 if hasattr(conn, 'tts_last_text_index') else 0
|
||||
text_index = (
|
||||
conn.tts_last_text_index + 1
|
||||
if hasattr(conn, "tts_last_text_index")
|
||||
else 0
|
||||
)
|
||||
conn.recode_first_last_text(text, text_index)
|
||||
future = conn.executor.submit(conn.speak_and_play, text, text_index)
|
||||
future = conn.executor.submit(
|
||||
conn.speak_and_play, text, text_index
|
||||
)
|
||||
conn.llm_finish_task = True
|
||||
conn.tts_queue.put(future)
|
||||
conn.dialogue.put(Message(role="assistant", content=text))
|
||||
@@ -120,10 +131,14 @@ def extract_text_in_brackets(s):
|
||||
:param s: 输入字符串
|
||||
:return: 中括号内的文字,如果不存在则返回空字符串
|
||||
"""
|
||||
left_bracket_index = s.find('[')
|
||||
right_bracket_index = s.find(']')
|
||||
left_bracket_index = s.find("[")
|
||||
right_bracket_index = s.find("]")
|
||||
|
||||
if left_bracket_index != -1 and right_bracket_index != -1 and left_bracket_index < right_bracket_index:
|
||||
return s[left_bracket_index + 1:right_bracket_index]
|
||||
if (
|
||||
left_bracket_index != -1
|
||||
and right_bracket_index != -1
|
||||
and left_bracket_index < right_bracket_index
|
||||
):
|
||||
return s[left_bracket_index + 1 : right_bracket_index]
|
||||
else:
|
||||
return ""
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import json
|
||||
import asyncio
|
||||
from config.logger import setup_logging
|
||||
from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType
|
||||
from plugins_func.register import (
|
||||
device_type_registry,
|
||||
register_function,
|
||||
ActionResponse,
|
||||
Action,
|
||||
ToolType,
|
||||
)
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -14,10 +20,13 @@ def wrap_async_function(async_func):
|
||||
try:
|
||||
# 获取连接对象(第一个参数)
|
||||
conn = args[0]
|
||||
if not hasattr(conn, 'loop'):
|
||||
if not hasattr(conn, "loop"):
|
||||
logger.bind(tag=TAG).error("Connection对象没有loop属性")
|
||||
return ActionResponse(Action.ERROR, "Connection对象没有loop属性",
|
||||
"执行操作时出错: Connection对象没有loop属性")
|
||||
return ActionResponse(
|
||||
Action.ERROR,
|
||||
"Connection对象没有loop属性",
|
||||
"执行操作时出错: Connection对象没有loop属性",
|
||||
)
|
||||
|
||||
# 使用conn对象中的事件循环
|
||||
loop = conn.loop
|
||||
@@ -37,11 +46,14 @@ def create_iot_function(device_name, method_name, method_info):
|
||||
根据IOT设备描述生成通用的控制函数
|
||||
"""
|
||||
|
||||
async def iot_control_function(conn, response_success=None, response_failure=None, **params):
|
||||
async def iot_control_function(
|
||||
conn, response_success=None, response_failure=None, **params
|
||||
):
|
||||
try:
|
||||
# 打印响应参数
|
||||
logger.bind(tag=TAG).info(
|
||||
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
|
||||
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
|
||||
)
|
||||
|
||||
# 发送控制命令
|
||||
await send_iot_conn(conn, device_name, method_name, params)
|
||||
@@ -51,14 +63,15 @@ def create_iot_function(device_name, method_name, method_info):
|
||||
# 生成结果信息
|
||||
result = f"{device_name}的{method_name}操作执行成功"
|
||||
|
||||
|
||||
# 处理响应中可能的占位符
|
||||
response = response_success
|
||||
# 替换{value}占位符
|
||||
for param_name, param_value in params.items():
|
||||
# 先尝试直接替换参数值
|
||||
if "{" + param_name + "}" in response:
|
||||
response = response.replace("{" + param_name + "}", str(param_value))
|
||||
response = response.replace(
|
||||
"{" + param_name + "}", str(param_value)
|
||||
)
|
||||
|
||||
# 如果有{value}占位符,用相关参数替换
|
||||
if "{value}" in response:
|
||||
@@ -86,7 +99,8 @@ def create_iot_query_function(device_name, prop_name, prop_info):
|
||||
try:
|
||||
# 打印响应参数
|
||||
logger.bind(tag=TAG).info(
|
||||
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
|
||||
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
|
||||
)
|
||||
|
||||
value = await get_iot_status(conn, device_name, prop_name)
|
||||
|
||||
@@ -126,7 +140,7 @@ class IotDescriptor:
|
||||
# 根据描述创建属性
|
||||
for key, value in properties.items():
|
||||
property_item = globals()[key] = {}
|
||||
property_item['name'] = key
|
||||
property_item["name"] = key
|
||||
property_item["description"] = value["description"]
|
||||
if value["type"] == "number":
|
||||
property_item["value"] = 0
|
||||
@@ -140,7 +154,7 @@ class IotDescriptor:
|
||||
for key, value in methods.items():
|
||||
method = globals()[key] = {}
|
||||
method["description"] = value["description"]
|
||||
method['name'] = key
|
||||
method["name"] = key
|
||||
for k, v in value["parameters"].items():
|
||||
method[k] = {}
|
||||
method[k]["description"] = v["description"]
|
||||
@@ -177,19 +191,21 @@ def register_device_type(descriptor):
|
||||
"properties": {
|
||||
"response_success": {
|
||||
"type": "string",
|
||||
"description": f"查询成功时的友好回复,必须使用{{value}}作为占位符表示查询到的值"
|
||||
"description": f"查询成功时的友好回复,必须使用{{value}}作为占位符表示查询到的值",
|
||||
},
|
||||
"response_failure": {
|
||||
"type": "string",
|
||||
"description": f"查询失败时的友好回复,例如:'无法获取{device_name}的{prop_info['description']}'"
|
||||
}
|
||||
"description": f"查询失败时的友好回复,例如:'无法获取{device_name}的{prop_info['description']}'",
|
||||
},
|
||||
},
|
||||
"required": ["response_success", "response_failure"]
|
||||
}
|
||||
}
|
||||
"required": ["response_success", "response_failure"],
|
||||
},
|
||||
},
|
||||
}
|
||||
query_func = create_iot_query_function(device_name, prop_name, prop_info)
|
||||
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func)
|
||||
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
|
||||
query_func
|
||||
)
|
||||
functions[func_name] = decorated_func
|
||||
|
||||
# 为每个方法创建控制函数
|
||||
@@ -200,22 +216,24 @@ def register_device_type(descriptor):
|
||||
parameters = {
|
||||
param_name: {
|
||||
"type": param_info["type"],
|
||||
"description": param_info["description"]
|
||||
"description": param_info["description"],
|
||||
}
|
||||
for param_name, param_info in method_info["parameters"].items()
|
||||
}
|
||||
|
||||
# 添加响应参数
|
||||
parameters.update({
|
||||
"response_success": {
|
||||
"type": "string",
|
||||
"description": "操作成功时的友好回复,关于该设备的操作结果,设备名称尽量使用description中的名称"
|
||||
},
|
||||
"response_failure": {
|
||||
"type": "string",
|
||||
"description": "操作失败时的友好回复,关于该设备的操作结果,设备名称尽量使用description中的名称"
|
||||
parameters.update(
|
||||
{
|
||||
"response_success": {
|
||||
"type": "string",
|
||||
"description": "操作成功时的友好回复,关于该设备的操作结果,设备名称尽量使用description中的名称",
|
||||
},
|
||||
"response_failure": {
|
||||
"type": "string",
|
||||
"description": "操作失败时的友好回复,关于该设备的操作结果,设备名称尽量使用description中的名称",
|
||||
},
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
# 构建必须参数列表(原有参数 + 响应参数)
|
||||
required_params = list(method_info["parameters"].keys())
|
||||
@@ -229,12 +247,14 @@ def register_device_type(descriptor):
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": parameters,
|
||||
"required": required_params
|
||||
}
|
||||
}
|
||||
"required": required_params,
|
||||
},
|
||||
},
|
||||
}
|
||||
control_func = create_iot_function(device_name, method_name, method_info)
|
||||
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func)
|
||||
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
|
||||
control_func
|
||||
)
|
||||
functions[func_name] = decorated_func
|
||||
|
||||
device_type_registry.register_device_type(type_id, functions)
|
||||
@@ -243,13 +263,24 @@ def register_device_type(descriptor):
|
||||
|
||||
# 用于接受前端设备推送的搜索iot描述
|
||||
async def handleIotDescriptors(conn, descriptors):
|
||||
wait_max_time = 5
|
||||
while conn.func_handler is None or not conn.func_handler.finish_init:
|
||||
await asyncio.sleep(1)
|
||||
wait_max_time -= 1
|
||||
if wait_max_time <= 0:
|
||||
logger.bind(tag=TAG).error("连接对象没有func_handler")
|
||||
return
|
||||
"""处理物联网描述"""
|
||||
functions_changed = False
|
||||
|
||||
for descriptor in descriptors:
|
||||
# 创建IOT设备描述符
|
||||
iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
|
||||
descriptor["methods"])
|
||||
iot_descriptor = IotDescriptor(
|
||||
descriptor["name"],
|
||||
descriptor["description"],
|
||||
descriptor["properties"],
|
||||
descriptor["methods"],
|
||||
)
|
||||
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
|
||||
|
||||
if conn.use_function_call_mode:
|
||||
@@ -258,18 +289,22 @@ async def handleIotDescriptors(conn, descriptors):
|
||||
device_functions = device_type_registry.get_device_functions(type_id)
|
||||
|
||||
# 在连接级注册设备函数
|
||||
if hasattr(conn, 'func_handler'):
|
||||
if hasattr(conn, "func_handler"):
|
||||
for func_name in device_functions:
|
||||
conn.func_handler.function_registry.register_function(func_name)
|
||||
logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}")
|
||||
logger.bind(tag=TAG).info(
|
||||
f"注册IOT函数到function handler: {func_name}"
|
||||
)
|
||||
functions_changed = True
|
||||
|
||||
# 如果注册了新函数,更新function描述列表
|
||||
if functions_changed and hasattr(conn, 'func_handler'):
|
||||
if functions_changed and hasattr(conn, "func_handler"):
|
||||
conn.func_handler.upload_functions_desc()
|
||||
func_names = conn.func_handler.current_support_functions()
|
||||
logger.bind(tag=TAG).info(f"设备类型: {type_id}")
|
||||
logger.bind(tag=TAG).info(f"更新function描述列表完成,当前支持的函数: {func_names}")
|
||||
logger.bind(tag=TAG).info(
|
||||
f"更新function描述列表完成,当前支持的函数: {func_names}"
|
||||
)
|
||||
|
||||
|
||||
async def handleIotStatus(conn, states):
|
||||
@@ -281,11 +316,15 @@ async def handleIotStatus(conn, states):
|
||||
for k, v in state["state"].items():
|
||||
if property_item["name"] == k:
|
||||
if type(v) != type(property_item["value"]):
|
||||
logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
|
||||
logger.bind(tag=TAG).error(
|
||||
f"属性{property_item['name']}的值类型不匹配"
|
||||
)
|
||||
break
|
||||
else:
|
||||
property_item["value"] = v
|
||||
logger.bind(tag=TAG).info(f"物联网状态更新: {key} , {property_item['name']} = {v}")
|
||||
logger.bind(tag=TAG).info(
|
||||
f"物联网状态更新: {key} , {property_item['name']} = {v}"
|
||||
)
|
||||
break
|
||||
break
|
||||
|
||||
@@ -308,10 +347,14 @@ async def set_iot_status(conn, name, property_name, value):
|
||||
for property_item in iot_descriptor.properties:
|
||||
if property_item["name"] == property_name:
|
||||
if type(value) != type(property_item["value"]):
|
||||
logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
|
||||
logger.bind(tag=TAG).error(
|
||||
f"属性{property_item['name']}的值类型不匹配"
|
||||
)
|
||||
return
|
||||
property_item["value"] = value
|
||||
logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}")
|
||||
logger.bind(tag=TAG).info(
|
||||
f"物联网状态更新: {name} , {property_name} = {value}"
|
||||
)
|
||||
return
|
||||
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
|
||||
|
||||
@@ -324,15 +367,19 @@ async def send_iot_conn(conn, name, method_name, parameters):
|
||||
for method in value.methods:
|
||||
# 找到了方法
|
||||
if method["name"] == method_name:
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"name": name,
|
||||
"method": method_name,
|
||||
"parameters": parameters
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name": name,
|
||||
"method": method_name,
|
||||
"parameters": parameters,
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
}))
|
||||
)
|
||||
)
|
||||
return
|
||||
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
|
||||
|
||||
@@ -21,7 +21,9 @@ async def handleAudioMessage(conn, audio):
|
||||
if have_voice == False and conn.client_have_voice == False:
|
||||
await no_voice_close_connect(conn)
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-5:] # 保留最新的5帧音频内容,解决ASR句首丢字问题
|
||||
conn.asr_audio = conn.asr_audio[
|
||||
-10:
|
||||
] # 保留最新的10帧音频内容,解决ASR句首丢字问题
|
||||
return
|
||||
conn.client_no_voice_last_time = 0.0
|
||||
conn.asr_audio.append(audio)
|
||||
@@ -30,10 +32,12 @@ async def handleAudioMessage(conn, audio):
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
# 音频太短了,无法识别
|
||||
if len(conn.asr_audio) < 10:
|
||||
if len(conn.asr_audio) < 15:
|
||||
conn.asr_server_receive = True
|
||||
else:
|
||||
text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
|
||||
text, file_path = await conn.asr.speech_to_text(
|
||||
conn.asr_audio, conn.session_id
|
||||
)
|
||||
logger.bind(tag=TAG).info(f"识别文本: {text}")
|
||||
text_len, _ = remove_punctuation_and_length(text)
|
||||
if text_len > 0:
|
||||
@@ -47,12 +51,12 @@ async def handleAudioMessage(conn, audio):
|
||||
async def startToChat(conn, text):
|
||||
# 首先进行意图分析
|
||||
intent_handled = await handle_user_intent(conn, text)
|
||||
|
||||
|
||||
if intent_handled:
|
||||
# 如果意图已被处理,不再进行聊天
|
||||
conn.asr_server_receive = True
|
||||
return
|
||||
|
||||
|
||||
# 意图未被处理,继续常规聊天流程
|
||||
await send_stt_message(conn, text)
|
||||
if conn.use_function_call_mode:
|
||||
@@ -67,10 +71,17 @@ async def no_voice_close_connect(conn):
|
||||
conn.client_no_voice_last_time = time.time() * 1000
|
||||
else:
|
||||
no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
|
||||
close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
|
||||
if not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time:
|
||||
close_connection_no_voice_time = conn.config.get(
|
||||
"close_connection_no_voice_time", 120
|
||||
)
|
||||
if (
|
||||
not conn.close_after_chat
|
||||
and no_voice_time > 1000 * close_connection_no_voice_time
|
||||
):
|
||||
conn.close_after_chat = True
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
prompt = "请你以“时间过得真快”未来头,用富有感情、依依不舍的话来结束这场对话吧。"
|
||||
prompt = (
|
||||
"请你以“时间过得真快”未来头,用富有感情、依依不舍的话来结束这场对话吧。"
|
||||
)
|
||||
await startToChat(conn, prompt)
|
||||
|
||||
@@ -2,7 +2,10 @@ from config.logger import setup_logging
|
||||
import json
|
||||
import asyncio
|
||||
import time
|
||||
from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
|
||||
from core.utils.util import (
|
||||
remove_punctuation_and_length,
|
||||
get_string_no_punctuation_or_emoji,
|
||||
)
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -21,10 +24,11 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
|
||||
|
||||
# 发送结束消息(如果是最后一个文本)
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
await send_tts_message(conn, "stop", None)
|
||||
if conn.close_after_chat:
|
||||
await conn.close()
|
||||
|
||||
|
||||
# 播放音频
|
||||
async def sendAudio(conn, audios):
|
||||
# 流控参数优化
|
||||
@@ -36,13 +40,12 @@ async def sendAudio(conn, audios):
|
||||
pre_buffer = min(3, len(audios))
|
||||
for i in range(pre_buffer):
|
||||
await conn.websocket.send(audios[i])
|
||||
conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms")
|
||||
|
||||
# 正常播放剩余帧
|
||||
for opus_packet in audios[pre_buffer:]:
|
||||
if conn.client_abort:
|
||||
return
|
||||
|
||||
|
||||
# 计算预期发送时间
|
||||
expected_time = start_time + (play_position / 1000)
|
||||
current_time = time.perf_counter()
|
||||
@@ -51,19 +54,13 @@ async def sendAudio(conn, audios):
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
await conn.websocket.send(opus_packet)
|
||||
conn.logger.bind(tag=TAG).debug(f"发送帧,位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms")
|
||||
|
||||
play_position += frame_duration
|
||||
|
||||
|
||||
|
||||
async def send_tts_message(conn, state, text=None):
|
||||
"""发送 TTS 状态消息"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": conn.session_id
|
||||
}
|
||||
message = {"type": "tts", "state": state, "session_id": conn.session_id}
|
||||
if text is not None:
|
||||
message["text"] = text
|
||||
|
||||
@@ -72,7 +69,9 @@ async def send_tts_message(conn, state, text=None):
|
||||
# 播放提示音
|
||||
tts_notify = conn.config.get("enable_stop_tts_notify", False)
|
||||
if tts_notify:
|
||||
stop_tts_notify_voice = conn.config.get("stop_tts_notify_voice", "config/assets/tts_notify.mp3")
|
||||
stop_tts_notify_voice = conn.config.get(
|
||||
"stop_tts_notify_voice", "config/assets/tts_notify.mp3"
|
||||
)
|
||||
audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
|
||||
await sendAudio(conn, audios)
|
||||
# 清除服务端讲话状态
|
||||
@@ -85,16 +84,17 @@ async def send_tts_message(conn, state, text=None):
|
||||
async def send_stt_message(conn, text):
|
||||
"""发送 STT 状态消息"""
|
||||
stt_text = get_string_no_punctuation_or_emoji(text)
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": stt_text,
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
await conn.websocket.send(
|
||||
json.dumps({
|
||||
"type": "llm",
|
||||
"text": "😊",
|
||||
"emotion": "happy",
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
|
||||
)
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "llm",
|
||||
"text": "😊",
|
||||
"emotion": "happy",
|
||||
"session_id": conn.session_id,
|
||||
}
|
||||
)
|
||||
)
|
||||
await send_tts_message(conn, "start")
|
||||
|
||||
@@ -6,6 +6,7 @@ from core.utils.util import remove_punctuation_and_length
|
||||
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
|
||||
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
|
||||
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
|
||||
import asyncio
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -34,7 +35,7 @@ async def handleTextMessage(conn, message):
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = True
|
||||
if len(conn.asr_audio) > 0:
|
||||
await handleAudioMessage(conn, b'')
|
||||
await handleAudioMessage(conn, b"")
|
||||
elif msg_json["state"] == "detect":
|
||||
conn.asr_server_receive = False
|
||||
conn.client_have_voice = False
|
||||
@@ -57,8 +58,8 @@ async def handleTextMessage(conn, message):
|
||||
await startToChat(conn, text)
|
||||
elif msg_json["type"] == "iot":
|
||||
if "descriptors" in msg_json:
|
||||
await handleIotDescriptors(conn, msg_json["descriptors"])
|
||||
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
|
||||
if "states" in msg_json:
|
||||
await handleIotStatus(conn, msg_json["states"])
|
||||
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
|
||||
except json.JSONDecodeError:
|
||||
await conn.websocket.send(message)
|
||||
|
||||
@@ -6,11 +6,12 @@ from core.providers.llm.base import LLMProviderBase
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.api_key = config["api_key"]
|
||||
self.mode = config.get("mode", "chat-messages")
|
||||
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
|
||||
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
|
||||
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
@@ -22,57 +23,58 @@ class LLMProvider(LLMProviderBase):
|
||||
# 发起流式请求
|
||||
if self.mode == "chat-messages":
|
||||
request_json = {
|
||||
"query": last_msg["content"],
|
||||
"response_mode": "streaming",
|
||||
"user": session_id,
|
||||
"inputs": {},
|
||||
"conversation_id": conversation_id
|
||||
}
|
||||
"query": last_msg["content"],
|
||||
"response_mode": "streaming",
|
||||
"user": session_id,
|
||||
"inputs": {},
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
elif self.mode == "workflows/run":
|
||||
request_json = {
|
||||
"inputs": {"query": last_msg["content"]},
|
||||
"response_mode": "streaming",
|
||||
"user": session_id
|
||||
"user": session_id,
|
||||
}
|
||||
elif self.mode == "completion-messages":
|
||||
request_json = {
|
||||
"inputs": {"query": last_msg["content"]},
|
||||
"response_mode": "streaming",
|
||||
"user": session_id
|
||||
"user": session_id,
|
||||
}
|
||||
|
||||
with requests.post(
|
||||
f"{self.base_url}/{self.mode}",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json=request_json,
|
||||
stream=True
|
||||
f"{self.base_url}/{self.mode}",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json=request_json,
|
||||
stream=True,
|
||||
) as r:
|
||||
if self.mode == "chat-messages":
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
if line.startswith(b"data: "):
|
||||
event = json.loads(line[6:])
|
||||
# 如果没有找到conversation_id,则获取此次conversation_id
|
||||
if not conversation_id:
|
||||
conversation_id = event.get('conversation_id')
|
||||
self.session_conversation_map[session_id] = conversation_id # 更新映射
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
conversation_id = event.get("conversation_id")
|
||||
self.session_conversation_map[session_id] = (
|
||||
conversation_id # 更新映射
|
||||
)
|
||||
if event.get("answer"):
|
||||
yield event["answer"]
|
||||
elif self.mode == "workflows/run":
|
||||
for line in r.iter_lines():
|
||||
# logger.bind(tag=TAG).info(f"chat message response: {line}")
|
||||
if line.startswith(b'data: '):
|
||||
if line.startswith(b"data: "):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('event') == "workflow_finished":
|
||||
if event['data']['status'] == "succeeded":
|
||||
yield event['data']['outputs']['answer']
|
||||
if event.get("event") == "workflow_finished":
|
||||
if event["data"]["status"] == "succeeded":
|
||||
yield event["data"]["outputs"]["answer"]
|
||||
else:
|
||||
yield "【服务响应异常】"
|
||||
elif self.mode == "completion-messages":
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
if line.startswith(b"data: "):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
if event.get("answer"):
|
||||
yield event["answer"]
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime, timezone
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.appid = config.get("appid")
|
||||
self.secret_id = config.get("secret_id")
|
||||
self.secret_key = config.get("secret_key")
|
||||
self.voice = config.get("voice")
|
||||
self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点
|
||||
self.region = config.get("region")
|
||||
self.output_file = config.get("output_dir")
|
||||
|
||||
def _get_auth_headers(self, request_body):
|
||||
"""生成鉴权请求头"""
|
||||
# 获取当前UTC时间戳
|
||||
timestamp = int(time.time())
|
||||
|
||||
# 使用UTC时间计算日期
|
||||
utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d')
|
||||
|
||||
# 服务名称必须是 "tts"
|
||||
service = "tts"
|
||||
|
||||
# 拼接凭证范围
|
||||
credential_scope = f"{utc_date}/{service}/tc3_request"
|
||||
|
||||
# 使用TC3-HMAC-SHA256签名方法
|
||||
algorithm = "TC3-HMAC-SHA256"
|
||||
|
||||
# 构建规范请求字符串
|
||||
http_request_method = "POST"
|
||||
canonical_uri = "/"
|
||||
canonical_querystring = ""
|
||||
|
||||
# 请求头必须包含host和content-type,且按字典序排列
|
||||
canonical_headers = (
|
||||
f"content-type:application/json\n"
|
||||
f"host:tts.tencentcloudapi.com\n"
|
||||
)
|
||||
signed_headers = "content-type;host"
|
||||
|
||||
# 请求体哈希值
|
||||
payload = json.dumps(request_body)
|
||||
payload_hash = hashlib.sha256(payload.encode('utf-8')).hexdigest()
|
||||
|
||||
# 构建规范请求字符串
|
||||
canonical_request = (
|
||||
f"{http_request_method}\n"
|
||||
f"{canonical_uri}\n"
|
||||
f"{canonical_querystring}\n"
|
||||
f"{canonical_headers}\n"
|
||||
f"{signed_headers}\n"
|
||||
f"{payload_hash}"
|
||||
)
|
||||
|
||||
# 计算规范请求的哈希值
|
||||
hashed_canonical_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
|
||||
|
||||
# 构建待签名字符串
|
||||
string_to_sign = (
|
||||
f"{algorithm}\n"
|
||||
f"{timestamp}\n"
|
||||
f"{credential_scope}\n"
|
||||
f"{hashed_canonical_request}"
|
||||
)
|
||||
|
||||
# 计算签名密钥
|
||||
secret_date = self._hmac_sha256(f"TC3{self.secret_key}".encode('utf-8'), utc_date)
|
||||
secret_service = self._hmac_sha256(secret_date, service)
|
||||
secret_signing = self._hmac_sha256(secret_service, "tc3_request")
|
||||
|
||||
# 计算签名
|
||||
signature = hmac.new(
|
||||
secret_signing,
|
||||
string_to_sign.encode('utf-8'),
|
||||
hashlib.sha256
|
||||
).hexdigest()
|
||||
|
||||
# 构建授权头
|
||||
authorization = (
|
||||
f"{algorithm} "
|
||||
f"Credential={self.secret_id}/{credential_scope}, "
|
||||
f"SignedHeaders={signed_headers}, "
|
||||
f"Signature={signature}"
|
||||
)
|
||||
|
||||
# 构建请求头
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Host": "tts.tencentcloudapi.com",
|
||||
"Authorization": authorization,
|
||||
"X-TC-Action": "TextToVoice",
|
||||
"X-TC-Timestamp": str(timestamp),
|
||||
"X-TC-Version": "2019-08-23",
|
||||
"X-TC-Region": self.region,
|
||||
"X-TC-Language": "zh-CN"
|
||||
}
|
||||
|
||||
return headers
|
||||
|
||||
def _hmac_sha256(self, key, msg):
|
||||
"""HMAC-SHA256加密"""
|
||||
if isinstance(msg, str):
|
||||
msg = msg.encode('utf-8')
|
||||
return hmac.new(key, msg, hashlib.sha256).digest()
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
# 构建请求体
|
||||
request_json = {
|
||||
"Text": text, # 合成语音的源文本
|
||||
"SessionId": str(uuid.uuid4()), # 会话ID,随机生成
|
||||
"VoiceType": int(self.voice), # 音色
|
||||
}
|
||||
|
||||
try:
|
||||
# 获取请求头(每次请求都重新生成,以确保时间戳和签名是最新的)
|
||||
headers = self._get_auth_headers(request_json)
|
||||
|
||||
# 发送请求
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=headers)
|
||||
|
||||
# 检查响应
|
||||
if resp.status_code == 200:
|
||||
response_data = resp.json()
|
||||
|
||||
# 检查是否成功
|
||||
if response_data.get("Response", {}).get("Error") is not None:
|
||||
error_info = response_data["Response"]["Error"]
|
||||
raise Exception(f"API返回错误: {error_info['Code']}: {error_info['Message']}")
|
||||
|
||||
# 提取音频数据
|
||||
audio_data = response_data["Response"].get("Audio")
|
||||
if audio_data:
|
||||
# 解码Base64音频数据并保存
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(base64.b64decode(audio_data))
|
||||
else:
|
||||
raise Exception(f"{__name__}: 没有返回音频数据: {response_data}")
|
||||
else:
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -12,50 +12,72 @@ class WebSocketServer:
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
self.logger = setup_logging()
|
||||
self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = self._create_processing_instances()
|
||||
self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = (
|
||||
self._create_processing_instances()
|
||||
)
|
||||
self.active_connections = set() # 添加全局连接记录
|
||||
|
||||
def _create_processing_instances(self):
|
||||
memory_cls_name = self.config["selected_module"].get("Memory", "nomem") # 默认使用nomem
|
||||
has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
|
||||
memory_cls_name = self.config["selected_module"].get(
|
||||
"Memory", "nomem"
|
||||
) # 默认使用nomem
|
||||
has_memory_cfg = (
|
||||
self.config.get("Memory") and memory_cls_name in self.config["Memory"]
|
||||
)
|
||||
memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
|
||||
|
||||
"""创建处理模块实例"""
|
||||
return (
|
||||
vad.create_instance(
|
||||
self.config["selected_module"]["VAD"],
|
||||
self.config["VAD"][self.config["selected_module"]["VAD"]]
|
||||
self.config["VAD"][self.config["selected_module"]["VAD"]],
|
||||
),
|
||||
asr.create_instance(
|
||||
self.config["selected_module"]["ASR"]
|
||||
if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
|
||||
else
|
||||
self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
|
||||
(
|
||||
self.config["selected_module"]["ASR"]
|
||||
if not "type"
|
||||
in self.config["ASR"][self.config["selected_module"]["ASR"]]
|
||||
else self.config["ASR"][self.config["selected_module"]["ASR"]][
|
||||
"type"
|
||||
]
|
||||
),
|
||||
self.config["ASR"][self.config["selected_module"]["ASR"]],
|
||||
self.config["delete_audio"]
|
||||
self.config["delete_audio"],
|
||||
),
|
||||
llm.create_instance(
|
||||
self.config["selected_module"]["LLM"]
|
||||
if not 'type' in self.config["LLM"][self.config["selected_module"]["LLM"]]
|
||||
else
|
||||
self.config["LLM"][self.config["selected_module"]["LLM"]]['type'],
|
||||
(
|
||||
self.config["selected_module"]["LLM"]
|
||||
if not "type"
|
||||
in self.config["LLM"][self.config["selected_module"]["LLM"]]
|
||||
else self.config["LLM"][self.config["selected_module"]["LLM"]][
|
||||
"type"
|
||||
]
|
||||
),
|
||||
self.config["LLM"][self.config["selected_module"]["LLM"]],
|
||||
),
|
||||
tts.create_instance(
|
||||
self.config["selected_module"]["TTS"]
|
||||
if not 'type' in self.config["TTS"][self.config["selected_module"]["TTS"]]
|
||||
else
|
||||
self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
|
||||
(
|
||||
self.config["selected_module"]["TTS"]
|
||||
if not "type"
|
||||
in self.config["TTS"][self.config["selected_module"]["TTS"]]
|
||||
else self.config["TTS"][self.config["selected_module"]["TTS"]][
|
||||
"type"
|
||||
]
|
||||
),
|
||||
self.config["TTS"][self.config["selected_module"]["TTS"]],
|
||||
self.config["delete_audio"]
|
||||
self.config["delete_audio"],
|
||||
),
|
||||
memory.create_instance(memory_cls_name, memory_cfg),
|
||||
intent.create_instance(
|
||||
self.config["selected_module"]["Intent"]
|
||||
if not 'type' in self.config["Intent"][self.config["selected_module"]["Intent"]]
|
||||
else
|
||||
self.config["Intent"][self.config["selected_module"]["Intent"]]["type"],
|
||||
self.config["Intent"][self.config["selected_module"]["Intent"]]
|
||||
(
|
||||
self.config["selected_module"]["Intent"]
|
||||
if not "type"
|
||||
in self.config["Intent"][self.config["selected_module"]["Intent"]]
|
||||
else self.config["Intent"][
|
||||
self.config["selected_module"]["Intent"]
|
||||
]["type"]
|
||||
),
|
||||
self.config["Intent"][self.config["selected_module"]["Intent"]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -64,19 +86,33 @@ class WebSocketServer:
|
||||
host = server_config["ip"]
|
||||
port = server_config["port"]
|
||||
|
||||
self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
|
||||
self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址,请勿用浏览器访问=======")
|
||||
async with websockets.serve(
|
||||
self._handle_connection,
|
||||
host,
|
||||
port
|
||||
):
|
||||
self.logger.bind(tag=TAG).info(
|
||||
"Server is running at ws://{}:{}/xiaozhi/v1/", get_local_ip(), port
|
||||
)
|
||||
self.logger.bind(tag=TAG).info(
|
||||
"=======上面的地址是websocket协议地址,请勿用浏览器访问======="
|
||||
)
|
||||
self.logger.bind(tag=TAG).info(
|
||||
"如想测试websocket请用谷歌浏览器打开test目录下的test_page.html"
|
||||
)
|
||||
self.logger.bind(tag=TAG).info(
|
||||
"=============================================================\n"
|
||||
)
|
||||
async with websockets.serve(self._handle_connection, host, port):
|
||||
await asyncio.Future()
|
||||
|
||||
async def _handle_connection(self, websocket):
|
||||
"""处理新连接,每次创建独立的ConnectionHandler"""
|
||||
# 创建ConnectionHandler时传入当前server实例
|
||||
handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._memory, self.intent)
|
||||
handler = ConnectionHandler(
|
||||
self.config,
|
||||
self._vad,
|
||||
self._asr,
|
||||
self._llm,
|
||||
self._tts,
|
||||
self._memory,
|
||||
self.intent,
|
||||
)
|
||||
self.active_connections.add(handler)
|
||||
try:
|
||||
await handler.handle_connection(websocket)
|
||||
|
||||
@@ -6,6 +6,7 @@ import asyncio
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def _get_device_status(conn, device_name, device_type, property_name):
|
||||
"""获取设备状态"""
|
||||
status = await get_iot_status(conn, device_type, property_name)
|
||||
@@ -13,6 +14,7 @@ async def _get_device_status(conn, device_name, device_type, property_name):
|
||||
raise Exception(f"你的设备不支持{device_name}控制")
|
||||
return status
|
||||
|
||||
|
||||
async def _set_device_property(conn, device_name, device_type, method_name, property_name, new_value=None, action=None, step=10):
|
||||
"""设置设备属性"""
|
||||
current_value = await _get_device_status(conn, device_name, device_type, property_name)
|
||||
@@ -32,9 +34,11 @@ async def _set_device_property(conn, device_name, device_type, method_name, prop
|
||||
await send_iot_conn(conn, device_type, method_name, {property_name: current_value})
|
||||
return current_value
|
||||
|
||||
|
||||
def _handle_device_action(conn, func, success_message, error_message, *args, **kwargs):
|
||||
"""处理设备操作的通用函数"""
|
||||
future = asyncio.run_coroutine_threadsafe(func(conn, *args, **kwargs), conn.loop)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
func(conn, *args, **kwargs), conn.loop)
|
||||
try:
|
||||
result = future.result()
|
||||
logger.bind(tag=TAG).info(f"{success_message}: {result}")
|
||||
@@ -45,6 +49,7 @@ def _handle_device_action(conn, func, success_message, error_message, *args, **k
|
||||
response = f"{error_message}: {e}"
|
||||
return ActionResponse(action=Action.RESPONSE, result=None, response=response)
|
||||
|
||||
|
||||
# 设备控制
|
||||
handle_device_function_desc = {
|
||||
"type": "function",
|
||||
@@ -52,9 +57,9 @@ handle_device_function_desc = {
|
||||
"name": "handle_device",
|
||||
"description": (
|
||||
"用户想要获取或者设置设备的音量/亮度大小,或者用户觉得声音/亮度过高或过低,或者用户想提高或降低音量/亮度。"
|
||||
"比如用户说现在亮度多少,参数为:device_type:Backlight,action:get。"
|
||||
"比如用户说现在亮度多少,参数为:device_type:Screen,action:get。"
|
||||
"比如用户说设置音量为50,参数为:device_type:Speaker,action:set,value:50。"
|
||||
"比如用户说亮度太高了,参数为:device_type:Backlight,action:lower。"
|
||||
"比如用户说亮度太高了,参数为:device_type:Screen,action:lower。"
|
||||
"比如用户说调大音量,参数为:device_type:Speaker,action:raise。"
|
||||
),
|
||||
"parameters": {
|
||||
@@ -62,7 +67,7 @@ handle_device_function_desc = {
|
||||
"properties": {
|
||||
"device_type": {
|
||||
"type": "string",
|
||||
"description": "设备类型,可选值:Speaker(音量),Backlight(亮度)"
|
||||
"description": "设备类型,可选值:Speaker(音量),Screen(亮度)"
|
||||
},
|
||||
"action": {
|
||||
"type": "string",
|
||||
@@ -78,18 +83,19 @@ handle_device_function_desc = {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@register_function('handle_device', handle_device_function_desc, ToolType.IOT_CTL)
|
||||
def handle_device(conn, device_type: str, action: str, value: int = None):
|
||||
if device_type == "Speaker":
|
||||
method_name, property_name, device_name = "SetVolume", "volume", "音量"
|
||||
elif device_type == "Backlight":
|
||||
elif device_type == "Screen":
|
||||
method_name, property_name, device_name = "SetBrightness", "brightness", "亮度"
|
||||
else:
|
||||
raise Exception(f"未识别的设备类型: {device_type}")
|
||||
|
||||
|
||||
if action not in ["get", "set", "raise", "lower"]:
|
||||
raise Exception(f"未识别的动作名称: {action}")
|
||||
|
||||
|
||||
if action == "get":
|
||||
# get
|
||||
return _handle_device_action(
|
||||
|
||||
@@ -23,5 +23,4 @@ bs4==0.0.2
|
||||
modelscope==1.23.2
|
||||
sherpa_onnx==1.11.0
|
||||
mcp==1.4.1
|
||||
|
||||
cnlunar==0.2.0
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user