diff --git a/Dockerfile-web b/Dockerfile-web index a347a88e..583d2e6b 100644 --- a/Dockerfile-web +++ b/Dockerfile-web @@ -18,9 +18,15 @@ FROM bellsoft/liberica-runtime-container:jre-21-glibc # 安装Nginx和字体库 RUN apk update && \ - apk add --no-cache nginx bash fontconfig ttf-dejavu && \ - apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ msttcorefonts-installer || true && \ - rm -rf /var/cache/apk/* + apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \ + nginx \ + bash \ + fontconfig \ + ttf-dejavu \ + msttcorefonts-installer \ + && ACCEPT_EULA=Y apk add --no-cache msttcorefonts-installer \ + && fc-cache -f -v \ + && rm -rf /var/cache/apk/* # 更新字体缓存 RUN (printf 'YES\n' | update-ms-fonts || true) && fc-cache -f -v diff --git a/docs/index-stream-integration.md b/docs/index-stream-integration.md index 875dbdd8..caf3ae60 100644 --- a/docs/index-stream-integration.md +++ b/docs/index-stream-integration.md @@ -1,9 +1,12 @@ # IndexStreamTTS 使用指南 ## 环境准备 -### 1. 克隆项目 （这里使用的为VLLM的版本） +### 1. 克隆项目 （注意这里使用的为VLLM1.0的Releases版本） ```bash -git clone https://github.com/Ksuriuri/index-tts-vllm.git +https://github.com/Ksuriuri/index-tts-vllm/releases/tag/IndexTTS-vLLM-1.0 +``` +进入解压后的目录 +```bash cd index-tts-vllm ``` @@ -13,7 +16,7 @@ conda create -n index-tts-vllm python=3.12 conda activate index-tts-vllm ``` -### 3. 安装PyTorch +### 3. 安装PyTorch 需要版本为2.8.0（最新版） #### 查看显卡最高支持的版本和实际安装的版本 ```bash nvidia-smi @@ -27,12 +30,11 @@ CUDA Version: 12.8 ```bash Cuda compilation tools, release 12.8, V12.8.89 ``` -#### 那么对应的安装命令 (请注意不要横跨大版本！！！) +#### 那么对应的安装命令（pytorch默认给的是12.8的驱动版本） ```bash -pip install torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 --index-url https://download.pytorch.org/whl/cu128 +pip install torch torchvision ``` -优先建议安装 pytorch 2.7.0（对应 vllm 0.9.0），具体安装指令请参考：[pytorch 官网](https://pytorch.org/get-started/locally/]\) -若显卡不支持，请安装 pytorch 2.5.1（对应 vllm 0.7.3），并将 requirements.txt 中 vllm==0.9.0 修改为 vllm==0.7.3 +需要 pytorch 版本 2.8.0（对应 vllm 0.10.2），具体安装指令请参考：[pytorch 官网](https://pytorch.org/get-started/locally/) ### 4. 安装依赖 ```bash @@ -54,7 +56,7 @@ git lfs install ``` 创建模型目录，并拉取模型 ```bash -mkkdir model_dir +mkdir model_dir cd model_dir git clone https://www.modelscope.cn/IndexTeam/IndexTTS-1.5.git ``` @@ -71,6 +73,9 @@ bash convert_hf_format.sh model_dir/IndexTTS-1.5 ### 6. 更改接口适配一下项目 接口返回数据与项目不适配需要调整一下，使其直接返回音频数据 +```bash +vi api_server.py +``` ```bash @app.post("/tts", responses={ 200: {"content": {"application/octet-stream": {}}}, @@ -104,7 +109,7 @@ async def tts_api(request: Request): vi start_api.sh ``` ### 将下面内容粘贴进去并按:输入wq保存 -#### 脚本中的/home/system/indexTTS/index-tts-vllm/model_dir/IndexTTS-1.5 请自行修改为实际路径 +#### 脚本中的/home/system/index-tts-vllm/model_dir/IndexTTS-1.5 请自行修改为实际路径 ```bash # 激活conda环境 conda activate index-tts-vllm @@ -129,17 +134,35 @@ else echo "已终止进程 $PID_VLLM" fi +# 查找占用VLLM::EngineCore进程 +GPU_PIDS=$(ps aux | grep -E "VLLM|EngineCore" | grep -v grep | awk '{print $2}') + +# 检查是否找到进程号 +if [ -z "$GPU_PIDS" ]; then + echo "没有找到VLLM相关进程" +else + echo "找到VLLM相关进程，进程号为: $GPU_PIDS" + # 先尝试普通kill，等待2秒 + kill $GPU_PIDS + sleep 2 + # 检查进程是否还在 + if ps -p $GPU_PIDS > /dev/null; then + echo "进程仍在运行，强制终止..." + kill -9 $GPU_PIDS + fi + echo "已终止进程 $GPU_PIDS" +fi + # 创建tmp目录（如果不存在） mkdir -p tmp # 后台运行api_server.py，日志重定向到tmp/server.log -export VLLM_USE_V1=0 -nohup python api_server.py --model_dir /home/system/indexTTS/index-tts-vllm/model_dir/IndexTTS-1.5 --port 11996 > tmp/server.log 2>&1 & +nohup python api_server.py --model_dir /home/system/index-tts-vllm/model_dir/IndexTTS-1.5 --port 11996 > tmp/server.log 2>&1 & echo "api_server.py 已在后台运行，日志请查看 tmp/server.log" ``` 给脚本执行权限并运行脚本 ```bash -chmod +x tmp +chmod +x start_api.sh ./start_api.sh ``` 日志会在tmp/server.log中输出，可以通过以下命令查看日志情况 @@ -148,7 +171,7 @@ tail -f tmp/server.log ``` ## 音色配置 index-tts-vllm支持通过配置文件注册自定义音色，支持单音色和混合音色配置。 -在项目根目录下的assets/speaker.json文件中配置自定义音色 +在项目根目录下的assets/speaker.json文件中配置自定义音色 ### 配置格式说明 ```bash { @@ -161,5 +184,5 @@ index-tts-vllm支持通过配置文件注册自定义音色，支持单音色和 ] } ``` -### 注意 +### 注意 （需重启服务进行注册） 添加后需在智控台中添加相应的说话人（单模块则更换相应的voice） \ No newline at end of file diff --git a/docs/mqtt-gateway-integration.md b/docs/mqtt-gateway-integration.md index 09fe83c1..99d183bf 100644 --- a/docs/mqtt-gateway-integration.md +++ b/docs/mqtt-gateway-integration.md @@ -1,11 +1,12 @@ # MQTT 网关部署教程 `xiaozhi-esp32-server`项目，可结合虾哥开源的[xiaozhi-mqtt-gateway](https://github.com/78/xiaozhi-mqtt-gateway) 项目进行简单改造，即可实现小智硬件MQTT+UDP连接。 +本教程分为三部分，你可以根据你是全模块部署还是单模块部署，选择对应的部分接入MQTT网关： +- 第一部分：部署MQTT网关 +- 第二部分：全模块运行实现小智硬件MQTT+UDP连接 +- 第三部分：单模块运行xiaozhi-server实现小智硬件MQTT+UDP连接 -## 准备工作 - -查看你智控台首页底部的版本号，确认你的智控台版本是否是`0.7.7`及以上版本。如果不是，需要升级智控台。 - +## 准备阶段 准备好你的`xiaozhi-server`的`mqtt-websocket`连接地址。在你原来的`websocket地址`基础上，添加`?from=mqtt_gateway`字符，就可以得到`mqtt-websocket`连接地址 1、如果你是源码部署，你的`mqtt-websocket`地址是： @@ -18,7 +19,16 @@ ws://127.0.0.1:8000/xiaozhi/v1?from=mqtt_gateway ws://你宿主机局域网IP:8000/xiaozhi/v1?from=mqtt_gateway ``` -## 第一步 部署MQTT网关 +## 重要提示 + +如果你是服务器部署，需要确保服务器`1883`、`8884`、`8007`端口都对外开放。`8884`选择的协议类型是`UDP`，其他是`TCP`。 + +如果你是服务器部署，需要确保服务器`1883`、`8884`、`8007`端口都对外开放。`8884`选择的协议类型是`UDP`，其他是`TCP`。 + +如果你是服务器部署，需要确保服务器`1883`、`8884`、`8007`端口都对外开放。`8884`选择的协议类型是`UDP`，其他是`TCP`。 + + +## 第一部分：部署MQTT网关 1. 克隆[改造后的xiaozhi-mqtt-gateway项目](https://github.com/xinnan-tech/xiaozhi-mqtt-gateway.git)： ```bash @@ -95,7 +105,9 @@ pm2 logs xz-mqtt pm2 restart xz-mqtt ``` -## 第二步 配置智控台 +## 第二部分：全模块运行实现小智硬件MQTT+UDP连接 + +查看你智控台首页底部的版本号，确认你的智控台版本是否是`0.7.7`及以上版本。如果不是，需要升级智控台。 1. 在智控台顶部，点击`参数管理`，搜索`server.mqtt_gateway`，点击编辑，填入你在`.env`文件中设置的`PUBLIC_IP`+`:`+`MQTT_PORT`。类似这样 ``` @@ -127,7 +139,38 @@ curl 'http://localhost:8002/xiaozhi/ota/' \ {"server_time":{"timestamp":1757567894012,"timeZone":"Asia/Shanghai","timezone_offset":480},"activation":{"code":"460609","message":"http://xiaozhi.server.com\n460609","challenge":"11:22:33:44:55:66"},"firmware":{"version":"1.0.1","url":"http://xiaozhi.server.com:8002/xiaozhi/otaMag/download/NOT_ACTIVATED_FIRMWARE_THIS_IS_A_INVALID_URL"},"websocket":{"url":"ws://192.168.4.23:8000/xiaozhi/v1/"},"mqtt":{"endpoint":"192.168.0.7:1883","client_id":"GID_default@@@11_22_33_44_55_66@@@7b94d69a-9808-4c59-9c9b-704333b38aff","username":"eyJpcCI6IjA6MDowOjA6MDowOjA6MSJ9","password":"Y8XP9xcUhVIN9OmbCHT9ETBiYNE3l3Z07Wk46wV9PE8=","publish_topic":"device-server","subscribe_topic":"devices/p2p/11_22_33_44_55_66"}} ``` -## 第三步 重启小智设备 +由于MQTT信息是需要靠OTA地址下发的，因此只有你保证能正常连接服务器的OTA地址，重启唤醒即可。 + +唤醒后留意mqtt-gateway的日志，确认是否有连接成功的日志。 +``` +pm2 logs xz-mqtt +``` + +## 第三部分：全模块运行实现小智硬件MQTT+UDP连接 + +打开你的`data/.config.yaml`文件，在`server`下找到`mqtt_gateway`填入你在`.env`文件中设置的`PUBLIC_IP`+`:`+`MQTT_PORT`。类似这样 +``` +192.168.0.7:1883 +``` +在`server`下找到`mqtt_signature_key`填入你在`.env`文件中设置的`MQTT_SIGNATURE_KEY`。 + +在`server`下找到`udp_gateway`填入你在`.env`文件中设置的`PUBLIC_IP`+`:`+`UDP_PORT`。类似这样 +``` +192.168.0.7:8884 +``` + +上面的配置完成后，你可以使用curl命令，验证你的ota地址是否会下发mqtt配置，把下面的`http://localhost:8002/xiaozhi/ota/`改成你的ota地址 +``` +curl 'http://localhost:8002/xiaozhi/ota/' \ + -H 'Device-Id: 11:22:33:44:55:66' \ + --data-raw $'{\n "application": {\n "version": "1.0.1",\n "elf_sha256": "1"\n },\n "board": {\n "mac": "11:22:33:44:55:66"\n }\n}' +``` + +如果返回的内容包含`mqtt`相关的配置，说明配置成功。类似这样 +``` +{"server_time":{"timestamp":1758781561083,"timeZone":"GMT+08:00","timezone_offset":480},"activation":{"code":"527111","message":"http://xiaozhi.server.com\n527111","challenge":"11:22:33:44:55:66"},"firmware":{"version":"1.0.1","url":"http://xiaozhi.server.com:8002/xiaozhi/otaMag/download/NOT_ACTIVATED_FIRMWARE_THIS_IS_A_INVALID_URL"},"websocket":{"url":"ws://192.168.1.15:8000/xiaozhi/v1/"},"mqtt":{"endpoint":"192.168.1.15:1883","client_id":"GID_default@@@11_22_33_44_55_66@@@11_22_33_44_55_66","username":"eyJpcCI6IjE5Mi4xNjguMS4xNSJ9","password":"fjAYs49zTJecWqJ3jBt+kqxVn/x7vkXRAc85ak/va7Y=","publish_topic":"device-server","subscribe_topic":"devices/p2p/11_22_33_44_55_66"}} +``` + 由于MQTT信息是需要靠OTA地址下发的，因此只有你保证能正常连接服务器的OTA地址，重启唤醒即可。 唤醒后留意mqtt-gateway的日志，确认是否有连接成功的日志。 diff --git a/docs/paddlespeech-deploy.md b/docs/paddlespeech-deploy.md index 17030fab..94621eb5 100644 --- a/docs/paddlespeech-deploy.md +++ b/docs/paddlespeech-deploy.md @@ -1,5 +1,9 @@ # PaddleSpeechTTS集成xiaozhi服务 +## 重点说明 +- 优点：本地离线部署、速度快 +- 缺点：截止2025年9月25日，默认的模型是中文模型，不支持英文转语音。如果含英文会发不出声音，如需同时支持中英文需要自己训练。 + ## 一、基础环境要求 操作系统：Windows / Linux / WSL 2 @@ -16,15 +20,21 @@ git clone https://github.com/PaddlePaddle/PaddleSpeech.git ``` ### 2.建立虚拟环境 ```bash -#请根据Paddle官方支持的python版本建立环境 ```https://www.paddlepaddle.org.cn/install``` + conda create -n paddle_env python=3.10 -y conda activate paddle_env ``` -### 3.进入paddlespeech目录 +### 3.安装paddle +因CPU架构、GPU架构不同，请根据Paddle官方支持的python版本建立环境 +``` +https://www.paddlepaddle.org.cn/install +``` + +### 4.进入paddlespeech目录 ```bash cd PaddleSpeech ``` -### 4.安装paddlespeech +### 5.安装paddlespeech ```bash pip install pytest-runner -i https://pypi.tuna.tsinghua.edu.cn/simple @@ -32,17 +42,17 @@ pip install pytest-runner -i https://pypi.tuna.tsinghua.edu.cn/simple pip install paddlepaddle -i https://mirror.baidu.com/pypi/simple pip install paddlespeech -i https://pypi.tuna.tsinghua.edu.cn/simple ``` -### 5.使用命令自动下载语音模型 +### 6.使用命令自动下载语音模型 ```bash paddlespeech tts --input "你好，这是一次测试" ``` 此步骤会自动下载模型缓存至本地 .paddlespeech/models 目录 -### 6.修改tts_online_application.yaml配置 +### 7.修改tts_online_application.yaml配置 参考目录 ```"PaddleSpeech\demos\streaming_tts_server\conf\tts_online_application.yaml"``` 选择```tts_online_application.yaml```文件用编辑器打开，设置```protocol```为```websocket``` -### 7.启动服务 +### 8.启动服务 ```yaml paddlespeech_server start --config_file ./demos/streaming_tts_server/conf/tts_online_application.yaml #官方默认启动命令： diff --git a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java index 5761ab7c..0da1699b 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java +++ b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java @@ -161,11 +161,8 @@ public interface ErrorCode { int MQTT_SECRET_LENGTH_INSECURE = 10123; // mqtt密钥长度不安全 int MQTT_SECRET_CHARACTER_INSECURE = 10124; // mqtt密钥必须同时包含大小写字母 int MQTT_SECRET_WEAK_PASSWORD = 10125; // mqtt密钥包含弱密码 - - // 字典相关错误码 int DICT_LABEL_DUPLICATE = 10128; // 字典标签重复 - - // SM2加密相关错误码 int SM2_KEY_NOT_CONFIGURED = 10129; // SM2密钥未配置 int SM2_DECRYPT_ERROR = 10130; // SM2解密失败 + int MODEL_TYPE_PROVIDE_CODE_NOT_NULL = 10131; // modelType和provideCode不能为空 } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index f0603519..a79d1f2e 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -134,6 +134,10 @@ public class DeviceServiceImpl extends BaseServiceImpl // 清理redis缓存 redisUtils.delete(cacheDeviceKey); redisUtils.delete(deviceKey); + + // 添加：清除智能体设备数量缓存 + redisUtils.delete(RedisKeys.getAgentDeviceCountById(agentId)); + return true; } @@ -225,6 +229,16 @@ public class DeviceServiceImpl extends BaseServiceImpl @Override public void unbindDevice(Long userId, String deviceId) { + // 先查询设备信息，获取agentId + DeviceEntity device = baseDao.selectById(deviceId); + if (device == null) { + return; + } + if (StringUtils.isNotBlank(device.getAgentId())) { + // 清除智能体设备数量缓存 + redisUtils.delete(RedisKeys.getAgentDeviceCountById(device.getAgentId())); + } + UpdateWrapper wrapper = new UpdateWrapper<>(); wrapper.eq("user_id", userId); wrapper.eq("id", deviceId); @@ -459,6 +473,9 @@ public class DeviceServiceImpl extends BaseServiceImpl entity.setUpdater(userId); entity.setAutoUpdate(1); baseDao.insert(entity); + + // 添加：清除智能体设备数量缓存 + redisUtils.delete(RedisKeys.getAgentDeviceCountById(dto.getAgentId())); } /** diff --git a/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelConfigServiceImpl.java index e309e45b..d66cf5aa 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelConfigServiceImpl.java @@ -105,7 +105,7 @@ public class ModelConfigServiceImpl extends BaseServiceImpl providerList = modelProviderService.getList(modelType, provideCode); if (CollectionUtil.isEmpty(providerList)) { @@ -124,7 +124,7 @@ public class ModelConfigServiceImpl extends BaseServiceImpl providerList = modelProviderService.getList(modelType, provideCode); if (CollectionUtil.isEmpty(providerList)) { diff --git a/main/manager-api/src/main/resources/db/changelog/202509191545.sql b/main/manager-api/src/main/resources/db/changelog/202509191545.sql new file mode 100644 index 00000000..a1992977 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509191545.sql @@ -0,0 +1,41 @@ +-- 添加讯飞流式TTS供应器 +delete from `ai_model_provider` where id = 'SYSTEM_TTS_XunFeiStreamTTS'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_TTS_XunFeiStreamTTS', 'TTS', 'xunfei_stream', '讯飞流式语音合成', '[{"key":"app_id","label":"APP_ID","type":"string"},{"key":"api_secret","label":"API_Secret","type":"string"},{"key":"api_key","label":"API密钥","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"},{"key":"voice","label":"音色","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key": "volume", "type": "number", "label": "音量"},{"key": "speed", "type": "number", "label": "语速"},{"key": "pitch", "type": "number", "label": "音调"},{"key": "oral_level", "type": "number", "label": "口语化等级"},{"key": "spark_assist", "type": "number", "label": "是否口语化"},{"key": "stop_split", "type": "number", "label": "服务端拆句"},{"key": "remain", "type": "number", "label": "保留书面语"}]', 20, 1, NOW(), 1, NOW()); + +-- 添加讯飞流式TTS模型配置 +delete from `ai_model_config` where id = 'TTS_XunFeiStreamTTS'; +INSERT INTO `ai_model_config` VALUES ('TTS_XunFeiStreamTTS', 'TTS', 'XunFeiStreamTTS', '讯飞流式语音合成', 0, 1, '{\"type\": \"xunfei_stream\", \"app_id\": \"\", \"api_secret\": \"\", \"api_key\": \"\", \"output_dir\": \"tmp/\", \"voice\": \"x5_lingxiaoxuan_flow\", \"format\": \"raw\", \"sample_rate\": 24000, \"volume\": 50, \"speed\": 50, \"pitch\": 50, \"oral_level\": \"mid\", \"spark_assist\": 1, \"stop_split\": 0, \"remain\": 0}', NULL, NULL, 23, NULL, NULL, NULL, NULL); + +-- 更新讯飞流式TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.xfyun.cn/app/myapp', +`remark` = '讯飞流式TTS说明： +1. 登录讯飞语音技术平台 https://console.xfyun.cn/app/myapp 创建相关应用 +2. 选择需要的服务获取api相关配置 https://console.xfyun.cn/services/uts +3. 为需要使用的应用(APPID)购买相关服务 例如：超拟人合成 https://console.xfyun.cn/services/uts +5. 支持实时双流式通信，具有较低的延迟 +6. 支持口语化设置和音频参数调整 注意：V5音色不支持相关口语化配置 +7. 支持实时调节音量、语速、音调等参数 +' WHERE `id` = 'TTS_XunFeiStreamTTS'; + +-- 添加讯飞流式TTS音色 +delete from `ai_tts_voice` where tts_model_id = 'TTS_XunFeiStreamTTS'; + +-- 基础角色 +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0001', 'TTS_XunFeiStreamTTS', '聆小璇', 'x5_lingxiaoxuan_flow', '中文', NULL, NULL, NULL, NULL, 1, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0002', 'TTS_XunFeiStreamTTS', '聆飞逸', 'x5_lingfeiyi_flow', '中文', NULL, NULL, NULL, NULL, 2, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0003', 'TTS_XunFeiStreamTTS', '聆小玥', 'x5_lingxiaoyue_flow', '中文', NULL, NULL, NULL, NULL, 3, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0004', 'TTS_XunFeiStreamTTS', '聆玉昭', 'x5_lingyuzhao_flow', '中文', NULL, NULL, NULL, NULL, 4, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0005', 'TTS_XunFeiStreamTTS', '聆玉言', 'x5_lingyuyan_flow', '中文', NULL, NULL, NULL, NULL, 5, NULL, NULL, NULL, NULL); + +-- 需要添加对应的角色音色 +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0006', 'TTS_XunFeiStreamTTS', '聆飞哲', 'x4_lingfeizhe_oral', '中文', NULL, NULL, NULL, NULL, 6, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0007', 'TTS_XunFeiStreamTTS', '聆小璃', 'x4_lingxiaoli_oral', '中文', NULL, NULL, NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0008', 'TTS_XunFeiStreamTTS', '聆小糖', 'x5_lingxiaotang_flow', '中文', NULL, NULL, NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0009', 'TTS_XunFeiStreamTTS', '聆小琪', 'x4_lingxiaoqi_oral', '中文', NULL, NULL, NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0010', 'TTS_XunFeiStreamTTS', '聆佑佑-童年女声', 'x4_lingyouyou_oral', '中文', NULL, NULL, NULL, NULL, 10, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0011', 'TTS_XunFeiStreamTTS', '子津', 'x4_zijin_oral', '天津话', NULL, NULL, NULL, NULL, 11, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0012', 'TTS_XunFeiStreamTTS', '子阳', 'x4_ziyang_oral', '东北话', NULL, NULL, NULL, NULL, 12, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0013', 'TTS_XunFeiStreamTTS', 'Grant', 'x5_EnUs_Grant_flow', '英文', NULL, NULL, NULL, NULL, 13, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_XunFeiStreamTTS_0014', 'TTS_XunFeiStreamTTS', 'Lila', 'x5_EnUs_Lila_flow', '英文', NULL, NULL, NULL, NULL, 14, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/202509220926.sql b/main/manager-api/src/main/resources/db/changelog/202509220926.sql new file mode 100644 index 00000000..060fa4d9 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509220926.sql @@ -0,0 +1,26 @@ +-- 添加讯飞流式语音识别服务配置 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_XunfeiStream'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_XunfeiStream', 'ASR', 'xunfei_stream', '讯飞流式语音识别', '[{"key":"app_id","label":"应用ID","type":"string"},{"key":"api_key","label":"API_KEY","type":"password"},{"key":"api_secret","label":"API_SECRET","type":"password"},{"key":"domain","label":"识别领域","type":"string"},{"key":"language","label":"识别语言","type":"string"},{"key":"accent","label":"方言","type":"string"},{"key":"dwa","label":"动态修正","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 18, 1, NOW(), 1, NOW()); + +delete from `ai_model_config` where id = 'ASR_XunfeiStream'; +INSERT INTO `ai_model_config` VALUES ('ASR_XunfeiStream', 'ASR', '讯飞流式语音识别', '讯飞流式语音识别服务', 0, 1, '{"type": "xunfei_stream", "app_id": "", "api_key": "", "api_secret": "", "domain": "slm", "language": "zh_cn", "accent": "mandarin", "dwa": "wpgs", "output_dir": "tmp/"}', 'https://www.xfyun.cn/doc/spark/spark_zh_iat.html', '支持实时流式语音识别，适用于中文普通话及多种方言识别', 21, NULL, NULL, NULL, NULL); + +-- 更新讯飞流式语音识别模型配置的说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://www.xfyun.cn/doc/spark/spark_zh_iat.html', +`remark` = '讯飞流式语音识别配置说明： +1. 登录讯飞开放平台 https://www.xfyun.cn/ +2. 创建语音识别应用获取APPID、APISecret、APIKey +3. 参数说明： + - app_id: 应用ID，在讯飞开放平台创建应用后获得 + - api_key: API密钥，用于接口鉴权 + - api_secret: API密钥，用于生成签名 + - domain: 识别领域，默认slm（智能化语音转写） + - language: 识别语言，默认zh_cn（中文） + - accent: 方言类型，默认mandarin（普通话），支持cantonese（粤语）等 + - dwa: 动态修正，默认wpgs（开启动态修正） + - output_dir: 音频文件输出目录，默认tmp/ +4. 支持实时流式识别，适用于实时语音交互场景 +5. 支持多种方言和语言识别 +' WHERE `id` = 'ASR_XunfeiStream'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202509220958.sql b/main/manager-api/src/main/resources/db/changelog/202509220958.sql new file mode 100644 index 00000000..b05faa1c --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509220958.sql @@ -0,0 +1,19 @@ +delete from `ai_model_config` where id = 'LLM_XunfeiSparkLLM'; +INSERT INTO `ai_model_config` VALUES ('LLM_XunfeiSparkLLM', 'LLM', '讯飞星火认知大模型', '讯飞星火认知大模型', 0, 1, '{"type": "openai", "model_name": "generalv3.5", "base_url": "https://spark-api-open.xf-yun.com/v1", "api_password": "你的api_password", "temperature": 0.5, "max_tokens": 2048, "top_p": 1.0, "frequency_penalty": 0.0}', 'https://www.xfyun.cn/doc/spark/HTTP%E8%B0%83%E7%94%A8%E6%96%87%E6%A1%A3.html', '讯飞星火认知大模型，支持多轮对话、文本生成等功能', 14, NULL, NULL, NULL, NULL); + +-- 更新讯飞星火认知大模型配置的说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://www.xfyun.cn/doc/spark/HTTP%E8%B0%83%E7%94%A8%E6%96%87%E6%A1%A3.html', +`remark` = '讯飞星火认知大模型配置说明： +1. 登录讯飞开放平台 https://www.xfyun.cn/，每一个模型对应每一个api_password,更改模型时需要查看对应模型的api_password +2. 创建星火认知大模型应用获取API Password +3. 参数说明： + - api_password: API Password，在讯飞开放平台创建应用后获得 + - model_name: 模型名称，支持generalv3.5、generalv3等版本 + - base_url: API地址，默认https://spark-api-open.xf-yun.com/v1 + - temperature: 温度参数，控制生成随机性，范围0-1，默认0.5 + - max_tokens: 最大输出token数，默认2048 + - top_p: 核心采样参数，控制词汇多样性，默认1.0 + - frequency_penalty: 频率惩罚，降低重复内容，默认0.0 +4. 每一个模型对应每一个api_password,更改模型时需要查看对应模型的api_password。 +' WHERE `id` = 'LLM_XunfeiSparkLLM'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index c660948e..3289d69f 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -366,3 +366,25 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202509221530.sql + - changeSet: + id: 202509191545 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202509191545.sql + - changeSet: + id: 202509220926 + author: fyb + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202509220926.sql + + - changeSet: + id: 202509220958 + author: fyb + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202509220958.sql diff --git a/main/manager-api/src/main/resources/i18n/messages.properties b/main/manager-api/src/main/resources/i18n/messages.properties index 30bdb8d1..63c9403d 100644 --- a/main/manager-api/src/main/resources/i18n/messages.properties +++ b/main/manager-api/src/main/resources/i18n/messages.properties @@ -134,4 +134,5 @@ 10123=您的mqtt密钥长度不安全，字符数需要至少8个字符且必须同时包含大小写字母 10124=您的mqtt密钥长度不安全，mqtt密钥必须同时包含大小写字母 10125=您的mqtt密钥包含弱密码 -10128=字典标签重复 \ No newline at end of file +10128=字典标签重复 +10129=modelType和provideCode不能为空 \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages_en_US.properties b/main/manager-api/src/main/resources/i18n/messages_en_US.properties index 2df3a45f..a566cab3 100644 --- a/main/manager-api/src/main/resources/i18n/messages_en_US.properties +++ b/main/manager-api/src/main/resources/i18n/messages_en_US.properties @@ -136,4 +136,5 @@ 10125=Your MQTT secret contains weak password 10128=Dictionary label is duplicated 10129=SM2 key not configured -10130=SM2 decryption failed \ No newline at end of file +10130=SM2 decryption failed +10131=modelType and provideCode cannot be empty \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties b/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties index 0c3df5cc..119326d0 100644 --- a/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties @@ -136,4 +136,5 @@ 10125=您的mqtt密钥包含弱密码 10128=字典标签重复 10129=SM2密钥未配置 -10130=SM2解密失败 \ No newline at end of file +10130=SM2解密失败 +10131=modelType和provideCode不能为空 \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties b/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties index 4a1cda76..5ad7bb6f 100644 --- a/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties +++ b/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties @@ -136,4 +136,5 @@ 10125=您的MQTT密鑰包含弱密碼 10128=字典標籤重複 10129=SM2密鑰未配置 -10130=SM2解密失敗 \ No newline at end of file +10130=SM2解密失敗 +10131=modelType和provideCode不能為空 \ No newline at end of file diff --git a/main/manager-web/src/i18n/en.js b/main/manager-web/src/i18n/en.js index 1e981a53..be5da633 100644 --- a/main/manager-web/src/i18n/en.js +++ b/main/manager-web/src/i18n/en.js @@ -7,7 +7,6 @@ export default { 'login.loginSuccess': 'Login successful!', // HeaderBar组件文本 - // 在header相关翻译中添加 'header.smartManagement': 'Agents', 'header.modelConfig': 'Models', 'header.userManagement': 'Users', @@ -588,43 +587,6 @@ export default { 'user.info': 'User Information', 'user.username': 'Username', 'user.mobile': 'Mobile Phone', - // Provider Management Page Text - 'providerManagement.categoryFilter': 'Category Filter', - 'providerManagement.searchPlaceholder': 'Please enter provider name to search', - 'providerManagement.category': 'Category', - 'providerManagement.providerCode': 'Provider Code', - 'providerManagement.fieldConfig': 'Field Configuration', - 'providerManagement.selectToDelete': 'Please select providers to delete first', - 'providerManagement.confirmDelete': 'Are you sure to delete the selected {count} providers?', - 'providerManagement.viewFields': 'View Fields', - // Common Text - 'common.all': 'All', - 'common.search': 'Search', - 'common.name': 'Name', - 'common.sort': 'Sort', - 'common.action': 'Action', - 'common.edit': 'Edit', - 'common.delete': 'Delete', - 'common.selectAll': 'Select All', - 'common.deselectAll': 'Deselect All', - 'common.add': 'Add', - 'common.perPage': '{number}/page', - 'common.firstPage': 'First Page', - 'common.prevPage': 'Previous Page', - 'common.nextPage': 'Next Page', - 'common.totalRecords': 'Total {number} records', - 'common.addProvider': 'Add Provider', - 'common.editProvider': 'Edit Provider', - 'common.updateSuccess': 'Update Success', - 'common.addSuccess': 'Add Success', - 'common.deleteSuccess': 'Delete Success', - 'common.deleteFailure': 'Delete Failed, Please Try Again', - 'common.deleteCancelled': 'Delete Cancelled', - 'common.warning': 'Warning', - 'common.tip': 'Tip', - 'common.confirm': 'Confirm', - 'common.cancel': 'Cancel', - 'common.sensitive': 'Sensitive', 'user.userid': 'User ID', 'user.deviceCount': 'Device Count', 'user.createDate': 'Registration Time', @@ -650,6 +612,7 @@ export default { 'user.deleteCancelled': 'Deletion cancelled', 'user.confirmResetPassword': 'A new password will be generated after reset. Continue?', 'user.resetPasswordSuccess': 'Password has been reset, please notify the user to log in with the new password', + 'user.generatedPassword': 'Generated Default Password', 'user.confirmDeleteUser': 'Are you sure you want to delete this user?', 'user.deleteUserSuccess': 'Deletion successful', 'user.operationFailed': 'Operation failed, please try again', @@ -659,6 +622,46 @@ export default { 'user.searchPhone': 'Please enter mobile phone number to search', 'user.search': 'Search', + // Provider Management Page Text + 'providerManagement.categoryFilter': 'Category Filter', + 'providerManagement.searchPlaceholder': 'Please enter provider name to search', + 'providerManagement.category': 'Category', + 'providerManagement.providerCode': 'Provider Code', + 'providerManagement.fieldConfig': 'Field Configuration', + 'providerManagement.selectToDelete': 'Please select providers to delete first', + 'providerManagement.confirmDelete': 'Are you sure to delete the selected {count} providers?', + 'providerManagement.viewFields': 'View Fields', + + // Common Text + 'common.all': 'All', + 'common.search': 'Search', + 'common.name': 'Name', + 'common.sort': 'Sort', + 'common.action': 'Action', + 'common.edit': 'Edit', + 'common.delete': 'Delete', + 'common.selectAll': 'Select All', + 'common.deselectAll': 'Deselect All', + 'common.add': 'Add', + 'common.perPage': '{number}/page', + 'common.firstPage': 'First Page', + 'common.prevPage': 'Previous Page', + 'common.nextPage': 'Next Page', + 'common.totalRecords': 'Total {number} records', + 'common.addProvider': 'Add Provider', + 'common.success': 'Success', + 'common.editProvider': 'Edit Provider', + 'common.updateSuccess': 'Update Success', + 'common.addSuccess': 'Add Success', + 'common.deleteSuccess': 'Delete Success', + 'common.deleteFailure': 'Delete Failed, Please Try Again', + 'common.deleteCancelled': 'Delete Cancelled', + 'common.warning': 'Warning', + 'common.tip': 'Tip', + 'common.confirm': 'Confirm', + 'common.cancel': 'Cancel', + 'common.sensitive': 'Sensitive', + // Language switch 'language.zhCN': '中文简体', 'language.zhTW': '中文繁體', @@ -675,7 +678,14 @@ export default { 'home.deviceManagement': 'Devices', 'home.chatHistory': 'Chat History', 'home.lastConversation': 'Last Conversation', - 'home.noConversation': 'No conversation yet', + 'home.noConversation': 'No conversation', + 'home.justNow': 'Just now', + 'home.minutesAgo': '{minutes} minutes ago', + 'home.hoursAgo': '{hours} hours {minutes} minutes ago', + 'home.confirmDeleteAgent': 'Are you sure you want to delete this agent?', + 'home.deleteSuccess': 'Delete successful', + 'home.deleteFailed': 'Delete failed', + 'home.enableMemory': 'Please enable memory in the "Config Role" page first', // Parameter management page text 'paramManagement.pageTitle': 'Parameter Management', @@ -711,14 +721,6 @@ export default { 'paramManagement.addSuccess': 'Add successful', 'paramManagement.updateFailed': 'Update failed', 'paramManagement.addFailed': 'Add failed', - 'home.noConversation': 'No conversation', - 'home.justNow': 'Just now', - 'home.minutesAgo': '{minutes} minutes ago', - 'home.hoursAgo': '{hours} hours {minutes} minutes ago', - 'home.confirmDeleteAgent': 'Are you sure you want to delete this agent?', - 'home.deleteSuccess': 'Delete successful', - 'home.deleteFailed': 'Delete failed', - 'home.enableMemory': 'Please enable memory in the "Config Role" page first', // Server side management page text 'serverSideManager.pageTitle': 'Server Management', diff --git a/main/manager-web/src/i18n/zh_CN.js b/main/manager-web/src/i18n/zh_CN.js index 3b9e6c24..2bc6e8d8 100644 --- a/main/manager-web/src/i18n/zh_CN.js +++ b/main/manager-web/src/i18n/zh_CN.js @@ -444,7 +444,6 @@ export default { 'device.input6DigitCode': '请输入6位数字验证码', 'device.bindSuccess': '设备绑定成功', 'device.bindFailed': '绑定失败', - // DeviceManagement页面扩展翻译 'device.searchPlaceholder': '请输入设备型号或Mac地址查询', 'device.model': '设备型号', 'device.macAddress': 'Mac地址', @@ -559,7 +558,7 @@ export default { 'dictManagement.dictValue': '字典值', 'dictManagement.sort': '排序', 'dictManagement.delete': '删除', - 'dictManagement.selectAll': '默认角色管理', + 'dictManagement.selectAll': '全选', 'dictManagement.deselectAll': '取消全选', 'dictManagement.addDictType': '新增字典类型', 'dictManagement.batchDeleteDictType': '批量删除字典类型', @@ -588,43 +587,6 @@ export default { 'user.info': '用户信息', 'user.username': '用户名', 'user.mobile': '手机号', - // 字段管理页面文本 - 'providerManagement.categoryFilter': '类别筛选', - 'providerManagement.searchPlaceholder': '请输入供应器名称查询', - 'providerManagement.category': '类别', - 'providerManagement.providerCode': '供应器编码', - 'providerManagement.fieldConfig': '字段配置', - 'providerManagement.selectToDelete': '请先选择需要删除的供应器', - 'providerManagement.confirmDelete': '确定要删除选中的{count}个供应器吗？', - 'providerManagement.viewFields': '查看字段', - // 公共文本 - 'common.all': '全部', - 'common.search': '搜索', - 'common.name': '名称', - 'common.sort': '排序', - 'common.action': '操作', - 'common.edit': '编辑', - 'common.delete': '删除', - 'common.selectAll': '全选', - 'common.deselectAll': '取消全选', - 'common.add': '新增', - 'common.perPage': '{number}条/页', - 'common.firstPage': '首页', - 'common.prevPage': '上一页', - 'common.nextPage': '下一页', - 'common.totalRecords': '共{number}条记录', - 'common.addProvider': '新增供应器', - 'common.editProvider': '编辑供应器', - 'common.updateSuccess': '修改成功', - 'common.addSuccess': '新增成功', - 'common.deleteSuccess': '删除成功', - 'common.deleteFailure': '删除失败，请重试', - 'common.deleteCancelled': '已取消删除', - 'common.warning': '警告', - 'common.tip': '提示', - 'common.confirm': '确定', - 'common.cancel': '取消', - 'common.sensitive': '敏感', 'user.userid': '用户Id', 'user.deviceCount': '设备数量', 'user.createDate': '注册时间', @@ -650,6 +612,7 @@ export default { 'user.deleteCancelled': '已取消删除', 'user.confirmResetPassword': '重置后将会生成新密码，是否继续？', 'user.resetPasswordSuccess': '密码已重置，请通知用户使用新密码登录', + 'user.generatedPassword': '生成的默认密码', 'user.confirmDeleteUser': '确定要删除该用户吗？', 'user.deleteUserSuccess': '删除成功', 'user.operationFailed': '操作失败，请重试', @@ -659,6 +622,46 @@ export default { 'user.searchPhone': '请输入手机号码查询', 'user.search': '搜索', + // 字段管理页面文本 + 'providerManagement.categoryFilter': '类别筛选', + 'providerManagement.searchPlaceholder': '请输入供应器名称查询', + 'providerManagement.category': '类别', + 'providerManagement.providerCode': '供应器编码', + 'providerManagement.fieldConfig': '字段配置', + 'providerManagement.selectToDelete': '请先选择需要删除的供应器', + 'providerManagement.confirmDelete': '确定要删除选中的{count}个供应器吗？', + 'providerManagement.viewFields': '查看字段', + + // 公共文本 + 'common.all': '全部', + 'common.search': '搜索', + 'common.name': '名称', + 'common.sort': '排序', + 'common.action': '操作', + 'common.edit': '编辑', + 'common.delete': '删除', + 'common.selectAll': '全选', + 'common.deselectAll': '取消全选', + 'common.add': '新增', + 'common.perPage': '{number}条/页', + 'common.firstPage': '首页', + 'common.prevPage': '上一页', + 'common.nextPage': '下一页', + 'common.totalRecords': '共{number}条记录', + 'common.addProvider': '新增供应器', + 'common.success': '成功', + 'common.editProvider': '编辑供应器', + 'common.updateSuccess': '修改成功', + 'common.addSuccess': '新增成功', + 'common.deleteSuccess': '删除成功', + 'common.deleteFailure': '删除失败，请重试', + 'common.deleteCancelled': '已取消删除', + 'common.warning': '警告', + 'common.tip': '提示', + 'common.confirm': '确定', + 'common.cancel': '取消', + 'common.sensitive': '敏感', + // 语言切换 'language.zhCN': '中文简体', 'language.zhTW': '中文繁體', @@ -676,6 +679,13 @@ export default { 'home.chatHistory': '聊天记录', 'home.lastConversation': '最近对话', 'home.noConversation': '暂未对话', + 'home.justNow': '刚刚', + 'home.minutesAgo': '{minutes}分钟前', + 'home.hoursAgo': '{hours}小时{minutes}分钟前', + 'home.confirmDeleteAgent': '确定要删除该智能体吗？', + 'home.deleteSuccess': '删除成功', + 'home.deleteFailed': '删除失败', + 'home.enableMemory': '请先在“配置角色”界面开启记忆', // 参数管理页面文本 'paramManagement.pageTitle': '参数管理', @@ -711,13 +721,6 @@ export default { 'paramManagement.addSuccess': '新增成功', 'paramManagement.updateFailed': '更新失败', 'paramManagement.addFailed': '新增失败', - 'home.justNow': '刚刚', - 'home.minutesAgo': '{minutes}分钟前', - 'home.hoursAgo': '{hours}小时{minutes}分钟前', - 'home.confirmDeleteAgent': '确定要删除该智能体吗？', - 'home.deleteSuccess': '删除成功', - 'home.deleteFailed': '删除失败', - 'home.enableMemory': '请先在“配置角色”界面开启记忆', // 服务端管理页面文本 'serverSideManager.pageTitle': '服务端管理', diff --git a/main/manager-web/src/i18n/zh_TW.js b/main/manager-web/src/i18n/zh_TW.js index 39916bec..886f000f 100644 --- a/main/manager-web/src/i18n/zh_TW.js +++ b/main/manager-web/src/i18n/zh_TW.js @@ -7,7 +7,6 @@ export default { 'login.loginSuccess': '登錄成功！', // HeaderBar组件文本 - // 在header相关翻译中添加 'header.smartManagement': '智能體管理', 'header.modelConfig': '模型配置', 'header.userManagement': '用戶管理', @@ -548,6 +547,42 @@ export default { 'voiceprint.add': '添加聲紋', 'voiceprint.delete': '刪除聲紋', + // 字典管理頁面文本 + 'dictManagement.pageTitle': '字典管理', + 'dictManagement.searchPlaceholder': '請輸入字典值標籤查詢', + 'dictManagement.search': '搜索', + 'dictManagement.dictTypeName': '字典類型名稱', + 'dictManagement.operation': '操作', + 'dictManagement.edit': '編輯', + 'dictManagement.dictLabel': '字典標籤', + 'dictManagement.dictValue': '字典值', + 'dictManagement.sort': '排序', + 'dictManagement.delete': '刪除', + 'dictManagement.selectAll': '全選', + 'dictManagement.deselectAll': '取消全選', + 'dictManagement.addDictType': '新增字典類型', + 'dictManagement.batchDeleteDictType': '批量刪除字典類型', + 'dictManagement.addDictData': '新增字典數據', + 'dictManagement.batchDeleteDictData': '批量刪除字典數據', + 'dictManagement.itemsPerPage': '{items}條/頁', + 'dictManagement.firstPage': '首頁', + 'dictManagement.prevPage': '上一頁', + 'dictManagement.nextPage': '下一頁', + 'dictManagement.totalRecords': '共{total}條記錄', + 'dictManagement.editDictType': '編輯字典類型', + 'dictManagement.editDictData': '編輯字典數據', + 'dictManagement.saveSuccess': '保存成功', + 'dictManagement.deleteSuccess': '刪除成功', + 'dictManagement.selectDictTypeToDelete': '請選擇要刪除的字典類型', + 'dictManagement.confirmDeleteDictType': '確定要刪除選中的字典類型嗎?', + 'dictManagement.confirm': '確定', + 'dictManagement.cancel': '取消', + 'dictManagement.selectDictTypeFirst': '請先選擇字典類型', + 'dictManagement.confirmDeleteDictData': '確定要刪除該字典數據嗎?', + 'dictManagement.selectDictDataToDelete': '請選擇要刪除的字典數據', + 'dictManagement.confirmBatchDeleteDictData': '確定要刪除選中的{count}個字典數據嗎?', + 'dictManagement.getDictDataFailed': '獲取字典數據失敗', + // 用户信息 'user.info': '用戶信息', 'user.username': '用戶名', @@ -577,6 +612,7 @@ export default { 'user.deleteCancelled': '已取消刪除', 'user.confirmResetPassword': '重置後將會生成新密碼，是否繼續？', 'user.resetPasswordSuccess': '密碼已重置，請通知用戶使用新密碼登錄', + 'user.generatedPassword': '生成的默認密碼', 'user.confirmDeleteUser': '確定要刪除該用戶嗎？', 'user.deleteUserSuccess': '刪除成功', 'user.operationFailed': '操作失敗，請重試', @@ -586,6 +622,46 @@ export default { 'user.searchPhone': '請輸入手機號碼查詢', 'user.search': '搜索', + // 字段管理頁面文本 + 'providerManagement.categoryFilter': '類別篩選', + 'providerManagement.searchPlaceholder': '請輸入供應器名稱查詢', + 'providerManagement.category': '類別', + 'providerManagement.providerCode': '供應器編碼', + 'providerManagement.fieldConfig': '字段配置', + 'providerManagement.selectToDelete': '請先選擇需要刪除的供應器', + 'providerManagement.confirmDelete': '確定要刪除選中的{count}個供應器嗎？', + 'providerManagement.viewFields': '查看字段', + + // 公共文本 + 'common.all': '全部', + 'common.search': '搜索', + 'common.name': '名稱', + 'common.sort': '排序', + 'common.action': '操作', + 'common.edit': '編輯', + 'common.delete': '刪除', + 'common.selectAll': '全選', + 'common.deselectAll': '取消全選', + 'common.add': '新增', + 'common.perPage': '{number}條/頁', + 'common.firstPage': '首頁', + 'common.prevPage': '上一頁', + 'common.nextPage': '下一頁', + 'common.totalRecords': '共{number}條記錄', + 'common.addProvider': '新增供應器', + 'common.success': '成功', + 'common.editProvider': '編輯供應器', + 'common.updateSuccess': '修改成功', + 'common.addSuccess': '新增成功', + 'common.deleteSuccess': '刪除成功', + 'common.deleteFailure': '刪除失敗，請重試', + 'common.deleteCancelled': '已取消刪除', + 'common.warning': '警告', + 'common.tip': '提示', + 'common.confirm': '確定', + 'common.cancel': '取消', + 'common.sensitive': '敏感', + // 語言切換 'language.zhCN': '中文简体', 'language.zhTW': '中文繁體', @@ -603,6 +679,13 @@ export default { 'home.chatHistory': '聊天記錄', 'home.lastConversation': '最近對話', 'home.noConversation': '暫未對話', + 'home.justNow': '剛剛', + 'home.minutesAgo': '{minutes}分鐘前', + 'home.hoursAgo': '{hours}小時{minutes}分鐘前', + 'home.confirmDeleteAgent': '確定要刪除該智能體嗎？', + 'home.deleteSuccess': '刪除成功', + 'home.deleteFailed': '刪除失敗', + 'home.enableMemory': '請先在「配置角色」介面開啟記憶', // 參數管理頁面文本 'paramManagement.pageTitle': '參數管理', @@ -623,44 +706,6 @@ export default { 'paramManagement.firstPage': '首頁', 'paramManagement.prevPage': '上一頁', 'paramManagement.nextPage': '下一頁', - - // 字段管理頁面文本 - 'providerManagement.categoryFilter': '類別篩選', - 'providerManagement.searchPlaceholder': '請輸入供應器名稱查詢', - 'providerManagement.category': '類別', - 'providerManagement.providerCode': '供應器編碼', - 'providerManagement.fieldConfig': '字段配置', - 'providerManagement.selectToDelete': '請先選擇需要刪除的供應器', - 'providerManagement.confirmDelete': '確定要刪除選中的{count}個供應器嗎？', - 'providerManagement.viewFields': '查看字段', - // 公共文本 - 'common.all': '全部', - 'common.search': '搜索', - 'common.name': '名稱', - 'common.sort': '排序', - 'common.action': '操作', - 'common.edit': '編輯', - 'common.delete': '刪除', - 'common.selectAll': '全選', - 'common.deselectAll': '取消全選', - 'common.add': '新增', - 'common.perPage': '{number}條/頁', - 'common.firstPage': '首頁', - 'common.prevPage': '上一頁', - 'common.nextPage': '下一頁', - 'common.totalRecords': '共{number}條記錄', - 'common.addProvider': '新增供應器', - 'common.editProvider': '編輯供應器', - 'common.updateSuccess': '修改成功', - 'common.addSuccess': '新增成功', - 'common.deleteSuccess': '刪除成功', - 'common.deleteFailure': '刪除失敗，請重試', - 'common.deleteCancelled': '已取消刪除', - 'common.warning': '警告', - 'common.tip': '提示', - 'common.confirm': '確定', - 'common.cancel': '取消', - 'common.sensitive': '敏感', 'paramManagement.totalRecords': '共{total}條記錄', 'paramManagement.addParam': '新增參數', 'paramManagement.editParam': '編輯參數', @@ -676,49 +721,6 @@ export default { 'paramManagement.addSuccess': '新增成功', 'paramManagement.updateFailed': '更新失敗', 'paramManagement.addFailed': '新增失敗', - 'home.justNow': '剛剛', - 'home.minutesAgo': '{minutes}分鐘前', - 'home.hoursAgo': '{hours}小時{minutes}分鐘前', - 'home.confirmDeleteAgent': '確定要刪除該智能體嗎？', - 'home.deleteSuccess': '刪除成功', - 'home.deleteFailed': '刪除失敗', - 'home.enableMemory': '請先在「配置角色」介面開啟記憶', - - // 字典管理頁面文本 - 'dictManagement.pageTitle': '字典管理', - 'dictManagement.searchPlaceholder': '請輸入字典名稱搜索', - 'dictManagement.search': '搜索', - 'dictManagement.addDictType': '新增字典類型', - 'dictManagement.batchDeleteDictType': '批量刪除字典類型', - 'dictManagement.dictName': '字典名稱', - 'dictManagement.dictCode': '字典代碼', - 'dictManagement.description': '描述', - 'dictManagement.operation': '操作', - 'dictManagement.edit': '修改', - 'dictManagement.delete': '刪除', - 'dictManagement.dictLabel': '字典標籤', - 'dictManagement.dictValue': '字典值', - 'dictManagement.sortOrder': '排序號', - 'dictManagement.status': '狀態', - 'dictManagement.itemsPerPage': '{items}條/頁', - 'dictManagement.firstPage': '首頁', - 'dictManagement.prevPage': '上一頁', - 'dictManagement.nextPage': '下一頁', - 'dictManagement.totalRecords': '共{total}條記錄', - 'dictManagement.addDictData': '新增字典數據', - 'dictManagement.editDictType': '修改字典類型', - 'dictManagement.confirmDelete': '確定要刪除此字典類型嗎？', - 'dictManagement.confirmBatchDelete': '確定要刪除選中的{count}個字典類型嗎？', - 'dictManagement.selectFirst': '請先選擇需要刪除的字典類型', - 'dictManagement.deleteSuccess': '刪除成功', - 'dictManagement.deleteFailed': '刪除失敗', - 'dictManagement.addSuccess': '新增成功', - 'dictManagement.editSuccess': '修改成功', - 'dictManagement.getDictListFailed': '獲取字典列表失敗', - 'dictManagement.getDictDataListFailed': '獲取字典數據列表失敗', - 'dictManagement.saveFailed': '保存失敗', - 'dictManagement.loading': '拼命加載中', - 'dictManagement.select': '選擇', // 服務端管理頁面文本 'serverSideManager.pageTitle': '服務端管理', @@ -1003,29 +1005,30 @@ export default { 'providerDialog.batchDeleteFieldsSuccess': '成功刪除{count}個字段', // 預設角色管理頁面文本 - 'agentTemplateManagement.title': '預設角色管理', - 'agentTemplateManagement.templateName': '模板名稱', - 'agentTemplateManagement.action': '操作', - 'agentTemplateManagement.createTemplate': '建立模板', - 'agentTemplateManagement.editTemplate': '編輯模板', - 'agentTemplateManagement.deleteTemplate': '刪除模板', - 'agentTemplateManagement.deleteSuccess': '模板刪除成功', - 'agentTemplateManagement.batchDelete': '批次刪除', - 'agentTemplateManagement.batchDeleteSuccess': '批次刪除成功', - 'agentTemplateManagement.selectTemplate': '請選擇模板', - 'agentTemplateManagement.select': '選擇', - 'agentTemplateManagement.searchPlaceholder': '請輸入模板名稱搜尋', - 'agentTemplateManagement.search': '搜尋', - 'agentTemplateManagement.serialNumber': '序號', - 'agentTemplateManagement.selectAll': '全選', - 'agentTemplateManagement.deselectAll': '取消全選', - 'agentTemplateManagement.loading': '拼命加載中', - 'agentTemplateManagement.confirmSingleDelete': '確定要刪除這個模板嗎？', - 'agentTemplateManagement.confirmBatchDelete': '確定要刪除選中的 {count} 個模板嗎？', - 'agentTemplateManagement.deleteFailed': '模板刪除失敗', - 'agentTemplateManagement.batchDeleteFailed': '批次刪除失敗', + 'agentTemplateManagement.title': '預設角色管理', + 'agentTemplateManagement.templateName': '模板名稱', + 'agentTemplateManagement.action': '操作', + 'templateQuickConfig.saveSuccess': '配置保存成功', + 'templateQuickConfig.saveFailed': '配置保存失敗', + 'agentTemplateManagement.createTemplate': '建立模板', + 'agentTemplateManagement.editTemplate': '編輯模板', + 'agentTemplateManagement.deleteTemplate': '刪除模板', + 'agentTemplateManagement.deleteSuccess': '模板刪除成功', + 'agentTemplateManagement.batchDelete': '批次刪除', + 'agentTemplateManagement.batchDeleteSuccess': '批次刪除成功', + 'agentTemplateManagement.selectTemplate': '請選擇模板', + 'agentTemplateManagement.select': '選擇', + 'agentTemplateManagement.searchPlaceholder': '請輸入模板名稱搜尋', + 'agentTemplateManagement.search': '搜尋', + 'agentTemplateManagement.serialNumber': '序號', + 'agentTemplateManagement.selectAll': '全選', + 'agentTemplateManagement.deselectAll': '取消全選', + 'agentTemplateManagement.loading': '拼命加載中', + 'agentTemplateManagement.confirmSingleDelete': '確定要刪除這個模板嗎？', + 'agentTemplateManagement.confirmBatchDelete': '確定要刪除選中的 {count} 個模板嗎？', + 'agentTemplateManagement.deleteFailed': '模板刪除失敗', + 'agentTemplateManagement.batchDeleteFailed': '模板批次刪除失敗', 'agentTemplateManagement.deleteBackendError': '刪除失敗，請檢查後端服務是否正常', - 'agentTemplateManagement.deleteCancelled': '已取消刪除', // 模板快速配置 'templateQuickConfig.title': '模組快速設定', @@ -1042,7 +1045,6 @@ export default { 'templateQuickConfig.cancel': '取消', 'templateQuickConfig.templateNotFound': '未找到指定模板', 'templateQuickConfig.newTemplate': '新模板', - 'error': '錯誤', 'warning': '警告', 'info': '提示', 'common.networkError': '網路請求失敗', diff --git a/main/manager-web/src/views/UserManagement.vue b/main/manager-web/src/views/UserManagement.vue index c61ca6df..8132ba01 100644 --- a/main/manager-web/src/views/UserManagement.vue +++ b/main/manager-web/src/views/UserManagement.vue @@ -257,11 +257,17 @@ export default { cancelButtonText: this.$t('common.cancel'), type: 'warning' }).then(() => { - resetPassword(row.userId).then(() => { - this.$message.success(this.$t('user.resetPasswordSuccess')); - this.getList(); - }).catch(() => { - this.$message.error(this.$t('user.operationFailed')); + Api.admin.resetUserPassword(row.userid, ({ data }) => { + if (data.code === 0) { + // 显示生成的默认密码 + this.$alert(this.$t('user.resetPasswordSuccess') + '\n\n' + this.$t('user.generatedPassword') + ': ' + data.data, this.$t('common.success'), { + confirmButtonText: this.$t('common.confirm'), + dangerouslyUseHTMLString: true + }); + this.fetchUsers(); + } else { + this.$message.error(data.msg || this.$t('user.operationFailed')); + } }); }).catch(() => { this.$message.info(this.$t('common.deleteCancelled')); @@ -273,11 +279,13 @@ export default { cancelButtonText: this.$t('common.cancel'), type: 'warning' }).then(() => { - deleteUser(row.userId).then(() => { - this.$message.success(this.$t('user.deleteUserSuccess')); - this.getList(); - }).catch(() => { - this.$message.error(this.$t('user.operationFailed')); + Api.admin.deleteUser(row.userid, ({ data }) => { + if (data.code === 0) { + this.$message.success(this.$t('user.deleteUserSuccess')); + this.fetchUsers(); + } else { + this.$message.error(data.msg || this.$t('user.operationFailed')); + } }); }).catch(() => { this.$message.info(this.$t('common.deleteCancelled')); @@ -338,67 +346,20 @@ export default { // 用户取消操作 }); }, + // 这个方法已被batchDelete替代，保留用于向后兼容 handleBatchDelete() { - if (!this.selection || this.selection.length === 0) { - this.$message.warning(this.$t('user.selectUsersFirst')); - return; - } - - const userIds = this.selection.map(item => item.userId); - this.$confirm(this.$t('user.confirmDeleteSelected', { count: this.selection.length }), this.$t('common.warning'), { - confirmButtonText: this.$t('common.confirm'), - cancelButtonText: this.$t('common.cancel'), - type: 'warning' - }).then(() => { - this.loading = true; - batchDeleteUsers(userIds).then(res => { - this.loading = false; - this.getList(); - this.selection = []; - if (res.successCount === userIds.length) { - this.$message.success(this.$t('user.deleteSuccess', { count: res.successCount })); - } else if (res.successCount === 0) { - this.$message.error(this.$t('user.deleteFailed')); - } else { - this.$message.warning(this.$t('user.partialDelete', { successCount: res.successCount, failCount: res.failCount })); - } - }).catch(() => { - this.loading = false; - this.$message.error(this.$t('user.deleteError')); - }); - }).catch(() => { - this.loading = false; - this.$message.info(this.$t('user.deleteCancelled')); - }); + this.batchDelete(); }, + // This method has been fixed to use existing functionality handleBatchStatusChange(status) { - if (!this.selection || this.selection.length === 0) { + const selectedUsers = this.userList.filter(user => user.selected); + if (selectedUsers.length === 0) { this.$message.warning(this.$t('user.selectUsersFirst')); return; } - - const actionText = status === 1 ? this.$t('user.enable') : this.$t('user.disable'); - const userIds = this.selection.map(item => item.userId); - - this.$confirm(this.$t('user.confirmStatusChange', { action: actionText, count: this.selection.length }), this.$t('common.warning'), { - confirmButtonText: this.$t('common.confirm'), - cancelButtonText: this.$t('common.cancel'), - type: 'warning' - }).then(() => { - this.loading = true; - updateUserStatus(userIds, status).then(() => { - this.loading = false; - this.getList(); - this.selection = []; - this.$message.success(this.$t('user.statusChangeSuccess', { action: actionText, count: userIds.length })); - }).catch(() => { - this.loading = false; - this.$message.error(this.$t('user.operationFailed')); - }); - }).catch(() => { - this.loading = false; - this.$message.info(this.$t('common.deleteCancelled')); - }); + + // Call the existing handleChangeStatus method which already handles both single and multiple users + this.handleChangeStatus(selectedUsers, status); }, }, }; diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a02de435..4acae788 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -41,6 +41,12 @@ server: # 可选:设备白名单，如果设置了白名单，那么白名单的机器无论是什么token都可以连接。 #allowed_devices: # - "24:0A:C4:1D:3B:F0" # MAC地址列表 + # MQTT网关配置，用于通过OTA下发到设备，根据mqtt_gateway的.env文件配置，格式为host:port + mqtt_gateway: null + # MQTT签名密钥，用于生成MQTT连接密码，根据mqtt_gateway的.env文件配置 + mqtt_signature_key: null + # UDP网关配置 + udp_gateway: null log: # 设置控制台输出的日志格式，时间、日志级别、标签、消息 log_format: "{time:YYMMDD HH:mm:ss}[{version}_{selected_module}][{extra[tag]}]-{level}-{message}" @@ -427,7 +433,26 @@ ASR: enable_itn: true # 逆文本归一化 #language: "zh" # 语种，支持zh、en、ja、ko等 context: "" # 上下文信息，用于提高识别准确率，不超过10000 Token - + XunfeiStreamASR: + # 讯飞流式语音识别服务 + # 需要先在讯飞开放平台创建应用，获取以下认证信息 + # 讯飞开放平台地址：https://www.xfyun.cn/ + # 创建应用后，在"我的应用"中获取： + # - APPID + # - APISecret + # - APIKey + type: xunfei_stream + # 必填参数 - 讯飞开放平台应用信息 + app_id: 你的APPID + api_key: 你的APIKey + api_secret: 你的APISecret + # 识别参数配置 + domain: slm # 识别领域，iat:日常用语，medical:医疗，finance:金融等 + language: zh_cn # 语言，zh_cn:中文，en_us:英文 + accent: mandarin # 方言，mandarin:普通话 + dwa: wpgs # 动态修正，wpgs:实时返回中间结果 + # 调整音频处理参数以提高长语音识别质量 + output_dir: tmp/ VAD: SileroVAD: @@ -585,6 +610,16 @@ VLLM: url: https://dashscope.aliyuncs.com/compatible-mode/v1 # 可在这里找到你的api key https://bailian.console.aliyun.com/?apiKey=1#/api-key api_key: 你的api_key + XunfeiSparkLLM: + # 定义LLM API类型 + type: openai + # 先新建应用，在下面的地址 + # 开通应用地址：https://console.xfyun.cn/app/myapp + # 有免费额度，但也要开通服务，才能获取api_key + # 每一个模型都需要单独开通，每一个模型的api_password都不同，例如Lite模型在https://console.xfyun.cn/services/cbm 开通 + base_url: https://ark.cn-beijing.volces.com/api/v3 + model_name: lite + api_key: 你的api_password TTS: # 当前支持的type为edge、doubao，可自行适配 EdgeTTS: @@ -936,4 +971,26 @@ TTS: # sample_rate: 24000 # 采样率：16000, 24000, 48000 # volume: 50 # 音量：0-100 # rate: 1 # 语速：0.5~2 - # pitch: 1 # 语调：0.5~2 \ No newline at end of file + # pitch: 1 # 语调：0.5~2 + XunFeiTTS: + # 讯飞TTS服务 官方网站：https://www.xfyun.cn/ + # 登录讯飞语音技术平台 https://console.xfyun.cn/app/myapp 创建相关应用 + # 选择需要的服务获取api相关配置 https://console.xfyun.cn/services/uts + # 为需要使用的应用(APPID)购买相关服务 例如：超拟人合成 https://console.xfyun.cn/services/uts + type: xunfei_stream + api_url: wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6 + app_id: 你的app_id + api_secret: 你的api_secret + api_key: 你的api_key + voice: x5_lingxiaoxuan_flow + output_dir: tmp/ + # 以下可不用设置，使用默认设置，注意V5音色不支持口语化配置 + # oral_level: mid # 口语化等级：high, mid, low + # spark_assist: 1 # 是否通过大模型进行口语化 开启:1, 关闭:0 + # stop_split: 0 # 关闭服务端拆句 不关闭：0，关闭：1 + # remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0 + # format: raw # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb + # sample_rate: 24000 # 采样率：16000, 8000, 24000 + # volume: 50 # 音量：0-100 + # speed: 50 # 语速：0-100 + # pitch: 50 # 语调：0-100 \ No newline at end of file diff --git a/main/xiaozhi-server/config/assets/wakeup_words/ed76d459636c2481aec828516c1b4f54.wav b/main/xiaozhi-server/config/assets/wakeup_words/ed76d459636c2481aec828516c1b4f54.wav new file mode 100644 index 00000000..1e7246be Binary files /dev/null and b/main/xiaozhi-server/config/assets/wakeup_words/ed76d459636c2481aec828516c1b4f54.wav differ diff --git a/main/xiaozhi-server/core/api/ota_handler.py b/main/xiaozhi-server/core/api/ota_handler.py index 763c9c49..b93a7ed3 100644 --- a/main/xiaozhi-server/core/api/ota_handler.py +++ b/main/xiaozhi-server/core/api/ota_handler.py @@ -1,5 +1,8 @@ import json import time +import base64 +import hashlib +import hmac from aiohttp import web from core.utils.util import get_local_ip from core.api.base_handler import BaseHandler @@ -10,6 +13,24 @@ TAG = __name__ class OTAHandler(BaseHandler): def __init__(self, config: dict): super().__init__(config) + + def generate_password_signature(self, content: str, secret_key: str) -> str: + """生成MQTT密码签名 + + Args: + content: 签名内容 (clientId + '|' + username) + secret_key: 密钥 + + Returns: + str: Base64编码的HMAC-SHA256签名 + """ + try: + hmac_obj = hmac.new(secret_key.encode('utf-8'), content.encode('utf-8'), hashlib.sha256) + signature = hmac_obj.digest() + return base64.b64encode(signature).decode('utf-8') + except Exception as e: + self.logger.bind(tag=TAG).error(f"生成MQTT密码签名失败: {e}") + return "" def _get_websocket_url(self, local_ip: str, port: int) -> str: """获取websocket地址 @@ -58,10 +79,66 @@ class OTAHandler(BaseHandler): "version": data_json["application"].get("version", "1.0.0"), "url": "", }, - "websocket": { - "url": self._get_websocket_url(local_ip, port), - }, } + + + mqtt_gateway_endpoint = server_config.get("mqtt_gateway") + + if mqtt_gateway_endpoint: # 如果配置了非空字符串 + # 尝试从请求数据中获取设备型号 + device_model = "default" + try: + if "device" in data_json and isinstance(data_json["device"], dict): + device_model = data_json["device"].get("model", "default") + elif "model" in data_json: + device_model = data_json["model"] + group_id = f"GID_{device_model}".replace(":", "_").replace(" ", "_") + except Exception as e: + self.logger.bind(tag=TAG).error(f"获取设备型号失败: {e}") + group_id = "GID_default" + + mac_address_safe = device_id.replace(":", "_") + mqtt_client_id = f"{group_id}@@@{mac_address_safe}@@@{mac_address_safe}" + + # 构建用户数据 + user_data = { + "ip": "unknown" + } + try: + user_data_json = json.dumps(user_data) + username = base64.b64encode(user_data_json.encode('utf-8')).decode('utf-8') + except Exception as e: + self.logger.bind(tag=TAG).error(f"生成用户名失败: {e}") + username = "" + + # 生成密码 + password = "" + signature_key = server_config.get("mqtt_signature_key", "") + if signature_key: + password = self.generate_password_signature(mqtt_client_id + "|" + username, signature_key) + if not password: + password = "" # 签名失败则留空，由设备决定是否允许无密码 + else: + self.logger.bind(tag=TAG).warning("缺少MQTT签名密钥，密码留空") + + # 构建MQTT配置（直接使用 mqtt_gateway 字符串） + return_json["mqtt_gateway"] = { + "endpoint": mqtt_gateway_endpoint, + "client_id": mqtt_client_id, + "username": username, + "password": password, + "publish_topic": "device-server", + "subscribe_topic": f"devices/p2p/{mac_address_safe}" + } + self.logger.bind(tag=TAG).info(f"为设备 {device_id} 下发MQTT网关配置") + + + else: # 未配置 mqtt_gateway，下发 WebSocket + return_json["websocket"] = { + "url": self._get_websocket_url(local_ip, port), + } + self.logger.bind(tag=TAG).info(f"未配置MQTT网关，为设备 {device_id} 下发WebSocket配置") + response = web.Response( text=json.dumps(return_json, separators=(",", ":")), content_type="application/json", @@ -90,4 +167,4 @@ class OTAHandler(BaseHandler): response = web.Response(text="OTA接口异常", content_type="text/plain") finally: self._add_cors_headers(response) - return response + return response \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/xunfei_stream.py b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py new file mode 100644 index 00000000..7f696428 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py @@ -0,0 +1,442 @@ +import json +import hmac +import base64 +import hashlib +import asyncio +import websockets +import opuslib_next +from time import mktime +from datetime import datetime +from urllib.parse import urlencode +from typing import List +from config.logger import setup_logging +from wsgiref.handlers import format_date_time +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.dto.dto import InterfaceType + +TAG = __name__ +logger = setup_logging() + +# 帧状态常量 +STATUS_FIRST_FRAME = 0 # 第一帧的标识 +STATUS_CONTINUE_FRAME = 1 # 中间帧标识 +STATUS_LAST_FRAME = 2 # 最后一帧的标识 + +class ASRProvider(ASRProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__() + self.interface_type = InterfaceType.STREAM + self.config = config + self.text = "" + self.decoder = opuslib_next.Decoder(16000, 1) + self.asr_ws = None + self.forward_task = None + self.is_processing = False + self.server_ready = False + self.last_frame_sent = False # 标记是否已发送最终帧 + self.best_text = "" # 保存最佳识别结果 + + # 讯飞配置 + self.app_id = config.get("app_id") + self.api_key = config.get("api_key") + self.api_secret = config.get("api_secret") + + if not all([self.app_id, self.api_key, self.api_secret]): + raise ValueError("必须提供app_id、api_key和api_secret") + + # 识别参数 + self.iat_params = { + "domain": config.get("domain", "slm"), + "language": config.get("language", "zh_cn"), + "accent": config.get("accent", "mandarin"), + "dwa": config.get("dwa", "wpgs"), + "result": { + "encoding": "utf8", + "compress": "raw", + "format": "plain" + } + } + + self.output_dir = config.get("output_dir", "tmp/") + self.delete_audio_file = delete_audio_file + + def create_url(self) -> str: + """生成认证URL""" + url = 'ws://iat.cn-huabei-1.xf-yun.com/v1' + # 生成RFC1123格式的时间戳 + now = datetime.now() + date = format_date_time(mktime(now.timetuple())) + + # 拼接字符串 + signature_origin = "host: " + "iat.cn-huabei-1.xf-yun.com" + "\n" + signature_origin += "date: " + date + "\n" + signature_origin += "GET " + "/v1 " + "HTTP/1.1" + + # 进行hmac-sha256进行加密 + signature_sha = hmac.new(self.api_secret.encode('utf-8'), signature_origin.encode('utf-8'), + digestmod=hashlib.sha256).digest() + signature_sha = base64.b64encode(signature_sha).decode(encoding='utf-8') + + authorization_origin = "api_key=\"%s\", algorithm=\"%s\", headers=\"%s\", signature=\"%s\"" % ( + self.api_key, "hmac-sha256", "host date request-line", signature_sha) + authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8') + + # 将请求的鉴权参数组合为字典 + v = { + "authorization": authorization, + "date": date, + "host": "iat.cn-huabei-1.xf-yun.com" + } + + # 拼接鉴权参数，生成url + url = url + '?' + urlencode(v) + return url + + async def open_audio_channels(self, conn): + await super().open_audio_channels(conn) + + async def receive_audio(self, conn, audio, audio_have_voice): + # 先调用父类方法处理基础逻辑 + await super().receive_audio(conn, audio, audio_have_voice) + + # 存储音频数据用于声纹识别 + if not hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + conn.asr_audio_for_voiceprint.append(audio) + + # 如果本次有声音，且之前没有建立连接 + if audio_have_voice and self.asr_ws is None and not self.is_processing: + try: + await self._start_recognition(conn) + except Exception as e: + logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}") + await self._cleanup(conn) + return + + # 发送当前音频数据 + if self.asr_ws and self.is_processing and self.server_ready: + try: + pcm_frame = self.decoder.decode(audio, 960) + await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}") + await self._cleanup(conn) + + async def _start_recognition(self, conn): + """开始识别会话""" + try: + self.is_processing = True + # 建立WebSocket连接 + ws_url = self.create_url() + logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...") + + self.asr_ws = await websockets.connect( + ws_url, + max_size=1000000000, + ping_interval=None, + ping_timeout=None, + close_timeout=10, + ) + + logger.bind(tag=TAG).info("ASR WebSocket连接已建立") + self.server_ready = False + self.last_frame_sent = False + self.best_text = "" + self.forward_task = asyncio.create_task(self._forward_results(conn)) + + # 发送首帧音频 + if conn.asr_audio and len(conn.asr_audio) > 0: + first_audio = conn.asr_audio[-1] if conn.asr_audio else b'' + pcm_frame = self.decoder.decode(first_audio, 960) if first_audio else b'' + await self._send_audio_frame(pcm_frame, STATUS_FIRST_FRAME) + self.server_ready = True + logger.bind(tag=TAG).info("已发送首帧，开始识别") + + # 发送缓存的音频数据 + for cached_audio in conn.asr_audio[-10:]: + try: + pcm_frame = self.decoder.decode(cached_audio, 960) + await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME) + except Exception as e: + logger.bind(tag=TAG).info(f"发送缓存音频数据时发生错误: {e}") + break + + except Exception as e: + logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}") + if hasattr(e, "__cause__") and e.__cause__: + logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}") + if self.asr_ws: + await self.asr_ws.close() + self.asr_ws = None + self.is_processing = False + raise + + async def _send_audio_frame(self, audio_data: bytes, status: int): + """发送音频帧""" + if not self.asr_ws: + return + + audio_b64 = base64.b64encode(audio_data).decode('utf-8') + + frame_data = { + "header": { + "status": status, + "app_id": self.app_id + }, + "parameter": { + "iat": self.iat_params + }, + "payload": { + "audio": { + "audio": audio_b64, + "sample_rate": 16000, + "encoding": "raw" + } + } + } + + await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False)) + + # 标记是否发送了最终帧 + if status == STATUS_LAST_FRAME: + self.last_frame_sent = True + logger.bind(tag=TAG).info("标记最终帧已发送") + + async def _forward_results(self, conn): + """转发识别结果""" + try: + while self.asr_ws and not conn.stop_event.is_set(): + # 获取当前连接的音频数据 + audio_data = getattr(conn, 'asr_audio_for_voiceprint', []) + try: + # 如果已发送最终帧，增加超时时间等待完整结果 + timeout = 3.0 if self.last_frame_sent else 30.0 + response = await asyncio.wait_for(self.asr_ws.recv(), timeout=timeout) + result = json.loads(response) + logger.bind(tag=TAG).debug(f"收到ASR结果: {result}") + + header = result.get("header", {}) + payload = result.get("payload", {}) + code = header.get("code", 0) + status = header.get("status", 0) + + if code != 0: + logger.bind(tag=TAG).error(f"识别错误，错误码: {code}, 消息: {header.get('message', '')}") + if code in [10114, 10160]: # 连接问题 + break + continue + + # 处理识别结果 + if payload and "result" in payload: + text_data = payload["result"]["text"] + if text_data: + # 解码base64文本 + decoded_text = base64.b64decode(text_data).decode('utf-8') + text_json = json.loads(decoded_text) + + # 提取文本内容 + text_ws = text_json.get('ws', []) + result_text = '' + for i in text_ws: + for j in i.get("cw", []): + w = j.get("w", "") + result_text += w + + # 更新识别文本 - 实时更新策略 + if result_text and result_text.strip() not in ['', '。', '.', ',', '，']: + # 实时更新：正常情况下都更新，提高响应速度 + should_update = True + + # 保存最佳文本（最长的有意义文本） + if (len(result_text) > len(self.best_text) and + result_text.strip() not in ['？', '?', '。', '.']): + self.best_text = result_text + logger.bind(tag=TAG).debug(f"保存最佳文本: {self.best_text}") + + # 如果已发送最终帧，只过滤明显的无效结果 + if self.last_frame_sent: + # 最终帧后拒绝问号等明显错误的结果 + if result_text.strip() in ['？', '?', '。', '.']: + should_update = False + logger.bind(tag=TAG).warning(f"最终帧后拒绝无效文本: {result_text}") + + if should_update: + self.text = result_text + logger.bind(tag=TAG).info(f"实时更新识别文本: {self.text} (最终帧已发送: {self.last_frame_sent})") + + # 识别完成，但如果还没发送最终帧，继续等待 + if status == 2: + logger.bind(tag=TAG).info(f"识别完成状态已到达，当前识别文本: {self.text}") + + # 如果还没发送最终帧，继续等待 + if not self.last_frame_sent: + logger.bind(tag=TAG).info("识别完成但最终帧未发送，继续等待...") + continue + + # 已发送最终帧且收到完成状态，使用最佳文本作为最终结果 + if self.best_text and len(self.best_text) > len(self.text): + logger.bind(tag=TAG).info(f"使用最佳文本作为最终结果: {self.text} -> {self.best_text}") + self.text = self.best_text + + logger.bind(tag=TAG).info(f"获取到最终完整文本: {self.text}") + conn.reset_vad_states() + if len(audio_data) > 15: # 确保有足够音频数据 + # 准备处理结果 + pass + break + + except asyncio.TimeoutError: + if self.last_frame_sent: + # 超时时也使用最佳文本 + if self.best_text and len(self.best_text) > len(self.text): + logger.bind(tag=TAG).info(f"超时，使用最佳文本: {self.text} -> {self.best_text}") + self.text = self.best_text + logger.bind(tag=TAG).info(f"最终帧后超时，使用结果: {self.text}") + break + # 如果还没发送最终帧，继续等待 + continue + except websockets.ConnectionClosed: + logger.bind(tag=TAG).info("ASR服务连接已关闭") + self.is_processing = False + break + except Exception as e: + logger.bind(tag=TAG).error(f"处理ASR结果时发生错误: {str(e)}") + if hasattr(e, "__cause__") and e.__cause__: + logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}") + self.is_processing = False + break + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR结果转发任务发生错误: {str(e)}") + if hasattr(e, "__cause__") and e.__cause__: + logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}") + finally: + if self.asr_ws: + await self.asr_ws.close() + self.asr_ws = None + self.is_processing = False + if conn: + if hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + if hasattr(conn, 'asr_audio'): + conn.asr_audio = [] + if hasattr(conn, 'has_valid_voice'): + conn.has_valid_voice = False + + async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]): + """处理语音停止，发送最后一帧并处理识别结果""" + try: + # 先发送最后一帧表示音频结束 + if self.asr_ws and self.is_processing: + try: + # 取最后一个有效的音频帧作为最后一帧数据 + last_frame = b'' + if asr_audio_task: + last_audio = asr_audio_task[-1] + last_frame = self.decoder.decode(last_audio, 960) + await self._send_audio_frame(last_frame, STATUS_LAST_FRAME) + logger.bind(tag=TAG).info("已发送最后一帧") + + # 发送最终帧后，给_forward_results适当时间处理最终结果 + # 减少等待时间，提高响应速度 + await asyncio.sleep(0.1) # 从3秒减少到1秒 + + logger.bind(tag=TAG).info(f"准备处理最终识别结果: {self.text}") + except Exception as e: + logger.bind(tag=TAG).error(f"发送最后一帧失败: {e}") + + # 调用父类的handle_voice_stop方法处理识别结果 + await super().handle_voice_stop(conn, asr_audio_task) + except Exception as e: + logger.bind(tag=TAG).error(f"处理语音停止失败: {e}") + import traceback + logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}") + + def stop_ws_connection(self): + if self.asr_ws: + asyncio.create_task(self.asr_ws.close()) + self.asr_ws = None + self.is_processing = False + + async def _cleanup(self, conn): + """清理资源""" + logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}") + + # 发送最后一帧 + if self.asr_ws and self.is_processing: + try: + await self._send_audio_frame(b'', STATUS_LAST_FRAME) + await asyncio.sleep(0.1) + logger.bind(tag=TAG).info("已发送最后一帧") + except Exception as e: + logger.bind(tag=TAG).error(f"发送最后一帧失败: {e}") + + # 状态重置 + self.is_processing = False + self.server_ready = False + self.last_frame_sent = False + self.best_text = "" + logger.bind(tag=TAG).info("ASR状态已重置") + + # 清理任务 + if self.forward_task and not self.forward_task.done(): + self.forward_task.cancel() + try: + await asyncio.wait_for(self.forward_task, timeout=1.0) + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}") + finally: + self.forward_task = None + + # 关闭连接 + if self.asr_ws: + try: + logger.bind(tag=TAG).debug("正在关闭WebSocket连接") + await asyncio.wait_for(self.asr_ws.close(), timeout=2.0) + logger.bind(tag=TAG).debug("WebSocket连接已关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}") + finally: + self.asr_ws = None + + # 清理连接的音频缓存 + if conn: + if hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + if hasattr(conn, 'asr_audio'): + conn.asr_audio = [] + if hasattr(conn, 'has_valid_voice'): + conn.has_valid_voice = False + + logger.bind(tag=TAG).info("ASR会话清理完成") + + async def speech_to_text(self, opus_data, session_id, audio_format): + """获取识别结果""" + result = self.text + self.text = "" + return result, None + + async def close(self): + """资源清理方法""" + if self.asr_ws: + await self.asr_ws.close() + self.asr_ws = None + if self.forward_task: + self.forward_task.cancel() + try: + await self.forward_task + except asyncio.CancelledError: + pass + self.forward_task = None + self.is_processing = False + # 清理所有连接的音频缓冲区 + if hasattr(self, '_connections'): + for conn in self._connections.values(): + if hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + if hasattr(conn, 'asr_audio'): + conn.asr_audio = [] + if hasattr(conn, 'has_valid_voice'): + conn.has_valid_voice = False \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py new file mode 100644 index 00000000..d06c9330 --- /dev/null +++ b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py @@ -0,0 +1,527 @@ +import os +import time +import uuid +import json +import hmac +import queue +import base64 +import hashlib +import asyncio +import traceback +import websockets +from asyncio import Task +from config.logger import setup_logging +from core.utils import opus_encoder_utils +from core.utils.tts import MarkdownCleaner +from urllib.parse import urlencode, urlparse +from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType + +TAG = __name__ +logger = setup_logging() + + +class XunfeiWSAuth: + @staticmethod + def create_auth_url(api_key, api_secret, api_url): + """生成讯飞WebSocket认证URL""" + parsed_url = urlparse(api_url) + host = parsed_url.netloc + path = parsed_url.path + + # 获取UTC时间，讯飞要求使用RFC1123格式 + now = time.gmtime() + date = time.strftime('%a, %d %b %Y %H:%M:%S GMT', now) + + # 构造签名字符串 + signature_origin = f"host: {host}\ndate: {date}\nGET {path} HTTP/1.1" + + # 计算签名 + signature_sha = hmac.new( + api_secret.encode('utf-8'), + signature_origin.encode('utf-8'), + digestmod=hashlib.sha256 + ).digest() + signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8') + + # 构造authorization + authorization_origin = f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"' + authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8') + + # 构造最终的WebSocket URL + v = { + "authorization": authorization, + "date": date, + "host": host + } + url = api_url + '?' + urlencode(v) + return url + + +class TTSProvider(TTSProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + + # 设置为流式接口类型 + self.interface_type = InterfaceType.DUAL_STREAM + + # 基础配置 + self.app_id = config.get("app_id") + self.api_key = config.get("api_key") + self.api_secret = config.get("api_secret") + + # 接口地址 + self.api_url = config.get("api_url", "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6") + + # 音色配置 + self.voice = config.get("voice", "x5_lingxiaoxuan_flow") + if config.get("private_voice"): + self.voice = config.get("private_voice") + + # 音频参数配置 + speed = config.get("speed", "50") + self.speed = int(speed) if speed else 50 + + volume = config.get("volume", "50") + self.volume = int(volume) if volume else 50 + + pitch = config.get("pitch", "50") + self.pitch = int(pitch) if pitch else 50 + + # 音频编码配置 + self.format = config.get("format", "raw") + + sample_rate = config.get("sample_rate", "24000") + self.sample_rate = int(sample_rate) if sample_rate else 24000 + + # 口语化配置 + self.oral_level = config.get("oral_level", "mid") + + spark_assist = config.get("spark_assist", "1") + self.spark_assist = int(spark_assist) if spark_assist else 1 + + stop_split = config.get("stop_split", "0") + self.stop_split = int(stop_split) if stop_split else 0 + + remain = config.get("remain", "0") + self.remain = int(remain) if remain else 0 + + # WebSocket配置 + self.ws = None + self._monitor_task = None + + # 序列号管理 + self.text_seq = 0 + + # 创建Opus编码器 + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=self.sample_rate, channels=1, frame_size_ms=60 + ) + + # 验证必需参数 + if not all([self.app_id, self.api_key, self.api_secret]): + raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret") + + async def _ensure_connection(self): + """确保WebSocket连接可用""" + try: + logger.bind(tag=TAG).info("开始建立新连接...") + + # 生成认证URL + auth_url = XunfeiWSAuth.create_auth_url( + self.api_key, self.api_secret, self.api_url + ) + + self.ws = await websockets.connect( + auth_url, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ) + logger.bind(tag=TAG).info("WebSocket连接建立成功") + return self.ws + except Exception as e: + logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}") + self.ws = None + raise + + def tts_text_priority_thread(self): + """流式文本处理线程""" + while not self.conn.stop_event.is_set(): + try: + message = self.tts_text_queue.get(timeout=1) + logger.bind(tag=TAG).debug( + f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}" + ) + + if message.sentence_type == SentenceType.FIRST: + # 重置序列号 + self.text_seq = 0 + self.conn.client_abort = False + # 增加序列号 + self.text_seq += 1 + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") + continue + + if message.sentence_type == SentenceType.FIRST: + # 初始化参数 + try: + if not getattr(self.conn, "sentence_id", None): + self.conn.sentence_id = uuid.uuid4().hex + logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") + + logger.bind(tag=TAG).info("开始启动TTS会话...") + future = asyncio.run_coroutine_threadsafe( + self.start_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + future.result() + self.before_stop_play_files.clear() + logger.bind(tag=TAG).info("TTS会话启动成功") + + except Exception as e: + logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}") + continue + + # 处理文本内容 + if ContentType.TEXT == message.content_type: + if message.content_detail: + try: + logger.bind(tag=TAG).debug( + f"开始发送TTS文本: {message.content_detail}" + ) + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(message.content_detail, None), + loop=self.conn.loop, + ) + future.result() + logger.bind(tag=TAG).debug("TTS文本发送成功") + except Exception as e: + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + # 不使用continue，确保后续处理不被中断 + + # 处理文件内容 + if ContentType.FILE == message.content_type: + logger.bind(tag=TAG).info( + f"添加音频文件到待播放列表: {message.content_file}" + ) + if message.content_file and os.path.exists(message.content_file): + # 先处理文件音频数据 + self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail)) + + # 处理会话结束 + if message.sentence_type == SentenceType.LAST: + try: + logger.bind(tag=TAG).info("开始结束TTS会话...") + asyncio.run_coroutine_threadsafe( + self.finish_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + except Exception as e: + logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}") + continue + + except queue.Empty: + continue + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" + ) + + async def text_to_speak(self, text, _): + """发送文本到TTS服务进行合成""" + try: + if self.ws is None: + logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本") + return + + filtered_text = MarkdownCleaner.clean_markdown(text) + + # 发送文本合成请求 + run_request = self._build_base_request(status=1,text=filtered_text) + await self.ws.send(json.dumps(run_request)) + return + + except Exception as e: + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + if self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + raise + + async def start_session(self, session_id): + logger.bind(tag=TAG).info(f"开始会话～～{session_id}") + try: + # 会话开始时检测上个会话的监听状态 + if ( + self._monitor_task is not None + and isinstance(self._monitor_task, Task) + and not self._monitor_task.done() + ): + logger.bind(tag=TAG).info( + "检测到未完成的上个会话，关闭监听任务和连接..." + ) + await self.close() + + # 建立新连接 + await self._ensure_connection() + + # 启动监听任务 + self._monitor_task = asyncio.create_task(self._start_monitor_tts_response()) + + # 发送会话启动请求 + start_request = self._build_base_request(status=0) + + await self.ws.send(json.dumps(start_request)) + logger.bind(tag=TAG).info("会话启动请求已发送") + except Exception as e: + logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise + + async def finish_session(self, session_id): + logger.bind(tag=TAG).info(f"关闭会话～～{session_id}") + try: + if self.ws: + # 发送会话结束请求 + stop_request = self._build_base_request(status=2) + await self.ws.send(json.dumps(stop_request)) + logger.bind(tag=TAG).info("会话结束请求已发送") + + if self._monitor_task: + try: + await self._monitor_task + except Exception as e: + logger.bind(tag=TAG).error(f"等待监听任务完成时发生错误: {str(e)}") + finally: + self._monitor_task = None + except Exception as e: + logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}") + await self.close() + raise + + async def close(self): + """资源清理""" + if self._monitor_task: + try: + self._monitor_task.cancel() + await self._monitor_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}") + self._monitor_task = None + + if self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + + async def _start_monitor_tts_response(self): + """监听TTS响应""" + try: + while not self.conn.stop_event.is_set(): + try: + msg = await self.ws.recv() + + # 检查客户端是否中止 + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") + break + + try: + data = json.loads(msg) + header = data.get("header", {}) + code = header.get("code") + + if code == 0: + payload = data.get("payload", {}) + audio_payload = payload.get("audio", {}) + + if audio_payload: + status = audio_payload.get("status", 0) + audio_data = audio_payload.get("audio", "") + if status == 0: + logger.bind(tag=TAG).debug("TTS合成已启动") + self.tts_audio_queue.put( + (SentenceType.FIRST, [], None) + ) + elif status == 2: + logger.bind(tag=TAG).debug("收到结束状态的音频数据，TTS合成完成") + self._process_before_stop_play_files() + break + else: + if self.conn.tts_MessageText: + logger.bind(tag=TAG).info( + f"句子语音生成成功： {self.conn.tts_MessageText}" + ) + self.tts_audio_queue.put( + (SentenceType.FIRST, [], self.conn.tts_MessageText) + ) + self.conn.tts_MessageText = None + try: + audio_bytes = base64.b64decode(audio_data) + self.opus_encoder.encode_pcm_to_opus_stream( + audio_bytes, False, self.handle_opus + ) + + except Exception as e: + logger.bind(tag=TAG).error(f"处理音频数据失败: {e}") + + else: + message = header.get("message", "未知错误") + logger.bind(tag=TAG).error(f"TTS合成错误: {code} - {message}") + break + + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + + except websockets.ConnectionClosed: + logger.bind(tag=TAG).warning("WebSocket连接已关闭") + break + + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS响应时出错: {e}\n{traceback.format_exc()}" + ) + break + + # 链接不可复用 + if self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + # 监听任务退出时清理引用 + finally: + self._monitor_task = None + + def to_tts(self, text: str) -> list: + """非流式TTS处理，用于测试及保存音频文件的场景""" + try: + # 创建新的事件循环 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # 存储音频数据 + audio_data = [] + + async def _generate_audio(): + # 生成认证URL + auth_url = XunfeiWSAuth.create_auth_url( + self.api_key, self.api_secret, self.api_url + ) + + # 建立WebSocket连接 + ws = await websockets.connect( + auth_url, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ) + + try: + filtered_text = MarkdownCleaner.clean_markdown(text) + + text_request = self._build_base_request(status=2,text=filtered_text) + + await ws.send(json.dumps(text_request)) + + task_finished = False + while not task_finished: + msg = await ws.recv() + + data = json.loads(msg) + header = data.get("header", {}) + code = header.get("code") + + if code == 0: + payload = data.get("payload", {}) + audio_payload = payload.get("audio", {}) + if audio_payload: + status = audio_payload.get("status", 0) + audio_base64 = audio_payload.get("audio", "") + if status == 1: + try: + audio_bytes = base64.b64decode(audio_base64) + self.opus_encoder.encode_pcm_to_opus_stream( + audio_bytes, + end_of_stream=False, + callback=lambda opus: audio_data.append(opus) + ) + except Exception as e: + logger.bind(tag=TAG).error(f"处理音频数据失败: {e}") + elif status == 2: + task_finished = True + logger.bind(tag=TAG).debug("TTS任务完成") + + else: + message = header.get("message", "未知错误") + raise Exception(f"合成失败: {code} - {message}") + + finally: + # 清理资源 + try: + await ws.close() + except: + pass + + loop.run_until_complete(_generate_audio()) + loop.close() + + return audio_data + except Exception as e: + logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") + return [] + + def _build_base_request(self, status,text=" "): + """构建基础请求结构""" + return { + "header": { + "app_id": self.app_id, + "status": status, + }, + "parameter": { + "oral": { + "oral_level": self.oral_level, + "spark_assist": self.spark_assist, + "stop_split": self.stop_split, + "remain": self.remain + }, + "tts": { + "vcn": self.voice, + "speed": self.speed, + "volume": self.volume, + "pitch": self.pitch, + "bgs": 0, + "reg": 0, + "rdn": 0, + "rhy": 0, + "audio": { + "encoding": self.format, + "sample_rate": self.sample_rate, + "channels": 1, + "bit_depth": 16, + "frame_size": 0 + } + } + }, + "payload": { + "text": { + "encoding": "utf8", + "compress": "raw", + "format": "plain", + "status": status, + "seq": self.text_seq, + "text": base64.b64encode(text.encode('utf-8')).decode('utf-8') + } + } + } diff --git a/main/xiaozhi-server/core/utils/tts.py b/main/xiaozhi-server/core/utils/tts.py index eb383667..8b68d195 100644 --- a/main/xiaozhi-server/core/utils/tts.py +++ b/main/xiaozhi-server/core/utils/tts.py @@ -6,6 +6,27 @@ import importlib logger = setup_logging() +punctuation_set = { + "，", + ",", # 中文逗号 + 英文逗号 + "。", + ".", # 中文句号 + 英文句号 + "！", + "!", # 中文感叹号 + 英文感叹号 + "“", + "”", + '"', # 中文双引号 + 英文引号 + "：", + ":", # 中文冒号 + 英文冒号 + "-", + "－", # 英文连字符 + 中文全角横线 + "、", # 中文顿号 + "[", + "]", # 方括号 + "【", + "】", # 中文方括号 + "~", # 波浪号 +} def create_instance(class_name, *args, **kwargs): # 创建TTS实例 @@ -107,6 +128,11 @@ class MarkdownCleaner: """ 主入口方法：依序执行所有正则，移除或替换 Markdown 元素 """ + # 检查文本是否全为英文和基本标点符号 + if text and all((c.isascii() or c.isspace() or c in punctuation_set) for c in text): + # 保留原始空格，直接返回 + return text + for regex, replacement in MarkdownCleaner.REGEXES: text = regex.sub(replacement, text) return text.strip() \ No newline at end of file