From 372dc8ecb045ae97c1eb12381c5ef48723b53f8d Mon Sep 17 00:00:00 2001 From: 3030332422 <3030332422@qq.com> Date: Tue, 19 Aug 2025 14:45:14 +0800 Subject: [PATCH 01/53] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E9=A1=B5=E9=9D=A2=E5=85=A8=E9=80=89=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E5=A4=B1=E6=95=88=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/views/DeviceManagement.vue | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/main/manager-web/src/views/DeviceManagement.vue b/main/manager-web/src/views/DeviceManagement.vue index 35238371..55f781fa 100644 --- a/main/manager-web/src/views/DeviceManagement.vue +++ b/main/manager-web/src/views/DeviceManagement.vue @@ -74,7 +74,7 @@
- {{ isAllSelected ? '取消全选' : '全选' }} + {{ isCurrentPageAllSelected ? '取消全选' : '全选' }} 验证码绑定 @@ -128,8 +128,6 @@ export default { return { addDeviceDialogVisible: false, manualAddDeviceDialogVisible: false, - selectedDevices: [], - isAllSelected: false, searchKeyword: "", activeSearchKeyword: "", currentAgentId: this.$route.query.agentId || '', @@ -160,6 +158,11 @@ export default { pageCount() { return Math.ceil(this.filteredDeviceList.length / this.pageSize); }, + // 计算当前页是否全选 + isCurrentPageAllSelected() { + return this.paginatedDeviceList.length > 0 && + this.paginatedDeviceList.every(device => device.selected); + }, visiblePages() { const pages = []; const maxVisible = 3; @@ -205,16 +208,15 @@ export default { }, handleSelectAll() { - this.isAllSelected = !this.isAllSelected; + const shouldSelectAll = !this.isCurrentPageAllSelected; this.paginatedDeviceList.forEach(row => { - row.selected = this.isAllSelected; + row.selected = shouldSelectAll; }); - this.selectedDevices = this.paginatedDeviceList.filter(device => device.selected); }, deleteSelected() { - this.selectedDevices = this.paginatedDeviceList.filter(device => device.selected); - if (this.selectedDevices.length === 0) { + const selectedDevices = this.paginatedDeviceList.filter(device => device.selected); + if (selectedDevices.length === 0) { this.$message.warning({ message: '请至少选择一条记录', showClose: true @@ -222,12 +224,12 @@ export default { return; } - this.$confirm(`确认要解绑选中的 ${this.selectedDevices.length} 台设备吗？`, '警告', { + this.$confirm(`确认要解绑选中的 ${selectedDevices.length} 台设备吗？`, '警告', { confirmButtonText: '确定', cancelButtonText: '取消', type: 'warning' }).then(() => { - const deviceIds = this.selectedDevices.map(device => device.device_id); + const deviceIds = selectedDevices.map(device => device.device_id); this.batchUnbindDevices(deviceIds); }); }, @@ -250,8 +252,6 @@ export default { showClose: true }); this.fetchBindDevices(this.currentAgentId); - this.selectedDevices = []; - this.isAllSelected = false; }) .catch(error => { this.$message.error({ @@ -355,7 +355,8 @@ export default { isEdit: false, _submitting: false, otaSwitch: device.autoUpdate === 1, - rawBindTime: new Date(device.createDate).getTime() + rawBindTime: new Date(device.createDate).getTime(), + selected: false }; }) .sort((a, b) => a.rawBindTime - b.rawBindTime); From 74826c1c598e6a532c7ecd1cfa62cb9b394616b0 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 20 Aug 2025 09:30:43 +0800 Subject: [PATCH 02/53] fix --- .../core/handle/sendAudioHandle.py | 9 +++++++-- main/xiaozhi-server/core/providers/tts/base.py | 7 +++---- .../core/utils/audio_flow_control.py | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index c3c50fbc..bb85c007 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -1,4 +1,5 @@ import json +import asyncio from core.providers.tts.dto.dto import SentenceType from core.utils import textUtils @@ -52,8 +53,12 @@ async def send_tts_message(conn, state, text=None): stop_tts_notify_voice = conn.config.get( "stop_tts_notify_voice", "config/assets/tts_notify.mp3" ) - audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice) - await sendAudio(conn, audios) + conn.tts.audio_to_opus_data_stream( + stop_tts_notify_voice, + callback=lambda audio_data: asyncio.run_coroutine_threadsafe( + sendAudio(conn, audio_data), conn.loop + ), + ) # 清除服务端讲话状态 conn.clearSpeakStatus() diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 0b942862..35d80fb6 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -11,7 +11,7 @@ from datetime import datetime from core.utils import textUtils from abc import ABC, abstractmethod from config.logger import setup_logging -from core.utils.audio_flow_control import FlowControlConfig +from core.utils.audio_flow_control import FlowControlConfig, simulate_device_consumption from core.utils.util import audio_bytes_to_data_stream, audio_to_data_stream from core.utils.tts import MarkdownCleaner from core.utils.output_counter import add_device_output @@ -356,9 +356,8 @@ class TTSProviderBase(ABC): # 模拟设备消费（实际应用中应该从设备获取反馈）防止音字不同步 if isinstance(audio_datas, bytes): - # 模拟设备播放延迟（60ms per frame）, 实际情况可以低一点（50ms），增加使用体验 - await asyncio.sleep(0.06) - self.flow_controller.update_device_consumption(1) + frame_count = 1 + asyncio.create_task(simulate_device_consumption(self.flow_controller, frame_count)) # 在类中添加流控制器重置方法 def reset_flow_controller(self): diff --git a/main/xiaozhi-server/core/utils/audio_flow_control.py b/main/xiaozhi-server/core/utils/audio_flow_control.py index 98c29676..958e68a7 100644 --- a/main/xiaozhi-server/core/utils/audio_flow_control.py +++ b/main/xiaozhi-server/core/utils/audio_flow_control.py @@ -151,6 +151,21 @@ class AudioFlowController: ) +async def simulate_device_consumption( + flow_controller: AudioFlowController, frame_count: int +): + """ + 模拟设备消费音频帧的过程 + 实际应用中应该根据设备反馈来更新消费情况 + Args: + flow_controller: 流控制器实例 + frame_count: 消费的帧数 + """ + # 模拟设备播放延迟（60ms per frame） + await asyncio.sleep(frame_count * 0.06) + flow_controller.update_device_consumption(frame_count) + + # 流控配置常量 class FlowControlConfig: """流控配置常量""" @@ -183,4 +198,4 @@ class FlowControlConfig: return AudioFlowController( max_device_buffer=max_buffer or cls.DEFAULT_MAX_DEVICE_BUFFER, refill_rate=refill_rate or cls.DEFAULT_REFILL_RATE - ) \ No newline at end of file + ) From 7d7b8ddfb114147650c7b9e12740f29637055b90 Mon Sep 17 00:00:00 2001 From: VanillaNahida Date: Wed, 20 Aug 2025 14:00:53 +0800 Subject: [PATCH 03/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E8=8E=B7=E5=8F=96=E4=B8=8D=E5=88=B0ws=E5=92=8C?= =?UTF-8?q?=E8=A7=86=E8=A7=89=E5=88=86=E6=9E=90=E6=8E=A5=E5=8F=A3=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=E7=9A=84bug=20docs:=20=E6=9B=B4=E6=96=B0README?= =?UTF-8?q?=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-setup.sh | 15 ++++++++------- docs/Deployment_all.md | 5 ++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docker-setup.sh b/docker-setup.sh index 938917f4..212f68ff 100755 --- a/docker-setup.sh +++ b/docker-setup.sh @@ -46,7 +46,7 @@ cat << "EOF" \/ \__,_||_| |_||_||_||_| \__,_| |_| \_| \__,_||_| |_||_| \__,_| \__,_| EOF echo -e "\e[0m" # 重置颜色 -echo -e "\e[1;36m 小智服务端全量部署一键安装脚本 Ver 0.2 \e[0m\n" +echo -e "\e[1;36m 小智服务端全量部署一键安装脚本 Ver 0.2 2025年8月20日更新 \e[0m\n" sleep 1 @@ -376,7 +376,7 @@ done echo "服务端启动成功！正在完成配置..." echo "正在启动服务..." - docker compose -f docker-compose_all.yml up -d + docker compose -f /opt/xiaozhi-server/docker-compose_all.yml up -d echo "服务启动完成！" ) @@ -402,13 +402,14 @@ fi # 获取并显示地址信息 LOCAL_IP=$(hostname -I | awk '{print $1}') -WEBSOCKET_ADDR=$(docker logs xiaozhi-esp32-server 2>&1 | tac | grep -m 1 -E -o "ws://[^ ]+") -VISION_ADDR=$(docker logs xiaozhi-esp32-server 2>&1 | tac | grep -m 1 "视觉" | grep -m 1 -E -o "http://[^ ]+") +# WEBSOCKET_ADDR=$(docker logs xiaozhi-esp32-server 2>&1 | tac | grep -m 1 -E -o "ws://[^ ]+") +# VISION_ADDR=$(docker logs xiaozhi-esp32-server 2>&1 | tac | grep -m 1 "视觉" | grep -m 1 -E -o "http://[^ ]+") +# 修复日志文件获取不到ws的问题，改为硬编码 whiptail --title "安装完成！" --msgbox "\ 服务端相关地址如下：\n\ 管理后台访问地址: http://$LOCAL_IP:8002\n\ -OTA 地址: http://$LOCAL_IP:8002/xiaozhi/ota/\n\ -视觉分析接口地址: $VISION_ADDR\n\ -WebSocket 地址: $WEBSOCKET_ADDR\n\ +OTA 地址: http://$LOCAL_IP:8003/xiaozhi/ota/\n\ +视觉分析接口地址: http://$LOCAL_IP:8003/mcp/vision/explain\n\ +WebSocket 地址: ws://$LOCAL_IP:8000/xiaozhi/v1/\n\ \n安装完毕！感谢您的使用！\n按Enter键退出..." 16 70 \ No newline at end of file diff --git a/docs/Deployment_all.md b/docs/Deployment_all.md index f93a266e..153a13fe 100644 --- a/docs/Deployment_all.md +++ b/docs/Deployment_all.md @@ -7,7 +7,10 @@ docker镜像已支持x86架构、arm64架构的CPU，支持在国产操作系统 如果您的电脑还没安装docker，可以按照这里的教程安装：[docker安装](https://www.runoob.com/docker/ubuntu-docker-install.html) -docker 安装全模块有两种方式，你可以[1.1使用懒人脚本](#1.1 懒人脚本)（作者[@VanillaNahida](https://github.com/VanillaNahida)）自动帮你下载所需的文件和配置文件，你可以使用[1.2手动部署](#1.2 手动部署)从零搭建。 +docker 安装全模块有两种方式，你可以[1.1使用懒人脚本](./docs/Deployment_all.md#1.1+%e6%87%92%e4%ba%ba%e8%84%9a%e6%9c%ac)（作者[@VanillaNahida](https://github.com/VanillaNahida)） +脚本会自动帮你下载所需的文件和配置文件，你可以使用[1.2 手动部署](./docs/Deployment_all.md#1.2+%e6%89%8b%e5%8a%a8%e9%83%a8%e7%bd%b2)从零搭建。 + + ### 1.1 懒人脚本 From 67d3a8f94f8db70f41de41a0113bb8ff2bdbc54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A6=99=E8=8D=89=E5=91=B3=E7=9A=84=E7=BA=B3=E8=A5=BF?= =?UTF-8?q?=E5=A6=B2=E5=96=B5?= <151599587+VanillaNahida@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:07:46 +0800 Subject: [PATCH 04/53] Update Deployment_all.md --- docs/Deployment_all.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Deployment_all.md b/docs/Deployment_all.md index 153a13fe..f48f6e10 100644 --- a/docs/Deployment_all.md +++ b/docs/Deployment_all.md @@ -7,13 +7,13 @@ docker镜像已支持x86架构、arm64架构的CPU，支持在国产操作系统 如果您的电脑还没安装docker，可以按照这里的教程安装：[docker安装](https://www.runoob.com/docker/ubuntu-docker-install.html) -docker 安装全模块有两种方式，你可以[1.1使用懒人脚本](./docs/Deployment_all.md#1.1+%e6%87%92%e4%ba%ba%e8%84%9a%e6%9c%ac)（作者[@VanillaNahida](https://github.com/VanillaNahida)） -脚本会自动帮你下载所需的文件和配置文件，你可以使用[1.2 手动部署](./docs/Deployment_all.md#1.2+%e6%89%8b%e5%8a%a8%e9%83%a8%e7%bd%b2)从零搭建。 +docker 安装全模块有两种方式，你可以[使用懒人脚本](./Deployment_all.md#11-懒人脚本)（作者[@VanillaNahida](https://github.com/VanillaNahida)） +脚本会自动帮你下载所需的文件和配置文件，你也可以使用[手动部署](./Deployment_all.md#12-手动部署)从零搭建。 ### 1.1 懒人脚本 - +**推荐使用**，**全自动部署**，只需**简单配置**即可使用，视频教程：[https://www.bilibili.com/video/BV17bbvzHExd/](https://www.bilibili.com/video/BV17bbvzHExd/) 你可以使用以下命令一键安装全模块版小智服务端： > [!NOTE] > 暂且只支持Ubuntu服务器一键部署，其他系统未尝试，可能会有一些奇怪的bug From d7dc636d3f9bfc0be52753c2711d7fde55591f9a Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Wed, 20 Aug 2025 14:30:47 +0800 Subject: [PATCH 05/53] =?UTF-8?q?update:=E6=9B=B4=E6=96=B0=E6=B5=81?= =?UTF-8?q?=E5=BC=8FTTS=E6=B5=8B=E8=AF=95=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/performance_tester.md | 4 +- .../performance_tester_asr.py | 307 +++++++++++--- .../performance_tester_stream_tts.py | 388 ++++++++++++++++++ .../performance_tester_tts.py | 66 ++- 4 files changed, 701 insertions(+), 64 deletions(-) create mode 100644 main/xiaozhi-server/performance_tester/performance_tester_stream_tts.py diff --git a/docs/performance_tester.md b/docs/performance_tester.md index abe57458..85987120 100644 --- a/docs/performance_tester.md +++ b/docs/performance_tester.md @@ -1,8 +1,8 @@ -# 语音识别、大语言模型、非流式语音合成、视觉模型的性能测试工具使用指南 +# 语音识别、大语言模型、非流式语音合成、流式语音合成、视觉模型的性能测试工具使用指南 1.在main/xiaozhi-server目录下创建data目录 2.在data目录下创建.config.yaml文件 -3.在.data/config.yaml中，写入你的语音识别、大语言模型、非流式语音合成、视觉模型的参数 +3.在.data/config.yaml中，写入你的语音识别、大语言模型、流式语音合成、视觉模型的参数 例如： ``` LLM: diff --git a/main/xiaozhi-server/performance_tester/performance_tester_asr.py b/main/xiaozhi-server/performance_tester/performance_tester_asr.py index e4d8c6d8..db359e28 100644 --- a/main/xiaozhi-server/performance_tester/performance_tester_asr.py +++ b/main/xiaozhi-server/performance_tester/performance_tester_asr.py @@ -2,35 +2,56 @@ import asyncio import logging import os import time -from typing import Dict - +import concurrent.futures +from typing import Dict, Optional import aiohttp from tabulate import tabulate from core.utils.asr import create_instance as create_stt_instance -from config.settings import load_config # 设置全局日志级别为WARNING，抑制INFO级别日志 logging.basicConfig(level=logging.WARNING) description = "语音识别模型性能测试" - class ASRPerformanceTester: def __init__(self): - self.config = load_config() + self.config = self._load_config_from_data_dir() self.test_wav_list = self._load_test_wav_files() self.results = {"stt": {}} - + # 调试日志 print(f"[DEBUG] 加载的ASR配置: {self.config.get('ASR', {})}") print(f"[DEBUG] 音频文件数量: {len(self.test_wav_list)}") + def _load_config_from_data_dir(self) -> Dict: + """从 data 目录加载所有 .config.yaml 文件的配置""" + config = {"ASR": {}} + data_dir = os.path.join(os.getcwd(), "data") + print(f"[DEBUG] 扫描配置文件目录: {data_dir}") + + for root, _, files in os.walk(data_dir): + for file in files: + if file.endswith(".config.yaml"): + file_path = os.path.join(root, file) + try: + with open(file_path, "r", encoding="utf-8") as f: + import yaml + file_config = yaml.safe_load(f) + # 兼容大小写的 ASR/asr 配置 + asr_config = file_config.get("ASR") or file_config.get("asr") + if asr_config: + config["ASR"].update(asr_config) + print(f"[DEBUG] 从 {file_path} 加载 ASR 配置成功") + except Exception as e: + print(f" 加载配置文件 {file_path} 失败: {str(e)}") + return config + def _load_test_wav_files(self) -> list: """加载测试用的音频文件（添加路径调试）""" wav_root = os.path.join(os.getcwd(), "config", "assets") print(f"[DEBUG] 音频文件目录: {wav_root}") test_wav_list = [] - + if os.path.exists(wav_root): file_list = os.listdir(wav_root) print(f"[DEBUG] 找到音频文件: {file_list}") @@ -43,18 +64,46 @@ class ASRPerformanceTester: print(f" 目录不存在: {wav_root}") return test_wav_list - async def _test_stt(self, stt_name: str, config: Dict) -> Dict: - """异步测试单个STT性能（跳过无效配置）""" + async def _test_single_audio(self, stt_name: str, stt, audio_data: bytes) -> Optional[float]: + """测试单个音频文件的性能""" try: + start_time = time.time() + text, _ = await stt.speech_to_text([audio_data], "1", stt.audio_format) + if text is None: + return None + + duration = time.time() - start_time + + # 检测0.000s的异常时间 + if abs(duration) < 0.001: # 小于1毫秒视为异常 + print(f"{stt_name} 检测到异常时间: {duration:.6f}s (视为错误)") + return None + + return duration + except Exception as e: + error_msg = str(e).lower() + if "502" in error_msg or "bad gateway" in error_msg: + print(f"{stt_name} 遇到502错误") + return None + return None + + async def _test_stt_with_timeout(self, stt_name: str, config: Dict) -> Dict: + """异步测试单个STT性能，带超时控制""" + try: + # 检查配置有效性 token_fields = ["access_token", "api_key", "token"] - # 忽略值为 "none" 的情况（需根据实际需求调整） if any( field in config - and str(config[field]).lower() in ["你的", "placeholder"] + and str(config[field]).lower() in ["你的", "placeholder", "none", "null", ""] for field in token_fields ): - print(f" STT {stt_name} 未配置access_token/api_key，已跳过") - return {"name": stt_name, "type": "stt", "errors": 1} + print(f" STT {stt_name} 未配置有效access_token/api_key，已跳过") + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "配置错误" + } module_type = config.get("type", stt_name) stt = create_stt_instance(module_type, config, delete_audio_file=True) @@ -62,56 +111,203 @@ class ASRPerformanceTester: print(f" 测试 STT: {stt_name}") - # 测试第一个音频文件 - text, _ = await stt.speech_to_text( - [self.test_wav_list[0]], "1", stt.audio_format - ) - if text is None: - print(f" {stt_name} 连接失败") - return {"name": stt_name, "type": "stt", "errors": 1} + # 使用线程池和超时控制 + loop = asyncio.get_event_loop() + + # 测试第一个音频文件作为连通性检查 + try: + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit( + lambda: asyncio.run(self._test_single_audio(stt_name, stt, self.test_wav_list[0])) + ) + first_result = await asyncio.wait_for( + asyncio.wrap_future(future), timeout=10.0 + ) + + if first_result is None: + print(f" {stt_name} 连接失败") + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "网络错误" + } + except asyncio.TimeoutError: + print(f" {stt_name} 连接超时（10秒），跳过") + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "超时连接" + } + except Exception as e: + error_msg = str(e).lower() + if "502" in error_msg or "bad gateway" in error_msg: + print(f" {stt_name} 遇到502错误，跳过") + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "502网络错误" + } + print(f" {stt_name} 连接异常: {str(e)}") + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "网络错误" + } - # 全量测试 + # 全量测试，带超时控制 total_time = 0 + valid_tests = 0 test_count = len(self.test_wav_list) - for i, sentence in enumerate(self.test_wav_list, 1): - start = time.time() - text, _ = await stt.speech_to_text([sentence], "1", stt.audio_format) - duration = time.time() - start - total_time += duration - print(f" {stt_name} [{i}/{test_count}] 耗时: {duration:.2f}s") + + for i, audio_data in enumerate(self.test_wav_list, 1): + try: + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit( + lambda: asyncio.run(self._test_single_audio(stt_name, stt, audio_data)) + ) + duration = await asyncio.wait_for( + asyncio.wrap_future(future), timeout=10.0 + ) + + if duration is not None and duration > 0.001: + total_time += duration + valid_tests += 1 + print(f" {stt_name} [{i}/{test_count}] 耗时: {duration:.2f}s") + else: + print(f" {stt_name} [{i}/{test_count}] 测试失败(含0.000s异常)") + + except asyncio.TimeoutError: + print(f" {stt_name} [{i}/{test_count}] 超时（10秒），跳过") + continue + except Exception as e: + error_msg = str(e).lower() + if "502" in error_msg or "bad gateway" in error_msg: + print(f" {stt_name} [{i}/{test_count}] 502错误，跳过") + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "502网络错误" + } + print(f" {stt_name} [{i}/{test_count}] 异常: {str(e)}") + continue + # 检查有效测试数量 + if valid_tests < test_count * 0.3: # 至少30%成功率 + print(f" {stt_name} 成功测试过少({valid_tests}/{test_count})，可能网络不稳定") + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "网络错误" + } + if valid_tests == 0: + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": "网络错误" + } + + avg_time = total_time / valid_tests return { "name": stt_name, "type": "stt", - "avg_time": total_time / test_count, + "avg_time": avg_time, + "success_rate": f"{valid_tests}/{test_count}", "errors": 0, } + except Exception as e: + error_msg = str(e).lower() + if "502" in error_msg or "bad gateway" in error_msg: + error_type = "502网络错误" + elif "timeout" in error_msg: + error_type = "超时连接" + else: + error_type = "网络错误" print(f"⚠️ {stt_name} 测试失败: {str(e)}") - return {"name": stt_name, "type": "stt", "errors": 1} + return { + "name": stt_name, + "type": "stt", + "errors": 1, + "error_type": error_type + } def _print_results(self): - """打印测试结果""" - stt_table = [] + """打印测试结果，按响应时间排序""" + print("\n" + "=" * 50) + print("ASR 性能测试结果") + print("=" * 50) + + if not self.results.get("stt"): + print("没有可用的测试结果") + return + + headers = ["模型名称", "平均耗时(s)", "成功率", "状态"] + table_data = [] + + # 收集所有数据并分类 + valid_results = [] + error_results = [] + for name, data in self.results["stt"].items(): if data["errors"] == 0: - stt_table.append([name, f"{data['avg_time']:.3f}秒"]) + # 正常结果 + avg_time = f"{data['avg_time']:.3f}" + success_rate = data.get("success_rate", "N/A") + status = "✅ 正常" + + # 保存用于排序的值 + sort_key = data["avg_time"] + + valid_results.append({ + "name": name, + "avg_time": avg_time, + "success_rate": success_rate, + "status": status, + "sort_key": sort_key, + }) + else: + # 错误结果 + avg_time = "-" + success_rate = "0/N" + + # 获取具体错误类型 + error_type = data.get("error_type", "网络错误") + status = f"❌ {error_type}" + + error_results.append([name, avg_time, success_rate, status]) - if stt_table: - print("\nASR 性能排行:\n") - print( - tabulate( - stt_table, - headers=["模型名称", "平均耗时"], - tablefmt="github", - colalign=("left", "right"), - ) - ) - else: - print("\n 没有可用的ASR模块进行测试。") + # 按响应时间升序排序（从快到慢） + valid_results.sort(key=lambda x: x["sort_key"]) + + # 将排序后的有效结果转换为表格数据 + for result in valid_results: + table_data.append([ + result["name"], + result["avg_time"], + result["success_rate"], + result["status"], + ]) + + # 将错误结果添加到表格数据末尾 + table_data.extend(error_results) + + print(tabulate(table_data, headers=headers, tablefmt="grid")) + print("\n测试说明:") + print("- 超时控制：单个音频最大等待时间为10秒") + print("- 错误处理：自动跳过502错误、超时和网络异常的模型") + print("- 成功率：成功识别的音频数量/总测试音频数量") + print("- 排序规则：按平均耗时从快到慢排序，错误模型排最后") + print("\n测试完成！") async def run(self): - """执行全量异步测试""" + """执行全量异步测试""" print("开始筛选可用ASR模块...") if not self.config.get("ASR"): print("配置中未找到 ASR 模块") @@ -119,24 +315,33 @@ class ASRPerformanceTester: all_tasks = [] for stt_name, config in self.config["ASR"].items(): - print(f"[DEBUG] 检查 ASR 模块: {stt_name}, 配置: {config}") - all_tasks.append(self._test_stt(stt_name, config)) + # 检查配置有效性 + token_fields = ["access_token", "api_key", "token"] + if any( + field in config + and str(config[field]).lower() in ["你的", "placeholder", "none", "null", ""] + for field in token_fields + ): + print(f"ASR {stt_name} 未配置有效access_token/api_key，已跳过") + continue + + print(f"添加 ASR 测试任务: {stt_name}") + all_tasks.append(self._test_stt_with_timeout(stt_name, config)) if not all_tasks: print("没有可用的ASR模块进行测试。") return + print(f"\n找到 {len(all_tasks)} 个可用ASR模块") print("\n开始并发测试所有ASR模块...") all_results = await asyncio.gather(*all_tasks, return_exceptions=True) # 处理结果 for result in all_results: if isinstance(result, dict) and result.get("type") == "stt": - if result["errors"] == 0: - self.results["stt"][result["name"]] = result + self.results["stt"][result["name"]] = result # 打印结果 - print("\n测试完成") self._print_results() @@ -146,4 +351,4 @@ async def main(): if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) \ No newline at end of file diff --git a/main/xiaozhi-server/performance_tester/performance_tester_stream_tts.py b/main/xiaozhi-server/performance_tester/performance_tester_stream_tts.py new file mode 100644 index 00000000..309ad339 --- /dev/null +++ b/main/xiaozhi-server/performance_tester/performance_tester_stream_tts.py @@ -0,0 +1,388 @@ +import asyncio +import time +import json +import uuid +import aiohttp +import websockets +from tabulate import tabulate +from config.settings import load_config + +description = "流式TTS语音合成首词耗时测试" +class StreamTTSPerformanceTester: + def __init__(self): + self.config = load_config() + self.test_texts = [ + "你好，这是一句话。" + ] + self.results = [] + + async def test_aliyun_tts(self, text=None, test_count=5): + """测试阿里云流式TTS首词延迟（测试多次取平均）""" + text = text or self.test_texts[0] + latencies = [] + + for i in range(test_count): + try: + tts_config = self.config["TTS"]["AliyunStreamTTS"] + appkey = tts_config["appkey"] + token = tts_config["token"] + voice = tts_config["voice"] + host = tts_config["host"] + ws_url = f"wss://{host}/ws/v1" + + start_time = time.time() + async with websockets.connect(ws_url, extra_headers={"X-NLS-Token": token}) as ws: + task_id = str(uuid.uuid4()) + message_id = str(uuid.uuid4()) + + start_request = { + "header": { + "message_id": message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": appkey, + }, + "payload": { + "voice": voice, + "format": "pcm", + "sample_rate": 16000, + "volume": 50, + "speech_rate": 0, + "pitch_rate": 0, + } + } + await ws.send(json.dumps(start_request)) + + start_response = json.loads(await ws.recv()) + if start_response["header"]["name"] != "SynthesisStarted": + raise Exception("启动合成失败") + + run_request = { + "header": { + "message_id": str(uuid.uuid4()), + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": appkey, + }, + "payload": {"text": text} + } + await ws.send(json.dumps(run_request)) + + while True: + response = await ws.recv() + if isinstance(response, bytes): + latency = time.time() - start_time + latencies.append(latency) + break + elif isinstance(response, str): + data = json.loads(response) + if data["header"]["name"] == "TaskFailed": + raise Exception(f"合成失败: {data['payload']['error_info']}") + + except Exception as e: + latencies.append(0) + + return self._calculate_result("阿里云TTS", latencies, test_count) + + async def test_doubao_tts(self, text=None, test_count=5): + """测试火山引擎流式TTS首词延迟（测试多次取平均）""" + text = text or self.test_texts[0] + latencies = [] + + for i in range(test_count): + try: + tts_config = self.config["TTS"]["HuoshanDoubleStreamTTS"] + ws_url = tts_config["ws_url"] + app_id = tts_config["appid"] + access_token = tts_config["access_token"] + resource_id = tts_config["resource_id"] + speaker = tts_config["speaker"] + + start_time = time.time() + ws_header = { + "X-Api-App-Key": app_id, + "X-Api-Access-Key": access_token, + "X-Api-Resource-Id": resource_id, + "X-Api-Connect-Id": str(uuid.uuid4()), + } + async with websockets.connect(ws_url, additional_headers=ws_header, max_size=1000000000) as ws: + session_id = uuid.uuid4().hex + + # 发送会话启动请求 + header = bytes([ + (0b0001 << 4) | 0b0001, + 0b0001 << 4 | 0b100, + 0b0001 << 4 | 0b0000, + 0 + ]) + optional = bytearray() + optional.extend((1).to_bytes(4, "big", signed=True)) + session_id_bytes = session_id.encode() + optional.extend(len(session_id_bytes).to_bytes(4, "big", signed=True)) + optional.extend(session_id_bytes) + payload = json.dumps({"speaker": speaker}).encode() + await ws.send(header + optional + len(payload).to_bytes(4, "big", signed=True) + payload) + + # 发送文本 + header = bytes([ + (0b0001 << 4) | 0b0001, + 0b0001 << 4 | 0b100, + 0b0001 << 4 | 0b0000, + 0 + ]) + optional = bytearray() + optional.extend((200).to_bytes(4, "big", signed=True)) + session_id_bytes = session_id.encode() + optional.extend(len(session_id_bytes).to_bytes(4, "big", signed=True)) + optional.extend(session_id_bytes) + payload = json.dumps({"text": text, "speaker": speaker}).encode() + await ws.send(header + optional + len(payload).to_bytes(4, "big", signed=True) + payload) + + first_chunk = await ws.recv() + latency = time.time() - start_time + latencies.append(latency) + + except Exception as e: + latencies.append(0) + + return self._calculate_result("火山引擎TTS", latencies, test_count) + + async def test_paddlespeech_tts(self, text=None, test_count=5): + """测试PaddleSpeech流式TTS首词延迟（测试多次取平均）""" + text = text or self.test_texts[0] + latencies = [] + + for i in range(test_count): + try: + tts_config = self.config["TTS"]["PaddleSpeechTTS"] + tts_url = tts_config["url"] + spk_id = tts_config["spk_id"] + speed = tts_config["speed"] + volume = tts_config["volume"] + + start_time = time.time() + async with websockets.connect(tts_url) as ws: + # 发送开始请求 + await ws.send(json.dumps({ + "task": "tts", + "signal": "start" + })) + + start_response = json.loads(await ws.recv()) + if start_response.get("status") != 0: + raise Exception("连接失败") + + # 发送文本数据 + await ws.send(json.dumps({ + "text": text, + "spk_id": spk_id, + "speed": speed, + "volume": volume + })) + + # 接收第一个数据块 + first_chunk = await ws.recv() + latency = time.time() - start_time + latencies.append(latency) + + # 发送结束请求 + end_request = { + "task": "tts", + "signal": "end" + } + await ws.send(json.dumps(end_request)) + + # 确保连接正常关闭 + try: + await ws.recv() + except websockets.exceptions.ConnectionClosedOK: + pass + + except Exception as e: + latencies.append(0) + + return self._calculate_result("PaddleSpeechTTS", latencies, test_count) + + async def test_indexstream_tts(self, text=None, test_count=5): + """测试IndexStream流式TTS首词延迟（测试多次取平均）""" + text = text or self.test_texts[0] + latencies = [] + + for i in range(test_count): + try: + tts_config = self.config["TTS"]["IndexStreamTTS"] + api_url = tts_config.get("api_url") + voice = tts_config.get("voice") + + start_time = time.time() + + async with aiohttp.ClientSession() as session: + payload = {"text": text, "character": voice} + async with session.post(api_url, json=payload, timeout=10) as resp: + if resp.status != 200: + raise Exception(f"请求失败: {resp.status}, {await resp.text()}") + + async for chunk in resp.content.iter_any(): + data = chunk[0] if isinstance(chunk, (list, tuple)) else chunk + if not data: + continue + + latency = time.time() - start_time + latencies.append(latency) + resp.close() + break + else: + latencies.append(0) + + except Exception as e: + latencies.append(0) + + return self._calculate_result("IndexStreamTTS", latencies, test_count) + + async def test_linkerai_tts(self, text=None, test_count=5): + """测试Linkerai流式TTS首词延迟（测试多次取平均）""" + text = text or self.test_texts[0] + latencies = [] + + for i in range(test_count): + try: + tts_config = self.config["TTS"]["LinkeraiTTS"] + api_url = tts_config["api_url"] + access_token = tts_config["access_token"] + voice = tts_config["voice"] + + start_time = time.time() + async with aiohttp.ClientSession() as session: + params = { + "tts_text": text, + "spk_id": voice, + "frame_durition": 60, + "stream": "true", + "target_sr": 16000, + "audio_format": "pcm", + "instruct_text": "请生成一段自然流畅的语音", + } + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + } + + async with session.get(api_url, params=params, headers=headers, timeout=10) as resp: + if resp.status != 200: + raise Exception(f"请求失败: {resp.status}, {await resp.text()}") + + # 接收第一个数据块 + async for _ in resp.content.iter_any(): + latency = time.time() - start_time + latencies.append(latency) + break + else: + latencies.append(0) + + except Exception as e: + latencies.append(0) + + return self._calculate_result("LinkeraiTTS", latencies, test_count) + + + def _calculate_result(self, service_name, latencies, test_count): + """计算测试结果""" + valid_latencies = [l for l in latencies if l > 0] + if valid_latencies: + avg_latency = sum(valid_latencies) / len(valid_latencies) + status = f"成功（{len(valid_latencies)}/{test_count}次有效）" + else: + avg_latency = 0 + status = "失败: 所有测试均失败" + return {"name": service_name, "latency": avg_latency, "status": status} + + def _print_results(self, test_text, test_count): + """打印测试结果""" + if not self.results: + print("没有有效的TTS测试结果") + return + + print(f"\n{'='*60}") + print("流式TTS首词延迟测试结果") + print(f"{'='*60}") + print(f"测试文本: {test_text}") + print(f"测试次数: 每个TTS服务测试 {test_count} 次") + + # 排序结果：成功优先，按延迟升序 + success_results = sorted( + [r for r in self.results if "成功" in r["status"]], + key=lambda x: x["latency"] + ) + failed_results = [r for r in self.results if "成功" not in r["status"]] + + table_data = [ + [r["name"], f"{r['latency']:.3f}", r["status"]] + for r in success_results + failed_results + ] + + print(tabulate(table_data, headers=["TTS服务", "首词延迟(秒)", "状态"], tablefmt="grid")) + print("\n测试说明：测量从发送请求到接收第一个音频数据块的时间，取多次测试平均值") + print("- 超时控制: 单个请求最大等待时间为10秒") + print("- 错误处理: 无法连接和超时的列为网络错误") + print("- 排序规则: 按平均耗时从快到慢排序") + + + async def run(self, test_text=None, test_count=5): + """执行测试 + + Args: + test_text: 要测试的文本，如果为None则使用默认文本 + test_count: 每个TTS服务的测试次数 + """ + test_text = test_text or self.test_texts[0] + print(f"开始流式TTS首词延迟测试...") + print(f"测试文本: {test_text}") + print(f"每个TTS服务测试次数: {test_count}次") + + if not self.config.get("TTS"): + print("配置文件中未找到TTS配置") + return + + # 测试每种TTS服务 + self.results = [] + + # 测试阿里云TTS + result = await self.test_aliyun_tts(test_text, test_count) + self.results.append(result) + + # 测试火山引擎TTS + result = await self.test_doubao_tts(test_text, test_count) + self.results.append(result) + + # 测试PaddleSpeech TTS + result = await self.test_paddlespeech_tts(test_text, test_count) + self.results.append(result) + + # 测试Linkerai TTS + result = await self.test_linkerai_tts(test_text, test_count) + self.results.append(result) + + # 测试IndexStreamTTS + result = await self.test_indexstream_tts(test_text, test_count) + self.results.append(result) + + # 打印结果 + self._print_results(test_text, test_count) + + +async def main(): + import argparse + + parser = argparse.ArgumentParser(description="流式TTS首词延迟测试工具") + parser.add_argument("--text", help="要测试的文本内容") + parser.add_argument("--count", type=int, default=5, help="每个TTS服务的测试次数") + + args = parser.parse_args() + await StreamTTSPerformanceTester().run(args.text, args.count) + + +if __name__ == "__main__": + import asyncio + asyncio.run(main()) \ No newline at end of file diff --git a/main/xiaozhi-server/performance_tester/performance_tester_tts.py b/main/xiaozhi-server/performance_tester/performance_tester_tts.py index 4702c5a9..61e49682 100644 --- a/main/xiaozhi-server/performance_tester/performance_tester_tts.py +++ b/main/xiaozhi-server/performance_tester/performance_tester_tts.py @@ -86,22 +86,67 @@ class TTSPerformanceTester: print("没有有效的TTS测试结果") return - table = [] + headers = ["TTS模块", "平均耗时(秒)", "测试句子数", "状态"] + table_data = [] + + # 收集所有数据并分类 + valid_results = [] + error_results = [] + for name, data in self.results.items(): if data["errors"] == 0: - table.append( - [name, f"{data['avg_time']:.3f}秒/句", len(self.test_sentences[:3])] - ) + # 正常结果 + avg_time = f"{data['avg_time']:.3f}" + test_count = len(self.test_sentences[:3]) + status = "✅ 正常" + + # 保存用于排序的值 + valid_results.append({ + "name": name, + "avg_time": avg_time, + "test_count": test_count, + "status": status, + "sort_key": data['avg_time'] + }) + else: + # 错误结果 + avg_time = "-" + test_count = "0/3" + + # 默认错误类型为网络错误 + error_type = "网络错误" + status = f"❌ {error_type}" + + error_results.append([name, avg_time, test_count, status]) + + # 按平均耗时升序排序 + valid_results.sort(key=lambda x: x["sort_key"]) + + # 将排序后的有效结果转换为表格数据 + for result in valid_results: + table_data.append([ + result["name"], + result["avg_time"], + result["test_count"], + result["status"] + ]) + + # 将错误结果添加到表格数据末尾 + table_data.extend(error_results) print("\nTTS性能测试结果:") print( tabulate( - table, - headers=["TTS模块", "平均耗时", "测试句子数"], - tablefmt="github", - colalign=("left", "right", "right"), + table_data, + headers=headers, + tablefmt="grid", + colalign=("left", "right", "right", "left"), ) ) + print("\n测试说明:") + print("- 超时控制: 单个请求最大等待时间为10秒") + print("- 错误处理: 无法连接和超时的列为网络错误") + print("- 排序规则: 按平均耗时从快到慢排序") async def run(self): """执行测试""" @@ -119,10 +164,9 @@ class TTSPerformanceTester: # 并发执行测试 results = await asyncio.gather(*tasks) - # 保存有效结果 + # 保存所有结果，包括错误 for result in results: - if result["errors"] == 0: - self.results[result["name"]] = result + self.results[result["name"]] = result # 打印结果 self._print_results() From 789d756ce193cc8a2e17ca42b673ef95adffc6c5 Mon Sep 17 00:00:00 2001 From: rainv123 <2148537152@qq.com> Date: Wed, 20 Aug 2025 14:36:42 +0800 Subject: [PATCH 06/53] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8Dmcp=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=E6=98=BE=E7=A4=BA=E6=85=A2=EF=BC=8C=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=AB=AF=E5=9C=B0=E5=9D=80=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-mobile/src/pages/agent/tools.vue | 28 ++++-- .../src/pages/settings/index.vue | 99 ++++++++++++++----- 2 files changed, 97 insertions(+), 30 deletions(-) diff --git a/main/manager-mobile/src/pages/agent/tools.vue b/main/manager-mobile/src/pages/agent/tools.vue index 88ca36fc..4e710b19 100644 --- a/main/manager-mobile/src/pages/agent/tools.vue +++ b/main/manager-mobile/src/pages/agent/tools.vue @@ -28,6 +28,11 @@ const agentId = computed(() => pluginStore.currentAgentId) const mcpAddress = ref('') const mcpTools = ref([]) +// 初始化时从本地存储加载MCP地址 +if (uni.getStorageSync('cachedMcpAddress_' + agentId.value)) { + mcpAddress.value = uni.getStorageSync('cachedMcpAddress_' + agentId.value) +} + // 参数编辑相关 const showParamDialog = ref(false) const currentFunction = ref(null) @@ -56,12 +61,23 @@ async function mergeFunctions() { ) if (agentId.value) { - const [address, tools] = await Promise.all([ - getMcpAddress(agentId.value), - getMcpTools(agentId.value), - ]) - mcpAddress.value = address - mcpTools.value = tools || [] + // 优先获取并显示MCP地址 + try { + const address = await getMcpAddress(agentId.value) + mcpAddress.value = address + // 缓存到本地存储，下次打开页面可以立即显示 + uni.setStorageSync('cachedMcpAddress_' + agentId.value, address) + } catch (error) { + console.error('获取MCP地址失败:', error) + } + + // 异步获取MCP工具列表，不阻塞UI显示 + try { + const tools = await getMcpTools(agentId.value) + mcpTools.value = tools || [] + } catch (error) { + console.error('获取MCP工具列表失败:', error) + } } } diff --git a/main/manager-mobile/src/pages/settings/index.vue b/main/manager-mobile/src/pages/settings/index.vue index cef02761..41d16c00 100644 --- a/main/manager-mobile/src/pages/settings/index.vue +++ b/main/manager-mobile/src/pages/settings/index.vue @@ -27,6 +27,7 @@ const cacheInfo = reactive({ // 服务端地址设置 const baseUrlInput = ref('') +const urlError = ref('') // 系统信息（保留） const systemInfo = computed(() => { @@ -52,10 +53,58 @@ function getCacheInfo() { } } +// 验证URL格式 +function validateUrl() { + urlError.value = '' + + if (!baseUrlInput.value) { + return + } + + if (!/^https?:\/\/.+\/xiaozhi$/.test(baseUrlInput.value)) { + urlError.value = '请输入有效的服务端地址（以 http 或 https 开头，并以 /xiaozhi 结尾）' + } +} + +// 测试服务端地址 +async function testServerBaseUrl() { + // 先清除错误信息 + urlError.value = '' + + if (!baseUrlInput.value || !/^https?:\/\/.+\/xiaozhi$/.test(baseUrlInput.value)) { + return false + } + + try { + const response = await uni.request({ + url: `${baseUrlInput.value}/api/ping`, + method: 'GET', + timeout: 3000 + }) + + if (response.statusCode === 200) { + return true + } else { + toast.error('无效地址，请检查服务端是否启动或网络连接是否正常') + return false + } + } catch (error) { + console.error('测试服务端地址失败:', error) + toast.error('无效地址，请检查服务端是否启动或网络连接是否正常') + return false + } +} + // 保存服务端地址 -function saveServerBaseUrl() { - if (!baseUrlInput.value || !/^https?:\/\//.test(baseUrlInput.value)) { - toast.warning('请输入有效的服务端地址（以 http 或 https 开头）') +async function saveServerBaseUrl() { + if (!baseUrlInput.value || !/^https?:\/\/.+\/xiaozhi$/.test(baseUrlInput.value)) { + toast.warning('请输入有效的服务端地址（以 http 或 https 开头，并以 /xiaozhi 结尾）') + return + } + + // 测试地址有效性 + const isServerValid = await testServerBaseUrl() + if (!isServerValid) { return } setServerBaseUrlOverride(baseUrlInput.value) @@ -64,20 +113,20 @@ function saveServerBaseUrl() { clearAllCacheAfterUrlChange() uni.showModal({ - title: '重启应用', - content: '服务端地址已保存并清空缓存，是否立即重启生效？', - confirmText: '立即重启', - cancelText: '稍后', - success: (res) => { - if (res.confirm) { - restartApp() - } - else { - toast.success('已保存，可稍后手动重启应用') - } - }, - }) -} + title: '重启应用', + content: '服务端地址已保存并清空缓存，是否立即重启生效？', + confirmText: '立即重启', + cancelText: '稍后', + success: (res) => { + if (res.confirm) { + restartApp() + } + else { + toast.success('已保存，可稍后手动重启应用') + } + }, + }) + } // 重置为 env 默认 function resetServerBaseUrl() { @@ -173,7 +222,7 @@ function showAbout() { title: `关于${import.meta.env.VITE_APP_TITLE}`, content: `${import.meta.env.VITE_APP_TITLE}\n\n基于 Vue.js 3 + uni-app 构建的跨平台移动端管理应用，为小智ESP32智能硬件提供设备管理、智能体配置等功能。\n\n© 2025 xiaozhi-esp32-server`, title: `关于小智智控台`, - content: `小智智控台\n\n基于 Vue.js 3 + uni-app 构建的跨平台移动端管理应用，为小智智控台ESP32智能硬件提供设备管理、智能体配置等功能。\n\n© 2025 xiaozhi-esp32-server 0.7.5`, + content: `小智智控台\n\n基于 Vue.js 3 + uni-app 构建的跨平台移动端管理应用，为小智ESP32智能硬件提供设备管理、智能体配置等功能。\n\n© 2025 xiaozhi-esp32-server 0.7.5`, showCancel: false, confirmText: '确定', }) @@ -201,7 +250,6 @@ onMounted(async () => { - @@ -214,20 +262,22 @@ onMounted(async () => { - + + {{ urlError }} + @@ -320,7 +370,8 @@ onMounted(async () => { - + + From 7f9989511602fef248b63bab5b152e68f7d8e466 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 20 Aug 2025 14:38:51 +0800 Subject: [PATCH 07/53] Update Deployment_all.md --- docs/Deployment_all.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/Deployment_all.md b/docs/Deployment_all.md index f48f6e10..135b7cf5 100644 --- a/docs/Deployment_all.md +++ b/docs/Deployment_all.md @@ -13,8 +13,7 @@ docker 安装全模块有两种方式，你可以[使用懒人脚本](./Deployme ### 1.1 懒人脚本 -**推荐使用**，**全自动部署**，只需**简单配置**即可使用，视频教程：[https://www.bilibili.com/video/BV17bbvzHExd/](https://www.bilibili.com/video/BV17bbvzHExd/) -你可以使用以下命令一键安装全模块版小智服务端： +部署简便，可以参考[视频教程](https://www.bilibili.com/video/BV17bbvzHExd/) ，文字版教程如下： > [!NOTE] > 暂且只支持Ubuntu服务器一键部署，其他系统未尝试，可能会有一些奇怪的bug From 31f596f96c905661db8457837cb28a2fd040b2dd Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 20 Aug 2025 14:41:22 +0800 Subject: [PATCH 08/53] Update docker-setup.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 全模块部署，OTA地址是在8002端口 --- docker-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-setup.sh b/docker-setup.sh index 212f68ff..eb91267c 100755 --- a/docker-setup.sh +++ b/docker-setup.sh @@ -409,7 +409,7 @@ LOCAL_IP=$(hostname -I | awk '{print $1}') whiptail --title "安装完成！" --msgbox "\ 服务端相关地址如下：\n\ 管理后台访问地址: http://$LOCAL_IP:8002\n\ -OTA 地址: http://$LOCAL_IP:8003/xiaozhi/ota/\n\ +OTA 地址: http://$LOCAL_IP:8002/xiaozhi/ota/\n\ 视觉分析接口地址: http://$LOCAL_IP:8003/mcp/vision/explain\n\ WebSocket 地址: ws://$LOCAL_IP:8000/xiaozhi/v1/\n\ -\n安装完毕！感谢您的使用！\n按Enter键退出..." 16 70 \ No newline at end of file +\n安装完毕！感谢您的使用！\n按Enter键退出..." 16 70 From 6a79d57a936a8ac0003f9285c6747821e9814756 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 20 Aug 2025 14:42:46 +0800 Subject: [PATCH 09/53] Update docker-setup.sh --- docker-setup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-setup.sh b/docker-setup.sh index eb91267c..8b2f1274 100755 --- a/docker-setup.sh +++ b/docker-setup.sh @@ -402,8 +402,6 @@ fi # 获取并显示地址信息 LOCAL_IP=$(hostname -I | awk '{print $1}') -# WEBSOCKET_ADDR=$(docker logs xiaozhi-esp32-server 2>&1 | tac | grep -m 1 -E -o "ws://[^ ]+") -# VISION_ADDR=$(docker logs xiaozhi-esp32-server 2>&1 | tac | grep -m 1 "视觉" | grep -m 1 -E -o "http://[^ ]+") # 修复日志文件获取不到ws的问题，改为硬编码 whiptail --title "安装完成！" --msgbox "\ From 0ea18d87a304b20ae14f96b8b80a0652885d354e Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 20 Aug 2025 16:22:32 +0800 Subject: [PATCH 10/53] =?UTF-8?q?fix:=20asr=E8=AF=86=E5=88=AB=E7=BC=BA?= =?UTF-8?q?=E5=AD=97=E7=8E=B0=E8=B1=A1=EF=BC=8C=E5=8E=BB=E9=99=A4=E6=97=A0?= =?UTF-8?q?=E7=94=A8=E7=9A=84=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/handle/receiveAudioHandle.py | 16 ---------------- main/xiaozhi-server/core/providers/asr/base.py | 4 +--- main/xiaozhi-server/core/providers/vad/silero.py | 4 ++-- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 7d46685b..e5b96be2 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -1,5 +1,4 @@ import time -import asyncio import json from core.handle.sendAudioHandle import send_stt_message from core.handle.intentHandler import handle_user_intent @@ -14,14 +13,6 @@ TAG = __name__ async def handleAudioMessage(conn, audio): # 当前片段是否有人说话 have_voice = conn.vad.is_vad(conn, audio) - # 如果设备刚刚被唤醒，短暂忽略VAD检测 - if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up: - have_voice = False - # 设置一个短暂延迟后恢复VAD检测 - conn.asr_audio.clear() - if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done(): - conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn)) - return if have_voice: if conn.client_is_speaking: @@ -31,13 +22,6 @@ async def handleAudioMessage(conn, audio): # 接收音频 await conn.asr.receive_audio(conn, audio, have_voice) - -async def resume_vad_detection(conn): - # 等待2秒后恢复VAD检测 - await asyncio.sleep(1) - conn.just_woken_up = False - - async def startToChat(conn, text): # 检查输入是否是JSON格式（包含说话人信息） speaker_name = None diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 972818d4..f4803d31 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -12,7 +12,7 @@ import time import concurrent.futures from abc import ABC, abstractmethod from config.logger import setup_logging -from typing import Optional, Tuple, List, Dict, Any +from typing import Optional, Tuple, List from core.handle.receiveAudioHandle import startToChat from core.handle.reportHandle import enqueue_asr_report from core.utils.util import remove_punctuation_and_length @@ -132,8 +132,6 @@ class ASRProviderBase(ABC): return None # 使用线程池执行器并行运行 - parallel_start_time = time.monotonic() - with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor: asr_future = thread_executor.submit(run_asr) diff --git a/main/xiaozhi-server/core/providers/vad/silero.py b/main/xiaozhi-server/core/providers/vad/silero.py index b516d8fb..95ab8ff3 100644 --- a/main/xiaozhi-server/core/providers/vad/silero.py +++ b/main/xiaozhi-server/core/providers/vad/silero.py @@ -33,8 +33,8 @@ class VADProvider(VADProviderBase): int(min_silence_duration_ms) if min_silence_duration_ms else 1000 ) - # 至少要多少帧才算有语音 - self.frame_window_threshold = 3 + # 至少要多少帧才算有语音,增加灵敏度 + self.frame_window_threshold = 1 def is_vad(self, conn, opus_packet): try: From a708382cfd411f2cdb155c717cc0320aee5464db Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Thu, 21 Aug 2025 11:54:06 +0800 Subject: [PATCH 11/53] =?UTF-8?q?update:=E6=9B=B4=E6=96=B0=E6=B5=81?= =?UTF-8?q?=E5=BC=8FASR=E9=A6=96=E8=AF=8D=E7=AD=89=E5=BE=85=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=B5=8B=E8=AF=95=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../performance_tester_stream_asr.py | 404 ++++++++++++++++++ 1 file changed, 404 insertions(+) create mode 100644 main/xiaozhi-server/performance_tester/performance_tester_stream_asr.py diff --git a/main/xiaozhi-server/performance_tester/performance_tester_stream_asr.py b/main/xiaozhi-server/performance_tester/performance_tester_stream_asr.py new file mode 100644 index 00000000..e0309685 --- /dev/null +++ b/main/xiaozhi-server/performance_tester/performance_tester_stream_asr.py @@ -0,0 +1,404 @@ +import asyncio +import time +import json +import uuid +import os +import websockets +import gzip +import hmac +import base64 +import hashlib +import random +from urllib import parse +from tabulate import tabulate +from config.settings import load_config +description = "流式ASR首词耗时测试" + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + query_string = AccessToken._encode_dict(parameters) + string_to_sign = ( + "GET" + "&" + AccessToken._encode_text("/") + "&" + AccessToken._encode_text(query_string) + ) + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + signature = AccessToken._encode_text(signature) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (signature, query_string) + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + if "Token" in root_obj: + return root_obj["Token"]["Id"], root_obj["Token"]["ExpireTime"] + return None, None + + +class DoubaoStreamASRPerformanceTester: + def __init__(self): + self.config = load_config() + self.test_audio_files = self._load_test_audio_files() + self.results = [] + + def _load_test_audio_files(self): + """加载测试用的音频文件""" + audio_root = os.path.join(os.getcwd(), "config", "assets") + test_files = [] + + if os.path.exists(audio_root): + for file_name in os.listdir(audio_root): + if file_name.endswith('.wav') or file_name.endswith('.pcm'): + with open(os.path.join(audio_root, file_name), 'rb') as f: + test_files.append(f.read()) + return test_files + + async def test_doubao_stream_asr(self, test_count=5): + """测试豆包流式ASR首词响应时间""" + if not self.test_audio_files: + print("没有找到测试音频文件") + return + + asr_config = self.config["ASR"]["DoubaoStreamASR"] + latencies = [] + + for i in range(test_count): + try: + ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel" + appid = asr_config["appid"] + access_token = asr_config["access_token"] + uid = asr_config.get("uid", "streaming_asr_service") + + start_time = time.time() + + headers = { + "X-Api-App-Key": appid, + "X-Api-Access-Key": access_token, + "X-Api-Resource-Id": "volc.bigasr.sauc.duration", + "X-Api-Connect-Id": str(uuid.uuid4()) + } + + async with websockets.connect( + ws_url, + additional_headers=headers, + max_size=1000000000, + ping_interval=None, + ping_timeout=None, + close_timeout=10 + ) as ws: + # 发送初始化请求 + request_params = { + "app": { + "appid": appid, + "token": access_token + }, + "user": {"uid": uid}, + "request": { + "reqid": str(uuid.uuid4()), + "workflow": "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate", + "show_utterances": True, + "result_type": "single", + "sequence": 1 + }, + "audio": { + "format": "pcm", + "codec": "pcm", + "rate": 16000, + "language": "zh-CN", + "bits": 16, + "channel": 1, + "sample_rate": 16000 + } + } + + payload_bytes = str.encode(json.dumps(request_params)) + payload_bytes = gzip.compress(payload_bytes) + full_client_request = self._generate_header() + full_client_request.extend((len(payload_bytes)).to_bytes(4, "big")) + full_client_request.extend(payload_bytes) + + await ws.send(full_client_request) + + init_res = await ws.recv() + result = self._parse_response(init_res) + + if "code" in result and result["code"] != 1000: + raise Exception(f"ASR服务初始化失败: {result.get('payload_msg', {}).get('error', '未知错误')}") + + # 发送音频数据 + audio_data = self.test_audio_files[0] + if audio_data.startswith(b'RIFF'): + audio_data = audio_data[44:] + + # 直接发送原始音频数据，不进行opus解码 + payload = gzip.compress(audio_data) + audio_request = bytearray(self._generate_audio_default_header()) + audio_request.extend(len(payload).to_bytes(4, "big")) + audio_request.extend(payload) + await ws.send(audio_request) + + # 等待第一个数据块 + first_chunk = await ws.recv() + latency = time.time() - start_time + latencies.append(latency) + await ws.close() + + except Exception as e: + print(f"第{i+1}次测试: {str(e)}") + latencies.append(0) + + return self._calculate_result("豆包流式ASR", latencies, test_count) + + async def test_aliyun_stream_asr(self, test_count=5): + """测试阿里云流式ASR首词响应时间""" + if not self.test_audio_files: + print("没有找到测试音频文件") + return + + asr_config = self.config["ASR"]["AliyunStreamASR"] + latencies = [] + + for i in range(test_count): + try: + access_key_id = asr_config["access_key_id"] + access_key_secret = asr_config["access_key_secret"] + appkey = asr_config["appkey"] + host = asr_config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com") + + # 获取Token + token, _ = AccessToken.create_token(access_key_id, access_key_secret) + if not token: + raise Exception("无法获取阿里云ASR Token") + + # 确定WebSocket URL + if "-internal." in host: + ws_url = f"ws://{host}/ws/v1" + else: + ws_url = f"wss://{host}/ws/v1" + + start_time = time.time() + async with websockets.connect( + ws_url, + additional_headers={"X-NLS-Token": token}, + max_size=1000000000, + ping_interval=None, + ping_timeout=None, + close_timeout=10 + ) as ws: + # 发送开始请求 + start_request = { + "header": { + "namespace": "SpeechTranscriber", + "name": "StartTranscription", + "status": 20000000, + "message_id": ''.join(random.choices('0123456789abcdef', k=32)), + "task_id": ''.join(random.choices('0123456789abcdef', k=32)), + "status_text": "Gateway:SUCCESS:Success.", + "appkey": appkey + }, + "payload": { + "format": "pcm", + "sample_rate": 16000, + "enable_intermediate_result": True, + "enable_punctuation_prediction": True, + "enable_inverse_text_normalization": True, + "max_sentence_silence": asr_config.get("max_sentence_silence", 8000), + "enable_voice_detection": False, + } + } + await ws.send(json.dumps(start_request, ensure_ascii=False)) + + # 等待服务器准备 + start_response = await ws.recv() + response_data = json.loads(start_response) + if response_data["header"]["name"] != "TranscriptionStarted": + raise Exception("阿里云ASR服务初始化失败") + + # 发送音频数据 + audio_data = self.test_audio_files[0] + if audio_data.startswith(b'RIFF'): + audio_data = audio_data[44:] # 去掉WAV头 + + await ws.send(audio_data) + + # 等待第一个结果 + while True: + response = await ws.recv() + if isinstance(response, str): + result = json.loads(response) + if result["header"]["name"] == "TranscriptionResultChanged": + latency = time.time() - start_time + latencies.append(latency) + break + elif result["header"]["name"] == "TaskFailed": + raise Exception(f"阿里云ASR识别失败: {result.get('payload', {}).get('error_info', '未知错误')}") + + # 发送停止请求 + stop_msg = { + "header": { + "namespace": "SpeechTranscriber", + "name": "StopTranscription", + "status": 20000000, + "message_id": ''.join(random.choices('0123456789abcdef', k=32)), + "status_text": "Client:Stop", + "appkey": appkey + } + } + await ws.send(json.dumps(stop_msg, ensure_ascii=False)) + await ws.close() + + except Exception as e: + print(f"第{i+1}次测试: {str(e)}") + latencies.append(0) + + return self._calculate_result("阿里云流式ASR", latencies, test_count) + + def _generate_header(self): + """生成请求头""" + header = bytearray() + header.append((0x01 << 4) | 0x01) + header.append((0x01 << 4) | 0x00) + header.append((0x01 << 4) | 0x01) + header.append(0x00) + return header + + def _generate_audio_default_header(self): + """生成音频请求头""" + return self._generate_header() + + def _parse_response(self, res: bytes) -> dict: + """解析响应""" + try: + if len(res) < 4: + return {"error": "响应数据长度不足"} + + header = res[:4] + message_type = header[1] >> 4 + + if message_type == 0x0F: + code = int.from_bytes(res[4:8], "big", signed=False) + msg_length = int.from_bytes(res[8:12], "big", signed=False) + error_msg = json.loads(res[12:].decode("utf-8")) + return { + "code": code, + "msg_length": msg_length, + "payload_msg": error_msg + } + + try: + json_data = res[12:].decode("utf-8") + return {"payload_msg": json.loads(json_data)} + except (UnicodeDecodeError, json.JSONDecodeError): + return {"error": "JSON解析失败"} + + except Exception: + return {"error": "解析响应失败"} + + def _calculate_result(self, service_name, latencies, test_count): + """计算结果""" + valid_latencies = [l for l in latencies if l > 0] + if valid_latencies: + avg_latency = sum(valid_latencies) / len(valid_latencies) + status = f"成功（{len(valid_latencies)}/{test_count}次有效）" + else: + avg_latency = 0 + status = "失败: 所有测试均失败" + return {"name": service_name, "latency": avg_latency, "status": status} + + def _print_results(self, test_count): + """打印测试结果""" + if not self.results: + print("没有有效的ASR测试结果") + return + + print(f"\n{'='*60}") + print("流式ASR首词响应时间测试结果") + print(f"{'='*60}") + print(f"测试次数: 每个ASR服务测试 {test_count} 次") + + # 排序结果：成功优先，按延迟升序 + success_results = sorted( + [r for r in self.results if "成功" in r["status"]], + key=lambda x: x["latency"] + ) + failed_results = [r for r in self.results if "成功" not in r["status"]] + + table_data = [ + [r["name"], f"{r['latency']:.3f}", r["status"]] + for r in success_results + failed_results + ] + + print(tabulate(table_data, headers=["ASR服务", "首词延迟(秒)", "状态"], tablefmt="grid")) + print("\n测试说明：测量从发送请求到接收第一个识别结果的时间，取多次测试平均值") + print("- 超时控制: 单个请求最大等待时间为10秒") + print("- 错误处理: 无法连接和超时的列为网络错误") + print("- 排序规则: 按平均耗时从快到慢排序") + + async def run(self, test_count=5): + """执行测试""" + print(f"开始流式ASR首词响应时间测试...") + print(f"每个ASR服务测试次数: {test_count}次") + + if not self.config.get("ASR"): + print("配置文件中未找到ASR配置") + return + + # 测试每种ASR服务 + self.results = [] + + # 测试豆包ASR + if self.config["ASR"].get("DoubaoStreamASR"): + result = await self.test_doubao_stream_asr(test_count) + self.results.append(result) + else: + print("配置文件中未找到豆包流式ASR配置，跳过测试") + + # 测试阿里云ASR + if self.config["ASR"].get("AliyunStreamASR"): + result = await self.test_aliyun_stream_asr(test_count) + self.results.append(result) + else: + print("配置文件中未找到阿里云流式ASR配置，跳过测试") + + # 打印结果 + self._print_results(test_count) + +async def main(): + import argparse + + parser = argparse.ArgumentParser(description="流式ASR首词响应时间测试工具") + parser.add_argument("--count", type=int, default=5, help="测试次数") + + args = parser.parse_args() + await DoubaoStreamASRPerformanceTester().run(args.count) + +if __name__ == "__main__": + import os + import gzip + import opuslib_next + asyncio.run(main()) \ No newline at end of file From 1988bced6068137e4179ac87b7a273eb4296cbdf Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 22 Aug 2025 17:10:36 +0800 Subject: [PATCH 12/53] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E6=B5=81?= =?UTF-8?q?=E6=8E=A7=E9=9F=B3=E9=A2=91=E6=92=AD=E6=94=BE=20wechat=E8=81=8A?= =?UTF-8?q?=E5=A4=A9=E6=A8=A1=E5=BC=8F=E9=94=99=E8=AF=AFSTT=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E5=8F=91=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/handle/sendAudioHandle.py | 56 ++++- .../core/providers/tts/aliyun_stream.py | 1 - .../xiaozhi-server/core/providers/tts/base.py | 92 ++------ .../providers/tts/huoshan_double_stream.py | 1 - .../core/providers/tts/index_stream.py | 1 - .../core/providers/tts/linkerai.py | 1 - .../core/utils/audio_flow_control.py | 201 ------------------ .../plugins_func/functions/play_music.py | 1 - 8 files changed, 67 insertions(+), 287 deletions(-) delete mode 100644 main/xiaozhi-server/core/utils/audio_flow_control.py diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index bb85c007..8604279a 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -1,5 +1,6 @@ import json import asyncio +import time from core.providers.tts.dto.dto import SentenceType from core.utils import textUtils @@ -29,13 +30,60 @@ async def sendAudioMessage(conn, sentenceType, audios, text): # 播放音频 -async def sendAudio(conn, audios): +async def sendAudio(conn, audios, pre_buffer=False): + """ + 发送单个opus包，支持流控 + Args: + conn: 连接对象 + opus_packet: 单个opus数据包 + pre_buffer: 快速发送音频 + """ if audios is None: return - # 如果audios不是opus数组，则不需要进行遍历，可以直接发送;这里需要进行流控管理，防止发送过快引发客户端溢出 + if isinstance(audios, bytes): + if conn.client_abort: + return + + # 短音频直接发送（例如：提示音） + if pre_buffer: + await conn.websocket.send(audios) + return + + # 重置没有声音的状态 + conn.last_activity_time = time.time() * 1000 + + # 流控逻辑：确保按60ms的帧时长间隔发送 + frame_duration = 60 # 毫秒 + + # 获取或初始化流控状态 + if not hasattr(conn, "audio_flow_control"): + conn.audio_flow_control = { + "last_send_time": 0, + "packet_count": 0, + "start_time": time.perf_counter(), + } + + flow_control = conn.audio_flow_control + current_time = time.perf_counter() + + # 计算期望的发送时间 + expected_time = flow_control["start_time"] + ( + flow_control["packet_count"] * frame_duration / 1000 + ) + + # 流控延迟 + delay = expected_time - current_time + if delay > 0: + await asyncio.sleep(delay) + + # 发送数据包 await conn.websocket.send(audios) + # 更新流控状态 + flow_control["packet_count"] += 1 + flow_control["last_send_time"] = time.perf_counter() + async def send_tts_message(conn, state, text=None): """发送 TTS 状态消息""" @@ -56,7 +104,7 @@ async def send_tts_message(conn, state, text=None): conn.tts.audio_to_opus_data_stream( stop_tts_notify_voice, callback=lambda audio_data: asyncio.run_coroutine_threadsafe( - sendAudio(conn, audio_data), conn.loop + sendAudio(conn, audio_data, True), conn.loop ), ) # 清除服务端讲话状态 @@ -77,7 +125,7 @@ async def send_stt_message(conn, text): display_text = text try: # 尝试解析JSON格式 - if text.strip().startswith('{') and text.strip().endswith('}'): + if text.strip().startswith("{") and text.strip().endswith("}"): parsed_data = json.loads(text) if isinstance(parsed_data, dict) and "content" in parsed_data: # 如果是包含说话人信息的JSON格式，只显示content部分 diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 32e4d50f..734681e2 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -216,7 +216,6 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.FIRST: self.conn.client_abort = False - self.reset_flow_controller() if self.conn.client_abort: logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 35d80fb6..6a29f1f5 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -11,7 +11,6 @@ from datetime import datetime from core.utils import textUtils from abc import ABC, abstractmethod from config.logger import setup_logging -from core.utils.audio_flow_control import FlowControlConfig, simulate_device_consumption from core.utils.util import audio_bytes_to_data_stream, audio_to_data_stream from core.utils.tts import MarkdownCleaner from core.utils.output_counter import add_device_output @@ -71,7 +70,6 @@ class TTSProviderBase(ABC): self.tts_stop_request = False self.processed_chars = 0 self.is_first_sentence = True - self.flow_controller = FlowControlConfig.create_flow_controller() def generate_filename(self, extension=".wav"): return os.path.join( @@ -225,7 +223,6 @@ class TTSProviderBase(ABC): self.tts_text_buff = [] self.is_first_sentence = True self.tts_audio_first_sentence = True - self.reset_flow_controller() elif ContentType.TEXT == message.content_type: self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() @@ -270,12 +267,19 @@ class TTSProviderBase(ABC): if self.conn.client_abort: logger.bind(tag=TAG).debug("收到打断信号，跳过当前音频数据") - # 打断时丢弃未上报的音频数据 enqueue_text, enqueue_audio = None, [] continue # 收到下一个文本开始或会话结束时进行上报 if sentence_type is not SentenceType.MIDDLE: + # 重置音频流控状态（新句子开始或者结束） + if hasattr(self.conn, 'audio_flow_control'): + self.conn.audio_flow_control = { + 'last_send_time': 0, + 'packet_count': 0, + 'start_time': time.perf_counter() + } + # 上报TTS数据 if enqueue_text is not None and enqueue_audio is not None: enqueue_tts_report(self.conn, enqueue_text, enqueue_audio) @@ -284,88 +288,22 @@ class TTSProviderBase(ABC): # 计算音频数据的帧数 if isinstance(audio_datas, bytes): - frame_count = 1 # 单个字节流作为一帧 enqueue_audio.append(audio_datas) - else: - frame_count = 0 + + # 发送音频 + future = asyncio.run_coroutine_threadsafe( + sendAudioMessage(self.conn, sentence_type, audio_datas, text), + self.conn.loop, + ) + future.result() # 记录输出和报告 if self.conn.max_output_size > 0 and text: add_device_output(self.conn.headers.get("device-id"), len(text)) - # 流控检查 - if frame_count > 0: - max_wait_time = FlowControlConfig.DEFAULT_MAX_WAIT_TIME - wait_start_time = time.time() - retry_interval = FlowControlConfig.DEFAULT_RETRY_INTERVAL - - while not self.flow_controller.can_send_frames(frame_count): - # 检查是否超时或需要停止 - if ( - time.time() - wait_start_time > max_wait_time - or self.conn.stop_event.is_set() - or self.conn.client_abort - ): - logger.bind(tag=TAG).debug( - "流控等待超时或收到停止信号，跳过音频发送" - ) - break - # 短暂等待后重试 - time.sleep(retry_interval) - else: - # 可以发送，记录发送的帧数 - self.flow_controller.record_sent_frames(frame_count) - - # 发送音频 - future = asyncio.run_coroutine_threadsafe( - self._send_audio_with_flow_control( - sentence_type, audio_datas, text - ), - self.conn.loop, - ) - future.result() - - # 输出流控状态（调试用） - # status = self.flow_controller.get_status() - # logger.bind(tag=TAG).debug( - # f"流控状态: 缓冲区使用率={status['buffer_usage_percent']:.1f}%, " - # f"可用令牌={status['available_tokens']}..." - # f"发送帧数={status['sent_frames']}..." - # f"消费帧数={status['consumed_frames']}..." - # f"代播放帧数={status['sent_frames'] - status['consumed_frames']}..." - # ) - else: - # 没有音频数据，直接发送 - future = asyncio.run_coroutine_threadsafe( - self._send_audio_with_flow_control( - sentence_type, audio_datas, text - ), - self.conn.loop, - ) - future.result() - except Exception as e: logger.bind(tag=TAG).error(f"audio_play_priority_thread: {text} {e}") - async def _send_audio_with_flow_control(self, sentence_type, audio_datas, text): - """ - 带流控的音频发送方法 模拟设备消费音频帧的过程 - 实际应用中应该根据设备反馈来更新消费情况 - """ - await sendAudioMessage(self.conn, sentence_type, audio_datas, text) - - # 模拟设备消费（实际应用中应该从设备获取反馈）防止音字不同步 - if isinstance(audio_datas, bytes): - frame_count = 1 - asyncio.create_task(simulate_device_consumption(self.flow_controller, frame_count)) - - # 在类中添加流控制器重置方法 - def reset_flow_controller(self): - """重置流控制器状态，通常在新会话开始时调用""" - if hasattr(self, "flow_controller"): - self.flow_controller.reset() - logger.bind(tag=TAG).info("流控制器状态已重置") - async def start_session(self, session_id): pass diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index ac6301f9..463c16fc 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -213,7 +213,6 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.FIRST: self.conn.client_abort = False - self.reset_flow_controller() if self.conn.client_abort: try: diff --git a/main/xiaozhi-server/core/providers/tts/index_stream.py b/main/xiaozhi-server/core/providers/tts/index_stream.py index d562e802..6f6b829c 100644 --- a/main/xiaozhi-server/core/providers/tts/index_stream.py +++ b/main/xiaozhi-server/core/providers/tts/index_stream.py @@ -45,7 +45,6 @@ class TTSProvider(TTSProviderBase): self.processed_chars = 0 self.tts_text_buff = [] self.before_stop_play_files.clear() - self.reset_flow_controller() elif ContentType.TEXT == message.content_type: self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index a855e8ed..875ed0b1 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -42,7 +42,6 @@ class TTSProvider(TTSProviderBase): self.processed_chars = 0 self.tts_text_buff = [] self.before_stop_play_files.clear() - self.reset_flow_controller() elif ContentType.TEXT == message.content_type: self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() diff --git a/main/xiaozhi-server/core/utils/audio_flow_control.py b/main/xiaozhi-server/core/utils/audio_flow_control.py deleted file mode 100644 index 958e68a7..00000000 --- a/main/xiaozhi-server/core/utils/audio_flow_control.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -音频流控模块 -包含令牌桶算法和音频流控制器的实现 -""" - -import asyncio -import time -import threading -from collections import deque -from typing import Optional, Dict, Any - - -class TokenBucket: - """令牌桶实现，用于限流控制""" - - def __init__(self, capacity: int, refill_rate: float, initial_tokens: Optional[int] = None): - """ - 初始化令牌桶 - - Args: - capacity: 桶容量（最大令牌数） - refill_rate: 令牌补充速率（每秒补充的令牌数） - initial_tokens: 初始令牌数，默认为桶容量 - """ - self.capacity = capacity - self.refill_rate = refill_rate - self.tokens = initial_tokens if initial_tokens is not None else capacity - self.last_refill_time = time.time() - self.lock = threading.Lock() - - def get_tokens(self, requested_tokens: int = 1) -> bool: - """ - 获取指定数量的令牌 - - Args: - requested_tokens: 请求的令牌数量 - - Returns: - bool: 是否成功获取到令牌 - """ - with self.lock: - self._refill_tokens() - - if self.tokens >= requested_tokens: - self.tokens -= requested_tokens - return True - else: - return False - - def get_available_tokens(self) -> int: - """获取当前可用令牌数""" - with self.lock: - self._refill_tokens() - return int(self.tokens) - - def _refill_tokens(self): - """内部方法：补充令牌""" - current_time = time.time() - time_passed = current_time - self.last_refill_time - tokens_to_add = time_passed * self.refill_rate - - self.tokens = min(self.capacity, self.tokens + tokens_to_add) - self.last_refill_time = current_time - - -class AudioFlowController: - """音频流控制器，基于令牌桶算法控制音频数据发送""" - - def __init__(self, max_device_buffer: int = 3000, refill_rate: float = 20): - """ - 初始化音频流控制器 - - Args: - max_device_buffer: 设备端最大缓冲区大小（Opus帧数） - refill_rate: 令牌补充速率（每秒允许发送的帧数） - """ - self.max_device_buffer = max_device_buffer - self.token_bucket = TokenBucket( - capacity=max_device_buffer, - refill_rate=refill_rate, - initial_tokens=max_device_buffer // 2 # 初始令牌为容量的一半 - ) - self.sent_frames_count = 0 # 已发送帧数计数 - self.device_consumed_frames = 0 # 设备端已消费帧数 - self.pending_queue = deque() # 等待发送的数据队列 - self._lock = threading.Lock() - - def can_send_frames(self, frame_count: int) -> bool: - """ - 检查是否可以发送指定数量的帧 - - Args: - frame_count: 要发送的帧数 - - Returns: - bool: 是否可以发送 - """ - with self._lock: - # 检查设备端缓冲区是否会溢出 - estimated_device_buffer = self.sent_frames_count - self.device_consumed_frames - if estimated_device_buffer + frame_count > self.max_device_buffer: - return False - - # 检查令牌桶是否有足够令牌 - return self.token_bucket.get_tokens(frame_count) - - def update_device_consumption(self, consumed_frames: int): - """ - 更新设备端消费的帧数 - - Args: - consumed_frames: 设备端消费的帧数 - """ - with self._lock: - self.device_consumed_frames += consumed_frames - - def record_sent_frames(self, frame_count: int): - """ - 记录已发送的帧数 - - Args: - frame_count: 发送的帧数 - """ - with self._lock: - self.sent_frames_count += frame_count - - def get_status(self) -> Dict[str, Any]: - """获取流控状态信息""" - with self._lock: - estimated_buffer = self.sent_frames_count - self.device_consumed_frames - return { - "sent_frames": self.sent_frames_count, - "consumed_frames": self.device_consumed_frames, - "estimated_device_buffer": estimated_buffer, - "available_tokens": self.token_bucket.get_available_tokens(), - "pending_queue_size": len(self.pending_queue), - "buffer_usage_percent": (estimated_buffer / self.max_device_buffer) * 100 - } - - def reset(self): - """重置流控状态""" - with self._lock: - self.sent_frames_count = 0 - self.device_consumed_frames = 0 - self.pending_queue.clear() - # 重新初始化令牌桶 - self.token_bucket = TokenBucket( - capacity=self.max_device_buffer, - refill_rate=self.token_bucket.refill_rate, - initial_tokens=self.max_device_buffer // 2 - ) - - -async def simulate_device_consumption( - flow_controller: AudioFlowController, frame_count: int -): - """ - 模拟设备消费音频帧的过程 - 实际应用中应该根据设备反馈来更新消费情况 - Args: - flow_controller: 流控制器实例 - frame_count: 消费的帧数 - """ - # 模拟设备播放延迟（60ms per frame） - await asyncio.sleep(frame_count * 0.06) - flow_controller.update_device_consumption(frame_count) - - -# 流控配置常量 -class FlowControlConfig: - """流控配置常量""" - # Opus 编码参数 - OPUS_FRAME_DURATION_MS = 60 # Opus帧时长（毫秒） - OPUS_FRAMES_PER_SECOND = 1000 / OPUS_FRAME_DURATION_MS # 每秒帧数 - - # 默认流控参数 - DEFAULT_MAX_DEVICE_BUFFER = 40 # 设备端最大缓冲帧数 - DEFAULT_REFILL_RATE = OPUS_FRAMES_PER_SECOND # 默认令牌补充速率（帧/秒） - DEFAULT_MAX_WAIT_TIME = 5.0 # 流控最大等待时间（秒） - DEFAULT_RETRY_INTERVAL = 0.06 # 流控重试间隔（秒） - - # 预缓冲参数 - PRE_BUFFER_FRAMES = 3 # 预缓冲帧数 - - @classmethod - def create_flow_controller(cls, max_buffer: Optional[int] = None, - refill_rate: Optional[float] = None) -> AudioFlowController: - """ - 创建流控制器的工厂方法 - - Args: - max_buffer: 最大缓冲区大小，使用默认值如果为None - refill_rate: 令牌补充速率，使用默认值如果为None - - Returns: - AudioFlowController: 配置好的流控制器实例 - """ - return AudioFlowController( - max_device_buffer=max_buffer or cls.DEFAULT_MAX_DEVICE_BUFFER, - refill_rate=refill_rate or cls.DEFAULT_REFILL_RATE - ) diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 2cbc4018..dd967d0a 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -212,7 +212,6 @@ async def play_local_music(conn, specific_file=None): conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}") return text = _get_random_play_prompt(selected_music) - await send_stt_message(conn, text) conn.dialogue.put(Message(role="assistant", content=text)) if conn.intent_type == "intent_llm": From d4b75c5d995be734aa644cdfea95b8bae2c7940e Mon Sep 17 00:00:00 2001 From: Minamiyama Date: Sat, 23 Aug 2025 07:47:40 +0800 Subject: [PATCH 13/53] =?UTF-8?q?refactor(=E9=9F=B3=E9=A2=91=E5=A4=84?= =?UTF-8?q?=E7=90=86):=20=E6=8F=90=E5=8F=96=E9=9F=B3=E9=A2=91=E4=B8=8A?= =?UTF-8?q?=E4=B8=8B=E6=96=87=E5=88=9B=E5=BB=BA=E9=80=BB=E8=BE=91=E5=88=B0?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将重复的音频上下文创建逻辑提取到独立的getAudioContextInstance函数中，减少代码重复并统一音频上下文配置 --- main/xiaozhi-server/test/test_page.html | 36 ++++++++++--------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index 6f8fdba9..983d3b9a 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -230,6 +230,16 @@ const conversationDiv = document.getElementById('conversation'); const logContainer = document.getElementById('logContainer'); + function getAudioContextInstance() { + if (!audioContext) { + audioContext = new (window.AudioContext || window.webkitAudioContext)({ + sampleRate: SAMPLE_RATE, + latencyHint: 'interactive' + }); + log('创建音频上下文，采样率: ' + SAMPLE_RATE + 'Hz', 'debug'); + } + return audioContext; + } // 初始化可视化器 function initVisualizer() { @@ -306,12 +316,7 @@ // 确保Opus解码器已初始化 try { // 确保音频上下文存在 - if (!audioContext) { - audioContext = new (window.AudioContext || window.webkitAudioContext)({ - sampleRate: SAMPLE_RATE - }); - log('创建音频上下文，采样率: ' + SAMPLE_RATE + 'Hz', 'debug'); - } + audioContext = getAudioContextInstance(); // 确保解码器已初始化 if (!opusDecoder) { @@ -617,10 +622,7 @@ log('已获取麦克风访问权限', 'success'); // 创建音频上下文 - audioContext = new (window.AudioContext || window.webkitAudioContext)({ - sampleRate: 16000, // 确保采样率与服务器期望的一致 - latencyHint: 'interactive' - }); + audioContext = getAudioContextInstance(); const source = audioContext.createMediaStreamSource(stream); // 获取实际音频轨道设置 @@ -1416,12 +1418,7 @@ // 创建音频处理器 async function createAudioProcessor() { - if (!audioContext) { - audioContext = new (window.AudioContext || window.webkitAudioContext)({ - sampleRate: 16000, - latencyHint: 'interactive' - }); - } + audioContext = getAudioContextInstance(); try { // 检查是否支持AudioWorklet @@ -1616,12 +1613,7 @@ }); // 创建音频上下文和分析器 - if (!audioContext) { - audioContext = new (window.AudioContext || window.webkitAudioContext)({ - sampleRate: 16000, - latencyHint: 'interactive' - }); - } + audioContext = getAudioContextInstance(); // 创建音频处理器 const processorResult = await createAudioProcessor(); From 5b7d613a35b15d7068404be5e712b11132829478 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 23 Aug 2025 23:25:19 +0800 Subject: [PATCH 14/53] =?UTF-8?q?update:=E6=98=BE=E7=A4=BA=E5=A4=A9?= =?UTF-8?q?=E6=B0=94=E6=8E=A5=E5=8F=A3=E9=94=99=E8=AF=AF=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/plugins_func/functions/get_weather.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main/xiaozhi-server/plugins_func/functions/get_weather.py b/main/xiaozhi-server/plugins_func/functions/get_weather.py index a3af6c03..38770a3f 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_weather.py +++ b/main/xiaozhi-server/plugins_func/functions/get_weather.py @@ -110,6 +110,11 @@ WEATHER_CODE_MAP = { def fetch_city_info(location, api_key, api_host): url = f"https://{api_host}/geo/v2/city/lookup?key={api_key}&location={location}&lang=zh" response = requests.get(url, headers=HEADERS).json() + if response.get("error") is not None: + logger.bind(tag=TAG).error( + f"获取天气失败，原因：{response.get('error', {}).get('detail')}" + ) + return None return response.get("location", [])[0] if response.get("location") else None From 18acec1a8144c535b6435aaceca65420da1ba27c Mon Sep 17 00:00:00 2001 From: Minamiyama Date: Sun, 24 Aug 2025 11:31:30 +0800 Subject: [PATCH 15/53] =?UTF-8?q?refactor(audio):=20=E5=B0=86=E6=B5=81?= =?UTF-8?q?=E5=BC=8F=E9=9F=B3=E9=A2=91=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E6=8F=90=E5=8F=96=E4=B8=BA=E7=8B=AC=E7=AB=8B=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将原本内联在test_page.html中的流式音频处理逻辑重构为独立的StreamingContext类，提高代码可维护性和复用性 --- .../test/js/StreamingContext.js | 149 ++++++++++++++++++ main/xiaozhi-server/test/test_page.html | 131 +-------------- 2 files changed, 152 insertions(+), 128 deletions(-) create mode 100644 main/xiaozhi-server/test/js/StreamingContext.js diff --git a/main/xiaozhi-server/test/js/StreamingContext.js b/main/xiaozhi-server/test/js/StreamingContext.js new file mode 100644 index 00000000..576d9761 --- /dev/null +++ b/main/xiaozhi-server/test/js/StreamingContext.js @@ -0,0 +1,149 @@ +import BlockingQueue from './utils/BlockingQueue.js'; +import { log } from './utils/logger.js'; + +// 音频流播放上下文类 +export class StreamingContext { + constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) { + this.opusDecoder = opusDecoder; + this.audioContext = audioContext; + + // 音频参数 + this.sampleRate = sampleRate; + this.channels = channels; + this.minAudioDuration = minAudioDuration; + + // 初始化队列和状态 + this.queue = []; // 已解码的PCM队列。正在播放 + this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放 + this.pendingAudioBufferQueue = []; // 待处理的缓存队列 + this.audioBufferQueue = new BlockingQueue(); // 缓存队列 + this.playing = false; // 是否正在播放 + this.endOfStream = false; // 是否收到结束信号 + this.source = null; // 当前音频源 + this.totalSamples = 0; // 累积的总样本数 + this.lastPlayTime = 0; // 上次播放的时间戳 + } + + // 缓存音频数组 + pushAudioBuffer(item) { + this.audioBufferQueue.enqueue(...item); + } + + // 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题 + async getPendingAudioBufferQueue() { + // 原子交换 + 清空 + [this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()]; + } + + // 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题 + async getQueue(minSamples) { + let TepArray = []; + const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1; + // 原子交换 + 清空 + [TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()]; + this.queue.push(...TepArray); + } + + // 将Int16音频数据转换为Float32音频数据 + convertInt16ToFloat32(int16Data) { + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + // 将[-32768,32767]范围转换为[-1,1] + float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF); + } + return float32Data; + } + + // 将Opus数据解码为PCM + async decodeOpusFrames() { + if (!this.opusDecoder) { + log('Opus解码器未初始化，无法解码', 'error'); + return; + } else { + log('Opus解码器启动', 'info'); + } + + while (true) { + let decodedSamples = []; + for (const frame of this.pendingAudioBufferQueue) { + try { + // 使用Opus解码器解码 + const frameData = this.opusDecoder.decode(frame); + if (frameData && frameData.length > 0) { + // 转换为Float32 + const floatData = this.convertInt16ToFloat32(frameData); + // 使用循环替代展开运算符 + for (let i = 0; i < floatData.length; i++) { + decodedSamples.push(floatData[i]); + } + } + } catch (error) { + log("Opus解码失败: " + error.message, 'error'); + } + } + + if (decodedSamples.length > 0) { + // 使用循环替代展开运算符 + for (let i = 0; i < decodedSamples.length; i++) { + this.activeQueue.enqueue(decodedSamples[i]); + } + this.totalSamples += decodedSamples.length; + } else { + log('没有成功解码的样本', 'warning'); + } + await this.getPendingAudioBufferQueue(); + } + } + + // 开始播放音频 + async startPlaying() { + while (true) { + // 如果累积了至少0.3秒的音频，开始播放 + const minSamples = this.sampleRate * this.minAudioDuration * 3; + if (!this.playing && this.queue.length < minSamples) { + await this.getQueue(minSamples); + } + this.playing = true; + while (this.playing && this.queue.length) { + // 创建新的音频缓冲区 + const minPlaySamples = Math.min(this.queue.length, this.sampleRate); + const currentSamples = this.queue.splice(0, minPlaySamples); + + const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate); + audioBuffer.copyToChannel(new Float32Array(currentSamples), 0); + + // 创建音频源 + this.source = this.audioContext.createBufferSource(); + this.source.buffer = audioBuffer; + + // 创建增益节点用于平滑过渡 + const gainNode = this.audioContext.createGain(); + + // 应用淡入淡出效果避免爆音 + const fadeDuration = 0.02; // 20毫秒 + gainNode.gain.setValueAtTime(0, this.audioContext.currentTime); + gainNode.gain.linearRampToValueAtTime(1, this.audioContext.currentTime + fadeDuration); + + const duration = audioBuffer.duration; + if (duration > fadeDuration * 2) { + gainNode.gain.setValueAtTime(1, this.audioContext.currentTime + duration - fadeDuration); + gainNode.gain.linearRampToValueAtTime(0, this.audioContext.currentTime + duration); + } + + // 连接节点并开始播放 + this.source.connect(gainNode); + gainNode.connect(this.audioContext.destination); + + this.lastPlayTime = this.audioContext.currentTime; + log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'info'); + this.source.start(); + } + await this.getQueue(minSamples); + } + } +} + +// 创建streamingContext实例的工厂函数 +export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) { + return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration); +} \ No newline at end of file diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index 983d3b9a..547f2b31 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -181,6 +181,7 @@ import { checkOpusLoaded, initOpusEncoder } from './js/opus.js'; import { addMessage } from './js/document.js' import BlockingQueue from './js/utils/BlockingQueue.js' + import { createStreamingContext } from './js/StreamingContext.js' // 需要加载的脚本列表 - 移除Opus依赖 const scriptFiles = []; @@ -336,125 +337,7 @@ // 创建流式播放上下文 if (!streamingContext) { - streamingContext = { - queue: [], // 已解码的PCM队列。正在播放 - activeQueue: new BlockingQueue(), // 已解码的PCM队列。准备播放 - pendingAudioBufferQueue: [], // 待处理的缓存队列 - audioBufferQueue: new BlockingQueue(), // 缓存队列 - playing: false, // 是否正在播放 - endOfStream: false, // 是否收到结束信号 - source: null, // 当前音频源 - totalSamples: 0, // 累积的总样本数 - lastPlayTime: 0, // 上次播放的时间戳 - - - // 缓存音频数组 - pushAudioBuffer: function (item) { - this.audioBufferQueue.enqueue(...item) - }, - - // 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题 - getPendingAudioBufferQueue: async function () { - // 原子交换 + 清空 - [this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()]; - - }, - // 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题 - getQueue: async function (minSamples) { - let TepArray = [] - const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1; - // 原子交换 + 清空 - [TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()]; - this.queue.push(...TepArray) - }, - // 将Opus数据解码为PCM - decodeOpusFrames: async function () { - if (!opusDecoder) { - log('Opus解码器未初始化，无法解码', 'error'); - return; - } else { - log('Opus解码器启动', 'info'); - } - - while (true) { - let decodedSamples = []; - for (const frame of this.pendingAudioBufferQueue) { - try { - // 使用Opus解码器解码 - const frameData = opusDecoder.decode(frame); - if (frameData && frameData.length > 0) { - // 转换为Float32 - const floatData = convertInt16ToFloat32(frameData); - // 使用循环替代展开运算符 - for (let i = 0; i < floatData.length; i++) { - decodedSamples.push(floatData[i]); - } - } - } catch (error) { - log("Opus解码失败: " + error.message, 'error'); - } - } - - if (decodedSamples.length > 0) { - // 使用循环替代展开运算符 - for (let i = 0; i < decodedSamples.length; i++) { - this.activeQueue.enqueue(decodedSamples[i]); - } - this.totalSamples += decodedSamples.length; - } else { - log('没有成功解码的样本', 'warning'); - } - await this.getPendingAudioBufferQueue(); - } - }, - - // 开始播放音频 - startPlaying: async function () { - while (true) { - // 如果累积了至少0.3秒的音频，开始播放 - const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION * 3; - if (!this.playing && this.queue.length < minSamples) { - await this.getQueue(minSamples) - } - this.playing = true; - while (this.playing && this.queue.length) { - // 创建新的音频缓冲区 - const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE); - const currentSamples = this.queue.splice(0, minPlaySamples); - - const audioBuffer = audioContext.createBuffer(CHANNELS, currentSamples.length, SAMPLE_RATE); - audioBuffer.copyToChannel(new Float32Array(currentSamples), 0); - - // 创建音频源 - this.source = audioContext.createBufferSource(); - this.source.buffer = audioBuffer; - - // 创建增益节点用于平滑过渡 - const gainNode = audioContext.createGain(); - - // 应用淡入淡出效果避免爆音 - const fadeDuration = 0.02; // 20毫秒 - gainNode.gain.setValueAtTime(0, audioContext.currentTime); - gainNode.gain.linearRampToValueAtTime(1, audioContext.currentTime + fadeDuration); - - const duration = audioBuffer.duration; - if (duration > fadeDuration * 2) { - gainNode.gain.setValueAtTime(1, audioContext.currentTime + duration - fadeDuration); - gainNode.gain.linearRampToValueAtTime(0, audioContext.currentTime + duration); - } - - // 连接节点并开始播放 - this.source.connect(gainNode); - gainNode.connect(audioContext.destination); - - this.lastPlayTime = audioContext.currentTime; - log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info'); - this.source.start(); - } - await this.getQueue(minSamples) - } - } - }; + streamingContext = createStreamingContext(opusDecoder, audioContext, SAMPLE_RATE, CHANNELS, MIN_AUDIO_DURATION); } streamingContext.decodeOpusFrames(); @@ -467,15 +350,7 @@ } } - // 将Int16音频数据转换为Float32音频数据 - function convertInt16ToFloat32(int16Data) { - const float32Data = new Float32Array(int16Data.length); - for (let i = 0; i < int16Data.length; i++) { - // 将[-32768,32767]范围转换为[-1,1] - float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF); - } - return float32Data; - } + // 初始化Opus解码器 - 确保完全初始化完成后才返回 async function initOpusDecoder() { From 5993470d3f304be545a8a1f5e7ee728dd23e4a05 Mon Sep 17 00:00:00 2001 From: Minamiyama Date: Sun, 24 Aug 2025 23:40:54 +0800 Subject: [PATCH 16/53] =?UTF-8?q?feat(=E6=A8=A1=E5=9E=8B=E7=AE=A1=E7=90=86?= =?UTF-8?q?):=20=E6=B7=BB=E5=8A=A0=E6=A8=A1=E5=9E=8B=E5=89=AF=E6=9C=AC?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在模型编辑对话框中增加副本创建功能，当选择创建副本时自动在原模型名称和代码后添加'_副本'后缀，并通过新增API接口保存为新模型 --- .../src/components/ModelEditDialog.vue | 7 +++- main/manager-web/src/views/ModelConfig.vue | 37 +++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/main/manager-web/src/components/ModelEditDialog.vue b/main/manager-web/src/components/ModelEditDialog.vue index 0bc2f7cf..ec29c30e 100644 --- a/main/manager-web/src/components/ModelEditDialog.vue +++ b/main/manager-web/src/components/ModelEditDialog.vue @@ -3,7 +3,7 @@ class="center-dialog" >
- 修改模型 + {{ modelData.duplicateMode ? '创建副本' : '修改模型' }}