From ee5a53d67d27d9362f6fe5abe1005767a2b42e47 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Mon, 9 Mar 2026 17:55:54 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=80=89=E7=94=A8?= =?UTF-8?q?seed-tts-2.0=E6=96=87=E6=9C=AC=E7=BC=BA=E5=A4=B1=20fix:=20?= =?UTF-8?q?=E8=AE=BE=E5=A4=87=E7=AB=AF=E8=AF=B4=E8=AF=9D=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E4=B8=8E=E6=9C=8D=E5=8A=A1=E7=AB=AF=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/handle/sendAudioHandle.py | 8 +++----- .../core/providers/tts/huoshan_double_stream.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 531d8958..fc2c6f6c 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -21,7 +21,6 @@ async def sendAudioMessage(conn: "ConnectionHandler", sentenceType, audios, text if conn.tts.tts_audio_first_sentence: conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}") conn.tts.tts_audio_first_sentence = False - await send_tts_message(conn, "start", None) if sentenceType == SentenceType.FIRST: # 同一句子的后续消息加入流控队列，其他情况立即发送 @@ -204,7 +203,6 @@ def _start_background_sender(conn: "ConnectionHandler", rate_controller, flow_co conn.last_activity_time = time.time() * 1000 await _do_send_audio(conn, packet, flow_control) - conn.client_is_speaking = True # 使用 start_sending 启动后台循环 rate_controller.start_sending(send_callback) @@ -232,12 +230,10 @@ async def _send_audio_with_rate_control( # 预缓冲：前N个包直接发送 if flow_control["packet_count"] < PRE_BUFFER_COUNT: await _do_send_audio(conn, packet, flow_control) - conn.client_is_speaking = True elif send_delay > 0: # 固定延迟模式 await asyncio.sleep(send_delay) await _do_send_audio(conn, packet, flow_control) - conn.client_is_speaking = True else: # 动态流控模式：仅添加到队列，由后台循环负责发送 rate_controller.add_audio(packet) @@ -267,7 +263,7 @@ async def _do_send_audio(conn: "ConnectionHandler", opus_packet, flow_control): async def send_tts_message(conn: "ConnectionHandler", state, text=None): """发送 TTS 状态消息""" if text is None and state == "sentence_start": - return + return message = {"type": "tts", "state": state, "session_id": conn.session_id} if text is not None: message["text"] = textUtils.check_emoji(text) @@ -318,3 +314,5 @@ async def send_stt_message(conn: "ConnectionHandler", text): json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id}) ) await send_tts_message(conn, "start") + # 发送start消息后客户端状态会处于说话中状态，同步服务端状态 + conn.client_is_speaking = True \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 152aa33f..a2680063 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -148,6 +148,7 @@ class TTSProvider(TTSProviderBase): self.access_token = config.get("access_token") self.cluster = config.get("cluster") self.resource_id = config.get("resource_id") + self.resource_type = True if self.resource_id == "seed-tts-2.0" else False self.activate_session = False if config.get("private_voice"): self.voice = config.get("private_voice") @@ -511,7 +512,7 @@ class TTSProvider(TTSProviderBase): if res.optional.event == EVENT_SessionCanceled: logger.bind(tag=TAG).debug(f"释放服务端资源成功～～") self.activate_session = False - elif res.optional.event == EVENT_TTSSentenceStart: + elif not self.resource_type and res.optional.event == EVENT_TTSSentenceStart: json_data = json.loads(res.payload.decode("utf-8")) self.tts_text = json_data.get("text", "") logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") @@ -522,8 +523,17 @@ class TTSProvider(TTSProviderBase): res.optional.event == EVENT_TTSResponse and res.header.message_type == AUDIO_ONLY_RESPONSE ): + # 处理seed-tts-2.0文本字幕 + if self.resource_type and self.conn.tts_MessageText: + logger.bind(tag=TAG).info( + f"句子语音生成成功： {self.conn.tts_MessageText}" + ) + self.tts_audio_queue.put( + (SentenceType.FIRST, [], self.conn.tts_MessageText) + ) + self.conn.tts_MessageText = None self.wav_to_opus_data_audio_raw_stream(res.payload, callback=self.handle_opus) - elif res.optional.event == EVENT_TTSSentenceEnd: + elif not self.resource_type and res.optional.event == EVENT_TTSSentenceEnd: logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}") elif res.optional.event == EVENT_SessionFinished: logger.bind(tag=TAG).debug(f"会话结束～～") From 7b2f6c2eee3f0394e28fc9558e87d0612b341dda Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Tue, 10 Mar 2026 10:53:24 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20=E5=8D=95=E4=B8=80=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E6=B5=81=E4=B8=8A=E6=8A=A5=E9=9F=B3=E9=A2=91=E4=B8=8D=E5=85=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/alibl_stream.py | 1 + .../core/providers/tts/aliyun_stream.py | 1 + .../xiaozhi-server/core/providers/tts/base.py | 25 +++++++++++++------ .../providers/tts/huoshan_double_stream.py | 1 + .../core/providers/tts/xunfei_stream.py | 1 + 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index 8eb0f708..1a639651 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -33,6 +33,7 @@ class TTSProvider(TTSProviderBase): self.api_key = config.get("api_key") if not self.api_key: raise ValueError("api_key is required for CosyVoice TTS") + self.report_on_last = True # WebSocket配置 self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/" diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index cf06e7ff..9548595e 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -104,6 +104,7 @@ class TTSProvider(TTSProviderBase): self.access_key_secret = config.get("access_key_secret") self.appkey = config.get("appkey") self.format = config.get("format", "pcm") + self.report_on_last = True # 音色配置 - CosyVoice大模型音色 if config.get("private_voice"): diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 4fef87dd..be42b14d 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -40,6 +40,7 @@ class TTSProviderBase(ABC): self.tts_audio_queue = queue.Queue() self.tts_audio_first_sentence = True self.before_stop_play_files = [] + self.report_on_last = False self.tts_text_buff = [] self.punctuations = ( @@ -322,7 +323,7 @@ class TTSProviderBase(ABC): def _audio_play_priority_thread(self): # 需要上报的文本和音频列表 enqueue_text = None - enqueue_audio = None + enqueue_audio = [] while not self.conn.stop_event.is_set(): text = None try: @@ -342,14 +343,24 @@ class TTSProviderBase(ABC): # 收到下一个文本开始或会话结束时进行上报 if sentence_type is not SentenceType.MIDDLE: - # 上报TTS数据 - if enqueue_text is not None and enqueue_audio is not None: - enqueue_tts_report(self.conn, enqueue_text, enqueue_audio) - enqueue_audio = [] - enqueue_text = text + if self.report_on_last: + # 累积模式：适用于全程只有一个语音流的TTS（如seed-tts-2.0） + # FIRST时只记录文本，音频持续累积，仅在LAST时统一上报 + if text: + enqueue_text = text + if sentence_type == SentenceType.LAST: + enqueue_tts_report(self.conn, enqueue_text, enqueue_audio) + enqueue_audio = [] + enqueue_text = None + else: + # 非累积模式：每个句子分别上报 + if enqueue_text is not None: + enqueue_tts_report(self.conn, enqueue_text, enqueue_audio) + enqueue_audio = [] + enqueue_text = text # 收集上报音频数据 - if isinstance(audio_datas, bytes) and enqueue_audio is not None: + if isinstance(audio_datas, bytes): enqueue_audio.append(audio_datas) # 发送音频 diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index a2680063..a237eedc 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -149,6 +149,7 @@ class TTSProvider(TTSProviderBase): self.cluster = config.get("cluster") self.resource_id = config.get("resource_id") self.resource_type = True if self.resource_id == "seed-tts-2.0" else False + self.report_on_last = self.resource_type self.activate_session = False if config.get("private_voice"): self.voice = config.get("private_voice") diff --git a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py index be53284e..ed0a58d6 100644 --- a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py @@ -76,6 +76,7 @@ class TTSProvider(TTSProviderBase): self.app_id = config.get("app_id") self.api_key = config.get("api_key") self.api_secret = config.get("api_secret") + self.report_on_last = True # 接口地址 self.api_url = config.get("api_url", "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6") From 42a132f5cd8a753275b4a144ff7b3e5dd2fcb278 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Tue, 10 Mar 2026 11:29:05 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E9=9F=B3=E8=89=B2=E6=97=B6=E7=9A=84=E9=BB=98=E8=AE=A4=E5=A1=AB?= =?UTF-8?q?=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-web/src/i18n/zh_CN.js | 2 +- main/manager-web/src/i18n/zh_TW.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main/manager-web/src/i18n/zh_CN.js b/main/manager-web/src/i18n/zh_CN.js index 82b31fc7..381c3399 100644 --- a/main/manager-web/src/i18n/zh_CN.js +++ b/main/manager-web/src/i18n/zh_CN.js @@ -189,7 +189,7 @@ export default { 'editVoiceDialog.remarkPlaceholder': '请输入备注内容', 'editVoiceDialog.generatePreview': '生成试听', 'editVoiceDialog.defaultVoiceName': '湾湾小何', - 'editVoiceDialog.defaultLanguageType': '中文', + 'editVoiceDialog.defaultLanguageType': '普通话', 'editVoiceDialog.requiredVoiceCode': '请输入音色编码', 'editVoiceDialog.requiredVoiceName': '请输入音色名称', diff --git a/main/manager-web/src/i18n/zh_TW.js b/main/manager-web/src/i18n/zh_TW.js index 5423da0e..c0349e27 100644 --- a/main/manager-web/src/i18n/zh_TW.js +++ b/main/manager-web/src/i18n/zh_TW.js @@ -189,7 +189,7 @@ export default { 'editVoiceDialog.remarkPlaceholder': '請輸入備註內容', 'editVoiceDialog.generatePreview': '生成試聽', 'editVoiceDialog.defaultVoiceName': '灣灣小何', - 'editVoiceDialog.defaultLanguageType': '中文', + 'editVoiceDialog.defaultLanguageType': '普通話', 'editVoiceDialog.requiredVoiceCode': '請輸入音色編碼', 'editVoiceDialog.requiredVoiceName': '請輸入音色名稱', From fbb0e02fdd9f6a78f2ddc4f56468971a241ed91d Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 11 Mar 2026 14:59:23 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E7=A9=BA=E6=A0=BC?= =?UTF-8?q?=EF=BC=8C=E6=9C=80=E4=B8=8B=E7=AB=AF=E7=95=99=E7=A9=BA=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/handle/sendAudioHandle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index fc2c6f6c..63a2411c 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -263,7 +263,7 @@ async def _do_send_audio(conn: "ConnectionHandler", opus_packet, flow_control): async def send_tts_message(conn: "ConnectionHandler", state, text=None): """发送 TTS 状态消息""" if text is None and state == "sentence_start": - return + return message = {"type": "tts", "state": state, "session_id": conn.session_id} if text is not None: message["text"] = textUtils.check_emoji(text) @@ -315,4 +315,4 @@ async def send_stt_message(conn: "ConnectionHandler", text): ) await send_tts_message(conn, "start") # 发送start消息后客户端状态会处于说话中状态，同步服务端状态 - conn.client_is_speaking = True \ No newline at end of file + conn.client_is_speaking = True