From 8b847a38265644ddd43daaa9166b6802d910e3c2 Mon Sep 17 00:00:00 2001 From: rainv123 <2148537152@qq.com> Date: Thu, 30 Apr 2026 14:32:36 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=B5=81=E5=BC=8F?= =?UTF-8?q?TTS=E6=9B=BF=E6=8D=A2=E8=AF=8D=E6=98=BE=E7=A4=BA/=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/alibl_stream.py | 2 ++ .../core/providers/tts/aliyun_stream.py | 2 ++ .../core/providers/tts/huoshan_double_stream.py | 13 +++++++++---- .../core/providers/tts/index_stream.py | 6 +++--- main/xiaozhi-server/core/providers/tts/linkerai.py | 10 +++++----- .../core/providers/tts/minimax_httpstream.py | 6 +++--- .../core/providers/tts/xunfei_stream.py | 2 ++ 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index 4ab30593..7618cbbd 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -145,6 +145,8 @@ class TTSProvider(TTSProviderBase): elif ContentType.TEXT == message.content_type: if message.content_detail: try: + # 保存原始文本用于流式响应时显示/上报 + self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 66a0df9c..0479cd15 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -251,6 +251,8 @@ class TTSProvider(TTSProviderBase): elif ContentType.TEXT == message.content_type: if message.content_detail: try: + # 保存原始文本用于流式响应时显示/上报 + self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index a6ecc21c..4dbacda9 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -321,6 +321,8 @@ class TTSProvider(TTSProviderBase): elif ContentType.TEXT == message.content_type: if message.content_detail: try: + # 保存原始文本用于流式响应时显示/上报 + self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) @@ -511,11 +513,14 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug(f"释放服务端资源成功～～") self.activate_session = False elif not self.resource_type and res.optional.event == EVENT_TTSSentenceStart: - json_data = json.loads(res.payload.decode("utf-8")) - self.tts_text = json_data.get("text", "") - logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") + # 使用保存的原始文本用于显示/上报（而不是服务端返回的替换后文本） + tts_text = self.get_tts_text(self.conn.sentence_id) + if not tts_text: + json_data = json.loads(res.payload.decode("utf-8")) + tts_text = json_data.get("text", "") + logger.bind(tag=TAG).debug(f"句子语音生成开始: {tts_text}") self.tts_audio_queue.put( - (SentenceType.FIRST, [], self.tts_text) + (SentenceType.FIRST, [], tts_text) ) elif ( res.optional.event == EVENT_TTSResponse diff --git a/main/xiaozhi-server/core/providers/tts/index_stream.py b/main/xiaozhi-server/core/providers/tts/index_stream.py index 7314ac9c..d1616874 100644 --- a/main/xiaozhi-server/core/providers/tts/index_stream.py +++ b/main/xiaozhi-server/core/providers/tts/index_stream.py @@ -97,7 +97,7 @@ class TTSProvider(TTSProviderBase): if self._correct_words_pattern: text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], text) try: - asyncio.run(self.text_to_speak(text, is_last)) + asyncio.run(self.text_to_speak(text, original_text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {original_text}，错误: {e}" @@ -117,7 +117,7 @@ class TTSProvider(TTSProviderBase): finally: return None - async def text_to_speak(self, text, is_last): + async def text_to_speak(self, text, original_text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" payload = {"text": text, "character": self.voice} @@ -140,7 +140,7 @@ class TTSProvider(TTSProviderBase): return self.pcm_buffer.clear() - self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + self.tts_audio_queue.put((SentenceType.FIRST, [], original_text)) # 处理音频流数据 async for chunk in resp.content.iter_any(): diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index ecfb8b38..145da1ae 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -89,7 +89,7 @@ class TTSProvider(TTSProviderBase): if self._correct_words_pattern: text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], text) try: - asyncio.run(self.text_to_speak(text, is_last)) + asyncio.run(self.text_to_speak(text, original_text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {original_text}，错误: {e}" @@ -109,9 +109,9 @@ class TTSProvider(TTSProviderBase): finally: return None - async def text_to_speak(self, text, is_last): + async def text_to_speak(self, text, original_text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" - await self._tts_request(text, is_last) + await self._tts_request(text, original_text, is_last) async def close(self): """资源清理""" @@ -119,7 +119,7 @@ class TTSProvider(TTSProviderBase): if hasattr(self, "opus_encoder"): self.opus_encoder.close() - async def _tts_request(self, text: str, is_last: bool) -> None: + async def _tts_request(self, text: str, original_text: str, is_last: bool) -> None: params = { "tts_text": text, "spk_id": self.voice, @@ -157,7 +157,7 @@ class TTSProvider(TTSProviderBase): return self.pcm_buffer.clear() - self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + self.tts_audio_queue.put((SentenceType.FIRST, [], original_text)) # 兼容 iter_chunked / iter_chunks / iter_any async for chunk in resp.content.iter_any(): diff --git a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py index f620adfb..b6ad1a9d 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py +++ b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py @@ -153,7 +153,7 @@ class TTSProvider(TTSProviderBase): if self._correct_words_pattern: text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], text) try: - asyncio.run(self.text_to_speak(text, is_last)) + asyncio.run(self.text_to_speak(text, original_text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {original_text}，错误: {e}" @@ -173,7 +173,7 @@ class TTSProvider(TTSProviderBase): finally: return None - async def text_to_speak(self, text, is_last): + async def text_to_speak(self, text, original_text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" payload = { "model": self.model, @@ -212,7 +212,7 @@ class TTSProvider(TTSProviderBase): return self.pcm_buffer.clear() - self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + self.tts_audio_queue.put((SentenceType.FIRST, [], original_text)) # 处理音频流数据 buffer = b"" diff --git a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py index 812c282b..9baed36c 100644 --- a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py @@ -197,6 +197,8 @@ class TTSProvider(TTSProviderBase): if ContentType.TEXT == message.content_type: if message.content_detail: try: + # 保存原始文本用于流式响应时显示/上报 + self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) From dd881071311118c92a9c44a9c14bb8daeeca368d Mon Sep 17 00:00:00 2001 From: rainv123 <2148537152@qq.com> Date: Thu, 30 Apr 2026 15:57:03 +0800 Subject: [PATCH 2/4] =?UTF-8?q?Revert=20"fix:=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=B5=81=E5=BC=8FTTS=E6=9B=BF=E6=8D=A2=E8=AF=8D=E6=98=BE?= =?UTF-8?q?=E7=A4=BA/=E4=B8=8A=E6=8A=A5=E9=97=AE=E9=A2=98"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 8b847a38265644ddd43daaa9166b6802d910e3c2. --- .../core/providers/tts/alibl_stream.py | 2 -- .../core/providers/tts/aliyun_stream.py | 2 -- .../core/providers/tts/huoshan_double_stream.py | 13 ++++--------- .../core/providers/tts/index_stream.py | 6 +++--- main/xiaozhi-server/core/providers/tts/linkerai.py | 10 +++++----- .../core/providers/tts/minimax_httpstream.py | 6 +++--- .../core/providers/tts/xunfei_stream.py | 2 -- 7 files changed, 15 insertions(+), 26 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index 7618cbbd..4ab30593 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -145,8 +145,6 @@ class TTSProvider(TTSProviderBase): elif ContentType.TEXT == message.content_type: if message.content_detail: try: - # 保存原始文本用于流式响应时显示/上报 - self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 0479cd15..66a0df9c 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -251,8 +251,6 @@ class TTSProvider(TTSProviderBase): elif ContentType.TEXT == message.content_type: if message.content_detail: try: - # 保存原始文本用于流式响应时显示/上报 - self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 4dbacda9..a6ecc21c 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -321,8 +321,6 @@ class TTSProvider(TTSProviderBase): elif ContentType.TEXT == message.content_type: if message.content_detail: try: - # 保存原始文本用于流式响应时显示/上报 - self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) @@ -513,14 +511,11 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug(f"释放服务端资源成功～～") self.activate_session = False elif not self.resource_type and res.optional.event == EVENT_TTSSentenceStart: - # 使用保存的原始文本用于显示/上报（而不是服务端返回的替换后文本） - tts_text = self.get_tts_text(self.conn.sentence_id) - if not tts_text: - json_data = json.loads(res.payload.decode("utf-8")) - tts_text = json_data.get("text", "") - logger.bind(tag=TAG).debug(f"句子语音生成开始: {tts_text}") + json_data = json.loads(res.payload.decode("utf-8")) + self.tts_text = json_data.get("text", "") + logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") self.tts_audio_queue.put( - (SentenceType.FIRST, [], tts_text) + (SentenceType.FIRST, [], self.tts_text) ) elif ( res.optional.event == EVENT_TTSResponse diff --git a/main/xiaozhi-server/core/providers/tts/index_stream.py b/main/xiaozhi-server/core/providers/tts/index_stream.py index d1616874..7314ac9c 100644 --- a/main/xiaozhi-server/core/providers/tts/index_stream.py +++ b/main/xiaozhi-server/core/providers/tts/index_stream.py @@ -97,7 +97,7 @@ class TTSProvider(TTSProviderBase): if self._correct_words_pattern: text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], text) try: - asyncio.run(self.text_to_speak(text, original_text, is_last)) + asyncio.run(self.text_to_speak(text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {original_text}，错误: {e}" @@ -117,7 +117,7 @@ class TTSProvider(TTSProviderBase): finally: return None - async def text_to_speak(self, text, original_text, is_last): + async def text_to_speak(self, text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" payload = {"text": text, "character": self.voice} @@ -140,7 +140,7 @@ class TTSProvider(TTSProviderBase): return self.pcm_buffer.clear() - self.tts_audio_queue.put((SentenceType.FIRST, [], original_text)) + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) # 处理音频流数据 async for chunk in resp.content.iter_any(): diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 145da1ae..ecfb8b38 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -89,7 +89,7 @@ class TTSProvider(TTSProviderBase): if self._correct_words_pattern: text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], text) try: - asyncio.run(self.text_to_speak(text, original_text, is_last)) + asyncio.run(self.text_to_speak(text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {original_text}，错误: {e}" @@ -109,9 +109,9 @@ class TTSProvider(TTSProviderBase): finally: return None - async def text_to_speak(self, text, original_text, is_last): + async def text_to_speak(self, text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" - await self._tts_request(text, original_text, is_last) + await self._tts_request(text, is_last) async def close(self): """资源清理""" @@ -119,7 +119,7 @@ class TTSProvider(TTSProviderBase): if hasattr(self, "opus_encoder"): self.opus_encoder.close() - async def _tts_request(self, text: str, original_text: str, is_last: bool) -> None: + async def _tts_request(self, text: str, is_last: bool) -> None: params = { "tts_text": text, "spk_id": self.voice, @@ -157,7 +157,7 @@ class TTSProvider(TTSProviderBase): return self.pcm_buffer.clear() - self.tts_audio_queue.put((SentenceType.FIRST, [], original_text)) + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) # 兼容 iter_chunked / iter_chunks / iter_any async for chunk in resp.content.iter_any(): diff --git a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py index b6ad1a9d..f620adfb 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py +++ b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py @@ -153,7 +153,7 @@ class TTSProvider(TTSProviderBase): if self._correct_words_pattern: text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], text) try: - asyncio.run(self.text_to_speak(text, original_text, is_last)) + asyncio.run(self.text_to_speak(text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {original_text}，错误: {e}" @@ -173,7 +173,7 @@ class TTSProvider(TTSProviderBase): finally: return None - async def text_to_speak(self, text, original_text, is_last): + async def text_to_speak(self, text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" payload = { "model": self.model, @@ -212,7 +212,7 @@ class TTSProvider(TTSProviderBase): return self.pcm_buffer.clear() - self.tts_audio_queue.put((SentenceType.FIRST, [], original_text)) + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) # 处理音频流数据 buffer = b"" diff --git a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py index 9baed36c..812c282b 100644 --- a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py @@ -197,8 +197,6 @@ class TTSProvider(TTSProviderBase): if ContentType.TEXT == message.content_type: if message.content_detail: try: - # 保存原始文本用于流式响应时显示/上报 - self.store_tts_text(self.conn.sentence_id, message.content_detail) logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) From a71ce53a687edf13c2aaf7b99f0cc8264abb529d Mon Sep 17 00:00:00 2001 From: rainv123 <2148537152@qq.com> Date: Thu, 30 Apr 2026 17:33:40 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E8=AF=8D=E6=97=B6=EF=BC=8C=E7=81=AB=E5=B1=B1?= =?UTF-8?q?=E5=BC=95=E6=93=8E=E5=8F=8C=E5=90=91=E6=B5=81=E5=BC=8Ftts?= =?UTF-8?q?=E7=9A=84=E6=96=87=E6=9C=AC=E6=98=BE=E7=A4=BA/=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/tts/base.py | 17 ++++++++++++++++- .../core/providers/tts/huoshan_double_stream.py | 6 +++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index c2333db3..479ecf41 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -56,10 +56,18 @@ class TTSProviderBase(ABC): if self.correct_words: # 按key长度降序排列，长的先匹配，避免短词部分干扰 sorted_keys = sorted(self.correct_words.keys(), key=len, reverse=True) - pattern_str = '|'.join(re.escape(k) for k in sorted_keys) + pattern_str = "|".join(re.escape(k) for k in sorted_keys) self._correct_words_pattern = re.compile(pattern_str) + # 构建反向替换正则，用于将TTS服务返回的替换后文本还原为原始文本（字幕显示） + reverse_map = {v: k for k, v in self.correct_words.items()} + sorted_reverse_keys = sorted(reverse_map.keys(), key=len, reverse=True) + reverse_pattern_str = "|".join(re.escape(k) for k in sorted_reverse_keys) + self._reverse_words_pattern = re.compile(reverse_pattern_str) + self._reverse_words_map = reverse_map else: self._correct_words_pattern = None + self._reverse_words_pattern = None + self._reverse_words_map = None self.tts_text_buff = [] self.punctuations = ( @@ -339,6 +347,13 @@ class TTSProviderBase(ABC): if sentence_id in self._sentence_text_map: del self._sentence_text_map[sentence_id] + def _restore_original_text(self, text): + if not self._reverse_words_pattern or not text: + return text + return self._reverse_words_pattern.sub( + lambda m: self._reverse_words_map[m.group(0)], text + ) + # 这里默认是非流式的处理方式 # 流式处理方式请在子类中重写 def tts_text_priority_thread(self): diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index a6ecc21c..27a734d0 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -514,9 +514,9 @@ class TTSProvider(TTSProviderBase): json_data = json.loads(res.payload.decode("utf-8")) self.tts_text = json_data.get("text", "") logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") - self.tts_audio_queue.put( - (SentenceType.FIRST, [], self.tts_text) - ) + # 将TTS服务返回的替换后文本还原为原始文本，用于字幕显示 + display_text = self._restore_original_text(self.tts_text) + self.tts_audio_queue.put((SentenceType.FIRST, [], display_text)) elif ( res.optional.event == EVENT_TTSResponse and res.header.message_type == AUDIO_ONLY_RESPONSE From 0a6f8114483f76a656d26ebfe9bcf21b5501fa61 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 7 May 2026 10:20:10 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=B5=81=E5=BC=8F?= =?UTF-8?q?=E6=83=85=E5=86=B5=E4=B8=8B=E6=9B=BF=E6=8D=A2=E8=AF=8D=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/alibl_stream.py | 30 ++++---- .../core/providers/tts/aliyun_stream.py | 35 ++++++---- .../xiaozhi-server/core/providers/tts/base.py | 70 +++++++++++++++++++ .../providers/tts/huoshan_double_stream.py | 14 ++-- .../core/providers/tts/xunfei_stream.py | 17 +++-- 5 files changed, 130 insertions(+), 36 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index 4ab30593..d7626c55 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -124,6 +124,8 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.FIRST: + # 重置流式处理状态 + self.reset_stream_state() # 初始化会话 try: if not getattr(self.conn, "sentence_id", None): @@ -194,22 +196,24 @@ class TTSProvider(TTSProviderBase): # 过滤Markdown filtered_text = MarkdownCleaner.clean_markdown(text) - if self._correct_words_pattern: - filtered_text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], filtered_text) if filtered_text: - # 发送continue-task消息 - continue_task_message = { - "header": { - "action": "continue-task", - "task_id": self.conn.sentence_id, - "streaming": "duplex", - }, - "payload": {"input": {"text": filtered_text}}, - } + # 使用滑动窗口匹配处理跨分片的替换词 + confirmed_texts, self._pending_prefix = self._match_stream_text(filtered_text) - await self.ws.send(json.dumps(continue_task_message)) - self.last_active_time = time.time() + # 发送每个确定的文本片段 + for txt in confirmed_texts: + if txt and self.ws: + continue_task_message = { + "header": { + "action": "continue-task", + "task_id": self.conn.sentence_id, + "streaming": "duplex", + }, + "payload": {"input": {"text": txt}}, + } + await self.ws.send(json.dumps(continue_task_message)) + self.last_active_time = time.time() return except Exception as e: logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 66a0df9c..7ae64d1b 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -233,6 +233,8 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.FIRST: + # 重置流式处理状态 + self.reset_stream_state() # 初始化参数 try: logger.bind(tag=TAG).debug("开始启动TTS会话...") @@ -295,21 +297,26 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本") return filtered_text = MarkdownCleaner.clean_markdown(text) - if self._correct_words_pattern: - filtered_text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], filtered_text) + if filtered_text: - run_request = { - "header": { - "message_id": uuid.uuid4().hex, - "task_id": self.task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "RunSynthesis", - "appkey": self.appkey, - }, - "payload": {"text": filtered_text}, - } - await self.ws.send(json.dumps(run_request)) - self.last_active_time = time.time() + # 使用滑动窗口匹配处理跨分片的替换词 + confirmed_texts, self._pending_prefix = self._match_stream_text(filtered_text) + + # 发送每个确定的文本片段 + for txt in confirmed_texts: + if txt and self.ws: + run_request = { + "header": { + "message_id": uuid.uuid4().hex, + "task_id": self.task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": {"text": txt}, + } + await self.ws.send(json.dumps(run_request)) + self.last_active_time = time.time() return except Exception as e: diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 479ecf41..f9b2e02e 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -64,11 +64,20 @@ class TTSProviderBase(ABC): reverse_pattern_str = "|".join(re.escape(k) for k in sorted_reverse_keys) self._reverse_words_pattern = re.compile(reverse_pattern_str) self._reverse_words_map = reverse_map + # 流式滑动窗口：按首字分组的替换词字典，用于快速查找 + self._words_by_first_char = {} + for key in sorted_keys: # 使用已按长度降序排列的keys，确保长词优先匹配 + first_char = key[0] if key else "" + if first_char not in self._words_by_first_char: + self._words_by_first_char[first_char] = [] + self._words_by_first_char[first_char].append(key) else: self._correct_words_pattern = None self._reverse_words_pattern = None self._reverse_words_map = None + # 流式滑动窗口：待匹配的缓存文本 + self._pending_prefix = "" self.tts_text_buff = [] self.punctuations = ( "。", @@ -564,3 +573,64 @@ class TTSProviderBase(ABC): if config_key in config: val = convert_percentage_to_range(config[config_key], min_val, max_val, base_val) setattr(self, attr_name, transform(val) if transform else val) + + def _match_stream_text(self, text): + """流式文本滑动窗口匹配，用于处理跨分片的替换词 + + Args: + text: 输入的文本片段 + + Returns: + tuple: (确定的文本列表, 剩余待匹配的前缀) + """ + if not self.correct_words or not text: + return [text] if text else [], "" + + result = [] + pending = self._pending_prefix + i = 0 + + while i < len(text): + char = text[i] + + # 尝试：pending + 当前字符 是否能匹配替换词 + test_text = pending + char + + matched = False + # 遍历可能匹配的替换词 + candidates = self._words_by_first_char.get(pending[0], []) if pending else self._words_by_first_char.get(char, []) + for key in candidates: + if test_text == key: + # 完整匹配，替换后发送 + result.append(self.correct_words[key]) + pending = "" + matched = True + break + elif key.startswith(test_text): + # 是替换词的前缀，继续等待 + pending = test_text + matched = True + break + + if matched: + i += 1 + continue + + # 没有匹配到更长的词，pending 的内容确定可以发送 + if pending: + result.append(pending) + pending = "" + + # 检查当前字符是否是某个替换词的开头 + if char in self._words_by_first_char: + pending = char + else: + result.append(char) + + i += 1 + + return result, pending + + def reset_stream_state(self): + """重置流式处理状态，用于会话开始时清理残留状态""" + self._pending_prefix = "" diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 27a734d0..9cea8144 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -300,6 +300,8 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.FIRST: + # 重置流式处理状态 + self.reset_stream_state() # 初始化参数 try: if not getattr(self.conn, "sentence_id", None): @@ -370,12 +372,16 @@ class TTSProvider(TTSProviderBase): # 过滤Markdown filtered_text = MarkdownCleaner.clean_markdown(text) - if self._correct_words_pattern: - filtered_text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], filtered_text) if filtered_text: - # 发送文本 - await self.send_text(self.voice, filtered_text, self.conn.sentence_id) + # 使用滑动窗口匹配处理跨分片的替换词 + confirmed_texts, self._pending_prefix = self._match_stream_text(filtered_text) + + # 发送每个确定的文本片段 + for txt in confirmed_texts: + if txt and self.ws: + await self.send_text(self.voice, txt, self.conn.sentence_id) + return except Exception as e: logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") diff --git a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py index 812c282b..8edc414b 100644 --- a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py @@ -168,6 +168,8 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.FIRST: + # 重置流式处理状态 + self.reset_stream_state() # 重置序列号 self.text_seq = 0 # 增加序列号 @@ -245,12 +247,17 @@ class TTSProvider(TTSProviderBase): return filtered_text = MarkdownCleaner.clean_markdown(text) - if self._correct_words_pattern: - filtered_text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], filtered_text) + if filtered_text: - # 发送文本合成请求 - run_request = self._build_base_request(status=1,text=filtered_text) - await self.ws.send(json.dumps(run_request)) + # 使用滑动窗口匹配处理跨分片的替换词 + confirmed_texts, self._pending_prefix = self._match_stream_text(filtered_text) + + # 发送每个确定的文本片段 + for txt in confirmed_texts: + if txt and self.ws: + # 发送文本合成请求 + run_request = self._build_base_request(status=1, text=txt) + await self.ws.send(json.dumps(run_request)) return except Exception as e: