From 3130044909fbd61d512286536540da933d9ef9f4 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 29 May 2025 09:05:08 +0800 Subject: [PATCH 1/7] test --- main/xiaozhi-server/config.yaml | 7 ++ .../core/providers/tts/linkerai.py | 95 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/tts/linkerai.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 25c8e017..473c095b 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -713,4 +713,11 @@ TTS: headers: # 自定义请求头 # Authorization: Bearer xxxx format: mp3 # 接口返回的音频格式 + output_dir: tmp/ + LinkeraiTTS: + #各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get + type: linkerai + api_url: https://tts.linkerai.top/tts + access_token: "test" + voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py new file mode 100644 index 00000000..8ad069bd --- /dev/null +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -0,0 +1,95 @@ +from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import ( + TTSMessageDTO, + SentenceType, + ContentType, + InterfaceType +) +from config.logger import setup_logging +from core.utils import opus_encoder_utils +import requests + +TAG = __name__ +logger = setup_logging() + + +class TTSProvider(TTSProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + self.interface_type = InterfaceType.SINGLE_STREAM + self.access_token = config.get("access_token") + self.voice = config.get("voice") + self.api_url = config.get("api_url") + + # 根据配置选择音频格式，优先使用opus + self.audio_format = config.get("audio_format", "opus") + + # 创建Opus编码器 + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=16000, + channels=1, + frame_size_ms=60 + ) + + async def text_to_speak(self, text, _): + """将文本转换为语音（流式）""" + await self.send_text(self.voice, text) + return + + async def send_text(self, speaker: str, text: str): + """向 TTS 服务发送文本并获取音频流""" + try: + # 构造请求参数 + params = { + "tts_text": text, + "spk_id": self.voice, + "frame_duration": 60, + "stream": "true", + "target_sr": 16000, + "audio_format": self.audio_format, + } + + # 构造请求头 + headers = { + "Authorization": f"Bearer {self.access_token}", + } + + # 发送流式请求 + response = requests.get( + self.api_url, + params=params, + headers=headers, + stream=True + ) + + # 检查响应状态 + if response.status_code != 200: + logger.bind(tag=TAG).error(f"TTS 请求失败: {response.status_code}, {response.text}") + return + + # 处理音频流 + audio_frames = [] + for chunk in response.iter_content(chunk_size=1024): + if chunk: + if self.audio_format == "pcm": + # 将PCM转换为Opus + opus_frames = self.opus_encoder.encode_pcm_to_opus(chunk, False) + audio_frames.extend(opus_frames) + else: + # 直接使用Opus帧 + audio_frames.append(chunk) + + # 将音频帧放入队列 + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_frames, text) + ) + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS 流式处理异常：{str(e)}") + raise + + async def close(self): + """资源清理方法""" + await super().close() + if hasattr(self, "opus_encoder"): + self.opus_encoder.close() From 92affd6e1345374285be93aaae69596d47daa77e Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 30 May 2025 15:22:41 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E5=BE=85=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 1 + .../xiaozhi-server/core/providers/tts/base.py | 11 +- .../core/providers/tts/linkerai.py | 198 ++++++++++++++---- 3 files changed, 162 insertions(+), 48 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 473c095b..4230715a 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -718,6 +718,7 @@ TTS: #各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get type: linkerai api_url: https://tts.linkerai.top/tts + audio_format: "pcm" access_token: "test" voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 0b17eef5..6bc77dff 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -331,10 +331,11 @@ class TTSProviderBase(ABC): segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: tts_file = self.to_tts(segment_text) - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, segment_text) - ) - self.processed_chars += len(full_text) + if tts_file: + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, segment_text) + ) + self.processed_chars += len(full_text) return True return False diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 8ad069bd..cdb7296e 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -1,3 +1,8 @@ +import asyncio +import traceback +import queue +import requests +from pathlib import Path from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import ( TTSMessageDTO, @@ -6,8 +11,7 @@ from core.providers.tts.dto.dto import ( InterfaceType ) from config.logger import setup_logging -from core.utils import opus_encoder_utils -import requests +from core.utils import opus_encoder_utils, textUtils TAG = __name__ logger = setup_logging() @@ -20,8 +24,6 @@ class TTSProvider(TTSProviderBase): self.access_token = config.get("access_token") self.voice = config.get("voice") self.api_url = config.get("api_url") - - # 根据配置选择音频格式，优先使用opus self.audio_format = config.get("audio_format", "opus") # 创建Opus编码器 @@ -31,15 +33,135 @@ class TTSProvider(TTSProviderBase): frame_size_ms=60 ) - async def text_to_speak(self, text, _): - """将文本转换为语音（流式）""" - await self.send_text(self.voice, text) + # 添加文本缓冲区 + self.text_buffer = "" + # 句子结束标点集合 + self.sentence_endings = ("。", "？", "！", "；", ":", ".", "?", "!", ";","……") + # 逗号类标点（用于第一句话分割） + self.comma_endings = ("，", "~", "、", ",", "。", ".", "？", "?", "！", "!", "；", ";", "：",) + + # PCM缓冲区 + self.pcm_buffer = bytearray() + + ################################################################################### + # linkerai单流式TTS重写父类的方法--开始 + ################################################################################### + + def tts_text_priority_thread(self): + """流式文本处理线程""" + while not self.conn.stop_event.is_set(): + try: + message = self.tts_text_queue.get(timeout=1) + logger.bind(tag=TAG).debug( + f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" + ) + + if message.sentence_type == SentenceType.FIRST: + # 初始化流式状态 + self.tts_audio_first_sentence = True + self.pcm_buffer = bytearray() + self.text_buffer = "" # 重置文本缓冲区 + + elif ContentType.TEXT == message.content_type: + # 将文本添加到缓冲区 + self.text_buffer += message.content_detail + # 尝试分割并发送完整句子 + self._process_text_buffer() + + elif ContentType.FILE == message.content_type: + # 先处理缓冲区中的剩余文本 + self._flush_text_buffer() + # 处理文件类型 + if message.content_file and Path(message.content_file).exists(): + audio_datas = self._process_audio_file(message.content_file) + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, message.content_detail) + ) + + if message.sentence_type == SentenceType.LAST: + # 处理缓冲区中的剩余文本 + self._flush_text_buffer() + # 发送结束帧 + if self.pcm_buffer: + opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) + self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, "")) + self.pcm_buffer = bytearray() + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + self.text_buffer = "" # 重置文本缓冲区 + + except queue.Empty: + continue + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" + ) + + def _process_text_buffer(self): + """处理文本缓冲区，分割并发送完整句子""" + while True: + # 查找最近的句子结束位置 + end_pos = -1 + for punct in self.sentence_endings: + pos = self.text_buffer.find(punct) + if pos != -1 and (end_pos == -1 or pos < end_pos): + end_pos = pos + + # 如果是第一句话，也允许在逗号处分隔 + if self.tts_audio_first_sentence and end_pos == -1: + for punct in self.comma_endings: + pos = self.text_buffer.find(punct) + if pos != -1 and (end_pos == -1 or pos < end_pos): + end_pos = pos + + # 找到分割点 + if end_pos != -1: + # 提取完整句子 + sentence = self.text_buffer[:end_pos + 1] + sentence = textUtils.get_string_no_punctuation_or_emoji(sentence) + + if not sentence.strip(): # 检查是否为空文本 + self.text_buffer = self.text_buffer[end_pos + 1:] + continue + + self.text_buffer = self.text_buffer[end_pos + 1:] + + # 发送句子 + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(sentence), + loop=self.conn.loop + ) + future.result() + + # 更新第一句话标志 + if self.tts_audio_first_sentence: + self.tts_audio_first_sentence = False + else: + break + + def _flush_text_buffer(self): + """处理缓冲区中剩余的文本""" + if self.text_buffer: + clean_text = textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) + if clean_text.strip(): # 检查是否为空文本 + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(clean_text), + loop=self.conn.loop + ) + future.result() + self.text_buffer = "" + + async def text_to_speak(self, text): + # 发送文本 + await self.send_text(text) return - async def send_text(self, speaker: str, text: str): - """向 TTS 服务发送文本并获取音频流""" + ################################################################################### + # linkerai单流式TTS重写父类的方法--结束 + ################################################################################### + + async def send_text(self, text: str): + """流式处理TTS音频""" try: - # 构造请求参数 params = { "tts_text": text, "spk_id": self.voice, @@ -48,48 +170,38 @@ class TTSProvider(TTSProviderBase): "target_sr": 16000, "audio_format": self.audio_format, } + headers = {"Authorization": f"Bearer {self.access_token}"} - # 构造请求头 - headers = { - "Authorization": f"Bearer {self.access_token}", - } + with requests.get(self.api_url, params=params, headers=headers, stream=True) as response: + if response.status_code != 200: + logger.error(f"TTS请求失败: {response.status_code}, {response.text}") + return - # 发送流式请求 - response = requests.get( - self.api_url, - params=params, - headers=headers, - stream=True - ) + logger.debug(f"处理TTS文本: {text}") - # 检查响应状态 - if response.status_code != 200: - logger.bind(tag=TAG).error(f"TTS 请求失败: {response.status_code}, {response.text}") - return + # 流式处理音频数据 + for chunk in response.iter_content(chunk_size=1024): + if chunk: + # 实时编码并发送音频帧 + self.pcm_buffer.extend(chunk) - # 处理音频流 - audio_frames = [] - for chunk in response.iter_content(chunk_size=1024): - if chunk: - if self.audio_format == "pcm": - # 将PCM转换为Opus - opus_frames = self.opus_encoder.encode_pcm_to_opus(chunk, False) - audio_frames.extend(opus_frames) - else: - # 直接使用Opus帧 - audio_frames.append(chunk) - - # 将音频帧放入队列 - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_frames, text) - ) + # 处理剩余缓冲区数据 + if self.pcm_buffer: + opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) + self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) + self.pcm_buffer = bytearray() except Exception as e: - logger.bind(tag=TAG).error(f"TTS 流式处理异常：{str(e)}") + logger.error(f"TTS流式处理异常：{str(e)}") raise + # 保持原有方法 + def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False): + opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end) + return opus_datas + async def close(self): - """资源清理方法""" + """资源清理""" await super().close() if hasattr(self, "opus_encoder"): self.opus_encoder.close() From 0bc609cac47c1b33b2d85dcb927d09330899bc6d Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Tue, 3 Jun 2025 11:20:54 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AE=9E=E6=97=B6?= =?UTF-8?q?=E7=BC=96=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/linkerai.py | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index cdb7296e..ab585ef3 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -35,10 +35,6 @@ class TTSProvider(TTSProviderBase): # 添加文本缓冲区 self.text_buffer = "" - # 句子结束标点集合 - self.sentence_endings = ("。", "？", "！", "；", ":", ".", "?", "!", ";","……") - # 逗号类标点（用于第一句话分割） - self.comma_endings = ("，", "~", "、", ",", "。", ".", "？", "?", "！", "!", "；", ";", "：",) # PCM缓冲区 self.pcm_buffer = bytearray() @@ -52,9 +48,9 @@ class TTSProvider(TTSProviderBase): while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) - logger.bind(tag=TAG).debug( - f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" - ) + # logger.bind(tag=TAG).debug( + # f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" + # ) if message.sentence_type == SentenceType.FIRST: # 初始化流式状态 @@ -83,7 +79,7 @@ class TTSProvider(TTSProviderBase): self._flush_text_buffer() # 发送结束帧 if self.pcm_buffer: - opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) + opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, end_of_stream=True) self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, "")) self.pcm_buffer = bytearray() self.tts_audio_queue.put((SentenceType.LAST, [], None)) @@ -98,17 +94,20 @@ class TTSProvider(TTSProviderBase): def _process_text_buffer(self): """处理文本缓冲区，分割并发送完整句子""" + # 使用父类的标点集合 + sentence_endings = self.punctuations + comma_endings = self.first_sentence_punctuations while True: # 查找最近的句子结束位置 end_pos = -1 - for punct in self.sentence_endings: + for punct in sentence_endings: pos = self.text_buffer.find(punct) if pos != -1 and (end_pos == -1 or pos < end_pos): end_pos = pos # 如果是第一句话，也允许在逗号处分隔 if self.tts_audio_first_sentence and end_pos == -1: - for punct in self.comma_endings: + for punct in comma_endings: pos = self.text_buffer.find(punct) if pos != -1 and (end_pos == -1 or pos < end_pos): end_pos = pos @@ -160,7 +159,7 @@ class TTSProvider(TTSProviderBase): ################################################################################### async def send_text(self, text: str): - """流式处理TTS音频""" + """流式处理TTS音频，每句只推送一次音频列表""" try: params = { "tts_text": text, @@ -172,32 +171,44 @@ class TTSProvider(TTSProviderBase): } headers = {"Authorization": f"Bearer {self.access_token}"} - with requests.get(self.api_url, params=params, headers=headers, stream=True) as response: + with requests.get(self.api_url, params=params, headers=headers, stream=True, timeout=5) as response: if response.status_code != 200: logger.error(f"TTS请求失败: {response.status_code}, {response.text}") + # 推送空LAST，防止播放端卡死 + self.tts_audio_queue.put((SentenceType.LAST, [], None)) return logger.debug(f"处理TTS文本: {text}") - # 流式处理音频数据 - for chunk in response.iter_content(chunk_size=1024): + pcm_buffer = bytearray() + frame_bytes = self.opus_encoder.frame_size * 4 # 每帧字节数（int16=2字节） + opus_datas = [] + for chunk in response.iter_content(chunk_size=960): if chunk: - # 实时编码并发送音频帧 - self.pcm_buffer.extend(chunk) - - # 处理剩余缓冲区数据 - if self.pcm_buffer: - opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) - self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) - self.pcm_buffer = bytearray() + pcm_buffer.extend(chunk) + # 只要够帧就编码 + while len(pcm_buffer) >= frame_bytes: + frame = pcm_buffer[:frame_bytes] + opus_chunk = self.opus_encoder.encode_pcm_to_opus(frame, end_of_stream=False) + if opus_chunk: + opus_datas.extend(opus_chunk) + pcm_buffer = pcm_buffer[frame_bytes:] # 剩余部分继续累积 + # 处理最后剩余数据 + if pcm_buffer: + opus_chunk = self.opus_encoder.encode_pcm_to_opus(pcm_buffer, end_of_stream=True) + if opus_chunk: + opus_datas.extend(opus_chunk) + # 推送本句所有音频帧 + self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) except Exception as e: logger.error(f"TTS流式处理异常：{str(e)}") - raise + # 推送空LAST，防止播放端卡死 + self.tts_audio_queue.put((SentenceType.LAST, [], None)) # 保持原有方法 - def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False): - opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end) + def wav_to_opus_data_audio_raw(self, raw_data_var): + opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, end_of_stream=True) return opus_datas async def close(self): From adf1a4794553ac5514202e69399d612baaf2b583 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 09:25:30 +0800 Subject: [PATCH 4/7] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 4230715a..eb3ec309 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -715,10 +715,12 @@ TTS: format: mp3 # 接口返回的音频格式 output_dir: tmp/ LinkeraiTTS: - #各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get type: linkerai api_url: https://tts.linkerai.top/tts audio_format: "pcm" - access_token: "test" + # 默认的access_token供大家测试时免费使用的，此access_token请勿用于商业用途 + # 如果效果不错，可自行申请token，申请地址：https://linkerai.top + # 各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get + access_token: "U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL" voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file From d7eecfdceaf2200a3d8432e7bf7b962fe683eafb Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 10:49:25 +0800 Subject: [PATCH 5/7] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 4 +- .../core/providers/tts/linkerai.py | 201 +++++++----------- 2 files changed, 77 insertions(+), 128 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 5d2c47a6..a7ff54f3 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -759,10 +759,10 @@ TTS: LinkeraiTTS: type: linkerai api_url: https://tts.linkerai.top/tts - audio_format: "pcm" + audio_format: "opus" # 默认的access_token供大家测试时免费使用的，此access_token请勿用于商业用途 # 如果效果不错，可自行申请token，申请地址：https://linkerai.top # 各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get access_token: "U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL" - voice: "OUeAo1mhq6IBExi" + voice: "7OEPouTL46bS2Qe" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index ab585ef3..f570cfb0 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -1,17 +1,14 @@ -import asyncio -import traceback +import os +import time import queue +import asyncio import requests -from pathlib import Path -from core.providers.tts.base import TTSProviderBase -from core.providers.tts.dto.dto import ( - TTSMessageDTO, - SentenceType, - ContentType, - InterfaceType -) +import traceback from config.logger import setup_logging +from core.utils.tts import MarkdownCleaner +from core.providers.tts.base import TTSProviderBase from core.utils import opus_encoder_utils, textUtils +from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType TAG = __name__ logger = setup_logging() @@ -24,13 +21,12 @@ class TTSProvider(TTSProviderBase): self.access_token = config.get("access_token") self.voice = config.get("voice") self.api_url = config.get("api_url") - self.audio_format = config.get("audio_format", "opus") + self.audio_format = "pcm" + self.before_stop_play_files = [] # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, - channels=1, - frame_size_ms=60 + sample_rate=16000, channels=1, frame_size_ms=60 ) # 添加文本缓冲区 @@ -48,42 +44,31 @@ class TTSProvider(TTSProviderBase): while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) - # logger.bind(tag=TAG).debug( - # f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" - # ) - if message.sentence_type == SentenceType.FIRST: - # 初始化流式状态 + # 初始化参数 + self.tts_stop_request = False + self.processed_chars = 0 + self.tts_text_buff = [] + self.is_first_sentence = True self.tts_audio_first_sentence = True - self.pcm_buffer = bytearray() - self.text_buffer = "" # 重置文本缓冲区 - + self.before_stop_play_files.clear() elif ContentType.TEXT == message.content_type: - # 将文本添加到缓冲区 - self.text_buffer += message.content_detail - # 尝试分割并发送完整句子 - self._process_text_buffer() + self.tts_text_buff.append(message.content_detail) + segment_text = self._get_segment_text() + if segment_text: + self.to_tts(segment_text) elif ContentType.FILE == message.content_type: - # 先处理缓冲区中的剩余文本 - self._flush_text_buffer() - # 处理文件类型 - if message.content_file and Path(message.content_file).exists(): - audio_datas = self._process_audio_file(message.content_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, message.content_detail) - ) + self._process_remaining_text() + logger.bind(tag=TAG).info( + f"添加音频文件到待播放列表: {message.content_file}" + ) + self.before_stop_play_files.append( + (message.content_file, message.content_detail) + ) if message.sentence_type == SentenceType.LAST: - # 处理缓冲区中的剩余文本 - self._flush_text_buffer() - # 发送结束帧 - if self.pcm_buffer: - opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, end_of_stream=True) - self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, "")) - self.pcm_buffer = bytearray() - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - self.text_buffer = "" # 重置文本缓冲区 + self._process_remaining_text() except queue.Empty: continue @@ -92,115 +77,77 @@ class TTSProvider(TTSProviderBase): f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" ) - def _process_text_buffer(self): - """处理文本缓冲区，分割并发送完整句子""" - # 使用父类的标点集合 - sentence_endings = self.punctuations - comma_endings = self.first_sentence_punctuations - while True: - # 查找最近的句子结束位置 - end_pos = -1 - for punct in sentence_endings: - pos = self.text_buffer.find(punct) - if pos != -1 and (end_pos == -1 or pos < end_pos): - end_pos = pos + def _process_remaining_text(self): + """处理剩余的文本并生成语音 - # 如果是第一句话，也允许在逗号处分隔 - if self.tts_audio_first_sentence and end_pos == -1: - for punct in comma_endings: - pos = self.text_buffer.find(punct) - if pos != -1 and (end_pos == -1 or pos < end_pos): - end_pos = pos + Returns: + bool: 是否成功处理了文本 + """ + full_text = "".join(self.tts_text_buff) + remaining_text = full_text[self.processed_chars :] + if remaining_text: + segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) + if segment_text: + self.to_tts(segment_text) + self.processed_chars += len(full_text) - # 找到分割点 - if end_pos != -1: - # 提取完整句子 - sentence = self.text_buffer[:end_pos + 1] - sentence = textUtils.get_string_no_punctuation_or_emoji(sentence) - - if not sentence.strip(): # 检查是否为空文本 - self.text_buffer = self.text_buffer[end_pos + 1:] - continue - - self.text_buffer = self.text_buffer[end_pos + 1:] - - # 发送句子 - future = asyncio.run_coroutine_threadsafe( - self.text_to_speak(sentence), - loop=self.conn.loop + def to_tts(self, text): + try: + max_repeat_time = 5 + text = MarkdownCleaner.clean_markdown(text) + try: + asyncio.run(self.text_to_speak(text, None)) + except Exception as e: + logger.bind(tag=TAG).warning( + f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" ) - future.result() + max_repeat_time -= 1 - # 更新第一句话标志 - if self.tts_audio_first_sentence: - self.tts_audio_first_sentence = False + if max_repeat_time > 0: + logger.bind(tag=TAG).info( + f"语音生成成功: {text}，重试{5 - max_repeat_time}次" + ) else: - break - - def _flush_text_buffer(self): - """处理缓冲区中剩余的文本""" - if self.text_buffer: - clean_text = textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) - if clean_text.strip(): # 检查是否为空文本 - future = asyncio.run_coroutine_threadsafe( - self.text_to_speak(clean_text), - loop=self.conn.loop + logger.bind(tag=TAG).error( + f"语音生成失败: {text}，请检查网络或服务是否正常" ) - future.result() - self.text_buffer = "" - - async def text_to_speak(self, text): - # 发送文本 - await self.send_text(text) - return + except Exception as e: + logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") + finally: + return None ################################################################################### # linkerai单流式TTS重写父类的方法--结束 ################################################################################### - async def send_text(self, text: str): + async def text_to_speak(self, text, _): """流式处理TTS音频，每句只推送一次音频列表""" + start_time = time.time() + logger.info(f"TTS请求: {text}") try: params = { "tts_text": text, "spk_id": self.voice, "frame_duration": 60, - "stream": "true", + "stream": True, "target_sr": 16000, "audio_format": self.audio_format, } headers = {"Authorization": f"Bearer {self.access_token}"} - with requests.get(self.api_url, params=params, headers=headers, stream=True, timeout=5) as response: + with requests.get( + self.api_url, params=params, headers=headers, timeout=5 + ) as response: if response.status_code != 200: - logger.error(f"TTS请求失败: {response.status_code}, {response.text}") + logger.error( + f"TTS请求失败: {response.status_code}, {response.text}" + ) # 推送空LAST，防止播放端卡死 self.tts_audio_queue.put((SentenceType.LAST, [], None)) return - - logger.debug(f"处理TTS文本: {text}") - - pcm_buffer = bytearray() - frame_bytes = self.opus_encoder.frame_size * 4 # 每帧字节数（int16=2字节） - opus_datas = [] - for chunk in response.iter_content(chunk_size=960): - if chunk: - pcm_buffer.extend(chunk) - # 只要够帧就编码 - while len(pcm_buffer) >= frame_bytes: - frame = pcm_buffer[:frame_bytes] - opus_chunk = self.opus_encoder.encode_pcm_to_opus(frame, end_of_stream=False) - if opus_chunk: - opus_datas.extend(opus_chunk) - pcm_buffer = pcm_buffer[frame_bytes:] # 剩余部分继续累积 - # 处理最后剩余数据 - if pcm_buffer: - opus_chunk = self.opus_encoder.encode_pcm_to_opus(pcm_buffer, end_of_stream=True) - if opus_chunk: - opus_datas.extend(opus_chunk) - # 推送本句所有音频帧 + logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") + opus_datas = self.wav_to_opus_data_audio_raw(response.content) self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) - except Exception as e: logger.error(f"TTS流式处理异常：{str(e)}") # 推送空LAST，防止播放端卡死 @@ -208,7 +155,9 @@ class TTSProvider(TTSProviderBase): # 保持原有方法 def wav_to_opus_data_audio_raw(self, raw_data_var): - opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, end_of_stream=True) + opus_datas = self.opus_encoder.encode_pcm_to_opus( + raw_data_var, end_of_stream=True + ) return opus_datas async def close(self): From 2654802bb079ac98662c1636d6b29c695629ece9 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 11:09:09 +0800 Subject: [PATCH 6/7] =?UTF-8?q?update:=E6=B5=81=E5=BC=8F=E6=92=AD=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/linkerai.py | 88 +++++++++++-------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index f570cfb0..767d2662 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -4,6 +4,7 @@ import queue import asyncio import requests import traceback +import aiohttp from config.logger import setup_logging from core.utils.tts import MarkdownCleaner from core.providers.tts.base import TTSProviderBase @@ -122,46 +123,59 @@ class TTSProvider(TTSProviderBase): async def text_to_speak(self, text, _): """流式处理TTS音频，每句只推送一次音频列表""" - start_time = time.time() - logger.info(f"TTS请求: {text}") - try: - params = { - "tts_text": text, - "spk_id": self.voice, - "frame_duration": 60, - "stream": True, - "target_sr": 16000, - "audio_format": self.audio_format, - } - headers = {"Authorization": f"Bearer {self.access_token}"} - - with requests.get( - self.api_url, params=params, headers=headers, timeout=5 - ) as response: - if response.status_code != 200: - logger.error( - f"TTS请求失败: {response.status_code}, {response.text}" - ) - # 推送空LAST，防止播放端卡死 - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - return - logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") - opus_datas = self.wav_to_opus_data_audio_raw(response.content) - self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) - except Exception as e: - logger.error(f"TTS流式处理异常：{str(e)}") - # 推送空LAST，防止播放端卡死 - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - # 保持原有方法 - def wav_to_opus_data_audio_raw(self, raw_data_var): - opus_datas = self.opus_encoder.encode_pcm_to_opus( - raw_data_var, end_of_stream=True - ) - return opus_datas + await self._tts_request(text) async def close(self): """资源清理""" await super().close() if hasattr(self, "opus_encoder"): self.opus_encoder.close() + + async def _tts_request(self, text: str) -> None: + """发送TTS请求""" + start_time = time.time() + params = { + "tts_text": text, + "spk_id": self.voice, + "frame_durition": 60, + "stream": "true", + "target_sr": 16000, + "audio_format": "pcm", + "instruct_text": "", + } + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + try: + async with aiohttp.ClientSession() as session: + async with session.get( + self.api_url, params=params, headers=headers, timeout=10 + ) as response: + if response.status != 200: + logger.error( + f"TTS请求失败: {response.status}, {await response.text()}" + ) + # 推送空LAST，防止播放端卡死 + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + return + + logger.info( + f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒" + ) + + # 流式处理音频数据 + async for chunk in response.content.iter_chunks(): + if chunk[0]: # 确保数据不为空 + opus_data = self.opus_encoder.encode_pcm_to_opus( + chunk[0], end_of_stream=True + ) + if opus_data: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_data, text) + ) + + except Exception as e: + logger.error(f"TTS请求异常: {str(e)}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) From 01eb416a034db4db541b1bce8bef16a5d615d0db Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 16:02:16 +0800 Subject: [PATCH 7/7] =?UTF-8?q?update:=E6=99=BA=E6=8E=A7=E5=8F=B0=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=85=8D=E8=B4=B9=E6=B5=81=E5=BC=8FTTS(linkerai)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202506051538.sql | 20 ++++ .../db/changelog/db.changelog-master.yaml | 9 +- main/xiaozhi-server/config.yaml | 13 +- .../xiaozhi-server/core/providers/tts/base.py | 9 ++ .../providers/tts/huoshan_double_stream.py | 10 +- .../core/providers/tts/linkerai.py | 112 ++++++++++++------ 6 files changed, 124 insertions(+), 49 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202506051538.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202506051538.sql b/main/manager-api/src/main/resources/db/changelog/202506051538.sql new file mode 100644 index 00000000..cfe65da7 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202506051538.sql @@ -0,0 +1,20 @@ +-- 增加LinkeraiTTS供应器和模型配置 +delete from `ai_model_provider` where id = 'SYSTEM_TTS_LinkeraiTTS'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_TTS_LinkeraiTTS', 'TTS', 'linkerai', 'Linkerai语音合成', '[{"key":"api_url","label":"API地址","type":"string"},{"key":"audio_format","label":"音频格式","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"voice","label":"默认音色","type":"string"}]', 14, 1, NOW(), 1, NOW()); + +delete from `ai_model_config` where id = 'TTS_LinkeraiTTS'; +INSERT INTO `ai_model_config` VALUES ('TTS_LinkeraiTTS', 'TTS', 'LinkeraiTTS', 'Linkerai语音合成', 0, 1, '{\"type\": \"linkerai\", \"api_url\": \"https://tts.linkerai.cn/tts\", \"audio_format\": \"pcm\", \"access_token\": \"U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL\", \"voice\": \"OUeAo1mhq6IBExi\"}', NULL, NULL, 17, NULL, NULL, NULL, NULL); + +-- LinkeraiTTS模型配置说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://tts.linkerai.cn/docs', +`remark` = 'Linkerai语音合成服务配置说明： +1. 访问 https://linkerai.cn 注册并获取访问令牌 +2. 默认的access_token供测试使用，请勿用于商业用途 +3. 支持声音克隆功能，可自行上传音频，填入voice参数 +4. 如果voice参数为空，将使用默认声音' WHERE `id` = 'TTS_LinkeraiTTS'; + + +delete from `ai_tts_voice` where tts_model_id = 'TTS_LinkeraiTTS'; +INSERT INTO `ai_tts_voice` VALUES ('TTS_LinkeraiTTS_0001', 'TTS_LinkeraiTTS', '芷若', 'OUeAo1mhq6IBExi', '中文', NULL, NULL, 1, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 0583fce9..cb3007aa 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -190,4 +190,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506032232.sql \ No newline at end of file + path: classpath:db/changelog/202506032232.sql + - changeSet: + id: 202506051538 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202506051538.sql \ No newline at end of file diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a7ff54f3..7721b132 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -324,7 +324,7 @@ VAD: type: silero threshold: 0.5 model_dir: models/snakers4_silero-vad - min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些 + min_silence_duration_ms: 200 # 如果说话停顿比较长，可以把这个值设置大一些 LLM: # 所有openai类型均可以修改超参，以AliLLM为例 @@ -758,11 +758,12 @@ TTS: output_dir: tmp/ LinkeraiTTS: type: linkerai - api_url: https://tts.linkerai.top/tts - audio_format: "opus" + api_url: https://tts.linkerai.cn/tts + audio_format: "pcm" # 默认的access_token供大家测试时免费使用的，此access_token请勿用于商业用途 - # 如果效果不错，可自行申请token，申请地址：https://linkerai.top - # 各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get + # 如果效果不错，可自行申请token，申请地址：https://linkerai.cn + # 各参数意义见开发文档：https://tts.linkerai.cn/docs + # 支持声音克隆，可自行上传音频，填入voice参数，voice参数为空时，使用默认声音 access_token: "U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL" - voice: "7OEPouTL46bS2Qe" + voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index e82a8725..c467e99a 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -37,6 +37,7 @@ class TTSProviderBase(ABC): self.tts_text_queue = queue.Queue() self.tts_audio_queue = queue.Queue() self.tts_audio_first_sentence = True + self.before_stop_play_files = [] self.tts_text_buff = [] self.punctuations = ( @@ -324,6 +325,14 @@ class TTSProviderBase(ABC): os.remove(tts_file) return audio_datas + def _process_before_stop_play_files(self): + for tts_file, text in self.before_stop_play_files: + if tts_file and os.path.exists(tts_file): + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) + self.before_stop_play_files.clear() + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + def _process_remaining_text(self): """处理剩余的文本并生成语音 diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index b9dc399b..b248f5ce 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -154,7 +154,6 @@ class TTSProvider(TTSProviderBase): self.header = {"Authorization": f"{self.authorization}{self.access_token}"} self.enable_two_way = True self.tts_text = "" - self.before_stop_play_files = [] self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 ) @@ -432,14 +431,7 @@ class TTSProvider(TTSProviderBase): is_first_sentence = False elif res.optional.event == EVENT_SessionFinished: logger.bind(tag=TAG).debug(f"会话结束～～") - for tts_file, text in self.before_stop_play_files: - if tts_file and os.path.exists(tts_file): - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, text) - ) - self.before_stop_play_files.clear() - self.tts_audio_queue.put((SentenceType.LAST, [], None)) + self._process_before_stop_play_files() break except websockets.ConnectionClosed: logger.bind(tag=TAG).warning("WebSocket连接已关闭") diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 767d2662..dd8d0e6b 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -1,8 +1,5 @@ -import os -import time import queue import asyncio -import requests import traceback import aiohttp from config.logger import setup_logging @@ -24,6 +21,7 @@ class TTSProvider(TTSProviderBase): self.api_url = config.get("api_url") self.audio_format = "pcm" self.before_stop_play_files = [] + self.segment_count = 0 # 添加片段计数器 # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( @@ -50,7 +48,7 @@ class TTSProvider(TTSProviderBase): self.tts_stop_request = False self.processed_chars = 0 self.tts_text_buff = [] - self.is_first_sentence = True + self.segment_count = 0 self.tts_audio_first_sentence = True self.before_stop_play_files.clear() elif ContentType.TEXT == message.content_type: @@ -60,7 +58,6 @@ class TTSProvider(TTSProviderBase): self.to_tts(segment_text) elif ContentType.FILE == message.content_type: - self._process_remaining_text() logger.bind(tag=TAG).info( f"添加音频文件到待播放列表: {message.content_file}" ) @@ -69,7 +66,8 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.LAST: - self._process_remaining_text() + # 处理剩余的文本 + self._process_remaining_text(True) except queue.Empty: continue @@ -78,7 +76,7 @@ class TTSProvider(TTSProviderBase): f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" ) - def _process_remaining_text(self): + def _process_remaining_text(self, is_last=False): """处理剩余的文本并生成语音 Returns: @@ -89,15 +87,17 @@ class TTSProvider(TTSProviderBase): if remaining_text: segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - self.to_tts(segment_text) + self.to_tts(segment_text, is_last) self.processed_chars += len(full_text) + else: + self._process_before_stop_play_files() - def to_tts(self, text): + def to_tts(self, text, is_last=False): try: max_repeat_time = 5 text = MarkdownCleaner.clean_markdown(text) try: - asyncio.run(self.text_to_speak(text, None)) + asyncio.run(self.text_to_speak(text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" @@ -121,9 +121,9 @@ class TTSProvider(TTSProviderBase): # linkerai单流式TTS重写父类的方法--结束 ################################################################################### - async def text_to_speak(self, text, _): + async def text_to_speak(self, text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" - await self._tts_request(text) + await self._tts_request(text, is_last) async def close(self): """资源清理""" @@ -131,9 +131,7 @@ class TTSProvider(TTSProviderBase): if hasattr(self, "opus_encoder"): self.opus_encoder.close() - async def _tts_request(self, text: str) -> None: - """发送TTS请求""" - start_time = time.time() + async def _tts_request(self, text: str, is_last: bool) -> None: params = { "tts_text": text, "spk_id": self.voice, @@ -141,41 +139,89 @@ class TTSProvider(TTSProviderBase): "stream": "true", "target_sr": 16000, "audio_format": "pcm", - "instruct_text": "", + "instruct_text": "请生成一段自然流畅的语音", } headers = { "Authorization": f"Bearer {self.access_token}", "Content-Type": "application/json", } + # 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels # 1 + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) # 16-bit = 2 bytes + try: async with aiohttp.ClientSession() as session: async with session.get( self.api_url, params=params, headers=headers, timeout=10 - ) as response: - if response.status != 200: - logger.error( - f"TTS请求失败: {response.status}, {await response.text()}" - ) - # 推送空LAST，防止播放端卡死 + ) as resp: + + if resp.status != 200: + logger.error(f"TTS请求失败: {resp.status}, {await resp.text()}") self.tts_audio_queue.put((SentenceType.LAST, [], None)) return - logger.info( - f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒" - ) + self.pcm_buffer.clear() + opus_datas_cache = [] - # 流式处理音频数据 - async for chunk in response.content.iter_chunks(): - if chunk[0]: # 确保数据不为空 - opus_data = self.opus_encoder.encode_pcm_to_opus( - chunk[0], end_of_stream=True + # 兼容 iter_chunked / iter_chunks / iter_any + async for chunk in resp.content.iter_any(): + data = chunk[0] if isinstance(chunk, (list, tuple)) else chunk + if not data: + continue + + # 拼到 buffer + self.pcm_buffer.extend(data) + + # 够一帧就编码 + while len(self.pcm_buffer) >= frame_bytes: + frame = bytes(self.pcm_buffer[:frame_bytes]) + del self.pcm_buffer[:frame_bytes] + + opus = self.opus_encoder.encode_pcm_to_opus( + frame, end_of_stream=False ) - if opus_data: + if opus: + if self.segment_count < 10: # 前10个片段直接发送 + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus, text) + ) + self.segment_count += 1 + else: + opus_datas_cache.extend(opus) + + # flush 剩余不足一帧的数据 + if self.pcm_buffer: + opus = self.opus_encoder.encode_pcm_to_opus( + bytes(self.pcm_buffer), end_of_stream=True + ) + if opus: + if self.segment_count < 10: # 前10个片段直接发送 + # 直接发送 self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_data, text) + (SentenceType.MIDDLE, opus, text) ) + self.segment_count += 1 + else: + # 后续片段缓存 + opus_datas_cache.extend(opus) + self.pcm_buffer.clear() + + # 如果不是前10个片段，发送缓存的数据 + if self.segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, text) + ) + + # 如果是最后一段，输出音频获取完毕 + if is_last: + self._process_before_stop_play_files() except Exception as e: - logger.error(f"TTS请求异常: {str(e)}") + logger.error(f"TTS请求异常: {e}") self.tts_audio_queue.put((SentenceType.LAST, [], None))