Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-28 10:03:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

update:流式优化暂不稳定，先回滚到早期代码

Browse Source
This commit is contained in:
hrz
2025-08-30 22:59:21 +08:00
parent b4f4995ff9
commit eaade698fb
19 changed files with 1116 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/tts/linkerai.py
+123 -12
View File
@@ -1,8 +1,10 @@
import os
import queue
import aiohttp
import asyncio
import traceback
import aiohttp
import requests
import time
from config.logger import setup_logging
from core.utils.tts import MarkdownCleaner
from core.providers.tts.base import TTSProviderBase
@@ -22,15 +24,23 @@ class TTSProvider(TTSProviderBase):
self.api_url = config.get("api_url")
self.audio_format = "pcm"
self.before_stop_play_files = []
self.segment_count = 0 # 添加片段计数器
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# 添加文本缓冲区
self.text_buffer = ""
# PCM缓冲区
self.pcm_buffer = bytearray()
###################################################################################
# linkerai单流式TTS重写父类的方法--开始
###################################################################################
def tts_text_priority_thread(self):
"""流式文本处理线程"""
while not self.conn.stop_event.is_set():
@@ -41,6 +51,7 @@ class TTSProvider(TTSProviderBase):
self.tts_stop_request = False
self.processed_chars = 0
self.tts_text_buff = []
self.segment_count = 0
self.before_stop_play_files.clear()
elif ContentType.TEXT == message.content_type:
self.tts_text_buff.append(message.content_detail)
@@ -54,10 +65,14 @@ class TTSProvider(TTSProviderBase):
)
if message.content_file and os.path.exists(message.content_file):
# 先处理文件音频数据
self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
file_audio = self._process_audio_file(message.content_file)
self.before_stop_play_files.append(
(file_audio, message.content_detail)
)
if message.sentence_type == SentenceType.LAST:
# 处理剩余的文本
self._process_remaining_text_stream(True)
self._process_remaining_text(True)
except queue.Empty:
continue
@@ -66,7 +81,7 @@ class TTSProvider(TTSProviderBase):
f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
)
def _process_remaining_text_stream(self, is_last=False):
def _process_remaining_text(self, is_last=False):
"""处理剩余的文本并生成语音
Returns:
@@ -109,6 +124,10 @@ class TTSProvider(TTSProviderBase):
finally:
return None
###################################################################################
# linkerai单流式TTS重写父类的方法--结束
###################################################################################
async def text_to_speak(self, text, is_last):
"""流式处理TTS音频，每句只推送一次音频列表"""
await self._tts_request(text, is_last)
@@ -157,6 +176,8 @@ class TTSProvider(TTSProviderBase):
return
self.pcm_buffer.clear()
opus_datas_cache = []
self.tts_audio_queue.put((SentenceType.FIRST, [], text))
# 兼容 iter_chunked / iter_chunks / iter_any
@@ -173,21 +194,41 @@ class TTSProvider(TTSProviderBase):
frame = bytes(self.pcm_buffer[:frame_bytes])
del self.pcm_buffer[:frame_bytes]
self.opus_encoder.encode_pcm_to_opus_stream(
frame,
end_of_stream=False,
callback=self.handle_opus
opus = self.opus_encoder.encode_pcm_to_opus(
frame, end_of_stream=False
)
if opus:
if self.segment_count < 10: # 前10个片段直接发送
self.tts_audio_queue.put(
(SentenceType.MIDDLE, opus, None)
)
self.segment_count += 1
else:
opus_datas_cache.extend(opus)
# flush 剩余不足一帧的数据
if self.pcm_buffer:
self.opus_encoder.encode_pcm_to_opus_stream(
bytes(self.pcm_buffer),
end_of_stream=True,
callback=self.handle_opus
opus = self.opus_encoder.encode_pcm_to_opus(
bytes(self.pcm_buffer), end_of_stream=True
)
if opus:
if self.segment_count < 10: # 前10个片段直接发送
# 直接发送
self.tts_audio_queue.put(
(SentenceType.MIDDLE, opus, None)
)
self.segment_count += 1
else:
# 后续片段缓存
opus_datas_cache.extend(opus)
self.pcm_buffer.clear()
# 如果不是前10个片段，发送缓存的数据
if self.segment_count >= 10 and opus_datas_cache:
self.tts_audio_queue.put(
(SentenceType.MIDDLE, opus_datas_cache, None)
)
# 如果是最后一段，输出音频获取完毕
if is_last:
self._process_before_stop_play_files()
@@ -195,3 +236,73 @@ class TTSProvider(TTSProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def to_tts(self, text: str) -> list:
"""非流式TTS处理，用于测试及保存音频文件的场景
Args:
text: 要转换的文本
Returns:
list: 返回opus编码后的音频数据列表
"""
start_time = time.time()
text = MarkdownCleaner.clean_markdown(text)
params = {
"tts_text": text,
"spk_id": self.voice,
"frame_duration": 60,
"stream": False,
"target_sr": 16000,
"audio_format": self.audio_format,
"instruct_text": "请生成一段自然流畅的语音",
}
headers = {
"Authorization": f"Bearer {self.access_token}",
"Content-Type": "application/json",
}
try:
with requests.get(
self.api_url, params=params, headers=headers, timeout=5
) as response:
if response.status_code != 200:
logger.bind(tag=TAG).error(
f"TTS请求失败: {response.status_code}, {response.text}"
)
return []
logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
# 使用opus编码器处理PCM数据
opus_datas = []
pcm_data = response.content
# 计算每帧的字节数
frame_bytes = int(
self.opus_encoder.sample_rate
* self.opus_encoder.channels
* self.opus_encoder.frame_size_ms
/ 1000
* 2
)
# 分帧处理PCM数据
for i in range(0, len(pcm_data), frame_bytes):
frame = pcm_data[i : i + frame_bytes]
if len(frame) < frame_bytes:
# 最后一帧可能不足，用0填充
frame = frame + b"\x00" * (frame_bytes - len(frame))
opus = self.opus_encoder.encode_pcm_to_opus(
frame, end_of_stream=(i + frame_bytes >= len(pcm_data))
)
if opus:
opus_datas.extend(opus)
return opus_datas
except Exception as e:
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
return []