mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-23 23:53:55 +08:00
310 lines
12 KiB
Python
310 lines
12 KiB
Python
import os
|
|
import queue
|
|
import asyncio
|
|
import traceback
|
|
import aiohttp
|
|
import requests
|
|
import time
|
|
from config.logger import setup_logging
|
|
from core.utils.tts import MarkdownCleaner
|
|
from core.providers.tts.base import TTSProviderBase
|
|
from core.utils import opus_encoder_utils, textUtils
|
|
from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
|
|
|
|
TAG = __name__
|
|
logger = setup_logging()
|
|
|
|
|
|
class TTSProvider(TTSProviderBase):
|
|
def __init__(self, config, delete_audio_file):
|
|
super().__init__(config, delete_audio_file)
|
|
self.interface_type = InterfaceType.SINGLE_STREAM
|
|
self.access_token = config.get("access_token")
|
|
self.voice = config.get("voice")
|
|
self.api_url = config.get("api_url")
|
|
self.audio_format = "pcm"
|
|
self.before_stop_play_files = []
|
|
self.segment_count = 0 # 添加片段计数器
|
|
|
|
# 创建Opus编码器
|
|
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
|
sample_rate=16000, channels=1, frame_size_ms=60
|
|
)
|
|
|
|
# 添加文本缓冲区
|
|
self.text_buffer = ""
|
|
|
|
# PCM缓冲区
|
|
self.pcm_buffer = bytearray()
|
|
|
|
###################################################################################
|
|
# linkerai单流式TTS重写父类的方法--开始
|
|
###################################################################################
|
|
|
|
def tts_text_priority_thread(self):
|
|
"""流式文本处理线程"""
|
|
while not self.conn.stop_event.is_set():
|
|
try:
|
|
message = self.tts_text_queue.get(timeout=1)
|
|
if message.sentence_type == SentenceType.FIRST:
|
|
# 初始化参数
|
|
self.tts_stop_request = False
|
|
self.processed_chars = 0
|
|
self.tts_text_buff = []
|
|
self.segment_count = 0
|
|
self.tts_audio_first_sentence = True
|
|
self.before_stop_play_files.clear()
|
|
elif ContentType.TEXT == message.content_type:
|
|
self.tts_text_buff.append(message.content_detail)
|
|
segment_text = self._get_segment_text()
|
|
if segment_text:
|
|
self.to_tts_single_stream(segment_text)
|
|
|
|
elif ContentType.FILE == message.content_type:
|
|
logger.bind(tag=TAG).info(
|
|
f"添加音频文件到待播放列表: {message.content_file}"
|
|
)
|
|
if message.content_file and os.path.exists(message.content_file):
|
|
# 先处理文件音频数据
|
|
file_audio = self._process_audio_file(message.content_file)
|
|
self.before_stop_play_files.append(
|
|
(file_audio, message.content_detail)
|
|
)
|
|
|
|
if message.sentence_type == SentenceType.LAST:
|
|
# 处理剩余的文本
|
|
self._process_remaining_text(True)
|
|
|
|
except queue.Empty:
|
|
continue
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(
|
|
f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
|
|
)
|
|
|
|
def _process_remaining_text(self, is_last=False):
|
|
"""处理剩余的文本并生成语音
|
|
|
|
Returns:
|
|
bool: 是否成功处理了文本
|
|
"""
|
|
full_text = "".join(self.tts_text_buff)
|
|
remaining_text = full_text[self.processed_chars :]
|
|
if remaining_text:
|
|
segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
|
|
if segment_text:
|
|
self.to_tts_single_stream(segment_text, is_last)
|
|
self.processed_chars += len(full_text)
|
|
else:
|
|
self._process_before_stop_play_files()
|
|
else:
|
|
self._process_before_stop_play_files()
|
|
|
|
def to_tts_single_stream(self, text, is_last=False):
|
|
try:
|
|
max_repeat_time = 5
|
|
text = MarkdownCleaner.clean_markdown(text)
|
|
try:
|
|
asyncio.run(self.text_to_speak(text, is_last))
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).warning(
|
|
f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
|
|
)
|
|
max_repeat_time -= 1
|
|
|
|
if max_repeat_time > 0:
|
|
logger.bind(tag=TAG).info(
|
|
f"语音生成成功: {text}，重试{5 - max_repeat_time}次"
|
|
)
|
|
else:
|
|
logger.bind(tag=TAG).error(
|
|
f"语音生成失败: {text}，请检查网络或服务是否正常"
|
|
)
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
|
|
finally:
|
|
return None
|
|
|
|
###################################################################################
|
|
# linkerai单流式TTS重写父类的方法--结束
|
|
###################################################################################
|
|
|
|
async def text_to_speak(self, text, is_last):
|
|
"""流式处理TTS音频，每句只推送一次音频列表"""
|
|
await self._tts_request(text, is_last)
|
|
|
|
async def close(self):
|
|
"""资源清理"""
|
|
await super().close()
|
|
if hasattr(self, "opus_encoder"):
|
|
self.opus_encoder.close()
|
|
|
|
async def _tts_request(self, text: str, is_last: bool) -> None:
|
|
params = {
|
|
"tts_text": text,
|
|
"spk_id": self.voice,
|
|
"frame_durition": 60,
|
|
"stream": "true",
|
|
"target_sr": 16000,
|
|
"audio_format": "pcm",
|
|
"instruct_text": "请生成一段自然流畅的语音",
|
|
}
|
|
headers = {
|
|
"Authorization": f"Bearer {self.access_token}",
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
# 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920
|
|
frame_bytes = int(
|
|
self.opus_encoder.sample_rate
|
|
* self.opus_encoder.channels # 1
|
|
* self.opus_encoder.frame_size_ms
|
|
/ 1000
|
|
* 2
|
|
) # 16-bit = 2 bytes
|
|
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(
|
|
self.api_url, params=params, headers=headers, timeout=10
|
|
) as resp:
|
|
|
|
if resp.status != 200:
|
|
logger.bind(tag=TAG).error(
|
|
f"TTS请求失败: {resp.status}, {await resp.text()}"
|
|
)
|
|
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
|
return
|
|
|
|
self.pcm_buffer.clear()
|
|
opus_datas_cache = []
|
|
|
|
self.tts_audio_queue.put((SentenceType.FIRST, [], text))
|
|
|
|
# 兼容 iter_chunked / iter_chunks / iter_any
|
|
async for chunk in resp.content.iter_any():
|
|
data = chunk[0] if isinstance(chunk, (list, tuple)) else chunk
|
|
if not data:
|
|
continue
|
|
|
|
# 拼到 buffer
|
|
self.pcm_buffer.extend(data)
|
|
|
|
# 够一帧就编码
|
|
while len(self.pcm_buffer) >= frame_bytes:
|
|
frame = bytes(self.pcm_buffer[:frame_bytes])
|
|
del self.pcm_buffer[:frame_bytes]
|
|
|
|
opus = self.opus_encoder.encode_pcm_to_opus(
|
|
frame, end_of_stream=False
|
|
)
|
|
if opus:
|
|
if self.segment_count < 10: # 前10个片段直接发送
|
|
self.tts_audio_queue.put(
|
|
(SentenceType.MIDDLE, opus, None)
|
|
)
|
|
self.segment_count += 1
|
|
else:
|
|
opus_datas_cache.extend(opus)
|
|
|
|
# flush 剩余不足一帧的数据
|
|
if self.pcm_buffer:
|
|
opus = self.opus_encoder.encode_pcm_to_opus(
|
|
bytes(self.pcm_buffer), end_of_stream=True
|
|
)
|
|
if opus:
|
|
if self.segment_count < 10: # 前10个片段直接发送
|
|
# 直接发送
|
|
self.tts_audio_queue.put(
|
|
(SentenceType.MIDDLE, opus, None)
|
|
)
|
|
self.segment_count += 1
|
|
else:
|
|
# 后续片段缓存
|
|
opus_datas_cache.extend(opus)
|
|
self.pcm_buffer.clear()
|
|
|
|
# 如果不是前10个片段，发送缓存的数据
|
|
if self.segment_count >= 10 and opus_datas_cache:
|
|
self.tts_audio_queue.put(
|
|
(SentenceType.MIDDLE, opus_datas_cache, None)
|
|
)
|
|
|
|
# 如果是最后一段，输出音频获取完毕
|
|
if is_last:
|
|
self._process_before_stop_play_files()
|
|
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
|
|
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
|
|
|
def to_tts(self, text: str) -> list:
|
|
"""非流式TTS处理，用于测试及保存音频文件的场景
|
|
|
|
Args:
|
|
text: 要转换的文本
|
|
|
|
Returns:
|
|
list: 返回opus编码后的音频数据列表
|
|
"""
|
|
start_time = time.time()
|
|
text = MarkdownCleaner.clean_markdown(text)
|
|
|
|
params = {
|
|
"tts_text": text,
|
|
"spk_id": self.voice,
|
|
"frame_duration": 60,
|
|
"stream": False,
|
|
"target_sr": 16000,
|
|
"audio_format": self.audio_format,
|
|
"instruct_text": "请生成一段自然流畅的语音",
|
|
}
|
|
headers = {
|
|
"Authorization": f"Bearer {self.access_token}",
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
try:
|
|
with requests.get(
|
|
self.api_url, params=params, headers=headers, timeout=5
|
|
) as response:
|
|
if response.status_code != 200:
|
|
logger.bind(tag=TAG).error(
|
|
f"TTS请求失败: {response.status_code}, {response.text}"
|
|
)
|
|
return []
|
|
|
|
logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
|
|
|
|
# 使用opus编码器处理PCM数据
|
|
opus_datas = []
|
|
pcm_data = response.content
|
|
|
|
# 计算每帧的字节数
|
|
frame_bytes = int(
|
|
self.opus_encoder.sample_rate
|
|
* self.opus_encoder.channels
|
|
* self.opus_encoder.frame_size_ms
|
|
/ 1000
|
|
* 2
|
|
)
|
|
|
|
# 分帧处理PCM数据
|
|
for i in range(0, len(pcm_data), frame_bytes):
|
|
frame = pcm_data[i : i + frame_bytes]
|
|
if len(frame) < frame_bytes:
|
|
# 最后一帧可能不足，用0填充
|
|
frame = frame + b"\x00" * (frame_bytes - len(frame))
|
|
|
|
opus = self.opus_encoder.encode_pcm_to_opus(
|
|
frame, end_of_stream=(i + frame_bytes >= len(pcm_data))
|
|
)
|
|
if opus:
|
|
opus_datas.extend(opus)
|
|
|
|
return opus_datas
|
|
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
|
|
return []
|