Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-27 09:33:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

update:流式优化暂不稳定，先回滚到早期代码

Browse Source
This commit is contained in:
hrz
2025-08-30 22:59:21 +08:00
parent b4f4995ff9
commit eaade698fb
19 changed files with 1116 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/tts/base.py
+61 -79
View File
@@ -4,14 +4,12 @@ import queue
import uuid
import asyncio
import threading
from typing import Callable, Any
from core.utils import p3
import time
from datetime import datetime
from core.utils import textUtils
from abc import ABC, abstractmethod
from config.logger import setup_logging
from core.utils.util import audio_bytes_to_data_stream, audio_to_data_stream
from core.utils.util import audio_to_data, audio_bytes_to_data
from core.utils.tts import MarkdownCleaner
from core.utils.output_counter import add_device_output
from core.handle.reportHandle import enqueue_tts_report
@@ -33,6 +31,7 @@ class TTSProviderBase(ABC):
def __init__(self, config, delete_audio_file):
self.interface_type = InterfaceType.NON_STREAM
self.conn = None
self.tts_timeout = 10
self.delete_audio_file = delete_audio_file
self.audio_file_type = "wav"
self.output_file = config.get("output_dir", "tmp/")
@@ -51,9 +50,11 @@ class TTSProviderBase(ABC):
"；",
";",
"：",
"~",
)
self.first_sentence_punctuations = (
"，",
"～",
"~",
"、",
",",
@@ -76,14 +77,7 @@ class TTSProviderBase(ABC):
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
)
def handle_opus(self, opus_data: bytes):
logger.bind(tag=TAG).debug(f"推送数据到队列里面帧数～～ {len(opus_data)}")
self.tts_audio_queue.put((SentenceType.MIDDLE, opus_data, None))
def handle_audio_file(self, file_audio: bytes, text):
self.before_stop_play_files.append((file_audio, text))
def to_tts_stream(self, text, opus_handler: Callable[[bytes], None] = None) -> None:
def to_tts(self, text):
text = MarkdownCleaner.clean_markdown(text)
max_repeat_time = 5
if self.delete_audio_file:
@@ -92,14 +86,10 @@ class TTSProviderBase(ABC):
try:
audio_bytes = asyncio.run(self.text_to_speak(text, None))
if audio_bytes:
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
audio_bytes_to_data_stream(
audio_bytes,
file_type=self.audio_file_type,
is_opus=True,
callback=opus_handler,
audio_datas, _ = audio_bytes_to_data(
audio_bytes, file_type=self.audio_file_type, is_opus=True
)
break
return audio_datas
else:
max_repeat_time -= 1
except Exception as e:
@@ -139,8 +129,8 @@ class TTSProviderBase(ABC):
logger.bind(tag=TAG).error(
f"语音生成失败: {text}，请检查网络或服务是否正常"
)
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
self._process_audio_file_stream(tmp_file, callback=opus_handler)
return tmp_file
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
return None
@@ -149,17 +139,13 @@ class TTSProviderBase(ABC):
async def text_to_speak(self, text, output_file):
pass
def audio_to_pcm_data_stream(
self, audio_file_path, callback: Callable[[Any], Any] = None
):
def audio_to_pcm_data(self, audio_file_path):
"""音频文件转换为PCM编码"""
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
return audio_to_data(audio_file_path, is_opus=False)
def audio_to_opus_data_stream(
self, audio_file_path, callback: Callable[[Any], Any] = None
):
def audio_to_opus_data(self, audio_file_path):
"""音频文件转换为Opus编码"""
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
return audio_to_data(audio_file_path, is_opus=True)
def tts_one_sentence(
self,
@@ -191,6 +177,7 @@ class TTSProviderBase(ABC):
async def open_audio_channels(self, conn):
self.conn = conn
self.tts_timeout = conn.config.get("tts_timeout", 10)
# tts 消化线程
self.tts_priority_thread = threading.Thread(
target=self.tts_text_priority_thread, daemon=True
@@ -225,16 +212,30 @@ class TTSProviderBase(ABC):
self.tts_text_buff.append(message.content_detail)
segment_text = self._get_segment_text()
if segment_text:
self.to_tts_stream(segment_text, opus_handler=self.handle_opus)
if self.delete_audio_file:
audio_datas = self.to_tts(segment_text)
if audio_datas:
self.tts_audio_queue.put(
(message.sentence_type, audio_datas, segment_text)
)
else:
tts_file = self.to_tts(segment_text)
if tts_file:
audio_datas = self._process_audio_file(tts_file)
self.tts_audio_queue.put(
(message.sentence_type, audio_datas, segment_text)
)
elif ContentType.FILE == message.content_type:
self._process_remaining_text_stream(opus_handler=self.handle_opus)
self._process_remaining_text()
tts_file = message.content_file
if tts_file and os.path.exists(tts_file):
self._process_audio_file_stream(
tts_file, callback=self.handle_opus
audio_datas = self._process_audio_file(tts_file)
self.tts_audio_queue.put(
(message.sentence_type, audio_datas, message.content_detail)
)
if message.sentence_type == SentenceType.LAST:
self._process_remaining_text_stream(opus_handler=self.handle_opus)
self._process_remaining_text()
self.tts_audio_queue.put(
(message.sentence_type, [], message.content_detail)
)
@@ -248,59 +249,29 @@ class TTSProviderBase(ABC):
continue
def _audio_play_priority_thread(self):
# 需要上报的文本和音频列表
enqueue_text = None
enqueue_audio = None
while not self.conn.stop_event.is_set():
text = None
try:
try:
sentence_type, audio_datas, text = self.tts_audio_queue.get(
timeout=0.1
timeout=1
)
except queue.Empty:
if self.conn.stop_event.is_set():
break
continue
if self.conn.client_abort:
logger.bind(tag=TAG).debug("收到打断信号，跳过当前音频数据")
enqueue_text, enqueue_audio = None, []
continue
# 收到下一个文本开始或会话结束时进行上报
if sentence_type is not SentenceType.MIDDLE:
# 重置音频流控状态（新句子开始或者结束）
if hasattr(self.conn, 'audio_flow_control'):
self.conn.audio_flow_control = {
'last_send_time': 0,
'packet_count': 0,
'start_time': time.perf_counter()
}
# 上报TTS数据
if enqueue_text is not None and enqueue_audio is not None:
enqueue_tts_report(self.conn, enqueue_text, enqueue_audio)
enqueue_audio = []
enqueue_text = text
# 收集上报音频数据
if isinstance(audio_datas, bytes) and enqueue_audio is not None:
enqueue_audio.append(audio_datas)
# 发送音频
future = asyncio.run_coroutine_threadsafe(
sendAudioMessage(self.conn, sentence_type, audio_datas, text),
self.conn.loop,
)
future.result()
# 记录输出和报告
if self.conn.max_output_size > 0 and text:
add_device_output(self.conn.headers.get("device-id"), len(text))
enqueue_tts_report(self.conn, text, audio_datas)
except Exception as e:
logger.bind(tag=TAG).error(f"audio_play_priority_thread: {text} {e}")
logger.bind(tag=TAG).error(
f"audio_play_priority priority_thread: {text} {e}"
)
async def start_session(self, session_id):
pass
@@ -352,21 +323,22 @@ class TTSProviderBase(ABC):
else:
return None
def _process_audio_file_stream(
self, tts_file, callback: Callable[[Any], Any]
) -> None:
def _process_audio_file(self, tts_file):
"""处理音频文件并转换为指定格式
Args:
tts_file: 音频文件路径
callback: 文件处理函数
content_detail: 内容详情
Returns:
tuple: (sentence_type, audio_datas, content_detail)
"""
if tts_file.endswith(".p3"):
p3.decode_opus_from_file_stream(tts_file, callback=callback)
audio_datas, _ = p3.decode_opus_from_file(tts_file)
elif self.conn.audio_format == "pcm":
self.audio_to_pcm_data_stream(tts_file, callback=callback)
audio_datas, _ = self.audio_to_pcm_data(tts_file)
else:
self.audio_to_opus_data_stream(tts_file, callback=callback)
audio_datas, _ = self.audio_to_opus_data(tts_file)
if (
self.delete_audio_file
@@ -375,6 +347,7 @@ class TTSProviderBase(ABC):
and tts_file.startswith(self.output_file)
):
os.remove(tts_file)
return audio_datas
def _process_before_stop_play_files(self):
for audio_datas, text in self.before_stop_play_files:
@@ -382,9 +355,7 @@ class TTSProviderBase(ABC):
self.before_stop_play_files.clear()
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def _process_remaining_text_stream(
self, opus_handler: Callable[[bytes], None] = None
):
def _process_remaining_text(self):
"""处理剩余的文本并生成语音
Returns:
@@ -395,7 +366,18 @@ class TTSProviderBase(ABC):
if remaining_text:
segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
if segment_text:
self.to_tts_stream(segment_text, opus_handler=opus_handler)
if self.delete_audio_file:
audio_datas = self.to_tts(segment_text)
if audio_datas:
self.tts_audio_queue.put(
(SentenceType.MIDDLE, audio_datas, segment_text)
)
else:
tts_file = self.to_tts(segment_text)
audio_datas = self._process_audio_file(tts_file)
self.tts_audio_queue.put(
(SentenceType.MIDDLE, audio_datas, segment_text)
)
self.processed_chars += len(full_text)
return True
return False