Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

TTS处理增加index信息，完善TTS出错时的处理 (#177)

Browse Source 
* TTS处理增加index信息，完善TTS出错时的处理

* 第一句语音index处理，歌曲播放index处理

* update:去掉了无用变量，修复continuebug

---------

Co-authored-by: hrz <1710360675@qq.com>
This commit is contained in:
HonestQiao
2025-03-04 00:35:51 +08:00
committed by GitHub
co-authored by hrz
parent 8dd045d9fd
commit bee0663040
4 changed files with 53 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -146,3 +146,4 @@ tmp
.private_config.yaml .private_config.yaml
.env.development .env.development
docker-compose.yml docker-compose.yml
web/vue/node_modules
core/connection.py
+46 -41
View File
@@ -22,6 +22,10 @@ from core.utils.auth_code_gen import AuthCodeGenerator
TAG = __name__ TAG = __name__
class TTSException(RuntimeError):
pass
class ConnectionHandler: class ConnectionHandler:
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music): def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music):
self.config = config self.config = config
@@ -68,10 +72,8 @@ class ConnectionHandler:
self.dialogue = Dialogue() self.dialogue = Dialogue()
# tts相关变量 # tts相关变量
self.tts_first_text = None self.tts_first_text_index = -1
self.tts_last_text = None self.tts_last_text_index = -1
self.tts_start_speak_time = None
self.tts_duration = 0
# iot相关变量 # iot相关变量
self.iot_descriptors = {} self.iot_descriptors = {}
@@ -221,6 +223,7 @@ class ConnectionHandler:
return None return None
self.llm_finish_task = False self.llm_finish_task = False
text_index = 0
for content in llm_responses: for content in llm_responses:
response_message.append(content) response_message.append(content)
if self.client_abort: if self.client_abort:
@@ -234,7 +237,7 @@ class ConnectionHandler:
current_text = full_text[processed_chars:] # 从未处理的位置开始 current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点 # 查找最后一个有效标点
punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：", "，") punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
last_punct_pos = -1 last_punct_pos = -1
for punct in punctuations: for punct in punctuations:
pos = current_text.rfind(punct) pos = current_text.rfind(punct)
@@ -246,8 +249,12 @@ class ConnectionHandler:
segment_text_raw = current_text[:last_punct_pos + 1] segment_text_raw = current_text[:last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
if segment_text: if segment_text:
self.recode_first_last_text(segment_text) # 强制设置空字符，测试TTS出错返回语音的健壮性
future = self.executor.submit(self.speak_and_play, segment_text) # if text_index % 2 == 0:
# segment_text = " "
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future) self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置 processed_chars += len(segment_text_raw) # 更新已处理字符位置
@@ -257,8 +264,9 @@ class ConnectionHandler:
if remaining_text: if remaining_text:
segment_text = get_string_no_punctuation_or_emoji(remaining_text) segment_text = get_string_no_punctuation_or_emoji(remaining_text)
if segment_text: if segment_text:
self.recode_first_last_text(segment_text) text_index += 1
future = self.executor.submit(self.speak_and_play, segment_text) self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future) self.tts_queue.put(future)
self.llm_finish_task = True self.llm_finish_task = True
@@ -274,30 +282,28 @@ class ConnectionHandler:
if future is None: if future is None:
continue continue
text = None text = None
opus_datas, text_index, tts_file = [], 0, None
try: try:
self.logger.bind(tag=TAG).debug("正在处理TTS任务...") self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
tts_file, text = future.result(timeout=10) tts_file, text, text_index = future.result(timeout=10)
if text is None or len(text) <= 0: if text is None or len(text) <= 0:
continue self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
if tts_file is None: elif tts_file is None:
self.logger.bind(tag=TAG).error(f"TTS文件生成失败: {text}") self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
continue
self.logger.bind(tag=TAG).debug(f"TTS文件生成完毕，文件路径: {tts_file}")
if os.path.exists(tts_file):
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
else: else:
self.logger.bind(tag=TAG).error(f"TTS文件不存在: {tts_file}") self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
opus_datas = [] if os.path.exists(tts_file):
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
else:
self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
except TimeoutError: except TimeoutError:
self.logger.bind(tag=TAG).error("TTS 任务超时") self.logger.bind(tag=TAG).error("TTS超时")
continue
except Exception as e: except Exception as e:
self.logger.bind(tag=TAG).error(f"TTS 任务出错: {e}") self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
continue
if not self.client_abort: if not self.client_abort:
# 如果没有中途打断就发送语音 # 如果没有中途打断就发送语音
self.audio_play_queue.put((opus_datas, text)) self.audio_play_queue.put((opus_datas, text, text_index))
if self.tts.delete_audio_file and os.path.exists(tts_file): if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
os.remove(tts_file) os.remove(tts_file)
except Exception as e: except Exception as e:
self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}") self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
@@ -306,42 +312,41 @@ class ConnectionHandler:
self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})), self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
self.loop self.loop
) )
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text}{e}") self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
def _audio_play_priority_thread(self): def _audio_play_priority_thread(self):
while not self.stop_event.is_set(): while not self.stop_event.is_set():
text = None text = None
try: try:
opus_datas, text = self.audio_play_queue.get() opus_datas, text, text_index = self.audio_play_queue.get()
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text), self.loop) future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
self.loop)
future.result() future.result()
except Exception as e: except Exception as e:
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text}{e}") self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
def speak_and_play(self, text): def speak_and_play(self, text, text_index=0):
if text is None or len(text) <= 0: if text is None or len(text) <= 0:
self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
return None, text return None, text, text_index
tts_file = self.tts.to_tts(text) tts_file = self.tts.to_tts(text)
if tts_file is None: if tts_file is None:
self.logger.bind(tag=TAG).error(f"tts转换失败，{text}") self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
return None, text return None, text, text_index
self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}") self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
return tts_file, text return tts_file, text, text_index
def clearSpeakStatus(self): def clearSpeakStatus(self):
self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态") self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
self.asr_server_receive = True self.asr_server_receive = True
self.tts_last_text = None self.tts_last_text_index = -1
self.tts_first_text = None self.tts_first_text_index = -1
self.tts_duration = 0
self.tts_start_speak_time = None
def recode_first_last_text(self, text): def recode_first_last_text(self, text, text_index=0):
if not self.tts_first_text: if self.tts_first_text_index == -1:
self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}") self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
self.tts_first_text = text self.tts_first_text_index = text_index
self.tts_last_text = text self.tts_last_text_index = text_index
async def close(self): async def close(self):
"""资源清理方法""" """资源清理方法"""
core/handle/musicHandler.py
+3 -3
View File
@@ -139,14 +139,14 @@ class MusicHandler:
return return
text = f"正在播放{selected_music}" text = f"正在播放{selected_music}"
await send_stt_message(conn, text) await send_stt_message(conn, text)
conn.tts_first_text = selected_music conn.tts_first_text_index = 0
conn.tts_last_text = selected_music conn.tts_last_text_index = 0
conn.llm_finish_task = True conn.llm_finish_task = True
if music_path.endswith(".p3"): if music_path.endswith(".p3"):
opus_packets, duration = p3.decode_opus_from_file(music_path) opus_packets, duration = p3.decode_opus_from_file(music_path)
else: else:
opus_packets, duration = conn.tts.wav_to_opus_data(music_path) opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
conn.audio_play_queue.put((opus_packets, selected_music)) conn.audio_play_queue.put((opus_packets, selected_music, 0))
except Exception as e: except Exception as e:
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
core/handle/sendAudioHandle.py
+3 -4
View File
@@ -15,11 +15,10 @@ async def isLLMWantToFinish(last_text):
return False return False
async def sendAudioMessage(conn, audios, text): async def sendAudioMessage(conn, audios, text, text_index=0):
# 发送句子开始消息 # 发送句子开始消息
if text == conn.tts_first_text: if text_index == conn.tts_first_text_index:
logger.bind(tag=TAG).info(f"发送第一段语音: {text}") logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
conn.tts_start_speak_time = time.perf_counter()
await send_tts_message(conn, "sentence_start", text) await send_tts_message(conn, "sentence_start", text)
# 初始化流控参数 # 初始化流控参数
@@ -45,7 +44,7 @@ async def sendAudioMessage(conn, audios, text):
play_position += frame_duration # 更新播放位置 play_position += frame_duration # 更新播放位置
await send_tts_message(conn, "sentence_end", text) await send_tts_message(conn, "sentence_end", text)
# 发送结束消息（如果是最后一个文本） # 发送结束消息（如果是最后一个文本）
if conn.llm_finish_task and text == conn.tts_last_text: if conn.llm_finish_task and text_index == conn.tts_last_text_index:
await send_tts_message(conn, 'stop', None) await send_tts_message(conn, 'stop', None)
if await isLLMWantToFinish(text): if await isLLMWantToFinish(text):
await conn.close() await conn.close()