Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-21 22:53:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

TTS处理增加index信息，完善TTS出错时的处理 (#177)

Browse Source 
* TTS处理增加index信息，完善TTS出错时的处理

* 第一句语音index处理，歌曲播放index处理

* update:去掉了无用变量，修复continuebug

---------

Co-authored-by: hrz <1710360675@qq.com>
This commit is contained in:
HonestQiao
2025-03-04 00:35:51 +08:00
committed by GitHub
co-authored by hrz
parent 8dd045d9fd
commit bee0663040
4 changed files with 53 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -146,3 +146,4 @@ tmp
.private_config.yaml
.env.development
docker-compose.yml
web/vue/node_modules
core/connection.py
+46 -41
View File
@@ -22,6 +22,10 @@ from core.utils.auth_code_gen import AuthCodeGenerator
TAG = __name__
class TTSException(RuntimeError):
pass
class ConnectionHandler:
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music):
self.config = config
@@ -68,10 +72,8 @@ class ConnectionHandler:
self.dialogue = Dialogue()
# tts相关变量
self.tts_first_text = None
self.tts_last_text = None
self.tts_start_speak_time = None
self.tts_duration = 0
self.tts_first_text_index = -1
self.tts_last_text_index = -1
# iot相关变量
self.iot_descriptors = {}
@@ -221,6 +223,7 @@ class ConnectionHandler:
return None
self.llm_finish_task = False
text_index = 0
for content in llm_responses:
response_message.append(content)
if self.client_abort:
@@ -234,7 +237,7 @@ class ConnectionHandler:
current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点
punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：", "，")
punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
last_punct_pos = -1
for punct in punctuations:
pos = current_text.rfind(punct)
@@ -246,8 +249,12 @@ class ConnectionHandler:
segment_text_raw = current_text[:last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
if segment_text:
self.recode_first_last_text(segment_text)
future = self.executor.submit(self.speak_and_play, segment_text)
# 强制设置空字符，测试TTS出错返回语音的健壮性
# if text_index % 2 == 0:
# segment_text = " "
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
@@ -257,8 +264,9 @@ class ConnectionHandler:
if remaining_text:
segment_text = get_string_no_punctuation_or_emoji(remaining_text)
if segment_text:
self.recode_first_last_text(segment_text)
future = self.executor.submit(self.speak_and_play, segment_text)
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
self.llm_finish_task = True
@@ -274,30 +282,28 @@ class ConnectionHandler:
if future is None:
continue
text = None
opus_datas, text_index, tts_file = [], 0, None
try:
self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
tts_file, text = future.result(timeout=10)
tts_file, text, text_index = future.result(timeout=10)
if text is None or len(text) <= 0:
continue
if tts_file is None:
self.logger.bind(tag=TAG).error(f"TTS文件生成失败: {text}")
continue
self.logger.bind(tag=TAG).debug(f"TTS文件生成完毕，文件路径: {tts_file}")
if os.path.exists(tts_file):
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
elif tts_file is None:
self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
else:
self.logger.bind(tag=TAG).error(f"TTS文件不存在: {tts_file}")
opus_datas = []
self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
if os.path.exists(tts_file):
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
else:
self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
except TimeoutError:
self.logger.bind(tag=TAG).error("TTS 任务超时")
continue
self.logger.bind(tag=TAG).error("TTS超时")
except Exception as e:
self.logger.bind(tag=TAG).error(f"TTS 任务出错: {e}")
continue
self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
if not self.client_abort:
# 如果没有中途打断就发送语音
self.audio_play_queue.put((opus_datas, text))
if self.tts.delete_audio_file and os.path.exists(tts_file):
self.audio_play_queue.put((opus_datas, text, text_index))
if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
os.remove(tts_file)
except Exception as e:
self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
@@ -306,42 +312,41 @@ class ConnectionHandler:
self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
self.loop
)
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text}{e}")
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
def _audio_play_priority_thread(self):
while not self.stop_event.is_set():
text = None
try:
opus_datas, text = self.audio_play_queue.get()
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text), self.loop)
opus_datas, text, text_index = self.audio_play_queue.get()
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
self.loop)
future.result()
except Exception as e:
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text}{e}")
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
def speak_and_play(self, text):
def speak_and_play(self, text, text_index=0):
if text is None or len(text) <= 0:
self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
return None, text
return None, text, text_index
tts_file = self.tts.to_tts(text)
if tts_file is None:
self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
return None, text
return None, text, text_index
self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
return tts_file, text
return tts_file, text, text_index
def clearSpeakStatus(self):
self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
self.asr_server_receive = True
self.tts_last_text = None
self.tts_first_text = None
self.tts_duration = 0
self.tts_start_speak_time = None
self.tts_last_text_index = -1
self.tts_first_text_index = -1
def recode_first_last_text(self, text):
if not self.tts_first_text:
def recode_first_last_text(self, text, text_index=0):
if self.tts_first_text_index == -1:
self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
self.tts_first_text = text
self.tts_last_text = text
self.tts_first_text_index = text_index
self.tts_last_text_index = text_index
async def close(self):
"""资源清理方法"""
core/handle/musicHandler.py
+3 -3
View File
@@ -139,14 +139,14 @@ class MusicHandler:
return
text = f"正在播放{selected_music}"
await send_stt_message(conn, text)
conn.tts_first_text = selected_music
conn.tts_last_text = selected_music
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
conn.llm_finish_task = True
if music_path.endswith(".p3"):
opus_packets, duration = p3.decode_opus_from_file(music_path)
else:
opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
conn.audio_play_queue.put((opus_packets, selected_music))
conn.audio_play_queue.put((opus_packets, selected_music, 0))
except Exception as e:
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
core/handle/sendAudioHandle.py
+3 -4
View File
@@ -15,11 +15,10 @@ async def isLLMWantToFinish(last_text):
return False
async def sendAudioMessage(conn, audios, text):
async def sendAudioMessage(conn, audios, text, text_index=0):
# 发送句子开始消息
if text == conn.tts_first_text:
if text_index == conn.tts_first_text_index:
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
conn.tts_start_speak_time = time.perf_counter()
await send_tts_message(conn, "sentence_start", text)
# 初始化流控参数
@@ -45,7 +44,7 @@ async def sendAudioMessage(conn, audios, text):
play_position += frame_duration # 更新播放位置
await send_tts_message(conn, "sentence_end", text)
# 发送结束消息（如果是最后一个文本）
if conn.llm_finish_task and text == conn.tts_last_text:
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
await send_tts_message(conn, 'stop', None)
if await isLLMWantToFinish(text):
await conn.close()