diff --git a/.gitignore b/.gitignore index c09248c5..cff679fb 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,4 @@ tmp .private_config.yaml .env.development docker-compose.yml +web/vue/node_modules diff --git a/core/connection.py b/core/connection.py index 5532e3a1..8b6d1285 100644 --- a/core/connection.py +++ b/core/connection.py @@ -22,6 +22,10 @@ from core.utils.auth_code_gen import AuthCodeGenerator TAG = __name__ +class TTSException(RuntimeError): + pass + + class ConnectionHandler: def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music): self.config = config @@ -68,10 +72,8 @@ class ConnectionHandler: self.dialogue = Dialogue() # tts相关变量 - self.tts_first_text = None - self.tts_last_text = None - self.tts_start_speak_time = None - self.tts_duration = 0 + self.tts_first_text_index = -1 + self.tts_last_text_index = -1 # iot相关变量 self.iot_descriptors = {} @@ -221,6 +223,7 @@ class ConnectionHandler: return None self.llm_finish_task = False + text_index = 0 for content in llm_responses: response_message.append(content) if self.client_abort: @@ -234,7 +237,7 @@ class ConnectionHandler: current_text = full_text[processed_chars:] # 从未处理的位置开始 # 查找最后一个有效标点 - punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：", "，") + punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：") last_punct_pos = -1 for punct in punctuations: pos = current_text.rfind(punct) @@ -246,8 +249,12 @@ class ConnectionHandler: segment_text_raw = current_text[:last_punct_pos + 1] segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) if segment_text: - self.recode_first_last_text(segment_text) - future = self.executor.submit(self.speak_and_play, segment_text) + # 强制设置空字符，测试TTS出错返回语音的健壮性 + # if text_index % 2 == 0: + # segment_text = " " + text_index += 1 + self.recode_first_last_text(segment_text, text_index) + future = self.executor.submit(self.speak_and_play, segment_text, text_index) self.tts_queue.put(future) processed_chars += len(segment_text_raw) # 更新已处理字符位置 @@ -257,8 +264,9 @@ class ConnectionHandler: if remaining_text: segment_text = get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - self.recode_first_last_text(segment_text) - future = self.executor.submit(self.speak_and_play, segment_text) + text_index += 1 + self.recode_first_last_text(segment_text, text_index) + future = self.executor.submit(self.speak_and_play, segment_text, text_index) self.tts_queue.put(future) self.llm_finish_task = True @@ -274,30 +282,28 @@ class ConnectionHandler: if future is None: continue text = None + opus_datas, text_index, tts_file = [], 0, None try: self.logger.bind(tag=TAG).debug("正在处理TTS任务...") - tts_file, text = future.result(timeout=10) + tts_file, text, text_index = future.result(timeout=10) if text is None or len(text) <= 0: - continue - if tts_file is None: - self.logger.bind(tag=TAG).error(f"TTS文件生成失败: {text}") - continue - self.logger.bind(tag=TAG).debug(f"TTS文件生成完毕，文件路径: {tts_file}") - if os.path.exists(tts_file): - opus_datas, duration = self.tts.wav_to_opus_data(tts_file) + self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty") + elif tts_file is None: + self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}") else: - self.logger.bind(tag=TAG).error(f"TTS文件不存在: {tts_file}") - opus_datas = [] + self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}") + if os.path.exists(tts_file): + opus_datas, duration = self.tts.wav_to_opus_data(tts_file) + else: + self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}") except TimeoutError: - self.logger.bind(tag=TAG).error("TTS 任务超时") - continue + self.logger.bind(tag=TAG).error("TTS超时") except Exception as e: - self.logger.bind(tag=TAG).error(f"TTS 任务出错: {e}") - continue + self.logger.bind(tag=TAG).error(f"TTS出错: {e}") if not self.client_abort: # 如果没有中途打断就发送语音 - self.audio_play_queue.put((opus_datas, text)) - if self.tts.delete_audio_file and os.path.exists(tts_file): + self.audio_play_queue.put((opus_datas, text, text_index)) + if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file): os.remove(tts_file) except Exception as e: self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}") @@ -306,42 +312,41 @@ class ConnectionHandler: self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})), self.loop ) - self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text}{e}") + self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}") def _audio_play_priority_thread(self): while not self.stop_event.is_set(): text = None try: - opus_datas, text = self.audio_play_queue.get() - future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text), self.loop) + opus_datas, text, text_index = self.audio_play_queue.get() + future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index), + self.loop) future.result() except Exception as e: - self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text}{e}") + self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}") - def speak_and_play(self, text): + def speak_and_play(self, text, text_index=0): if text is None or len(text) <= 0: self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") - return None, text + return None, text, text_index tts_file = self.tts.to_tts(text) if tts_file is None: self.logger.bind(tag=TAG).error(f"tts转换失败，{text}") - return None, text + return None, text, text_index self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}") - return tts_file, text + return tts_file, text, text_index def clearSpeakStatus(self): self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态") self.asr_server_receive = True - self.tts_last_text = None - self.tts_first_text = None - self.tts_duration = 0 - self.tts_start_speak_time = None + self.tts_last_text_index = -1 + self.tts_first_text_index = -1 - def recode_first_last_text(self, text): - if not self.tts_first_text: + def recode_first_last_text(self, text, text_index=0): + if self.tts_first_text_index == -1: self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}") - self.tts_first_text = text - self.tts_last_text = text + self.tts_first_text_index = text_index + self.tts_last_text_index = text_index async def close(self): """资源清理方法""" diff --git a/core/handle/musicHandler.py b/core/handle/musicHandler.py index f1eb37ed..53ec49c4 100644 --- a/core/handle/musicHandler.py +++ b/core/handle/musicHandler.py @@ -139,14 +139,14 @@ class MusicHandler: return text = f"正在播放{selected_music}" await send_stt_message(conn, text) - conn.tts_first_text = selected_music - conn.tts_last_text = selected_music + conn.tts_first_text_index = 0 + conn.tts_last_text_index = 0 conn.llm_finish_task = True if music_path.endswith(".p3"): opus_packets, duration = p3.decode_opus_from_file(music_path) else: opus_packets, duration = conn.tts.wav_to_opus_data(music_path) - conn.audio_play_queue.put((opus_packets, selected_music)) + conn.audio_play_queue.put((opus_packets, selected_music, 0)) except Exception as e: logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") diff --git a/core/handle/sendAudioHandle.py b/core/handle/sendAudioHandle.py index 07b24fb9..d0dcb82f 100644 --- a/core/handle/sendAudioHandle.py +++ b/core/handle/sendAudioHandle.py @@ -15,11 +15,10 @@ async def isLLMWantToFinish(last_text): return False -async def sendAudioMessage(conn, audios, text): +async def sendAudioMessage(conn, audios, text, text_index=0): # 发送句子开始消息 - if text == conn.tts_first_text: + if text_index == conn.tts_first_text_index: logger.bind(tag=TAG).info(f"发送第一段语音: {text}") - conn.tts_start_speak_time = time.perf_counter() await send_tts_message(conn, "sentence_start", text) # 初始化流控参数 @@ -45,7 +44,7 @@ async def sendAudioMessage(conn, audios, text): play_position += frame_duration # 更新播放位置 await send_tts_message(conn, "sentence_end", text) # 发送结束消息（如果是最后一个文本） - if conn.llm_finish_task and text == conn.tts_last_text: + if conn.llm_finish_task and text_index == conn.tts_last_text_index: await send_tts_message(conn, 'stop', None) if await isLLMWantToFinish(text): await conn.close()