From c271b88c8727c8d7f330840794681774ad266845 Mon Sep 17 00:00:00 2001 From: lizhongxiang Date: Thu, 13 Mar 2025 17:44:10 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=B5=81=E5=BC=8Ftts=EF=BC=8Cfunction?= =?UTF-8?q?=E8=81=8A=E5=A4=A9=E6=97=B6=E5=80=99=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 14 ++-- main/xiaozhi-server/core/connection.py | 81 +++++++++++++------ .../core/providers/llm/openai/openai.py | 3 +- 3 files changed, 65 insertions(+), 33 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a2fa3446..ebb482e9 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -72,14 +72,14 @@ selected_module: # 将根据配置名称对应的type调用实际的LLM适配器 LLM: ChatGLMLLM # TTS将根据配置名称对应的type调用实际的TTS适配器 - TTS: EdgeTTS + TTS: FishSpeech # 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short Memory: nomem # 意图识别模块，默认不开启。开启后，可以播放音乐、控制音量、识别退出指令 # 意图识别使用intent_llm，优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间 # 意图识别使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快 # 如果意图识别设置成 function_call，建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028 - Intent: nointent + Intent: function_call # 意图识别，是用于理解用户意图的模块，例如：播放音乐 Intent: @@ -157,7 +157,7 @@ LLM: # 可在这里找到你的api key https://bigmodel.cn/usercenter/proj-mgmt/apikeys model_name: glm-4-flash url: https://open.bigmodel.cn/api/paas/v4/ - api_key: 你的chat-glm web key + api_key: f04500d80a234c9085be2b4020dce25f.gnnnWr3BRLuqeDHr OllamaLLM: # 定义LLM API类型 type: ollama @@ -208,8 +208,8 @@ LLM: k2: "v2" TTS_SET: - TTS_STREAM: false #是否启动流响应 true/false，默认false - MAX_WORKERS: 4 #并发请求tts的数量，这个调太大，本地tts会变慢 + TTS_STREAM: true #是否启动流响应 true/false，默认false + MAX_WORKERS: 5 #并发请求tts的数量，这个调太大，本地tts会变慢 TTS: # 当前支持的type为edge、doubao，可自行适配 @@ -283,8 +283,8 @@ TTS: seed: null channels: 1 rate: 44100 - api_key: "你的api_key" - api_url: "http://127.0.0.1:8080/v1/tts" + api_key: "xxxx" + api_url: "http://192.168.101.242:10000/tts-fishspeech/v1/tts" GPT_SOVITS_V2: # 定义TTS API类型 #启动tts方法： diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index fd7fe0d6..f53aa1e8 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -257,7 +257,7 @@ class ConnectionHandler: current_text = full_text[processed_chars:] # 从未处理的位置开始 # 查找最后一个有效标点 - punctuations = ("。", "？", "！", "；", "：", ".", "?", "!", ";", ":"," ") + punctuations = ("。", "？", "！", "；", "：", ".", "?", "!", ";", ":", " ") last_punct_pos = -1 for punct in punctuations: pos = current_text.rfind(punct) @@ -328,7 +328,7 @@ class ConnectionHandler: response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 - + try: start_time = time.time() @@ -336,7 +336,7 @@ class ConnectionHandler: future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop) memory_str = future.result() - #self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") + # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") # 使用支持functions的streaming接口 llm_responses = self.llm.response_with_functions( @@ -359,8 +359,8 @@ class ConnectionHandler: content_arguments = "" for response in llm_responses: content, tools_call = response - if content is not None and len(content)>0: - if len(response_message)<=0 and content=="```": + if content is not None and len(content) > 0: + if len(response_message) <= 0 and content == "```": tool_call_flag = True if tools_call is not None: @@ -374,7 +374,7 @@ class ConnectionHandler: if content is not None and len(content) > 0: if tool_call_flag: - content_arguments+=content + content_arguments += content else: response_message.append(content) @@ -390,7 +390,7 @@ class ConnectionHandler: current_text = full_text[processed_chars:] # 从未处理的位置开始 # 查找最后一个有效标点 - punctuations = ("。", "？", "！", "；", "：") + punctuations = ("。", "？", "！", "；", "：", ".", "?", "!", ";", ":", " ") last_punct_pos = -1 for punct in punctuations: pos = current_text.rfind(punct) @@ -404,8 +404,17 @@ class ConnectionHandler: if segment_text: text_index += 1 self.recode_first_last_text(segment_text, text_index) - future = self.executor.submit(self.speak_and_play, segment_text, text_index) - self.tts_queue.put(future) + if self.tts_stream: + stream_queue = queue.Queue() + self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue) + self.tts_queue_stream.put({ + "text": segment_text, + "chunk_queque": stream_queue, + "text_index": text_index + }) + else: + future = self.executor.submit(self.speak_and_play, segment_text, text_index) + self.tts_queue.put(future) processed_chars += len(segment_text_raw) # 更新已处理字符位置 # 处理最后剩余的文本 @@ -415,12 +424,21 @@ class ConnectionHandler: segment_text = get_string_no_punctuation_or_emoji(remaining_text) if segment_text: text_index += 1 - self.recode_first_last_text(segment_text, text_index) - future = self.executor.submit(self.speak_and_play, segment_text, text_index) + if self.tts_stream: + stream_queue = queue.Queue() + self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue, text_index) + self.tts_queue_stream.put({ + "text": segment_text, + "chunk_queque": stream_queue, + "text_index": text_index + }) + else: + self.recode_first_last_text(segment_text, text_index) + future = self.executor.submit(self.speak_and_play, segment_text, text_index) self.tts_queue.put(future) # 存储对话内容 - if len(response_message)>0: + if len(response_message) > 0: self.dialogue.put(Message(role="assistant", content="".join(response_message))) # 处理function call @@ -435,14 +453,15 @@ class ConnectionHandler: else: return [] function_arguments = json.loads(function_arguments) - self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}") + self.logger.bind(tag=TAG).info( + f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}") function_call_data = { "name": function_name, "id": function_id, "arguments": function_arguments } result = handle_llm_function_call(self, function_call_data) - self._handle_function_result(result, function_call_data, text_index+1) + self._handle_function_result(result, function_call_data, text_index + 1) self.llm_finish_task = True self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)) @@ -450,19 +469,26 @@ class ConnectionHandler: return True def _handle_function_result(self, result, function_call_data, text_index): - if result.action == Action.RESPONSE: # 直接回复前端 + if result.action == Action.RESPONSE: # 直接回复前端 text = result.response self.recode_first_last_text(text, text_index) - future = self.executor.submit(self.speak_and_play, text, text_index) - self.tts_queue.put(future) + if self.tts_stream: + stream_queue = queue.Queue() + self.executor.submit(self.speak_and_play_stream, text, stream_queue, text_index) + self.tts_queue_stream.put({ + "text": text, + "chunk_queque": stream_queue, + "text_index": text_index + }) + else: + future = self.executor.submit(self.speak_and_play, text, text_index) + self.tts_queue.put(future) self.dialogue.put(Message(role="assistant", content=text)) - if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 + if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.response if result.action == Action.NOTFOUND: text = result.response - - def _tts_priority_thread(self): if self.tts_stream: self._tts_priority_thread_stream() @@ -540,11 +566,16 @@ class ConnectionHandler: text = None try: if self.tts_stream: - chunk_queque, text, text_index = self.audio_play_queue.get() - future = asyncio.run_coroutine_threadsafe( - sendAudioMessageStream(self, chunk_queque, text, text_index), - self.loop) - future.result() + data, text, text_index = self.audio_play_queue.get() + if isinstance(data, list): + future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, data, text, text_index), + self.loop) + future.result() + else: + future = asyncio.run_coroutine_threadsafe( + sendAudioMessageStream(self, data, text, text_index), + self.loop) + future.result() else: opus_datas, text, text_index = self.audio_play_queue.get() future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index), diff --git a/main/xiaozhi-server/core/providers/llm/openai/openai.py b/main/xiaozhi-server/core/providers/llm/openai/openai.py index 95cd85f9..0cce1da6 100644 --- a/main/xiaozhi-server/core/providers/llm/openai/openai.py +++ b/main/xiaozhi-server/core/providers/llm/openai/openai.py @@ -2,6 +2,7 @@ import openai from core.utils.util import check_model_key from core.providers.llm.base import LLMProviderBase +TAG = __name__ class LLMProvider(LLMProviderBase): def __init__(self, config): @@ -42,7 +43,7 @@ class LLMProvider(LLMProviderBase): yield content except Exception as e: - logger.bind(tag=TAG).error(f"Error in response generation: {e}") + self.logger.bind(tag=TAG).error(f"Error in response generation: {e}") def response_with_functions(self, session_id, dialogue, functions=None): try: