From 98e2526f8a09fa58b2d591a587a615a82d60da5b Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 24 May 2025 02:15:06 +0800 Subject: [PATCH] =?UTF-8?q?update:=E5=90=88=E5=B9=B6chat=E5=92=8Cchat=5Fwi?= =?UTF-8?q?th=5Ffunction=5Fcalling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 153 +++++------------- .../core/handle/receiveAudioHandle.py | 10 +- 2 files changed, 40 insertions(+), 123 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 74b612fe..e73b340d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -506,106 +506,20 @@ class ConnectionHandler: # 更新系统prompt至上下文 self.dialogue.update_system_message(self.prompt) - def chat(self, query): - - self.dialogue.put(Message(role="user", content=query)) - - response_message = [] - processed_chars = 0 # 跟踪已处理的字符位置 - try: - # 使用带记忆的对话 - memory_str = None - if self.memory is not None: - future = asyncio.run_coroutine_threadsafe( - self.memory.query_memory(query), self.loop - ) - memory_str = future.result() - - self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}") - llm_responses = self.llm.response( - self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str) - ) - except Exception as e: - self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}") - return None - - self.llm_finish_task = False - text_index = 0 - for content in llm_responses: - response_message.append(content) - if self.client_abort: - break - - # 合并当前全部文本并处理未分割部分 - full_text = "".join(response_message) - current_text = full_text[processed_chars:] # 从未处理的位置开始 - - # 查找最后一个有效标点 - punctuations = ("。", ".", "？", "?", "！", "!", "；", ";", "：") - last_punct_pos = -1 - number_flag = True - for punct in punctuations: - pos = current_text.rfind(punct) - prev_char = current_text[pos - 1] if pos - 1 >= 0 else "" - # 如果.前面是数字统一判断为小数 - if prev_char.isdigit() and punct == ".": - number_flag = False - if pos > last_punct_pos and number_flag: - last_punct_pos = pos - - # 找到分割点则处理 - if last_punct_pos != -1: - segment_text_raw = current_text[: last_punct_pos + 1] - segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) - if segment_text: - # 强制设置空字符，测试TTS出错返回语音的健壮性 - # if text_index % 2 == 0: - # segment_text = " " - text_index += 1 - self.recode_first_last_text(segment_text, text_index) - future = self.executor.submit( - self.speak_and_play, segment_text, text_index - ) - self.tts_queue.put((future, text_index)) - processed_chars += len(segment_text_raw) # 更新已处理字符位置 - - # 处理最后剩余的文本 - full_text = "".join(response_message) - remaining_text = full_text[processed_chars:] - if remaining_text: - segment_text = get_string_no_punctuation_or_emoji(remaining_text) - if segment_text: - text_index += 1 - self.recode_first_last_text(segment_text, text_index) - future = self.executor.submit( - self.speak_and_play, segment_text, text_index - ) - self.tts_queue.put((future, text_index)) - - self.llm_finish_task = True - self.dialogue.put(Message(role="assistant", content="".join(response_message))) - self.logger.bind(tag=TAG).debug( - json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False) - ) - return True - - def chat_with_function_calling(self, query, tool_call=False): - self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}") - """Chat with function calling for intent detection using streaming""" + def chat(self, query, tool_call=False): + self.logger.bind(tag=TAG).debug(f"Chat: {query}") if not tool_call: self.dialogue.put(Message(role="user", content=query)) # Define intent functions functions = None - if hasattr(self, "func_handler"): + if self.intent_type == "function_call" and hasattr(self, "func_handler"): functions = self.func_handler.get_functions() response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 try: - start_time = time.time() - # 使用带记忆的对话 memory_str = None if self.memory is not None: @@ -614,14 +528,18 @@ class ConnectionHandler: ) memory_str = future.result() - # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") - - # 使用支持functions的streaming接口 - llm_responses = self.llm.response_with_functions( - self.session_id, - self.dialogue.get_llm_dialogue_with_memory(memory_str), - functions=functions, - ) + if functions is not None: + # 使用支持functions的streaming接口 + llm_responses = self.llm.response_with_functions( + self.session_id, + self.dialogue.get_llm_dialogue_with_memory(memory_str), + functions=functions, + ) + else: + llm_responses = self.llm.response( + self.session_id, + self.dialogue.get_llm_dialogue_with_memory(memory_str), + ) except Exception as e: self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}") return None @@ -637,27 +555,28 @@ class ConnectionHandler: content_arguments = "" for response in llm_responses: - content, tools_call = response + if self.intent_type == "function_call": + content, tools_call = response + if "content" in response: + content = response["content"] + tools_call = None + if content is not None and len(content) > 0: + content_arguments += content - if "content" in response: - content = response["content"] - tools_call = None - if content is not None and len(content) > 0: - content_arguments += content - - if not tool_call_flag and content_arguments.startswith(""): - # print("content_arguments", content_arguments) - tool_call_flag = True - - if tools_call is not None: - tool_call_flag = True - if tools_call[0].id is not None: - function_id = tools_call[0].id - if tools_call[0].function.name is not None: - function_name = tools_call[0].function.name - if tools_call[0].function.arguments is not None: - function_arguments += tools_call[0].function.arguments + if not tool_call_flag and content_arguments.startswith(""): + # print("content_arguments", content_arguments) + tool_call_flag = True + if tools_call is not None: + tool_call_flag = True + if tools_call[0].id is not None: + function_id = tools_call[0].id + if tools_call[0].function.name is not None: + function_name = tools_call[0].function.name + if tools_call[0].function.arguments is not None: + function_arguments += tools_call[0].function.arguments + else: + content = response if content is not None and len(content) > 0: if not tool_call_flag: response_message.append(content) @@ -853,7 +772,7 @@ class ConnectionHandler: content=text, ) ) - self.chat_with_function_calling(text, tool_call=True) + self.chat(text, tool_call=True) elif result.action == Action.NOTFOUND or result.action == Action.ERROR: text = result.result self.recode_first_last_text(text, text_index) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 9c693751..2584ab83 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -39,7 +39,9 @@ async def handleAudioMessage(conn, audio): if len(conn.asr_audio) < 15: conn.asr_server_receive = True else: - raw_text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) # 确保ASR模块返回原始文本 + raw_text, _ = await conn.asr.speech_to_text( + conn.asr_audio, conn.session_id + ) # 确保ASR模块返回原始文本 conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}") text_len, _ = remove_punctuation_and_length(raw_text) if text_len > 0: @@ -76,11 +78,7 @@ async def startToChat(conn, text): # 意图未被处理，继续常规聊天流程 await send_stt_message(conn, text) - if conn.intent_type == "function_call": - # 使用支持function calling的聊天方法 - conn.executor.submit(conn.chat_with_function_calling, text) - else: - conn.executor.submit(conn.chat, text) + conn.executor.submit(conn.chat, text) async def no_voice_close_connect(conn):