diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 570c3105..80c62336 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -11,7 +11,7 @@ import websockets from typing import Dict, Any from core.utils.dialogue import Message, Dialogue from core.handle.textHandle import handleTextMessage -from core.utils.util import get_string_no_punctuation_or_emoji +from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string from concurrent.futures import ThreadPoolExecutor, TimeoutError from core.handle.sendAudioHandle import sendAudioMessage from core.handle.receiveAudioHandle import handleAudioMessage @@ -307,8 +307,7 @@ class ConnectionHandler: response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 - function_call_data = None # 存储function call数据 - + try: start_time = time.time() @@ -316,7 +315,7 @@ class ConnectionHandler: future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop) memory_str = future.result() - # self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}") + #self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") # 使用支持functions的streaming接口 llm_responses = self.llm.response_with_functions( @@ -332,48 +331,61 @@ class ConnectionHandler: text_index = 0 # 处理流式响应 + tool_call_flag = False + function_name = None + function_id = None + function_arguments = "" + content_arguments = "" for response in llm_responses: - if response["type"] == "content": - content = response["content"] - response_message.append(content) + content, tools_call = response + if content is not None and len(content)>0: + if len(response_message)<=0 and content=="```": + tool_call_flag = True - if self.client_abort: - break + if tools_call is not None: + tool_call_flag = True + if tools_call[0].id is not None: + function_id = tools_call[0].id + if tools_call[0].function.name is not None: + function_name = tools_call[0].function.name + if tools_call[0].function.arguments is not None: + function_arguments += tools_call[0].function.arguments - end_time = time.time() - self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") + if content is not None and len(content) > 0: + if tool_call_flag: + content_arguments+=content + else: + response_message.append(content) - # 处理文本分段和TTS逻辑 - # 合并当前全部文本并处理未分割部分 - full_text = "".join(response_message) - current_text = full_text[processed_chars:] # 从未处理的位置开始 + if self.client_abort: + break - # 查找最后一个有效标点 - punctuations = ("。", "？", "！", "；", "：") - last_punct_pos = -1 - for punct in punctuations: - pos = current_text.rfind(punct) - if pos > last_punct_pos: - last_punct_pos = pos + end_time = time.time() + self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") - # 找到分割点则处理 - if last_punct_pos != -1: - segment_text_raw = current_text[:last_punct_pos + 1] - segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) - if segment_text: - text_index += 1 - self.recode_first_last_text(segment_text, text_index) - future = self.executor.submit(self.speak_and_play, segment_text, text_index) - self.tts_queue.put(future) - processed_chars += len(segment_text_raw) # 更新已处理字符位置 + # 处理文本分段和TTS逻辑 + # 合并当前全部文本并处理未分割部分 + full_text = "".join(response_message) + current_text = full_text[processed_chars:] # 从未处理的位置开始 - elif response["type"] == "function_call": - # Extract function call data - function_call_data = { - "name": response["function_call"]["function"]["name"], - "arguments": response["function_call"]["function"]["arguments"] - } - self.logger.bind(tag=TAG).info(f"Function call detected: {function_call_data}") + # 查找最后一个有效标点 + punctuations = ("。", "？", "！", "；", "：") + last_punct_pos = -1 + for punct in punctuations: + pos = current_text.rfind(punct) + if pos > last_punct_pos: + last_punct_pos = pos + + # 找到分割点则处理 + if last_punct_pos != -1: + segment_text_raw = current_text[:last_punct_pos + 1] + segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) + if segment_text: + text_index += 1 + self.recode_first_last_text(segment_text, text_index) + future = self.executor.submit(self.speak_and_play, segment_text, text_index) + self.tts_queue.put(future) + processed_chars += len(segment_text_raw) # 更新已处理字符位置 # 处理最后剩余的文本 full_text = "".join(response_message) @@ -387,23 +399,46 @@ class ConnectionHandler: self.tts_queue.put(future) # 存储对话内容 - self.dialogue.put(Message(role="assistant", content="".join(response_message))) + if len(response_message)>0: + self.dialogue.put(Message(role="assistant", content="".join(response_message))) # 处理function call - if function_call_data: + if tool_call_flag: + if function_id is None: + a = extract_json_from_string(content_arguments) + if a is not None: + content_arguments_json = json.loads(a) + function_name = content_arguments_json["function_name"] + function_arguments = json.dumps(content_arguments_json["args"], ensure_ascii=False) + function_id = str(uuid.uuid4().hex) + else: + return [] + function_arguments = json.loads(function_arguments) + self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}") + function_call_data = { + "name": function_name, + "id": function_id, + "arguments": function_arguments + } result = handle_llm_function_call(self, function_call_data) - if result.action == Action.RESPONSE: - text = result.response - text_index += 1 - self.recode_first_last_text(text, text_index) - future = self.executor.submit(self.speak_and_play, text, text_index) - self.tts_queue.put(future) + self._handle_function_result(result, function_call_data, text_index+1) self.llm_finish_task = True self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)) return True + def _handle_function_result(self, result, function_call_data, text_index): + if result.action == Action.RESPONSE: # 直接回复前端 + text = result.response + self.recode_first_last_text(text, text_index) + future = self.executor.submit(self.speak_and_play, text, text_index) + self.tts_queue.put(future) + self.dialogue.put(Message(role="assistant", content=text)) + if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 + text = result.response + + def _tts_priority_thread(self): while not self.stop_event.is_set(): text = None diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index cae22b2c..2dde6255 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -122,12 +122,10 @@ async def analyze_intent_with_llm(conn, text): logger.bind(tag=TAG).warning("意图识别服务未初始化") return None - # 创建对话历史记录 + # 对话历史记录 dialogue = conn.dialogue - dialogue.put(Message(role="user", content=text)) - try: - intent_result = await conn.intent.detect_intent(dialogue.dialogue) + intent_result = await conn.intent.detect_intent(dialogue.dialogue, text) logger.bind(tag=TAG).info(f"意图识别结果: {intent_result}") # 尝试解析JSON结果 diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index e97c17ca..bf6d4c36 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -67,9 +67,9 @@ async def no_voice_close_connect(conn): else: no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120) - if no_voice_time > 1000 * close_connection_no_voice_time: + if not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time: + conn.close_after_chat = True conn.client_abort = False conn.asr_server_receive = False - conn.close_after_chat = True - prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾" + prompt = "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。" await startToChat(conn, prompt) diff --git a/main/xiaozhi-server/core/providers/intent/base.py b/main/xiaozhi-server/core/providers/intent/base.py index 1333f620..9726f1be 100644 --- a/main/xiaozhi-server/core/providers/intent/base.py +++ b/main/xiaozhi-server/core/providers/intent/base.py @@ -5,6 +5,7 @@ from config.logger import setup_logging TAG = __name__ logger = setup_logging() + class IntentProviderBase(ABC): def __init__(self, config): self.config = config @@ -17,9 +18,9 @@ class IntentProviderBase(ABC): def set_llm(self, llm): self.llm = llm logger.bind(tag=TAG).debug("Set LLM for intent provider") - + @abstractmethod - async def detect_intent(self, dialogue_history: List[Dict]) -> str: + async def detect_intent(self, dialogue_history: List[Dict], text: str) -> str: """ 检测用户最后一句话的意图 Args: diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index c3930296..191f73df 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -37,7 +37,7 @@ class IntentProvider(IntentProviderBase): ) return prompt - async def detect_intent(self, dialogue_history: List[Dict]) -> str: + async def detect_intent(self, dialogue_history: List[Dict], text:str) -> str: if not self.llm: raise ValueError("LLM provider not set") @@ -48,9 +48,8 @@ class IntentProvider(IntentProviderBase): msgStr += f"User: {msg.content}\n" elif msg.role== "assistant": msgStr += f"Assistant: {msg.content}\n" - + msgStr += f"User: {text}\n" user_prompt = f"请分析用户的意图：\n{msgStr}" - # 使用LLM进行意图识别 intent = self.llm.response_no_stream( system_prompt=self.promot, diff --git a/main/xiaozhi-server/core/providers/intent/nointent/nointent.py b/main/xiaozhi-server/core/providers/intent/nointent/nointent.py index 9feadaad..70ea96fb 100644 --- a/main/xiaozhi-server/core/providers/intent/nointent/nointent.py +++ b/main/xiaozhi-server/core/providers/intent/nointent/nointent.py @@ -5,12 +5,14 @@ from config.logger import setup_logging TAG = __name__ logger = setup_logging() + class IntentProvider(IntentProviderBase): - async def detect_intent(self, dialogue_history: List[Dict]) -> str: + async def detect_intent(self, dialogue_history: List[Dict], text: str) -> str: """ 默认的意图识别实现，始终返回继续聊天 Args: dialogue_history: 对话历史记录列表 + text: 本次对话记录 Returns: 固定返回"继续聊天" """ diff --git a/main/xiaozhi-server/core/providers/llm/ollama/ollama.py b/main/xiaozhi-server/core/providers/llm/ollama/ollama.py index 86d0f6ee..aa5184a4 100644 --- a/main/xiaozhi-server/core/providers/llm/ollama/ollama.py +++ b/main/xiaozhi-server/core/providers/llm/ollama/ollama.py @@ -51,30 +51,8 @@ class LLMProvider(LLMProviderBase): tools=functions, ) - current_function_call = None - current_content = "" - for chunk in stream: - delta = chunk.choices[0].delta - - if delta.content: - current_content += delta.content - yield {"type": "content", "content": delta.content} - - if delta.tool_calls: - tool_call = delta.tool_calls[0] - # Handle the function call data using proper attribute access - if not current_function_call: - current_function_call = { - "function": { - "name": tool_call.function.name, - "arguments": tool_call.function.arguments - } - } - - if current_function_call: - logger.bind(tag=TAG).debug(f"ollama Function call detected: {current_function_call}") - yield {"type": "function_call", "function_call": current_function_call} + yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls except Exception as e: logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}") diff --git a/main/xiaozhi-server/core/providers/llm/openai/openai.py b/main/xiaozhi-server/core/providers/llm/openai/openai.py index 33ef10eb..95cd85f9 100644 --- a/main/xiaozhi-server/core/providers/llm/openai/openai.py +++ b/main/xiaozhi-server/core/providers/llm/openai/openai.py @@ -53,30 +53,8 @@ class LLMProvider(LLMProviderBase): tools=functions, ) - current_function_call = None - current_content = "" - for chunk in stream: - delta = chunk.choices[0].delta - - if delta.content: - current_content += delta.content - yield {"type": "content", "content": delta.content} - - if delta.tool_calls: - tool_call = delta.tool_calls[0] - # Handle the function call data using proper attribute access - if not current_function_call: - current_function_call = { - "function": { - "name": tool_call.function.name, - "arguments": tool_call.function.arguments - } - } - - if current_function_call: - logger.bind(tag=TAG).debug(f"openai Function call detected: {current_function_call}") - yield {"type": "function_call", "function_call": current_function_call} + yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls except Exception as e: self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}") diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 8a8dac95..60b98665 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -4,6 +4,7 @@ import yaml import socket import subprocess import logging +import re def get_project_dir(): @@ -119,3 +120,11 @@ def check_ffmpeg_installed(): error_msg += "1、按照项目的安装文档，正确进入conda环境\n" error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n" raise ValueError(error_msg) + +def extract_json_from_string(input_string): + """提取字符串中的 JSON 部分""" + pattern = r'(\{.*\})' + match = re.search(pattern, input_string) + if match: + return match.group(1) # 返回提取的 JSON 字符串 + return None \ No newline at end of file