fix:function call bug (#268)

* 优化function call消息处理

* fix:用户说话重复记录bug

* 2025-3-10-优化llm intent (#254)

Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com>

* update:回复上版提示词，无需再二次识别歌曲名

---------

Co-authored-by: 玄凤科技 <eric230308@gmail.com>
Co-authored-by: hrz <1710360675@qq.com>
Co-authored-by: Jiao Haoyang <108573524+XuSenfeng@users.noreply.github.com>

main/xiaozhi-server/core/connection.py

@@ -11,7 +11,7 @@ import websockets
 from typing import Dict, Any
 from core.utils.dialogue import Message, Dialogue
 from core.handle.textHandle import handleTextMessage
-from core.utils.util import get_string_no_punctuation_or_emoji
+from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage
 from core.handle.receiveAudioHandle import handleAudioMessage
@@ -307,7 +307,6 @@ class ConnectionHandler:
 
         response_message = []
         processed_chars = 0  # 跟踪已处理的字符位置
-        function_call_data = None  # 存储function call数据
    
         try:
             start_time = time.time()
@@ -316,7 +315,7 @@ class ConnectionHandler:
             future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
             memory_str = future.result()
 
-            # self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
+            #self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
 
             # 使用支持functions的streaming接口
             llm_responses = self.llm.response_with_functions(
@@ -332,48 +331,61 @@ class ConnectionHandler:
         text_index = 0
 
         # 处理流式响应
+        tool_call_flag = False
+        function_name = None
+        function_id = None
+        function_arguments = ""
+        content_arguments = ""
         for response in llm_responses:
-            if response["type"] == "content":
+            content, tools_call = response
-                content = response["content"]
+            if content is not None and len(content)>0:
-                response_message.append(content)
+                if len(response_message)<=0 and content=="```":
+                    tool_call_flag = True
 
-                if self.client_abort:
+            if tools_call is not None:
-                    break
+                tool_call_flag = True
+                if tools_call[0].id is not None:
+                    function_id = tools_call[0].id
+                if tools_call[0].function.name is not None:
+                    function_name = tools_call[0].function.name
+                if tools_call[0].function.arguments is not None:
+                    function_arguments += tools_call[0].function.arguments
 
-                end_time = time.time()
+            if content is not None and len(content) > 0:
-                self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+                if tool_call_flag:
+                    content_arguments+=content
+                else:
+                    response_message.append(content)
 
-                # 处理文本分段和TTS逻辑
+                    if self.client_abort:
-                # 合并当前全部文本并处理未分割部分
+                        break
-                full_text = "".join(response_message)
-                current_text = full_text[processed_chars:]  # 从未处理的位置开始
 
-                # 查找最后一个有效标点
+                    end_time = time.time()
-                punctuations = ("。", "？", "！", "；", "：")
+                    self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
-                last_punct_pos = -1
-                for punct in punctuations:
-                    pos = current_text.rfind(punct)
-                    if pos > last_punct_pos:
-                        last_punct_pos = pos
 
-                # 找到分割点则处理
+                    # 处理文本分段和TTS逻辑
-                if last_punct_pos != -1:
+                    # 合并当前全部文本并处理未分割部分
-                    segment_text_raw = current_text[:last_punct_pos + 1]
+                    full_text = "".join(response_message)
-                    segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                    current_text = full_text[processed_chars:]  # 从未处理的位置开始
-                    if segment_text:
-                        text_index += 1
-                        self.recode_first_last_text(segment_text, text_index)
-                        future = self.executor.submit(self.speak_and_play, segment_text, text_index)
-                        self.tts_queue.put(future)
-                        processed_chars += len(segment_text_raw)  # 更新已处理字符位置
 
-            elif response["type"] == "function_call":
+                    # 查找最后一个有效标点
-                # Extract function call data
+                    punctuations = ("。", "？", "！", "；", "：")
-                function_call_data = {
+                    last_punct_pos = -1
-                    "name": response["function_call"]["function"]["name"],
+                    for punct in punctuations:
-                    "arguments": response["function_call"]["function"]["arguments"]
+                        pos = current_text.rfind(punct)
-                }
+                        if pos > last_punct_pos:
-                self.logger.bind(tag=TAG).info(f"Function call detected: {function_call_data}")
+                            last_punct_pos = pos
+
+                    # 找到分割点则处理
+                    if last_punct_pos != -1:
+                        segment_text_raw = current_text[:last_punct_pos + 1]
+                        segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                        if segment_text:
+                            text_index += 1
+                            self.recode_first_last_text(segment_text, text_index)
+                            future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                            self.tts_queue.put(future)
+                            processed_chars += len(segment_text_raw)  # 更新已处理字符位置
 
         # 处理最后剩余的文本
         full_text = "".join(response_message)
@@ -387,23 +399,46 @@ class ConnectionHandler:
                 self.tts_queue.put(future)
 
         # 存储对话内容
-        self.dialogue.put(Message(role="assistant", content="".join(response_message)))
+        if len(response_message)>0:
+            self.dialogue.put(Message(role="assistant", content="".join(response_message)))
 
         # 处理function call
-        if function_call_data:
+        if tool_call_flag:
+            if function_id is None:
+                a = extract_json_from_string(content_arguments)
+                if a is not None:
+                    content_arguments_json = json.loads(a)
+                    function_name = content_arguments_json["function_name"]
+                    function_arguments = json.dumps(content_arguments_json["args"], ensure_ascii=False)
+                    function_id = str(uuid.uuid4().hex)
+                else:
+                    return []
+                function_arguments = json.loads(function_arguments)
+            self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
+            function_call_data = {
+                "name": function_name,
+                "id": function_id,
+                "arguments": function_arguments
+            }
             result = handle_llm_function_call(self, function_call_data)
-            if result.action == Action.RESPONSE:
+            self._handle_function_result(result, function_call_data, text_index+1)
-                text = result.response
-                text_index += 1
-                self.recode_first_last_text(text, text_index)
-                future = self.executor.submit(self.speak_and_play, text, text_index)
-                self.tts_queue.put(future)
 
         self.llm_finish_task = True
         self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
 
         return True
 
+    def _handle_function_result(self, result, function_call_data, text_index):
+        if result.action == Action.RESPONSE: # 直接回复前端
+            text = result.response
+            self.recode_first_last_text(text, text_index)
+            future = self.executor.submit(self.speak_and_play, text, text_index)
+            self.tts_queue.put(future)
+            self.dialogue.put(Message(role="assistant", content=text))
+        if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
+            text = result.response
+        
+
     def _tts_priority_thread(self):
         while not self.stop_event.is_set():
             text = None

main/xiaozhi-server/core/handle/intentHandler.py

@@ -122,12 +122,10 @@ async def analyze_intent_with_llm(conn, text):
         logger.bind(tag=TAG).warning("意图识别服务未初始化")
         return None
 
-    # 创建对话历史记录
+    # 对话历史记录
     dialogue = conn.dialogue
-    dialogue.put(Message(role="user", content=text))
-
     try:
-        intent_result = await conn.intent.detect_intent(dialogue.dialogue)
+        intent_result = await conn.intent.detect_intent(dialogue.dialogue, text)
         logger.bind(tag=TAG).info(f"意图识别结果: {intent_result}")
 
         # 尝试解析JSON结果

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -67,9 +67,9 @@ async def no_voice_close_connect(conn):
     else:
         no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
         close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
-        if no_voice_time > 1000 * close_connection_no_voice_time:
+        if not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time:
+            conn.close_after_chat = True
             conn.client_abort = False
             conn.asr_server_receive = False
-            conn.close_after_chat = True
+            prompt = "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
-            prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
             await startToChat(conn, prompt)

main/xiaozhi-server/core/providers/intent/base.py

@@ -5,6 +5,7 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+
 class IntentProviderBase(ABC):
     def __init__(self, config):
         self.config = config
@@ -19,7 +20,7 @@ class IntentProviderBase(ABC):
         logger.bind(tag=TAG).debug("Set LLM for intent provider")
 
     @abstractmethod
-    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+    async def detect_intent(self, dialogue_history: List[Dict], text: str) -> str:
         """
         检测用户最后一句话的意图
         Args:

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -37,7 +37,7 @@ class IntentProvider(IntentProviderBase):
         )
         return prompt
     
-    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+    async def detect_intent(self, dialogue_history: List[Dict], text:str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
 
@@ -48,9 +48,8 @@ class IntentProvider(IntentProviderBase):
                 msgStr += f"User: {msg.content}\n"
             elif msg.role== "assistant":
                 msgStr += f"Assistant: {msg.content}\n"
-
+        msgStr += f"User: {text}\n"
         user_prompt = f"请分析用户的意图：\n{msgStr}"
-        
         # 使用LLM进行意图识别
         intent = self.llm.response_no_stream(
             system_prompt=self.promot,

main/xiaozhi-server/core/providers/intent/nointent/nointent.py

@@ -5,12 +5,14 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+
 class IntentProvider(IntentProviderBase):
-    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+    async def detect_intent(self, dialogue_history: List[Dict], text: str) -> str:
         """
         默认的意图识别实现，始终返回继续聊天
         Args:
             dialogue_history: 对话历史记录列表
+            text: 本次对话记录
         Returns:
             固定返回"继续聊天"
         """

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -51,30 +51,8 @@ class LLMProvider(LLMProviderBase):
                 tools=functions,
             )
             
-            current_function_call = None
-            current_content = ""
-            
             for chunk in stream:
-                delta = chunk.choices[0].delta
+                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
-                
-                if delta.content:
-                    current_content += delta.content
-                    yield {"type": "content", "content": delta.content}
-                
-                if delta.tool_calls:
-                    tool_call = delta.tool_calls[0]
-                    # Handle the function call data using proper attribute access
-                    if not current_function_call:
-                        current_function_call = {
-                            "function": {
-                                "name": tool_call.function.name,
-                                "arguments": tool_call.function.arguments
-                            }
-                        }
-            
-            if current_function_call:
-                logger.bind(tag=TAG).debug(f"ollama Function call detected: {current_function_call}")
-                yield {"type": "function_call", "function_call": current_function_call}
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -53,30 +53,8 @@ class LLMProvider(LLMProviderBase):
                 tools=functions,
             )
             
-            current_function_call = None
-            current_content = ""
-            
             for chunk in stream:
-                delta = chunk.choices[0].delta
+                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
-                
-                if delta.content:
-                    current_content += delta.content
-                    yield {"type": "content", "content": delta.content}
-                
-                if delta.tool_calls:
-                    tool_call = delta.tool_calls[0]
-                    # Handle the function call data using proper attribute access
-                    if not current_function_call:
-                        current_function_call = {
-                            "function": {
-                                "name": tool_call.function.name,
-                                "arguments": tool_call.function.arguments
-                            }
-                        }
-            
-            if current_function_call:
-                logger.bind(tag=TAG).debug(f"openai Function call detected: {current_function_call}")
-                yield {"type": "function_call", "function_call": current_function_call}
                 
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")

main/xiaozhi-server/core/utils/util.py

@@ -4,6 +4,7 @@ import yaml
 import socket
 import subprocess
 import logging
+import re
 
 
 def get_project_dir():
@@ -119,3 +120,11 @@ def check_ffmpeg_installed():
         error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
         error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
         raise ValueError(error_msg)
+    
+def extract_json_from_string(input_string):
+    """提取字符串中的 JSON 部分"""
+    pattern = r'(\{.*\})'
+    match = re.search(pattern, input_string)
+    if match:
+        return match.group(1)  # 返回提取的 JSON 字符串
+    return None