Merge branch 'main' into py-test

main/xiaozhi-server/core/connection.py

@@ -60,9 +60,9 @@ class ConnectionHandler:
         self.config = copy.deepcopy(config)
         self.session_id = str(uuid.uuid4())
         self.logger = setup_logging()
-        self.auth = AuthMiddleware(config)
         self.server = server  # 保存server实例的引用
 
+        self.auth = AuthMiddleware(config)
         self.need_bind = False
         self.bind_code = None
         self.read_config_from_api = self.config.get("read_config_from_api", False)
@@ -130,8 +130,10 @@ class ConnectionHandler:
             if len(cmd) > self.max_cmd_length:
                 self.max_cmd_length = len(cmd)
 
-        self.close_after_chat = False  # 是否在聊天结束后关闭连接
-        self.use_function_call_mode = False
+        # 是否在聊天结束后关闭连接
+        self.close_after_chat = False
+        self.load_function_plugin = False
+        self.intent_type = "nointent"
 
         self.timeout_task = None
         self.timeout_seconds = (
@@ -410,11 +412,11 @@ class ConnectionHandler:
         self.memory.init_memory(self.device_id, self.llm)
 
     def _initialize_intent(self):
-        if (
-            self.config["Intent"][self.config["selected_module"]["Intent"]]["type"]
-            == "function_call"
-        ):
-            self.use_function_call_mode = True
+        self.intent_type = self.config["Intent"][
+            self.config["selected_module"]["Intent"]
+        ]["type"]
+        if self.intent_type == "function_call" or self.intent_type == "intent_llm":
+            self.load_function_plugin = True
         """初始化意图识别模块"""
         # 获取意图识别配置
         intent_config = self.config["Intent"]
@@ -802,7 +804,13 @@ class ConnectionHandler:
                 )
 
                 self.dialogue.put(
-                    Message(role="tool", tool_call_id=function_id, content=text)
+                    Message(
+                        role="tool",
+                        tool_call_id=(
+                            str(uuid.uuid4()) if function_id is None else function_id
+                        ),
+                        content=text,
+                    )
                 )
                 self.chat_with_function_calling(text, tool_call=True)
         elif result.action == Action.NOTFOUND or result.action == Action.ERROR:

main/xiaozhi-server/core/handle/intentHandler.py

@@ -5,6 +5,8 @@ from core.handle.sendAudioHandle import send_stt_message
 from core.handle.helloHandle import checkWakeupWords
 from core.utils.util import remove_punctuation_and_length
 from core.utils.dialogue import Message
+from plugins_func.register import Action
+from loguru import logger
 
 TAG = __name__
 
@@ -17,7 +19,7 @@ async def handle_user_intent(conn, text):
     if await checkWakeupWords(conn, text):
         return True
 
-    if conn.use_function_call_mode:
+    if conn.intent_type == "function_call":
         # 使用支持function calling的聊天方法,不再进行意图分析
         return False
     # 使用LLM进行意图分析
@@ -100,24 +102,35 @@ async def process_intent_result(conn, intent_result, original_text):
                 result = conn.func_handler.handle_llm_function_call(
                     conn, function_call_data
                 )
-                if result and function_name != "play_music":
-                    # 获取当前最新的文本索引
-                    text = result.response
-                    if text is None:
+                logger.bind(tag=TAG).debug(f"检测到Action : {result.action}")
+
+                if result:
+                    if result.action == Action.RESPONSE:  # 直接回复前端
+                        text = result.response
+                        if text is not None:
+                            speak_and_play(conn, text)
+                    elif result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
                         text = result.result
-                    if text is not None:
-                        text_index = (
-                            conn.tts_last_text_index + 1
-                            if hasattr(conn, "tts_last_text_index")
-                            else 0
-                        )
-                        conn.recode_first_last_text(text, text_index)
-                        future = conn.executor.submit(
-                            conn.speak_and_play, text, text_index
-                        )
-                        conn.llm_finish_task = True
-                        conn.tts_queue.put((future, text_index))
-                        conn.dialogue.put(Message(role="assistant", content=text))
+                        conn.dialogue.put(Message(role="tool", content=text))
+                        llm_result = conn.intent.replyResult(text, original_text)
+                        if llm_result is None:
+                            llm_result = text
+                        speak_and_play(conn, llm_result)
+                    elif (
+                        result.action == Action.NOTFOUND
+                        or result.action == Action.ERROR
+                    ):
+                        text = result.result
+                        if text is not None:
+                            speak_and_play(conn, text)
+                    elif function_name != "play_music":
+                        # For backward compatibility with original code
+                        # 获取当前最新的文本索引
+                        text = result.response
+                        if text is None:
+                            text = result.result
+                        if text is not None:
+                            speak_and_play(conn, text)
 
             # 将函数执行放在线程池中
             conn.executor.submit(process_function_call)
@@ -128,21 +141,12 @@ async def process_intent_result(conn, intent_result, original_text):
         return False
 
 
-def extract_text_in_brackets(s):
-    """
-    从字符串中提取中括号内的文字
-
-    :param s: 输入字符串
-    :return: 中括号内的文字，如果不存在则返回空字符串
-    """
-    left_bracket_index = s.find("[")
-    right_bracket_index = s.find("]")
-
-    if (
-        left_bracket_index != -1
-        and right_bracket_index != -1
-        and left_bracket_index < right_bracket_index
-    ):
-        return s[left_bracket_index + 1 : right_bracket_index]
-    else:
-        return ""
+def speak_and_play(conn, text):
+    text_index = (
+        conn.tts_last_text_index + 1 if hasattr(conn, "tts_last_text_index") else 0
+    )
+    conn.recode_first_last_text(text, text_index)
+    future = conn.executor.submit(conn.speak_and_play, text, text_index)
+    conn.llm_finish_task = True
+    conn.tts_queue.put((future, text_index))
+    conn.dialogue.put(Message(role="assistant", content=text))

main/xiaozhi-server/core/handle/iotHandle.py

@@ -317,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors):
         )
         conn.iot_descriptors[descriptor["name"]] = iot_descriptor
 
-        if conn.use_function_call_mode:
+        if conn.load_function_plugin:
             # 注册或获取设备类型
             type_id = register_device_type(descriptor)
             device_functions = device_type_registry.get_device_functions(type_id)

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -76,7 +76,7 @@ async def startToChat(conn, text):
 
     # 意图未被处理，继续常规聊天流程
     await send_stt_message(conn, text)
-    if conn.use_function_call_mode:
+    if conn.intent_type == "function_call":
         # 使用支持function calling的聊天方法
         conn.executor.submit(conn.chat_with_function_calling, text)
     else:

main/xiaozhi-server/core/providers/intent/base.py

@@ -9,18 +9,6 @@ logger = setup_logging()
 class IntentProviderBase(ABC):
     def __init__(self, config):
         self.config = config
-        self.intent_options = [
-            {
-                "name": "handle_exit_intent",
-                "desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
-            },
-            {
-                "name": "play_music",
-                "desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
-            },
-            {"name": "get_time", "desc": "获取今天日期或者当前时间信息"},
-            {"name": "continue_chat", "desc": "继续聊天"},
-        ]
 
     def set_llm(self, llm):
         self.llm = llm

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -15,57 +15,72 @@ class IntentProvider(IntentProviderBase):
     def __init__(self, config):
         super().__init__(config)
         self.llm = None
-        self.promot = self.get_intent_system_prompt()
+        self.promot = ""
         # 添加缓存管理
         self.intent_cache = {}  # 缓存意图识别结果
         self.cache_expiry = 600  # 缓存有效期10分钟
         self.cache_max_size = 100  # 最多缓存100个意图
+        self.history_count = 4  # 默认使用最近4条对话记录
 
-    def get_intent_system_prompt(self) -> str:
+    def get_intent_system_prompt(self, functions_list: str) -> str:
         """
-        根据配置的意图选项动态生成系统提示词
+        根据配置的意图选项和可用函数动态生成系统提示词
+        Args:
+            functions: 可用的函数列表，JSON格式字符串
         Returns:
             格式化后的系统提示词
         """
 
+        # 构建函数说明部分
+        functions_desc = "可用的函数列表：\n"
+        for func in functions_list:
+            func_info = func.get("function", {})
+            name = func_info.get("name", "")
+            desc = func_info.get("description", "")
+            params = func_info.get("parameters", {})
+
+            functions_desc += f"\n函数名: {name}\n"
+            functions_desc += f"描述: {desc}\n"
+
+            if params:
+                functions_desc += "参数:\n"
+                for param_name, param_info in params.get("properties", {}).items():
+                    param_desc = param_info.get("description", "")
+                    param_type = param_info.get("type", "")
+                    functions_desc += f"- {param_name} ({param_type}): {param_desc}\n"
+
+            functions_desc += "---\n"
+
         prompt = (
-            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
-            "<start>"
-            f"{str(self.intent_options)}"
-            "<end>\n"
-            "处理步骤:"
-            "1. 思考意图类型，生成function_call格式"
-            "\n\n"
-            "返回格式示例：\n"
-            '1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n'
-            '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
-            '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n'
-            '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n'
-            "\n"
-            "注意:\n"
-            '- 播放音乐：无歌名时，song_name设为"random"\n'
-            "- 如果没有明显的意图，应按照继续聊天意图处理\n"
-            "- 只返回纯JSON，不要任何其他内容\n"
-            "\n"
-            "示例分析:\n"
+            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            f"{functions_desc}\n"
+            "处理步骤:\n"
+            "1. 分析用户输入，确定用户意图\n"
+            "2. 从可用函数列表中选择最匹配的函数\n"
+            "3. 如果找到匹配的函数，生成对应的function_call 格式\n"
+            '4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
+            "返回格式要求：\n"
+            "1. 必须返回纯JSON格式\n"
+            "2. 必须包含function_call字段\n"
+            "3. function_call必须包含name字段\n"
+            "4. 如果函数需要参数，必须包含arguments字段\n\n"
+            "示例：\n"
             "```\n"
-            "用户: 你也太搞笑了\n"
-            '返回: {"function_call": {"name": "continue_chat"}}\n'
-            "```\n"
-            "```\n"
-            "用户: 现在是几号了?现在几点了？\n"
+            "用户: 现在几点了？\n"
             '返回: {"function_call": {"name": "get_time"}}\n'
             "```\n"
             "```\n"
-            "用户: 我们明天再聊吧\n"
-            '返回: {"function_call": {"name": "handle_exit_intent"}}\n'
+            "用户: 我想结束对话\n"
+            '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
             "```\n"
             "```\n"
-            "用户: 播放中秋月\n"
-            '返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n'
-            "```\n"
-            "```\n"
-            "可用的音乐名称:\n"
+            "用户: 你好啊\n"
+            '返回: {"function_call": {"name": "continue_chat"}}\n'
+            "```\n\n"
+            "注意：\n"
+            "1. 只返回JSON格式，不要包含任何其他文字\n"
+            '2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
+            "3. 确保返回的JSON格式正确，包含所有必要的字段\n"
         )
         return prompt
 
@@ -90,6 +105,14 @@ class IntentProvider(IntentProviderBase):
             for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]:
                 del self.intent_cache[key]
 
+    def replyResult(self, text: str, original_text: str):
+        llm_result = self.llm.response_no_stream(
+            system_prompt=text,
+            user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+            + original_text,
+        )
+        return llm_result
+
     async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
@@ -118,22 +141,35 @@ class IntentProvider(IntentProviderBase):
         # 清理缓存
         self.clean_cache()
 
-        # 构建用户最后一句话的提示
-        msgStr = ""
+        if self.promot == "":
+            if hasattr(conn, "func_handler"):
+                functions = conn.func_handler.get_functions()
+                self.promot = self.get_intent_system_prompt(functions)
 
-        # 只使用最后两句即可
-        if len(dialogue_history) >= 2:
-            # 保证最少有两句话的时候处理
-            msgStr += f"{dialogue_history[-2].role}: {dialogue_history[-2].content}\n"
-        msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n"
-
-        msgStr += f"User: {text}\n"
-        user_prompt = f"当前的对话如下：\n{msgStr}"
         music_config = initialize_music_handler(conn)
         music_file_names = music_config["music_file_names"]
-        prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
+        prompt_music = f"{self.promot}\n<musicNames>{music_file_names}\n</musicNames>"
+
+        devices = conn.config["plugins"]["home_assistant"].get("devices", [])
+        if len(devices) > 0:
+            hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
+            for device in devices:
+                hass_prompt += device + "\n"
+            prompt_music += hass_prompt
+
         logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
 
+        # 构建用户对话历史的提示
+        msgStr = ""
+
+        # 获取最近的对话历史
+        start_idx = max(0, len(dialogue_history) - self.history_count)
+        for i in range(start_idx, len(dialogue_history)):
+            msgStr += f"{dialogue_history[i].role}: {dialogue_history[i].content}\n"
+
+        msgStr += f"User: {text}\n"
+        user_prompt = f"current dialogue:\n{msgStr}"
+
         # 记录预处理完成时间
         preprocess_time = time.time() - total_start_time
         logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")

main/xiaozhi-server/core/utils/dialogue.py

@@ -33,7 +33,13 @@ class Dialogue:
             dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
         elif m.role == "tool":
             dialogue.append(
-                {"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content}
+                {
+                    "role": m.role,
+                    "tool_call_id": (
+                        str(uuid.uuid4()) if m.tool_call_id is None else m.tool_call_id
+                    ),
+                    "content": m.content,
+                }
             )
         else:
             dialogue.append({"role": m.role, "content": m.content})