Function插件自动装载 (#351)

* function call功能完善，增加天气查询，支持插件式扩展

* 增加角色切换功能，通过切换system提示词，修改角色认知

* 增加插件管理系统，可以通过语音加载和卸载插件

* docs: 添加命令操作 (#329)

* docs: 添加命令操作

* feat: 添加 docker-setup.sh 脚本以简化服务端部署

- 新增 docker-setup.sh 脚本，自动创建目录结构、下载语音识别模型和配置文件，并检查文件完整性。
- 更新 Deployment.md 文档，提供一键执行脚本的说明和使用示例。

* docs: 更新 Deployment.md，添加环境访问 GitHub 的注意事项

* refactor: 更新 docker-setup.sh 脚本以支持多操作系统下载命令

- 修改脚本以检测操作系统类型，并根据不同系统选择合适的下载命令（curl 或 wget）。
- 优化错误处理，确保在下载失败时提供清晰的提示信息。
- 更新 Deployment.md 文档，调整懒人脚本的使用说明，增加手动部署的步骤。

* Update docker-setup.sh

* Update docker-setup.sh

---------

Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com>

* update:优化插件加载的配置提示

* update:增加自动安装脚本的操作说明

* update:优化插件配置，去掉旧版本的时间设定

---------

Co-authored-by: 玄凤科技 <eric230308@gmail.com>
Co-authored-by: TinsFox <fox@tinsfox.com>
Co-authored-by: hrz <1710360675@qq.com>

main/xiaozhi-server/core/connection.py

@@ -15,10 +15,12 @@ from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_fro
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage
 from core.handle.receiveAudioHandle import handleAudioMessage
-from core.handle.intentHandler import Action, get_functions, handle_llm_function_call
+from core.handle.functionHandler import FunctionHandler
+from plugins_func.register import Action
 from config.private_config import PrivateConfig
 from core.auth import AuthMiddleware, AuthenticationError
 from core.utils.auth_code_gen import AuthCodeGenerator
+import plugins_func.loadplugins  
 
 TAG = __name__
 
@@ -94,6 +96,8 @@ class ConnectionHandler:
         self.use_function_call_mode = False
         if self.config["selected_module"]["Intent"] == 'function_call':
             self.use_function_call_mode = True
+        
+        self.func_handler = FunctionHandler(self.config)
 
     async def handle_connection(self, ws):
         try:
@@ -185,11 +189,14 @@ class ConnectionHandler:
         self.prompt = self.config["prompt"]
         if self.private_config:
             self.prompt = self.private_config.private_config.get("prompt", self.prompt)
-        # 赋予LLM时间观念
-        if "{date_time}" in self.prompt:
-            date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
-            self.prompt = self.prompt.replace("{date_time}", date_time)
         self.dialogue.put(Message(role="system", content=self.prompt))
+    
+    def change_system_prompt(self, prompt):
+        self.prompt = prompt
+        # 找到原来的role==system，替换原来的系统提示
+        for m in self.dialogue.dialogue:
+            if m.role == "system":
+                m.content = prompt
 
     async def _check_and_broadcast_auth_code(self):
         """检查设备绑定状态并广播认证码"""
@@ -289,7 +296,7 @@ class ConnectionHandler:
         self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
         return True
 
-    def chat_with_function_calling(self, query):
+    def chat_with_function_calling(self, query, tool_call = False):
         self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
         """Chat with function calling for intent detection using streaming"""
         if self.isNeedAuth():
@@ -297,11 +304,12 @@ class ConnectionHandler:
             future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
             future.result()
             return True
-
-        self.dialogue.put(Message(role="user", content=query))
+        
+        if not tool_call:
+            self.dialogue.put(Message(role="user", content=query))
 
         # Define intent functions
-        functions = get_functions()
+        functions = self.func_handler.get_functions()
 
         response_message = []
         processed_chars = 0  # 跟踪已处理的字符位置
@@ -337,7 +345,7 @@ class ConnectionHandler:
         for response in llm_responses:
             content, tools_call = response
             if content is not None and len(content)>0:
-                if len(response_message)<=0 and content=="```":
+                if len(response_message)<=0 and (content=="```" or "<tool_call>" in content):
                     tool_call_flag = True
 
             if tools_call is not None:
@@ -385,7 +393,38 @@ class ConnectionHandler:
                             self.tts_queue.put(future)
                             processed_chars += len(segment_text_raw)  # 更新已处理字符位置
 
-        # 处理最后剩余的文本
+        # 处理function call
+        if tool_call_flag:
+            bHasError = False
+            if function_id is None:
+                a = extract_json_from_string(content_arguments)
+                if a is not None:
+                    try:
+                        content_arguments_json = json.loads(a)
+                        function_name = content_arguments_json["name"]
+                        function_arguments = json.dumps(content_arguments_json["arguments"], ensure_ascii=False)
+                        function_id = str(uuid.uuid4().hex)
+                    except Exception as e:
+                        bHasError = True
+                        response_message.append(a)
+                else:
+                    bHasError = True
+                    response_message.append(content_arguments)
+                if bHasError:
+                    self.logger.bind(tag=TAG).error(f"function call error: {content_arguments}")
+                else:
+                    function_arguments = json.loads(function_arguments)
+            if not bHasError:
+                self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
+                function_call_data = {
+                    "name": function_name,
+                    "id": function_id,
+                    "arguments": function_arguments
+                }
+                result = self.func_handler.handle_llm_function_call(self, function_call_data)
+                self._handle_function_result(result, function_call_data, text_index+1)
+
+         # 处理最后剩余的文本
         full_text = "".join(response_message)
         remaining_text = full_text[processed_chars:]
         if remaining_text:
@@ -400,27 +439,6 @@ class ConnectionHandler:
         if len(response_message)>0:
             self.dialogue.put(Message(role="assistant", content="".join(response_message)))
 
-        # 处理function call
-        if tool_call_flag:
-            if function_id is None:
-                a = extract_json_from_string(content_arguments)
-                if a is not None:
-                    content_arguments_json = json.loads(a)
-                    function_name = content_arguments_json["function_name"]
-                    function_arguments = json.dumps(content_arguments_json["args"], ensure_ascii=False)
-                    function_id = str(uuid.uuid4().hex)
-                else:
-                    return []
-                function_arguments = json.loads(function_arguments)
-            self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
-            function_call_data = {
-                "name": function_name,
-                "id": function_id,
-                "arguments": function_arguments
-            }
-            result = handle_llm_function_call(self, function_call_data)
-            self._handle_function_result(result, function_call_data, text_index+1)
-
         self.llm_finish_task = True
         self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
 
@@ -434,7 +452,20 @@ class ConnectionHandler:
             self.tts_queue.put(future)
             self.dialogue.put(Message(role="assistant", content=text))
         if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
-            text = result.response
+            
+            text = result.result
+            if text is not None and len(text) > 0:
+                function_id = function_call_data["id"]
+                function_name = function_call_data["name"]
+                function_arguments = function_call_data["arguments"]
+                self.dialogue.put(Message(role='assistant',
+                                            tool_calls=[{"id": function_id, 
+                                                        "function": {"arguments": function_arguments,"name": function_name},
+                                                        "type": 'function', 
+                                                        "index": 0}]))
+
+                self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
+                self.chat_with_function_calling(text, tool_call=True)
         if result.action == Action.NOTFOUND:
             text = result.response
 

main/xiaozhi-server/core/handle/functionHandler.py

@@ -0,0 +1,83 @@
+import asyncio
+from enum import Enum
+
+from config.logger import setup_logging
+import json
+from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+TAG = __name__
+logger = setup_logging()
+
+
+class FunctionHandler:
+    def __init__(self, config):
+        self.config = config
+        self.function_registry = FunctionRegistry()
+        self.register_nessary_functions()
+        self.register_config_functions()
+        self.functions_desc = self.function_registry.get_all_function_desc()
+        func_names = self.current_support_functions()
+        self.modify_plugin_loader_des(func_names)
+
+    def modify_plugin_loader_des(self, func_names):
+        if "plugin_loader" not in func_names:
+            return
+        # 可编辑的列表中去掉plugin_loader
+        surport_plugins = [func for func in func_names if func != "plugin_loader"]
+        func_names = ",".join(surport_plugins)
+        for function_desc in self.functions_desc:
+            if function_desc["function"]["name"] == "plugin_loader":
+                function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names)
+                break
+        
+    def upload_functions_desc(self):
+        self.functions_desc = self.function_registry.get_all_function_desc()
+
+    def current_support_functions(self):
+        func_names = []
+        for func in self.functions_desc:
+            func_names.append(func["function"]["name"])
+        # 打印当前支持的函数列表
+        logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
+        return func_names
+
+    def get_functions(self):
+        """获取功能调用配置"""
+        return self.functions_desc
+
+    def register_nessary_functions(self):
+        """注册必要的函数"""
+        self.function_registry.register_function("handle_exit_intent")
+        self.function_registry.register_function("play_music")
+        self.function_registry.register_function("plugin_loader")
+        self.function_registry.register_function("get_time")
+
+    def register_config_functions(self):
+        """注册配置中的函数,可以不同客户端使用不同的配置"""
+        for func in self.config["Intent"]["function_call"].get("functions", []):
+            self.function_registry.register_function(func)
+
+    def get_function(self, name):
+        return self.function_registry.get_function(name)
+
+    def handle_llm_function_call(self, conn, function_call_data):
+        try:
+            function_name = function_call_data["name"]
+            funcItem = self.get_function(function_name)
+            if not funcItem:
+                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+            func = funcItem.func
+            arguments = function_call_data["arguments"]
+            arguments = json.loads(arguments) if arguments else {}
+            logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
+            if funcItem.type == ToolType.SYSTEM_CTL:
+                return func(conn, **arguments)
+            elif funcItem.type == ToolType.WAIT:
+                return func(**arguments)
+            elif funcItem.type == ToolType.CHANGE_SYS_PROMPT:
+                return func(conn, **arguments)
+            else:
+                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+
+        return None

main/xiaozhi-server/core/handle/intentHandler.py

@@ -1,80 +1,12 @@
 from config.logger import setup_logging
 import json
 from core.handle.sendAudioHandle import send_stt_message
-from core.utils.dialogue import Message
 from core.utils.util import remove_punctuation_and_length
-from config.functionCallConfig import FunctionCallConfig
-import asyncio
-from enum import Enum
 
 TAG = __name__
 logger = setup_logging()
 
 
-class Action(Enum):
-    NOTFOUND = (0, "没有找到函数")
-    NONE = (1, "啥也不干")
-    RESPONSE = (2, "直接回复")
-    REQLLM = (3, "调用函数后再请求llm生成回复")
-
-    def __init__(self, code, message):
-        self.code = code
-        self.message = message
-
-
-class ActionResponse:
-    def __init__(self, action: Action, result, response):
-        self.action = action  # 动作类型
-        self.result = result  # 动作产生的结果
-        self.response = response  # 直接回复的内容
-
-
-def get_functions():
-    """获取功能调用配置"""
-    return FunctionCallConfig
-
-
-def handle_llm_function_call(conn, function_call_data):
-    try:
-        function_name = function_call_data["name"]
-
-        if function_name == "handle_exit_intent":
-            # 处理退出意图
-            try:
-                say_goodbye = json.loads(function_call_data["arguments"]).get("say_goodbye", "再见")
-                conn.close_after_chat = True
-                logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
-                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
-            except Exception as e:
-                logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
-
-        elif function_name == "play_music":
-            # 处理音乐播放意图
-            try:
-                song_name = "random"
-                arguments = function_call_data["arguments"]
-                if arguments is not None and len(arguments) > 0:
-                    args = json.loads(arguments)
-                    song_name = args.get("song_name", "random")
-                music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
-
-                # 执行音乐播放命令
-                future = asyncio.run_coroutine_threadsafe(
-                    conn.music_handler.handle_music_command(conn, music_intent),
-                    conn.loop
-                )
-                future.result()
-                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
-            except Exception as e:
-                logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
-        else:
-            return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="没有找到对应的函数处理相对于的功能呢，你可以需要添加预设的对应函数处理呢")
-    except Exception as e:
-        logger.bind(tag=TAG).error(f"处理function call错误: {e}")
-
-    return None
-
-
 async def handle_user_intent(conn, text):
     """
     Handle user intent before starting chat

main/xiaozhi-server/core/utils/dialogue.py

@@ -4,10 +4,12 @@ from datetime import datetime
 
 
 class Message:
-    def __init__(self, role: str, content: str = None, uniq_id: str = None):
+    def __init__(self, role: str, content: str = None, uniq_id: str = None, tool_calls = None, tool_call_id=None):
         self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
         self.role = role
         self.content = content
+        self.tool_calls = tool_calls
+        self.tool_call_id = tool_call_id
 
 
 class Dialogue:
@@ -19,10 +21,18 @@ class Dialogue:
     def put(self, message: Message):
         self.dialogue.append(message)
 
+    def getMessages(self, m, dialogue):
+        if m.tool_calls is not None:
+            dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
+        elif m.role == "tool":
+            dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content})
+        else:
+            dialogue.append({"role": m.role, "content": m.content})
+
     def get_llm_dialogue(self) -> List[Dict[str, str]]:
         dialogue = []
         for m in self.dialogue:
-            dialogue.append({"role": m.role, "content": m.content})
+            self.getMessages(m, dialogue)
         return dialogue
 
     def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
@@ -46,8 +56,8 @@ class Dialogue:
             dialogue.append({"role": "system", "content": enhanced_system_prompt})
 
         # 添加用户和助手的对话
-        for msg in self.dialogue:
-            if msg.role != "system":  # 跳过原始的系统消息
-                dialogue.append({"role": msg.role, "content": msg.content})
+        for m in self.dialogue:
+            if m.role != "system":  # 跳过原始的系统消息
+                self.getMessages(m, dialogue)
 
         return dialogue

main/xiaozhi-server/core/utils/util.py

@@ -1,5 +1,6 @@
 import os
 import json
+from datetime import datetime
 import yaml
 import socket
 import subprocess

main/xiaozhi-server/core/websocket_server.py

@@ -66,7 +66,7 @@ class WebSocketServer:
         host = server_config["ip"]
         port = server_config["port"]
         selected_module = self.config.get("selected_module")
-        self.logger.bind(tag=TAG).info(f"selected_module: {selected_module}")
+        self.logger.bind(tag=TAG).info(f"selected_module values: {', '.join(selected_module.values())}")
 
         self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
         self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")