diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ad794965..98c3d058 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -175,6 +175,13 @@ Intent: # 如果这里不填,则会默认使用selected_module.LLM的模型作为意图识别的思考模型 # 如果你的不想使用selected_module.LLM意图识别,这里最好使用独立的LLM作为意图识别,例如使用免费的ChatGLMLLM llm: ChatGLMLLM + # plugins_func/functions下的模块,可以通过配置,选择加载哪个模块,加载后对话支持相应的function调用 + # 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件,请勿重复加载 + # 下面是加载查天气、角色切换、加载查新闻的插件示例 + functions: + - get_weather + - get_news_from_newsnow + - play_music function_call: # 不需要动type type: function_call diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index b49917ba..0a5cd8e4 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -58,9 +58,9 @@ class ConnectionHandler: self.config = copy.deepcopy(config) self.session_id = str(uuid.uuid4()) self.logger = setup_logging() - self.auth = AuthMiddleware(config) self.server = server # 保存server实例的引用 + self.auth = AuthMiddleware(config) self.need_bind = False self.bind_code = None self.read_config_from_api = self.config.get("read_config_from_api", False) @@ -128,8 +128,10 @@ class ConnectionHandler: if len(cmd) > self.max_cmd_length: self.max_cmd_length = len(cmd) - self.close_after_chat = False # 是否在聊天结束后关闭连接 - self.use_function_call_mode = False + # 是否在聊天结束后关闭连接 + self.close_after_chat = False + self.load_function_plugin = False + self.intent_type = "nointent" self.timeout_task = None self.timeout_seconds = ( @@ -365,11 +367,11 @@ class ConnectionHandler: self.memory.init_memory(self.device_id, self.llm) def _initialize_intent(self): - if ( - self.config["Intent"][self.config["selected_module"]["Intent"]]["type"] - == "function_call" - ): - self.use_function_call_mode = True + self.intent_type = self.config["Intent"][ + self.config["selected_module"]["Intent"] + ]["type"] + if self.intent_type == "function_call" or self.intent_type == "intent_llm": + self.load_function_plugin = True """初始化意图识别模块""" # 获取意图识别配置 intent_config = self.config["Intent"] @@ -757,7 +759,13 @@ class ConnectionHandler: ) self.dialogue.put( - Message(role="tool", tool_call_id=function_id, content=text) + Message( + role="tool", + tool_call_id=( + str(uuid.uuid4()) if function_id is None else function_id + ), + content=text, + ) ) self.chat_with_function_calling(text, tool_call=True) elif result.action == Action.NOTFOUND or result.action == Action.ERROR: diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 5b00f2ef..383e4e65 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -19,7 +19,7 @@ async def handle_user_intent(conn, text): if await checkWakeupWords(conn, text): return True - if conn.use_function_call_mode: + if conn.intent_type == "function_call": # 使用支持function calling的聊天方法,不再进行意图分析 return False # 使用LLM进行意图分析 @@ -103,7 +103,7 @@ async def process_intent_result(conn, intent_result, original_text): conn, function_call_data ) logger.bind(tag=TAG).debug(f"检测到Action : {result.action}") - + if result: if result.action == Action.RESPONSE: # 直接回复前端 text = result.response @@ -118,14 +118,19 @@ async def process_intent_result(conn, intent_result, original_text): conn.speak_and_play, text, text_index ) conn.llm_finish_task = True - conn.tts_queue.put(future) + conn.tts_queue.put((future, text_index)) conn.dialogue.put(Message(role="assistant", content=text)) elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result if text is not None and len(text) > 0: conn.dialogue.put(Message(role="tool", content=text)) - conn.executor.submit(conn.chat_with_function_calling, text, True) - elif result.action == Action.NOTFOUND or result.action == Action.ERROR: + conn.executor.submit( + conn.chat_with_function_calling, text, True + ) + elif ( + result.action == Action.NOTFOUND + or result.action == Action.ERROR + ): text = result.result if text is not None: text_index = ( @@ -157,7 +162,7 @@ async def process_intent_result(conn, intent_result, original_text): conn.speak_and_play, text, text_index ) conn.llm_finish_task = True - conn.tts_queue.put(future) + conn.tts_queue.put((future, text_index)) conn.dialogue.put(Message(role="assistant", content=text)) # 将函数执行放在线程池中 diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index 12432102..5b1bb9bb 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -317,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors): ) conn.iot_descriptors[descriptor["name"]] = iot_descriptor - if conn.use_function_call_mode: + if conn.load_function_plugin: # 注册或获取设备类型 type_id = register_device_type(descriptor) device_functions = device_type_registry.get_device_functions(type_id) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 383616c3..48ec24b2 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -76,7 +76,7 @@ async def startToChat(conn, text): # 意图未被处理,继续常规聊天流程 await send_stt_message(conn, text) - if conn.use_function_call_mode: + if conn.intent_type == "function_call": # 使用支持function calling的聊天方法 conn.executor.submit(conn.chat_with_function_calling, text) else: diff --git a/main/xiaozhi-server/core/providers/intent/base.py b/main/xiaozhi-server/core/providers/intent/base.py index 040dd025..f6df10dd 100644 --- a/main/xiaozhi-server/core/providers/intent/base.py +++ b/main/xiaozhi-server/core/providers/intent/base.py @@ -9,18 +9,6 @@ logger = setup_logging() class IntentProviderBase(ABC): def __init__(self, config): self.config = config - self.intent_options = [ - { - "name": "handle_exit_intent", - "desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候", - }, - { - "name": "play_music", - "desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图", - }, - {"name": "get_time", "desc": "获取今天日期或者当前时间信息"}, - {"name": "continue_chat", "desc": "继续聊天"}, - ] def set_llm(self, llm): self.llm = llm diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index 05d01c10..d8b18916 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -15,57 +15,71 @@ class IntentProvider(IntentProviderBase): def __init__(self, config): super().__init__(config) self.llm = None - self.promot = self.get_intent_system_prompt() + self.promot = "" # 添加缓存管理 self.intent_cache = {} # 缓存意图识别结果 self.cache_expiry = 600 # 缓存有效期10分钟 self.cache_max_size = 100 # 最多缓存100个意图 - def get_intent_system_prompt(self) -> str: + def get_intent_system_prompt(self, functions_list: str) -> str: """ - 根据配置的意图选项动态生成系统提示词 + 根据配置的意图选项和可用函数动态生成系统提示词 + Args: + functions: 可用的函数列表,JSON格式字符串 Returns: 格式化后的系统提示词 """ + # 构建函数说明部分 + functions_desc = "可用的函数列表:\n" + for func in functions_list: + func_info = func.get("function", {}) + name = func_info.get("name", "") + desc = func_info.get("description", "") + params = func_info.get("parameters", {}) + + functions_desc += f"\n函数名: {name}\n" + functions_desc += f"描述: {desc}\n" + + if params: + functions_desc += "参数:\n" + for param_name, param_info in params.get("properties", {}).items(): + param_desc = param_info.get("description", "") + param_type = param_info.get("type", "") + functions_desc += f"- {param_name} ({param_type}): {param_desc}\n" + + functions_desc += "---\n" + prompt = ( - "你是一个意图识别助手。请分析用户的最后一句话,判断用户意图属于以下哪一类:\n" - "" - f"{str(self.intent_options)}" - "\n" - "处理步骤:" - "1. 思考意图类型,生成function_call格式" - "\n\n" - "返回格式示例:\n" - '1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n' - '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' - '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n' - '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n' - "\n" - "注意:\n" - '- 播放音乐:无歌名时,song_name设为"random"\n' - "- 如果没有明显的意图,应按照继续聊天意图处理\n" - "- 只返回纯JSON,不要任何其他内容\n" - "\n" - "示例分析:\n" + "你是一个意图识别助手。请分析用户的最后一句话,判断用户意图并调用相应的函数。\n\n" + f"{functions_desc}\n" + "处理步骤:\n" + "1. 分析用户输入,确定用户意图\n" + "2. 从可用函数列表中选择最匹配的函数\n" + "3. 如果找到匹配的函数,生成对应的function_call 格式\n" + '4. 如果没有找到匹配的函数,返回{"function_call": {"name": "continue_chat"}}\n\n' + "返回格式要求:\n" + "1. 必须返回纯JSON格式\n" + "2. 必须包含function_call字段\n" + "3. function_call必须包含name字段\n" + "4. 如果函数需要参数,必须包含arguments字段\n\n" + "示例:\n" "```\n" - "用户: 你也太搞笑了\n" - '返回: {"function_call": {"name": "continue_chat"}}\n' - "```\n" - "```\n" - "用户: 现在是几号了?现在几点了?\n" + "用户: 现在几点了?\n" '返回: {"function_call": {"name": "get_time"}}\n' "```\n" "```\n" - "用户: 我们明天再聊吧\n" - '返回: {"function_call": {"name": "handle_exit_intent"}}\n' + "用户: 我想结束对话\n" + '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' "```\n" "```\n" - "用户: 播放中秋月\n" - '返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n' - "```\n" - "```\n" - "可用的音乐名称:\n" + "用户: 你好啊\n" + '返回: {"function_call": {"name": "continue_chat"}}\n' + "```\n\n" + "注意:\n" + "1. 只返回JSON格式,不要包含任何其他文字\n" + '2. 如果没有找到匹配的函数,返回{"function_call": {"name": "continue_chat"}}\n' + "3. 确保返回的JSON格式正确,包含所有必要的字段\n" ) return prompt @@ -118,6 +132,24 @@ class IntentProvider(IntentProviderBase): # 清理缓存 self.clean_cache() + if self.promot == "": + if hasattr(conn, "func_handler"): + functions = conn.func_handler.get_functions() + self.promot = self.get_intent_system_prompt(functions) + + music_config = initialize_music_handler(conn) + music_file_names = music_config["music_file_names"] + prompt_music = f"{self.promot}\n{music_file_names}\n" + + devices = conn.config["plugins"]["home_assistant"].get("devices", []) + if len(devices) > 0: + hass_prompt = "\n下面是我家智能设备列表(位置,设备名,entity_id),可以通过homeassistant控制\n" + for device in devices: + hass_prompt += device + "\n" + prompt_music += hass_prompt + + logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}") + # 构建用户最后一句话的提示 msgStr = "" @@ -128,11 +160,7 @@ class IntentProvider(IntentProviderBase): msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n" msgStr += f"User: {text}\n" - user_prompt = f"当前的对话如下:\n{msgStr}" - music_config = initialize_music_handler(conn) - music_file_names = music_config["music_file_names"] - prompt_music = f"{self.promot}\n{music_file_names}\n" - logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}") + user_prompt = f"current dialogue:\n{msgStr}" # 记录预处理完成时间 preprocess_time = time.time() - total_start_time diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index 8b79a35a..b6fa541b 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -33,7 +33,13 @@ class Dialogue: dialogue.append({"role": m.role, "tool_calls": m.tool_calls}) elif m.role == "tool": dialogue.append( - {"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content} + { + "role": m.role, + "tool_call_id": ( + str(uuid.uuid4()) if m.tool_call_id is None else m.tool_call_id + ), + "content": m.content, + } ) else: dialogue.append({"role": m.role, "content": m.content}) diff --git a/main/xiaozhi-server/plugins_func/functions/hass_init.py b/main/xiaozhi-server/plugins_func/functions/hass_init.py index 06d7c8bc..3a5458d3 100644 --- a/main/xiaozhi-server/plugins_func/functions/hass_init.py +++ b/main/xiaozhi-server/plugins_func/functions/hass_init.py @@ -8,7 +8,7 @@ HASS_CACHE = {} def append_devices_to_prompt(conn): - if conn.use_function_call_mode: + if conn.intent_type == "function_call": funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( "functions", [] ) @@ -27,7 +27,7 @@ def append_devices_to_prompt(conn): def initialize_hass_handler(conn): global HASS_CACHE if HASS_CACHE == {}: - if conn.use_function_call_mode: + if conn.load_function_plugin: funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( "functions", [] )