From 6ba63db47fa07d7e243e667c40f7e2a791751f35 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Mon, 31 Mar 2025 22:54:37 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E5=B0=86=E8=80=97=E6=97=B6=E6=9C=80?= =?UTF-8?q?=E5=A4=A7=E7=9A=84iot=E5=A4=84=E7=90=86=E6=94=B9=E6=88=90?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=202=E3=80=81iot=E5=BE=AA=E7=8E=AF5=E6=AC=A1?= =?UTF-8?q?=E7=AD=89=E5=BE=85func=5Fhandler=E5=8A=A0=E8=BD=BD=E5=AE=8C?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 1 + .../core/handle/functionHandler.py | 19 ++- .../xiaozhi-server/core/handle/helloHandle.py | 22 ++- main/xiaozhi-server/core/handle/iotHandle.py | 147 ++++++++++++------ .../core/handle/receiveAudioHandle.py | 25 ++- main/xiaozhi-server/core/handle/textHandle.py | 7 +- 6 files changed, 150 insertions(+), 71 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index eac944a3..df0c2aa3 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -93,6 +93,7 @@ class ConnectionHandler: # iot相关变量 self.iot_descriptors = {} + self.func_handler = None self.cmd_exit = self.config["CMD_exit"] self.max_cmd_length = 0 diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index 9cad9499..cec9bd25 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -17,6 +17,7 @@ class FunctionHandler: self.functions_desc = self.function_registry.get_all_function_desc() func_names = self.current_support_functions() self.modify_plugin_loader_des(func_names) + self.finish_init = True def modify_plugin_loader_des(self, func_names): if "plugin_loader" not in func_names: @@ -26,8 +27,9 @@ class FunctionHandler: func_names = ",".join(surport_plugins) for function_desc in self.functions_desc: if function_desc["function"]["name"] == "plugin_loader": - function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", - func_names) + function_desc["function"]["description"] = function_desc["function"][ + "description" + ].replace("[plugins]", func_names) break def upload_functions_desc(self): @@ -69,19 +71,26 @@ class FunctionHandler: function_name = function_call_data["name"] funcItem = self.get_function(function_name) if not funcItem: - return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="") + return ActionResponse( + action=Action.NOTFOUND, result="没有找到对应的函数", response="" + ) func = funcItem.func arguments = function_call_data["arguments"] arguments = json.loads(arguments) if arguments else {} logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}") - if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL: + if ( + funcItem.type == ToolType.SYSTEM_CTL + or funcItem.type == ToolType.IOT_CTL + ): return func(conn, **arguments) elif funcItem.type == ToolType.WAIT: return func(**arguments) elif funcItem.type == ToolType.CHANGE_SYS_PROMPT: return func(conn, **arguments) else: - return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="") + return ActionResponse( + action=Action.NOTFOUND, result="没有找到对应的函数", response="" + ) except Exception as e: logger.bind(tag=TAG).error(f"处理function call错误: {e}") diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 41e2b6b0..6cc0bceb 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -10,8 +10,14 @@ import time logger = setup_logging() -WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10, - "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"], "text": ""} +WAKEUP_CONFIG = { + "dir": "config/assets/", + "file_name": "wakeup_words", + "create_time": time.time(), + "refresh_time": 10, + "words": ["你好小智", "你好啊小智", "小智你好", "小智"], + "text": "", +} async def handleHelloMessage(conn): @@ -19,7 +25,9 @@ async def handleHelloMessage(conn): async def checkWakeupWords(conn, text): - enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"] + enable_wakeup_words_response_cache = conn.config[ + "enable_wakeup_words_response_cache" + ] """是否开启唤醒词加速""" if not enable_wakeup_words_response_cache: return False @@ -69,12 +77,14 @@ async def wakeupWordsResponse(conn): if tts_file is not None and os.path.exists(tts_file): file_type = os.path.splitext(tts_file)[1] if file_type: - file_type = file_type.lstrip('.') + file_type = file_type.lstrip(".") old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) if old_file is not None: os.remove(old_file) """将文件挪到"wakeup_words.mp3""" - shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" + - WAKEUP_CONFIG["file_name"] + "." + file_type) + shutil.move( + tts_file, + WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type, + ) WAKEUP_CONFIG["create_time"] = time.time() WAKEUP_CONFIG["text"] = result diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index 1f75330d..db84886a 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -1,7 +1,13 @@ import json import asyncio from config.logger import setup_logging -from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType +from plugins_func.register import ( + device_type_registry, + register_function, + ActionResponse, + Action, + ToolType, +) TAG = __name__ logger = setup_logging() @@ -14,10 +20,13 @@ def wrap_async_function(async_func): try: # 获取连接对象（第一个参数） conn = args[0] - if not hasattr(conn, 'loop'): + if not hasattr(conn, "loop"): logger.bind(tag=TAG).error("Connection对象没有loop属性") - return ActionResponse(Action.ERROR, "Connection对象没有loop属性", - "执行操作时出错: Connection对象没有loop属性") + return ActionResponse( + Action.ERROR, + "Connection对象没有loop属性", + "执行操作时出错: Connection对象没有loop属性", + ) # 使用conn对象中的事件循环 loop = conn.loop @@ -37,11 +46,14 @@ def create_iot_function(device_name, method_name, method_info): 根据IOT设备描述生成通用的控制函数 """ - async def iot_control_function(conn, response_success=None, response_failure=None, **params): + async def iot_control_function( + conn, response_success=None, response_failure=None, **params + ): try: # 打印响应参数 logger.bind(tag=TAG).info( - f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'") + f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'" + ) # 发送控制命令 await send_iot_conn(conn, device_name, method_name, params) @@ -51,14 +63,15 @@ def create_iot_function(device_name, method_name, method_info): # 生成结果信息 result = f"{device_name}的{method_name}操作执行成功" - # 处理响应中可能的占位符 response = response_success # 替换{value}占位符 for param_name, param_value in params.items(): # 先尝试直接替换参数值 if "{" + param_name + "}" in response: - response = response.replace("{" + param_name + "}", str(param_value)) + response = response.replace( + "{" + param_name + "}", str(param_value) + ) # 如果有{value}占位符，用相关参数替换 if "{value}" in response: @@ -86,7 +99,8 @@ def create_iot_query_function(device_name, prop_name, prop_info): try: # 打印响应参数 logger.bind(tag=TAG).info( - f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'") + f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'" + ) value = await get_iot_status(conn, device_name, prop_name) @@ -126,7 +140,7 @@ class IotDescriptor: # 根据描述创建属性 for key, value in properties.items(): property_item = globals()[key] = {} - property_item['name'] = key + property_item["name"] = key property_item["description"] = value["description"] if value["type"] == "number": property_item["value"] = 0 @@ -140,7 +154,7 @@ class IotDescriptor: for key, value in methods.items(): method = globals()[key] = {} method["description"] = value["description"] - method['name'] = key + method["name"] = key for k, v in value["parameters"].items(): method[k] = {} method[k]["description"] = v["description"] @@ -177,19 +191,21 @@ def register_device_type(descriptor): "properties": { "response_success": { "type": "string", - "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值" + "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值", }, "response_failure": { "type": "string", - "description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'" - } + "description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'", + }, }, - "required": ["response_success", "response_failure"] - } - } + "required": ["response_success", "response_failure"], + }, + }, } query_func = create_iot_query_function(device_name, prop_name, prop_info) - decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func) + decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)( + query_func + ) functions[func_name] = decorated_func # 为每个方法创建控制函数 @@ -200,22 +216,24 @@ def register_device_type(descriptor): parameters = { param_name: { "type": param_info["type"], - "description": param_info["description"] + "description": param_info["description"], } for param_name, param_info in method_info["parameters"].items() } # 添加响应参数 - parameters.update({ - "response_success": { - "type": "string", - "description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称" - }, - "response_failure": { - "type": "string", - "description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称" + parameters.update( + { + "response_success": { + "type": "string", + "description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称", + }, + "response_failure": { + "type": "string", + "description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称", + }, } - }) + ) # 构建必须参数列表（原有参数 + 响应参数） required_params = list(method_info["parameters"].keys()) @@ -229,12 +247,14 @@ def register_device_type(descriptor): "parameters": { "type": "object", "properties": parameters, - "required": required_params - } - } + "required": required_params, + }, + }, } control_func = create_iot_function(device_name, method_name, method_info) - decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func) + decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)( + control_func + ) functions[func_name] = decorated_func device_type_registry.register_device_type(type_id, functions) @@ -243,13 +263,24 @@ def register_device_type(descriptor): # 用于接受前端设备推送的搜索iot描述 async def handleIotDescriptors(conn, descriptors): + wait_max_time = 5 + while conn.func_handler is None or not conn.func_handler.finish_init: + await asyncio.sleep(1) + wait_max_time -= 1 + if wait_max_time <= 0: + logger.bind(tag=TAG).error("连接对象没有func_handler") + return """处理物联网描述""" functions_changed = False for descriptor in descriptors: # 创建IOT设备描述符 - iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"], - descriptor["methods"]) + iot_descriptor = IotDescriptor( + descriptor["name"], + descriptor["description"], + descriptor["properties"], + descriptor["methods"], + ) conn.iot_descriptors[descriptor["name"]] = iot_descriptor if conn.use_function_call_mode: @@ -258,18 +289,22 @@ async def handleIotDescriptors(conn, descriptors): device_functions = device_type_registry.get_device_functions(type_id) # 在连接级注册设备函数 - if hasattr(conn, 'func_handler'): + if hasattr(conn, "func_handler"): for func_name in device_functions: conn.func_handler.function_registry.register_function(func_name) - logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}") + logger.bind(tag=TAG).info( + f"注册IOT函数到function handler: {func_name}" + ) functions_changed = True # 如果注册了新函数，更新function描述列表 - if functions_changed and hasattr(conn, 'func_handler'): + if functions_changed and hasattr(conn, "func_handler"): conn.func_handler.upload_functions_desc() func_names = conn.func_handler.current_support_functions() logger.bind(tag=TAG).info(f"设备类型: {type_id}") - logger.bind(tag=TAG).info(f"更新function描述列表完成，当前支持的函数: {func_names}") + logger.bind(tag=TAG).info( + f"更新function描述列表完成，当前支持的函数: {func_names}" + ) async def handleIotStatus(conn, states): @@ -281,11 +316,15 @@ async def handleIotStatus(conn, states): for k, v in state["state"].items(): if property_item["name"] == k: if type(v) != type(property_item["value"]): - logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配") + logger.bind(tag=TAG).error( + f"属性{property_item['name']}的值类型不匹配" + ) break else: property_item["value"] = v - logger.bind(tag=TAG).info(f"物联网状态更新: {key} , {property_item['name']} = {v}") + logger.bind(tag=TAG).info( + f"物联网状态更新: {key} , {property_item['name']} = {v}" + ) break break @@ -308,10 +347,14 @@ async def set_iot_status(conn, name, property_name, value): for property_item in iot_descriptor.properties: if property_item["name"] == property_name: if type(value) != type(property_item["value"]): - logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配") + logger.bind(tag=TAG).error( + f"属性{property_item['name']}的值类型不匹配" + ) return property_item["value"] = value - logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}") + logger.bind(tag=TAG).info( + f"物联网状态更新: {name} , {property_name} = {value}" + ) return logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") @@ -324,15 +367,19 @@ async def send_iot_conn(conn, name, method_name, parameters): for method in value.methods: # 找到了方法 if method["name"] == method_name: - await conn.websocket.send(json.dumps({ - "type": "iot", - "commands": [ + await conn.websocket.send( + json.dumps( { - "name": name, - "method": method_name, - "parameters": parameters + "type": "iot", + "commands": [ + { + "name": name, + "method": method_name, + "parameters": parameters, + } + ], } - ] - })) + ) + ) return logger.bind(tag=TAG).error(f"未找到方法{method_name}") diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index a082ada8..19e346e6 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -21,7 +21,9 @@ async def handleAudioMessage(conn, audio): if have_voice == False and conn.client_have_voice == False: await no_voice_close_connect(conn) conn.asr_audio.append(audio) - conn.asr_audio = conn.asr_audio[-10:] # 保留最新的10帧音频内容，解决ASR句首丢字问题 + conn.asr_audio = conn.asr_audio[ + -10: + ] # 保留最新的10帧音频内容，解决ASR句首丢字问题 return conn.client_no_voice_last_time = 0.0 conn.asr_audio.append(audio) @@ -33,7 +35,9 @@ async def handleAudioMessage(conn, audio): if len(conn.asr_audio) < 15: conn.asr_server_receive = True else: - text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) + text, file_path = await conn.asr.speech_to_text( + conn.asr_audio, conn.session_id + ) logger.bind(tag=TAG).info(f"识别文本: {text}") text_len, _ = remove_punctuation_and_length(text) if text_len > 0: @@ -47,12 +51,12 @@ async def handleAudioMessage(conn, audio): async def startToChat(conn, text): # 首先进行意图分析 intent_handled = await handle_user_intent(conn, text) - + if intent_handled: # 如果意图已被处理，不再进行聊天 conn.asr_server_receive = True return - + # 意图未被处理，继续常规聊天流程 await send_stt_message(conn, text) if conn.use_function_call_mode: @@ -67,10 +71,17 @@ async def no_voice_close_connect(conn): conn.client_no_voice_last_time = time.time() * 1000 else: no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time - close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120) - if not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time: + close_connection_no_voice_time = conn.config.get( + "close_connection_no_voice_time", 120 + ) + if ( + not conn.close_after_chat + and no_voice_time > 1000 * close_connection_no_voice_time + ): conn.close_after_chat = True conn.client_abort = False conn.asr_server_receive = False - prompt = "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。" + prompt = ( + "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。" + ) await startToChat(conn, prompt) diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index 3f60592a..43e980e8 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -6,6 +6,7 @@ from core.utils.util import remove_punctuation_and_length from core.handle.receiveAudioHandle import startToChat, handleAudioMessage from core.handle.sendAudioHandle import send_stt_message, send_tts_message from core.handle.iotHandle import handleIotDescriptors, handleIotStatus +import asyncio TAG = __name__ logger = setup_logging() @@ -34,7 +35,7 @@ async def handleTextMessage(conn, message): conn.client_have_voice = True conn.client_voice_stop = True if len(conn.asr_audio) > 0: - await handleAudioMessage(conn, b'') + await handleAudioMessage(conn, b"") elif msg_json["state"] == "detect": conn.asr_server_receive = False conn.client_have_voice = False @@ -57,8 +58,8 @@ async def handleTextMessage(conn, message): await startToChat(conn, text) elif msg_json["type"] == "iot": if "descriptors" in msg_json: - await handleIotDescriptors(conn, msg_json["descriptors"]) + asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"])) if "states" in msg_json: - await handleIotStatus(conn, msg_json["states"]) + asyncio.create_task(handleIotStatus(conn, msg_json["states"])) except json.JSONDecodeError: await conn.websocket.send(message)