From de8c762d79eee4e5c3d5cb83cea2c4a8daacc0f6 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 13 Apr 2025 18:10:17 +0800 Subject: [PATCH] =?UTF-8?q?fix:manager=E6=84=8F=E5=9B=BE=E8=AF=86=E5=88=AB?= =?UTF-8?q?bug=20(#762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix:连接manager后无法使用functioncallbug * fix:意图识别使用llm无法播放音乐bug * fix：manager第一图识别使用独立llm无法初始化llm的bug --- .../agent/service/impl/AgentServiceImpl.java | 2 +- .../service/impl/ConfigServiceImpl.java | 22 ++++++++- .../service/impl/DeviceServiceImpl.java | 2 +- .../service/impl/SysParamsServiceImpl.java | 2 +- .../service/impl/TimbreServiceImpl.java | 2 +- .../resources/db/changelog/202504131542.sql | 3 ++ .../db/changelog/db.changelog-master.yaml | 9 +++- main/xiaozhi-server/core/connection.py | 27 ++++++---- .../core/handle/functionHandler.py | 4 +- .../core/handle/intentHandler.py | 5 ++ main/xiaozhi-server/core/handle/iotHandle.py | 4 +- .../core/providers/llm/openai/openai.py | 10 +++- .../core/providers/vad/silero.py | 3 +- .../functions/handle_exit_intent.py | 49 +++++++++++-------- .../plugins_func/functions/hass_init.py | 8 ++- 15 files changed, 107 insertions(+), 45 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202504131542.sql diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java index bb2a3ed0..ff284ef0 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java @@ -44,7 +44,7 @@ public class AgentServiceImpl extends BaseServiceImpl imp @Override public PageData adminAgentList(Map params) { IPage page = agentDao.selectPage( - getPage(params, "sort", true), + getPage(params, "agent_name", true), new QueryWrapper<>()); return new PageData<>(page.getRecords(), page.getTotal()); } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index a139289d..78a7b511 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -231,8 +231,9 @@ public class ConfigServiceImpl implements ConfigService { boolean isCache) { Map selectedModule = new HashMap<>(); - String[] modelTypes = { "VAD", "ASR", "LLM", "TTS", "Memory", "Intent" }; - String[] modelIds = { vadModelId, asrModelId, llmModelId, ttsModelId, memModelId, intentModelId }; + String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM" }; + String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId }; + String intentLLMModelId = null; for (int i = 0; i < modelIds.length; i++) { if (modelIds[i] == null) { @@ -246,10 +247,27 @@ public class ConfigServiceImpl implements ConfigService { if ("TTS".equals(modelTypes[i]) && voice != null) { ((Map) model.getConfigJson()).put("private_voice", voice); } + // 如果是Intent类型，且type=intent_llm，则给他添加附加模型 + if ("Intent".equals(modelTypes[i])) { + Map map = (Map) model.getConfigJson(); + if ("intent_llm".equals(map.get("type"))) { + intentLLMModelId = (String) map.get("llm"); + if (intentLLMModelId != null && intentLLMModelId.equals(llmModelId)) { + intentLLMModelId = null; + } + } + } + // 如果是LLM类型，且intentLLMModelId不为空，则添加附加模型 + if ("LLM".equals(modelTypes[i]) && intentLLMModelId != null) { + ModelConfigEntity intentLLM = modelConfigService.getModelById(intentLLMModelId, isCache); + typeConfig.put(intentLLM.getId(), intentLLM.getConfigJson()); + } } result.put(modelTypes[i], typeConfig); + selectedModule.put(modelTypes[i], model.getId()); } + result.put("selected_module", selectedModule); if (StringUtils.isNotBlank(prompt)) { prompt = prompt.replace("{{assistant_name}}", "小智"); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index f2821053..f2351800 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -221,7 +221,7 @@ public class DeviceServiceImpl extends BaseServiceImpl params.put(Constant.PAGE, dto.getPage()); params.put(Constant.LIMIT, dto.getLimit()); IPage page = baseDao.selectPage( - getPage(params, "sort", true), + getPage(params, "mac_address", true), // 定义查询条件 new QueryWrapper() // 必须设备关键词查找 diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java index 5ca59f6b..15f62816 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java @@ -37,7 +37,7 @@ public class SysParamsServiceImpl extends BaseServiceImpl page(Map params) { IPage page = baseDao.selectPage( - getPage(params, Constant.CREATE_DATE, false), + getPage(params, null, false), getWrapper(params)); return getPageData(page, SysParamsDTO.class); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/timbre/service/impl/TimbreServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/timbre/service/impl/TimbreServiceImpl.java index fe5cb555..2ed5b1ab 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/timbre/service/impl/TimbreServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/timbre/service/impl/TimbreServiceImpl.java @@ -47,7 +47,7 @@ public class TimbreServiceImpl extends BaseServiceImpl params.put(Constant.PAGE, dto.getPage()); params.put(Constant.LIMIT, dto.getLimit()); IPage page = baseDao.selectPage( - getPage(params, "sort", true), + getPage(params, null, true), // 定义查询条件 new QueryWrapper() // 必须按照ttsID查找 diff --git a/main/manager-api/src/main/resources/db/changelog/202504131542.sql b/main/manager-api/src/main/resources/db/changelog/202504131542.sql new file mode 100644 index 00000000..383ba187 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202504131542.sql @@ -0,0 +1,3 @@ +-- 对0.3.0版本之前的参数进行修改 +update `sys_params` set param_value = '.mp3;.wav;.p3' where param_code = 'plugins.play_music.music_ext'; +update `ai_model_config` set config_json = '{\"type\": \"intent_llm\", \"llm\": \"LLM_ChatGLMLLM\"}' where id = 'Intent_intent_llm'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 438b1cf4..41f65488 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -50,4 +50,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202504112058.sql \ No newline at end of file + path: classpath:db/changelog/202504112058.sql + - changeSet: + id: 202504131542 + author: John + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202504131542.sql \ No newline at end of file diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 5a9fac3e..fc191b6f 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -104,8 +104,6 @@ class ConnectionHandler: self.close_after_chat = False # 是否在聊天结束后关闭连接 self.use_function_call_mode = False - if self.config["selected_module"]["Intent"] == "function_call": - self.use_function_call_mode = True async def handle_connection(self, ws): try: @@ -222,37 +220,37 @@ class ConnectionHandler: ) if private_config.get("VAD", None) is not None: init_vad = True - self.config["vad"] = private_config["VAD"] + self.config["VAD"] = private_config["VAD"] self.config["selected_module"]["VAD"] = private_config["selected_module"][ "VAD" ] if private_config.get("ASR", None) is not None: init_asr = True - self.config["asr"] = private_config["ASR"] + self.config["ASR"] = private_config["ASR"] self.config["selected_module"]["ASR"] = private_config["selected_module"][ "ASR" ] if private_config.get("LLM", None) is not None: init_llm = True - self.config["llm"] = private_config["LLM"] + self.config["LLM"] = private_config["LLM"] self.config["selected_module"]["LLM"] = private_config["selected_module"][ "LLM" ] if private_config.get("TTS", None) is not None: init_tts = True - self.config["tts"] = private_config["TTS"] + self.config["TTS"] = private_config["TTS"] self.config["selected_module"]["TTS"] = private_config["selected_module"][ "TTS" ] if private_config.get("Memory", None) is not None: init_memory = True - self.config["memory"] = private_config["Memory"] + self.config["Memory"] = private_config["Memory"] self.config["selected_module"]["Memory"] = private_config[ "selected_module" ]["Memory"] if private_config.get("Intent", None) is not None: init_intent = True - self.config["intent"] = private_config["Intent"] + self.config["Intent"] = private_config["Intent"] self.config["selected_module"]["Intent"] = private_config[ "selected_module" ]["Intent"] @@ -287,17 +285,26 @@ class ConnectionHandler: self.memory.init_memory(device_id, self.llm) def _initialize_intent(self): + if ( + self.config["Intent"][self.config["selected_module"]["Intent"]]["type"] + == "function_call" + ): + self.use_function_call_mode = True """初始化意图识别模块""" # 获取意图识别配置 intent_config = self.config["Intent"] - intent_type = self.config["selected_module"]["Intent"] + intent_type = self.config["Intent"][self.config["selected_module"]["Intent"]][ + "type" + ] # 如果使用 nointent，直接返回 if intent_type == "nointent": return # 使用 intent_llm 模式 elif intent_type == "intent_llm": - intent_llm_name = intent_config["intent_llm"]["llm"] + intent_llm_name = intent_config[self.config["selected_module"]["Intent"]][ + "llm" + ] if intent_llm_name and intent_llm_name in self.config["LLM"]: # 如果配置了专用LLM，则创建独立的LLM实例 diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index cec9bd25..d1bf9b44 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -57,7 +57,9 @@ class FunctionHandler: def register_config_functions(self): """注册配置中的函数,可以不同客户端使用不同的配置""" - for func in self.config["Intent"]["function_call"].get("functions", []): + for func in self.config["Intent"][self.config["selected_module"]["Intent"]].get( + "functions", [] + ): self.function_registry.register_function(func) """home assistant需要初始化提示词""" diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 419f6081..90d8e1d5 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -76,6 +76,11 @@ async def process_intent_result(conn, intent_result, original_text): if function_name == "continue_chat": return False + if function_name == "play_music": + funcItem = conn.func_handler.get_function(function_name) + if not funcItem: + conn.func_handler.function_registry.register_function("play_music") + function_args = None if "arguments" in intent_data["function_call"]: function_args = intent_data["function_call"]["arguments"] diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index 76d8b4b7..0af0bcde 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -269,14 +269,12 @@ def register_device_type(descriptor): # 用于接受前端设备推送的搜索iot描述 async def handleIotDescriptors(conn, descriptors): - if not conn.use_function_call_mode: - return wait_max_time = 5 while conn.func_handler is None or not conn.func_handler.finish_init: await asyncio.sleep(1) wait_max_time -= 1 if wait_max_time <= 0: - logger.bind(tag=TAG).error("连接对象没有func_handler") + logger.bind(tag=TAG).debug("连接对象没有func_handler") return """处理物联网描述""" functions_changed = False diff --git a/main/xiaozhi-server/core/providers/llm/openai/openai.py b/main/xiaozhi-server/core/providers/llm/openai/openai.py index 9a9748ca..d6e66dcf 100644 --- a/main/xiaozhi-server/core/providers/llm/openai/openai.py +++ b/main/xiaozhi-server/core/providers/llm/openai/openai.py @@ -15,7 +15,15 @@ class LLMProvider(LLMProviderBase): self.base_url = config.get("base_url") else: self.base_url = config.get("url") - self.max_tokens = config.get("max_tokens", 500) + max_tokens = config.get("max_tokens") + if max_tokens is None or max_tokens == "": + max_tokens = 500 + + try: + max_tokens = int(max_tokens) + except (ValueError, TypeError): + max_tokens = 500 + self.max_tokens = max_tokens check_model_key("LLM", self.api_key) self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url) diff --git a/main/xiaozhi-server/core/providers/vad/silero.py b/main/xiaozhi-server/core/providers/vad/silero.py index e1ad9691..df05cd6d 100644 --- a/main/xiaozhi-server/core/providers/vad/silero.py +++ b/main/xiaozhi-server/core/providers/vad/silero.py @@ -42,7 +42,8 @@ class VADProvider(VADProviderBase): audio_tensor = torch.from_numpy(audio_float32) # 检测语音活动 - speech_prob = self.model(audio_tensor, 16000).item() + with torch.no_grad(): + speech_prob = self.model(audio_tensor, 16000).item() client_have_voice = speech_prob >= self.vad_threshold # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话 diff --git a/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py b/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py index e4769d00..affa8af2 100644 --- a/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py +++ b/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py @@ -1,34 +1,43 @@ -from plugins_func.register import register_function,ToolType, ActionResponse, Action +from plugins_func.register import register_function, ToolType, ActionResponse, Action from config.logger import setup_logging TAG = __name__ logger = setup_logging() handle_exit_intent_function_desc = { - "type": "function", - "function": { - "name": "handle_exit_intent", - "description": "当用户想结束对话或需要退出系统时调用", - "parameters": { - "type": "object", - "properties": { - "say_goodbye": { - "type": "string", - "description": "和用户友好结束对话的告别语" - } - }, - "required": ["say_goodbye"] - } + "type": "function", + "function": { + "name": "handle_exit_intent", + "description": "当用户想结束对话或需要退出系统时调用", + "parameters": { + "type": "object", + "properties": { + "say_goodbye": { + "type": "string", + "description": "和用户友好结束对话的告别语", } - } + }, + "required": ["say_goodbye"], + }, + }, +} -@register_function('handle_exit_intent', handle_exit_intent_function_desc, ToolType.SYSTEM_CTL) -def handle_exit_intent(conn, say_goodbye: str): + +@register_function( + "handle_exit_intent", handle_exit_intent_function_desc, ToolType.SYSTEM_CTL +) +def handle_exit_intent(conn, say_goodbye: str | None = None): # 处理退出意图 try: + if say_goodbye is None: + say_goodbye = "再见，祝您生活愉快！" conn.close_after_chat = True logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}") - return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye) + return ActionResponse( + action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye + ) except Exception as e: logger.bind(tag=TAG).error(f"处理退出意图错误: {e}") - return ActionResponse(action=Action.NONE, result="退出意图处理失败", response="") \ No newline at end of file + return ActionResponse( + action=Action.NONE, result="退出意图处理失败", response="" + ) diff --git a/main/xiaozhi-server/plugins_func/functions/hass_init.py b/main/xiaozhi-server/plugins_func/functions/hass_init.py index 17846cfa..4bd2f2ec 100644 --- a/main/xiaozhi-server/plugins_func/functions/hass_init.py +++ b/main/xiaozhi-server/plugins_func/functions/hass_init.py @@ -9,7 +9,9 @@ HASS_CACHE = {} def append_devices_to_prompt(conn): if conn.use_function_call_mode: - funcs = conn.config["Intent"]["function_call"].get("functions", []) + funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( + "functions", [] + ) if "hass_get_state" in funcs or "hass_set_state" in funcs: prompt = "下面是我家智能设备，可以通过homeassistant控制\n" devices = conn.config["plugins"]["home_assistant"].get("devices", []) @@ -26,7 +28,9 @@ def initialize_hass_handler(conn): global HASS_CACHE if HASS_CACHE == {}: if conn.use_function_call_mode: - funcs = conn.config["Intent"]["function_call"].get("functions", []) + funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( + "functions", [] + ) if "hass_get_state" in funcs or "hass_set_state" in funcs: HASS_CACHE["base_url"] = conn.config["plugins"]["home_assistant"].get( "base_url"