From ce776f210ca773a95236a8520fd36058c1dc90da Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 1 Jun 2025 13:34:32 +0800 Subject: [PATCH] =?UTF-8?q?update:=E6=99=BA=E6=8E=A7=E5=8F=B0=EF=BC=8C?= =?UTF-8?q?=E5=AE=8C=E6=88=90mcp=E6=8B=8D=E7=85=A7=E8=AF=86=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi/modules/agent/dto/AgentDTO.java | 3 + .../modules/agent/entity/AgentEntity.java | 3 + .../agent/entity/AgentTemplateEntity.java | 5 ++ .../agent/service/impl/AgentServiceImpl.java | 3 + .../service/impl/ConfigServiceImpl.java | 7 +- .../resources/db/changelog/202506010920.sql | 29 ++++++++ .../db/changelog/db.changelog-master.yaml | 9 ++- .../manager-web/src/components/DeviceItem.vue | 4 +- main/manager-web/src/views/ModelConfig.vue | 4 + main/manager-web/src/views/roleConfig.vue | 47 +++++++++++- main/xiaozhi-server/app.py | 6 +- main/xiaozhi-server/config/config_loader.py | 4 + main/xiaozhi-server/config_from_api.yaml | 9 +++ .../xiaozhi-server/core/api/vision_handler.py | 74 +++++++++++++++++-- main/xiaozhi-server/core/connection.py | 4 + main/xiaozhi-server/core/handle/mcpHandle.py | 22 +++--- main/xiaozhi-server/core/http_server.py | 17 ++++- main/xiaozhi-server/core/utils/auth.py | 43 +++++++++++ main/xiaozhi-server/requirements.txt | 1 + 19 files changed, 267 insertions(+), 27 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202506010920.sql create mode 100644 main/xiaozhi-server/core/utils/auth.py diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java index 075db842..218ee1eb 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java @@ -27,6 +27,9 @@ public class AgentDTO { @Schema(description = "大语言模型名称", example = "llm_model_01") private String llmModelName; + @Schema(description = "视觉模型名称", example = "vllm_model_01") + private String vllmModelName; + @Schema(description = "记忆模型ID", example = "mem_model_01") private String memModelId; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentEntity.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentEntity.java index 1ffc5550..d92d49b8 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentEntity.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentEntity.java @@ -36,6 +36,9 @@ public class AgentEntity { @Schema(description = "大语言模型标识") private String llmModelId; + @Schema(description = "VLLM模型标识") + private String vllmModelId; + @Schema(description = "语音合成模型标识") private String ttsModelId; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentTemplateEntity.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentTemplateEntity.java index 2528ca2c..a7704e5b 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentTemplateEntity.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentTemplateEntity.java @@ -49,6 +49,11 @@ public class AgentTemplateEntity implements Serializable { */ private String llmModelId; + /** + * VLLM模型标识 + */ + private String vllmModelId; + /** * 语音合成模型标识 */ diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java index 37bb7d93..ad6e75c3 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java @@ -102,6 +102,9 @@ public class AgentServiceImpl extends BaseServiceImpl imp // 获取 LLM 模型名称 dto.setLlmModelName(modelConfigService.getModelNameById(agent.getLlmModelId())); + // 获取 VLLM 模型名称 + dto.setVllmModelName(modelConfigService.getModelNameById(agent.getVllmModelId())); + // 获取记忆模型名称 dto.setMemModelId(agent.getMemModelId()); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index f28c4e55..b9a7918c 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -72,6 +72,7 @@ public class ConfigServiceImpl implements ConfigService { null, null, null, + null, result, isCache); @@ -140,6 +141,7 @@ public class ConfigServiceImpl implements ConfigService { agent.getVadModelId(), agent.getAsrModelId(), agent.getLlmModelId(), + agent.getVllmModelId(), agent.getTtsModelId(), agent.getMemModelId(), agent.getIntentModelId(), @@ -241,6 +243,7 @@ public class ConfigServiceImpl implements ConfigService { String vadModelId, String asrModelId, String llmModelId, + String vllmModelId, String ttsModelId, String memModelId, String intentModelId, @@ -248,8 +251,8 @@ public class ConfigServiceImpl implements ConfigService { boolean isCache) { Map selectedModule = new HashMap<>(); - String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM" }; - String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId }; + String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM" }; + String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId }; String intentLLMModelId = null; String memLocalShortLLMModelId = null; diff --git a/main/manager-api/src/main/resources/db/changelog/202506010920.sql b/main/manager-api/src/main/resources/db/changelog/202506010920.sql new file mode 100644 index 00000000..79898eed --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202506010920.sql @@ -0,0 +1,29 @@ +-- VLLM模型供应器 +delete from `ai_model_provider` where id = 'SYSTEM_VLLM_openai'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_VLLM_openai', 'VLLM', 'openai', 'OpenAI接口', '[{"key":"base_url","label":"基础URL","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"api_key","label":"API密钥","type":"string"}]', 9, 1, NOW(), 1, NOW()); + +-- VLLM模型配置 +delete from `ai_model_config` where id = 'VLLM_ChatGLMVLLM'; +INSERT INTO `ai_model_config` VALUES ('VLLM_ChatGLMVLLM', 'VLLM', 'ChatGLMVLLM', '智谱视觉AI', 1, 1, '{\"type\": \"openai\", \"model_name\": \"glm-4v-flash\", \"base_url\": \"https://open.bigmodel.cn/api/paas/v4/\", \"api_key\": \"你的api_key\"}', NULL, NULL, 1, NULL, NULL, NULL, NULL); + +-- 更新文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://bigmodel.cn/usercenter/proj-mgmt/apikeys', +`remark` = '智谱视觉AI配置说明： +1. 访问 https://bigmodel.cn/usercenter/proj-mgmt/apikeys +2. 注册并获取API密钥 +3. 填入配置文件中' WHERE `id` = 'VLLM_ChatGLMVLLM'; + + +-- 添加参数 +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (113, 'server.http_port', '8003', 'number', 1, 'http服务的端口，用于启动视觉分析接口'); +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (114, 'server.vision_explain', 'null', 'string', 1, '视觉分析接口地址，用于下发到设备，多个用;分隔'); + +-- 智能体表增加VLLM模型配置 +ALTER TABLE `ai_agent` +ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`; + +-- 智能体模版表增加VLLM模型配置 +ALTER TABLE `ai_agent_template` +ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 55b7cd4c..437e8ea6 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -169,4 +169,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505271414.sql \ No newline at end of file + path: classpath:db/changelog/202505271414.sql + - changeSet: + id: 202506010920 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202506010920.sql \ No newline at end of file diff --git a/main/manager-web/src/components/DeviceItem.vue b/main/manager-web/src/components/DeviceItem.vue index eeab90a1..6a782d24 100644 --- a/main/manager-web/src/components/DeviceItem.vue +++ b/main/manager-web/src/components/DeviceItem.vue @@ -14,10 +14,10 @@
- 设备型号：{{ device.ttsModelName }} + 语言模型：{{ device.llmModelName }}
- 音色模型：{{ device.ttsVoiceName }} + 音色模型：{{ device.ttsModelName }} ({{ device.ttsVoiceName }})
diff --git a/main/manager-web/src/views/ModelConfig.vue b/main/manager-web/src/views/ModelConfig.vue index 8abf8bb6..6e5a76f8 100644 --- a/main/manager-web/src/views/ModelConfig.vue +++ b/main/manager-web/src/views/ModelConfig.vue @@ -30,6 +30,9 @@ 大语言模型 + + 视觉大语言模型 + 意图识别 @@ -173,6 +176,7 @@ export default { vad: '语言活动检测模型(VAD)', asr: '语音识别模型(ASR)', llm: '大语言模型（LLM）', + vllm: '视觉大语言模型（VLLM）', intent: '意图识别模型(Intent)', tts: '语音合成模型(TTS)', memory: '记忆模型(Memory)' diff --git a/main/manager-web/src/views/roleConfig.vue b/main/manager-web/src/views/roleConfig.vue index 03ec8060..50cf278a 100644 --- a/main/manager-web/src/views/roleConfig.vue +++ b/main/manager-web/src/views/roleConfig.vue @@ -64,7 +64,27 @@
- + +
+ + + +
+ + +
+ + + +
+ +
+
dict: """创建统一的错误响应格式""" return {"success": False, "message": message} + def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]: + """验证认证token""" + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + return False, None + + token = auth_header[7:] # 移除"Bearer "前缀 + return self.auth.verify_token(token) + async def handle_post(self, request): """处理 MCP Vision POST 请求""" try: + # 验证token + is_valid, token_device_id = self._verify_auth_token(request) + if not is_valid: + return web.Response( + text=json.dumps( + self._create_error_response("无效的认证token或token已过期") + ), + content_type="application/json", + status=401, + ) + # 获取请求头信息 device_id = request.headers.get("Device-Id", "") client_id = request.headers.get("Client-Id", "") - self.logger.bind(tag=TAG).debug(f"Device-Id: {device_id}") - self.logger.bind(tag=TAG).debug(f"Client-Id: {client_id}") - + if device_id != token_device_id: + return web.Response( + text=json.dumps(self._create_error_response("设备ID与token不匹配")), + content_type="application/json", + status=401, + ) # 解析multipart/form-data请求 reader = await request.multipart() @@ -64,9 +93,34 @@ class VisionHandler: # 将图片转换为base64编码 image_base64 = base64.b64encode(image_data).decode("utf-8") - vllm_config = self.config["VLLM"]["ChatGLMVLLM"] - provider = create_instance("openai", vllm_config) - response = provider.response(question, image_base64) + # 如果开启了智控台，则从智控台获取模型配置 + current_config = copy.deepcopy(self.config) + read_config_from_api = current_config.get("read_config_from_api", False) + if read_config_from_api: + current_config = get_private_config_from_api( + current_config, + device_id, + client_id, + ) + + select_vllm_module = current_config["selected_module"].get("VLLM") + if not select_vllm_module: + raise ValueError("您还未设置默认的视觉分析模块") + + vllm_type = ( + select_vllm_module + if "type" not in current_config["VLLM"][select_vllm_module] + else current_config["VLLM"][select_vllm_module]["type"] + ) + + if not vllm_type: + raise ValueError(f"无法找到VLLM模块对应的供应器{vllm_type}") + + vllm = create_instance( + vllm_type, current_config["VLLM"][select_vllm_module] + ) + + response = vllm.response(question, image_base64) return_json = { "success": True, @@ -99,7 +153,13 @@ class VisionHandler: """处理 MCP Vision GET 请求""" try: vision_explain = get_vision_url(self.config) - message = f"MCP Vision 接口运行正常，视觉解释接口地址是：{vision_explain}" + if vision_explain and len(vision_explain) > 0 and "null" != vision_explain: + message = ( + f"MCP Vision 接口运行正常，视觉解释接口地址是：{vision_explain}" + ) + else: + message = "MCP Vision 接口运行不正常，请打开data目录下的.config.yaml文件，找到【server.vision_explain】，设置好地址" + response = web.Response(text=message, content_type="text/plain") except Exception as e: self.logger.bind(tag=TAG).error(f"MCP Vision GET请求异常: {e}") diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 9d8f67a6..238eb562 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -478,6 +478,8 @@ class ConnectionHandler: self.memory = modules["memory"] def _initialize_memory(self): + if self.memory is None: + return """初始化记忆模块""" self.memory.init_memory( role_id=self.device_id, @@ -518,6 +520,8 @@ class ConnectionHandler: self.logger.bind(tag=TAG).info("使用主LLM作为意图识别模型") def _initialize_intent(self): + if self.intent is None: + return self.intent_type = self.config["Intent"][ self.config["selected_module"]["Intent"] ]["type"] diff --git a/main/xiaozhi-server/core/handle/mcpHandle.py b/main/xiaozhi-server/core/handle/mcpHandle.py index bf8ff36b..cab8a96b 100644 --- a/main/xiaozhi-server/core/handle/mcpHandle.py +++ b/main/xiaozhi-server/core/handle/mcpHandle.py @@ -2,6 +2,7 @@ import json import asyncio from concurrent.futures import Future from core.utils.util import get_vision_url +from core.utils.auth import AuthToken TAG = __name__ @@ -207,15 +208,18 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict): async def send_mcp_initialize_message(conn): """发送MCP初始化消息""" - # 智控台暂时不启动视觉分析 - if conn.read_config_from_api: - vision = {} - else: - vision_url = get_vision_url(conn.config) - vision = { - "url": vision_url, - "token": "test_token", - } + vision_url = get_vision_url(conn.config) + + # 密钥生成token + auth = AuthToken(conn.config["server"]["auth_key"]) + token = auth.generate_token(conn.headers.get("device-id")) + + vision = { + "url": vision_url, + "token": token, + } + + conn.logger.bind(tag=TAG).info(f"视觉服务信息: {vision}") payload = { "jsonrpc": "2.0", diff --git a/main/xiaozhi-server/core/http_server.py b/main/xiaozhi-server/core/http_server.py index 6acccfec..0ed831e8 100644 --- a/main/xiaozhi-server/core/http_server.py +++ b/main/xiaozhi-server/core/http_server.py @@ -35,16 +35,25 @@ class SimpleHttpServer: async def start(self): server_config = self.config["server"] host = server_config.get("ip", "0.0.0.0") - port = int(server_config.get("http_port")) + port = int(server_config.get("http_port", 8003)) if port: app = web.Application() + + read_config_from_api = server_config.get("read_config_from_api", False) + + if not read_config_from_api: + # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口 + app.add_routes( + [ + web.get("/xiaozhi/ota/", self.ota_handler.handle_get), + web.post("/xiaozhi/ota/", self.ota_handler.handle_post), + web.options("/xiaozhi/ota/", self.ota_handler.handle_post), + ] + ) # 添加路由 app.add_routes( [ - web.get("/xiaozhi/ota/", self.ota_handler.handle_get), - web.post("/xiaozhi/ota/", self.ota_handler.handle_post), - web.options("/xiaozhi/ota/", self.ota_handler.handle_post), web.get("/mcp/vision/explain", self.vision_handler.handle_get), web.post("/mcp/vision/explain", self.vision_handler.handle_post), web.options("/mcp/vision/explain", self.vision_handler.handle_post), diff --git a/main/xiaozhi-server/core/utils/auth.py b/main/xiaozhi-server/core/utils/auth.py new file mode 100644 index 00000000..33b2aab0 --- /dev/null +++ b/main/xiaozhi-server/core/utils/auth.py @@ -0,0 +1,43 @@ +import jwt +import time +from datetime import datetime, timedelta, timezone +from typing import Optional, Tuple + + +class AuthToken: + def __init__(self, secret_key: str): + self.secret_key = secret_key + + def generate_token(self, device_id: str) -> str: + """ + 生成JWT token + :param device_id: 设备ID + :return: JWT token字符串 + """ + # 设置过期时间为1小时后 + expire_time = datetime.now(timezone.utc) + timedelta(hours=1) + + # 创建payload + payload = {"device_id": device_id, "exp": expire_time.timestamp()} + + # 使用JWT进行编码 + token = jwt.encode(payload, self.secret_key, algorithm="HS256") + return token + + def verify_token(self, token: str) -> Tuple[bool, Optional[str]]: + """ + 验证token + :param token: JWT token字符串 + :return: (是否有效, 设备ID) + """ + try: + # 解码token + payload = jwt.decode(token, self.secret_key, algorithms=["HS256"]) + + # 检查是否过期 + if payload["exp"] < time.time(): + return False, None + + return True, payload["device_id"] + except jwt.InvalidTokenError: + return False, None diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index 9df7dde1..910ab4cd 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -31,3 +31,4 @@ chardet==5.2.0 aioconsole==0.8.1 markitdown==0.1.1 mcp-proxy==0.6.0 +PyJWT==2.8.0 \ No newline at end of file