update:智控台，完成mcp拍照识图

2026-07-22 07:03:53 +08:00 · 2025-06-01 13:34:32 +08:00
parent c4f2411fee
commit ce776f210c
19 changed files with 267 additions and 27 deletions
@@ -27,6 +27,9 @@ public class AgentDTO {
    @Schema(description = "大语言模型名称", example = "llm_model_01")
    private String llmModelName;

+    @Schema(description = "视觉模型名称", example = "vllm_model_01")
+    private String vllmModelName;
+
    @Schema(description = "记忆模型ID", example = "mem_model_01")
    private String memModelId;

@@ -36,6 +36,9 @@ public class AgentEntity {
    @Schema(description = "大语言模型标识")
    private String llmModelId;

+    @Schema(description = "VLLM模型标识")
+    private String vllmModelId;
+
    @Schema(description = "语音合成模型标识")
    private String ttsModelId;

@@ -49,6 +49,11 @@ public class AgentTemplateEntity implements Serializable {
     */
    private String llmModelId;

+    /**
+     * VLLM模型标识
+     */
+    private String vllmModelId;
+
    /**
     * 语音合成模型标识
     */
@@ -102,6 +102,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
            // 获取 LLM 模型名称
            dto.setLlmModelName(modelConfigService.getModelNameById(agent.getLlmModelId()));

+            // 获取 VLLM 模型名称
+            dto.setVllmModelName(modelConfigService.getModelNameById(agent.getVllmModelId()));
+
            // 获取记忆模型名称
            dto.setMemModelId(agent.getMemModelId());

@@ -72,6 +72,7 @@ public class ConfigServiceImpl implements ConfigService {
                null,
                null,
                null,
+                null,
                result,
                isCache);

@@ -140,6 +141,7 @@ public class ConfigServiceImpl implements ConfigService {
                agent.getVadModelId(),
                agent.getAsrModelId(),
                agent.getLlmModelId(),
+                agent.getVllmModelId(),
                agent.getTtsModelId(),
                agent.getMemModelId(),
                agent.getIntentModelId(),
@@ -241,6 +243,7 @@ public class ConfigServiceImpl implements ConfigService {
            String vadModelId,
            String asrModelId,
            String llmModelId,
+            String vllmModelId,
            String ttsModelId,
            String memModelId,
            String intentModelId,
@@ -248,8 +251,8 @@ public class ConfigServiceImpl implements ConfigService {
            boolean isCache) {
        Map<String, String> selectedModule = new HashMap<>();

-        String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM" };
-        String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId };
+        String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM" };
+        String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId };
        String intentLLMModelId = null;
        String memLocalShortLLMModelId = null;

@@ -0,0 +1,29 @@
+-- VLLM模型供应器
+delete from `ai_model_provider` where id = 'SYSTEM_VLLM_openai';
+INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
+('SYSTEM_VLLM_openai', 'VLLM', 'openai', 'OpenAI接口', '[{"key":"base_url","label":"基础URL","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"api_key","label":"API密钥","type":"string"}]', 9, 1, NOW(), 1, NOW());
+
+-- VLLM模型配置
+delete from `ai_model_config` where id = 'VLLM_ChatGLMVLLM';
+INSERT INTO `ai_model_config` VALUES ('VLLM_ChatGLMVLLM', 'VLLM', 'ChatGLMVLLM', '智谱视觉AI', 1, 1, '{\"type\": \"openai\", \"model_name\": \"glm-4v-flash\", \"base_url\": \"https://open.bigmodel.cn/api/paas/v4/\", \"api_key\": \"你的api_key\"}', NULL, NULL, 1, NULL, NULL, NULL, NULL);
+
+-- 更新文档
+UPDATE `ai_model_config` SET 
+`doc_link` = 'https://bigmodel.cn/usercenter/proj-mgmt/apikeys',
+`remark` = '智谱视觉AI配置说明：
+1. 访问 https://bigmodel.cn/usercenter/proj-mgmt/apikeys
+2. 注册并获取API密钥
+3. 填入配置文件中' WHERE `id` = 'VLLM_ChatGLMVLLM';
+
+
+-- 添加参数
+INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (113, 'server.http_port', '8003', 'number', 1, 'http服务的端口，用于启动视觉分析接口');
+INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (114, 'server.vision_explain', 'null', 'string', 1, '视觉分析接口地址，用于下发到设备，多个用;分隔');
+
+-- 智能体表增加VLLM模型配置
+ALTER TABLE `ai_agent` 
+ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`;
+
+-- 智能体模版表增加VLLM模型配置
+ALTER TABLE `ai_agent_template` 
+ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`;
@@ -169,4 +169,11 @@ databaseChangeLog:
      changes:
        - sqlFile:
            encoding: utf8
-            path: classpath:db/changelog/202505271414.sql
+            path: classpath:db/changelog/202505271414.sql
+  - changeSet:
+      id: 202506010920
+      author: hrz
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202506010920.sql
@@ -14,10 +14,10 @@
      </div>
    </div>
    <div class="device-name">
-      设备型号：{{ device.ttsModelName }}
+      语言模型：{{ device.llmModelName }}
    </div>
    <div class="device-name">
-      音色模型：{{ device.ttsVoiceName }}
+      音色模型：{{ device.ttsModelName }} ({{ device.ttsVoiceName }})
    </div>
    <div style="display: flex;gap: 10px;align-items: center;">
      <div class="settings-btn" @click="handleConfigure">
@@ -30,6 +30,9 @@
          <el-menu-item index="llm">
            <span class="menu-text">大语言模型</span>
          </el-menu-item>
+          <el-menu-item index="vllm">
+            <span class="menu-text">视觉大语言模型</span>
+          </el-menu-item>
          <el-menu-item index="intent">
            <span class="menu-text">意图识别</span>
          </el-menu-item>
@@ -173,6 +176,7 @@ export default {
        vad: '语言活动检测模型(VAD)',
        asr: '语音识别模型(ASR)',
        llm: '大语言模型（LLM）',
+        vllm: '视觉大语言模型（VLLM）',
        intent: '意图识别模型(Intent)',
        tts: '语音合成模型(TTS)',
        memory: '记忆模型(Memory)'
@@ -64,7 +64,27 @@
                    </el-form-item>
                  </div>
                  <div class="form-column">
-                    <el-form-item v-for="(model, index) in models" :key="`model-${index}`" :label="model.label"
+                    <div class="model-row">
+                      <el-form-item label="语音活动检测(VAD)" class="model-item">
+                        <div class="model-select-wrapper">
+                          <el-select v-model="form.model.vadModelId" filterable placeholder="请选择" class="form-select"
+                            @change="handleModelChange('VAD', $event)">
+                            <el-option v-for="(item, optionIndex) in modelOptions['VAD']"
+                              :key="`option-vad-${optionIndex}`" :label="item.label" :value="item.value" />
+                          </el-select>
+                        </div>
+                      </el-form-item>
+                      <el-form-item label="语音识别(ASR)" class="model-item">
+                        <div class="model-select-wrapper">
+                          <el-select v-model="form.model.asrModelId" filterable placeholder="请选择" class="form-select"
+                            @change="handleModelChange('ASR', $event)">
+                            <el-option v-for="(item, optionIndex) in modelOptions['ASR']"
+                              :key="`option-asr-${optionIndex}`" :label="item.label" :value="item.value" />
+                          </el-select>
+                        </div>
+                      </el-form-item>
+                    </div>
+                    <el-form-item v-for="(model, index) in models.slice(2)" :key="`model-${index}`" :label="model.label"
                      class="model-item">
                      <div class="model-select-wrapper">
                        <el-select v-model="form.model[model.key]" filterable placeholder="请选择" class="form-select"
@@ -148,6 +168,7 @@ export default {
          vadModelId: "",
          asrModelId: "",
          llmModelId: "",
+          vllmModelId: "",
          memModelId: "",
          intentModelId: "",
        }
@@ -156,6 +177,7 @@ export default {
        { label: '语音活动检测(VAD)', key: 'vadModelId', type: 'VAD' },
        { label: '语音识别(ASR)', key: 'asrModelId', type: 'ASR' },
        { label: '大语言模型(LLM)', key: 'llmModelId', type: 'LLM' },
+        { label: '视觉大语言模型(VLLM)', key: 'vllmModelId', type: 'VLLM' },
        { label: '意图识别(Intent)', key: 'intentModelId', type: 'Intent' },
        { label: '记忆(Memory)', key: 'memModelId', type: 'Memory' },
        { label: '语音合成(TTS)', key: 'ttsModelId', type: 'TTS' },
@@ -189,6 +211,7 @@ export default {
        asrModelId: this.form.model.asrModelId,
        vadModelId: this.form.model.vadModelId,
        llmModelId: this.form.model.llmModelId,
+        vllmModelId: this.form.model.vllmModelId,
        ttsModelId: this.form.model.ttsModelId,
        ttsVoiceId: this.form.ttsVoiceId,
        chatHistoryConf: this.form.chatHistoryConf,
@@ -236,6 +259,7 @@ export default {
            vadModelId: "",
            asrModelId: "",
            llmModelId: "",
+            vllmModelId: "",
            memModelId: "",
            intentModelId: "",
          }
@@ -289,6 +313,7 @@ export default {
          vadModelId: templateData.vadModelId || this.form.model.vadModelId,
          asrModelId: templateData.asrModelId || this.form.model.asrModelId,
          llmModelId: templateData.llmModelId || this.form.model.llmModelId,
+          vllmModelId: templateData.vllmModelId || this.form.model.vllmModelId,
          memModelId: templateData.memModelId || this.form.model.memModelId,
          intentModelId: templateData.intentModelId || this.form.model.intentModelId
        }
@@ -305,6 +330,7 @@ export default {
              vadModelId: data.data.vadModelId,
              asrModelId: data.data.asrModelId,
              llmModelId: data.data.llmModelId,
+              vllmModelId: data.data.vllmModelId,
              memModelId: data.data.memModelId,
              intentModelId: data.data.intentModelId
            }
@@ -587,6 +613,25 @@ export default {
  width: 100%;
 }

+.model-row {
+  display: flex;
+  gap: 20px;
+  margin-bottom: 6px;
+}
+
+.model-row .model-item {
+  flex: 1;
+  margin-bottom: 0;
+}
+
+.model-row .el-form-item__label {
+  font-size: 12px !important;
+  color: #3d4566 !important;
+  font-weight: 400;
+  line-height: 22px;
+  padding-bottom: 2px;
+}
+
 .function-icons {
  display: flex;
  align-items: center;
@@ -1,4 +1,5 @@
 import sys
+import uuid
 import signal
 import asyncio
 from aioconsole import ainput
@@ -45,13 +46,16 @@ async def main():
    check_ffmpeg_installed()
    config = load_config()

+    # 生成随机密钥并添加到配置中
+    config["server"]["auth_key"] = str(uuid.uuid4().hex)
+
    # 添加 stdin 监控任务
    stdin_task = asyncio.create_task(monitor_stdin())

    # 启动 WebSocket 服务器
    ws_server = WebSocketServer(config)
    ws_task = asyncio.create_task(ws_server.start())
-    # 启动 Simple OTA 服务器
+    # 启动 Simple http 服务器
    ota_server = SimpleHttpServer(config)
    ota_task = asyncio.create_task(ota_server.start())

@@ -59,10 +59,14 @@ def get_config_from_api(config):
        "url": config["manager-api"].get("url", ""),
        "secret": config["manager-api"].get("secret", ""),
    }
+    # server的配置以本地为准
    if config.get("server"):
        config_data["server"] = {
            "ip": config["server"].get("ip", ""),
            "port": config["server"].get("port", ""),
+            "http_port": config["server"].get("http_port", ""),
+            "vision_explain": config["server"].get("vision_explain", ""),
+            "auth_key": config["server"].get("auth_key", ""),
        }
    return config_data

@@ -8,6 +8,15 @@
 server:
  ip: 0.0.0.0
  port: 8000
+  # http服务的端口，用于视觉分析接口
+  http_port: 8003
+  # 视觉分析接口地址
+  # 向设备发送的视觉分析的接口地址
+  # 如果按下面默认的写法，系统会自动生成视觉识别地址，并输出在启动日志里，这个地址你可以直接用浏览器访问确认一下
+  # 当你使用docker部署或使用公网部署(使用ssl、域名)时，不一定准确
+  # 所以如果你使用docker部署时，将vision_explain设置成局域网地址
+  # 如果你使用公网部署时，将vision_explain设置成公网地址
+  vision_explain: http://你的ip或者域名:端口号/mcp/vision/explain
 manager-api:
  # 你的manager-api的地址，最好使用局域网ip
  url: http://127.0.0.1:8002/xiaozhi
@@ -1,9 +1,13 @@
 import json
+import copy
 from aiohttp import web
 from config.logger import setup_logging
 from core.utils.util import get_vision_url, is_valid_image_file
 from core.utils.vllm import create_instance
+from config.config_loader import get_private_config_from_api
+from core.utils.auth import AuthToken
 import base64
+from typing import Tuple, Optional

 TAG = __name__

@@ -15,20 +19,45 @@ class VisionHandler:
    def __init__(self, config: dict):
        self.config = config
        self.logger = setup_logging()
+        # 初始化认证工具
+        self.auth = AuthToken(config["server"]["auth_key"])

    def _create_error_response(self, message: str) -> dict:
        """创建统一的错误响应格式"""
        return {"success": False, "message": message}

+    def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]:
+        """验证认证token"""
+        auth_header = request.headers.get("Authorization", "")
+        if not auth_header.startswith("Bearer "):
+            return False, None
+
+        token = auth_header[7:]  # 移除"Bearer "前缀
+        return self.auth.verify_token(token)
+
    async def handle_post(self, request):
        """处理 MCP Vision POST 请求"""
        try:
+            # 验证token
+            is_valid, token_device_id = self._verify_auth_token(request)
+            if not is_valid:
+                return web.Response(
+                    text=json.dumps(
+                        self._create_error_response("无效的认证token或token已过期")
+                    ),
+                    content_type="application/json",
+                    status=401,
+                )
+
            # 获取请求头信息
            device_id = request.headers.get("Device-Id", "")
            client_id = request.headers.get("Client-Id", "")
-            self.logger.bind(tag=TAG).debug(f"Device-Id: {device_id}")
-            self.logger.bind(tag=TAG).debug(f"Client-Id: {client_id}")
-
+            if device_id != token_device_id:
+                return web.Response(
+                    text=json.dumps(self._create_error_response("设备ID与token不匹配")),
+                    content_type="application/json",
+                    status=401,
+                )
            # 解析multipart/form-data请求
            reader = await request.multipart()

@@ -64,9 +93,34 @@ class VisionHandler:
            # 将图片转换为base64编码
            image_base64 = base64.b64encode(image_data).decode("utf-8")

-            vllm_config = self.config["VLLM"]["ChatGLMVLLM"]
-            provider = create_instance("openai", vllm_config)
-            response = provider.response(question, image_base64)
+            # 如果开启了智控台，则从智控台获取模型配置
+            current_config = copy.deepcopy(self.config)
+            read_config_from_api = current_config.get("read_config_from_api", False)
+            if read_config_from_api:
+                current_config = get_private_config_from_api(
+                    current_config,
+                    device_id,
+                    client_id,
+                )
+
+            select_vllm_module = current_config["selected_module"].get("VLLM")
+            if not select_vllm_module:
+                raise ValueError("您还未设置默认的视觉分析模块")
+
+            vllm_type = (
+                select_vllm_module
+                if "type" not in current_config["VLLM"][select_vllm_module]
+                else current_config["VLLM"][select_vllm_module]["type"]
+            )
+
+            if not vllm_type:
+                raise ValueError(f"无法找到VLLM模块对应的供应器{vllm_type}")
+
+            vllm = create_instance(
+                vllm_type, current_config["VLLM"][select_vllm_module]
+            )
+
+            response = vllm.response(question, image_base64)

            return_json = {
                "success": True,
@@ -99,7 +153,13 @@ class VisionHandler:
        """处理 MCP Vision GET 请求"""
        try:
            vision_explain = get_vision_url(self.config)
-            message = f"MCP Vision 接口运行正常，视觉解释接口地址是：{vision_explain}"
+            if vision_explain and len(vision_explain) > 0 and "null" != vision_explain:
+                message = (
+                    f"MCP Vision 接口运行正常，视觉解释接口地址是：{vision_explain}"
+                )
+            else:
+                message = "MCP Vision 接口运行不正常，请打开data目录下的.config.yaml文件，找到【server.vision_explain】，设置好地址"
+
            response = web.Response(text=message, content_type="text/plain")
        except Exception as e:
            self.logger.bind(tag=TAG).error(f"MCP Vision GET请求异常: {e}")
@@ -478,6 +478,8 @@ class ConnectionHandler:
            self.memory = modules["memory"]

    def _initialize_memory(self):
+        if self.memory is None:
+            return
        """初始化记忆模块"""
        self.memory.init_memory(
            role_id=self.device_id,
@@ -518,6 +520,8 @@ class ConnectionHandler:
                self.logger.bind(tag=TAG).info("使用主LLM作为意图识别模型")

    def _initialize_intent(self):
+        if self.intent is None:
+            return
        self.intent_type = self.config["Intent"][
            self.config["selected_module"]["Intent"]
        ]["type"]
@@ -2,6 +2,7 @@ import json
 import asyncio
 from concurrent.futures import Future
 from core.utils.util import get_vision_url
+from core.utils.auth import AuthToken

 TAG = __name__

@@ -207,15 +208,18 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict):
 async def send_mcp_initialize_message(conn):
    """发送MCP初始化消息"""

-    # 智控台暂时不启动视觉分析
-    if conn.read_config_from_api:
-        vision = {}
-    else:
-        vision_url = get_vision_url(conn.config)
-        vision = {
-            "url": vision_url,
-            "token": "test_token",
-        }
+    vision_url = get_vision_url(conn.config)
+
+    # 密钥生成token
+    auth = AuthToken(conn.config["server"]["auth_key"])
+    token = auth.generate_token(conn.headers.get("device-id"))
+
+    vision = {
+        "url": vision_url,
+        "token": token,
+    }
+
+    conn.logger.bind(tag=TAG).info(f"视觉服务信息: {vision}")

    payload = {
        "jsonrpc": "2.0",
@@ -35,16 +35,25 @@ class SimpleHttpServer:
    async def start(self):
        server_config = self.config["server"]
        host = server_config.get("ip", "0.0.0.0")
-        port = int(server_config.get("http_port"))
+        port = int(server_config.get("http_port", 8003))

        if port:
            app = web.Application()
+
+            read_config_from_api = server_config.get("read_config_from_api", False)
+
+            if not read_config_from_api:
+                # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
+                app.add_routes(
+                    [
+                        web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
+                        web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
+                        web.options("/xiaozhi/ota/", self.ota_handler.handle_post),
+                    ]
+                )
            # 添加路由
            app.add_routes(
                [
-                    web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
-                    web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
-                    web.options("/xiaozhi/ota/", self.ota_handler.handle_post),
                    web.get("/mcp/vision/explain", self.vision_handler.handle_get),
                    web.post("/mcp/vision/explain", self.vision_handler.handle_post),
                    web.options("/mcp/vision/explain", self.vision_handler.handle_post),
@@ -0,0 +1,43 @@
+import jwt
+import time
+from datetime import datetime, timedelta, timezone
+from typing import Optional, Tuple
+
+
+class AuthToken:
+    def __init__(self, secret_key: str):
+        self.secret_key = secret_key
+
+    def generate_token(self, device_id: str) -> str:
+        """
+        生成JWT token
+        :param device_id: 设备ID
+        :return: JWT token字符串
+        """
+        # 设置过期时间为1小时后
+        expire_time = datetime.now(timezone.utc) + timedelta(hours=1)
+
+        # 创建payload
+        payload = {"device_id": device_id, "exp": expire_time.timestamp()}
+
+        # 使用JWT进行编码
+        token = jwt.encode(payload, self.secret_key, algorithm="HS256")
+        return token
+
+    def verify_token(self, token: str) -> Tuple[bool, Optional[str]]:
+        """
+        验证token
+        :param token: JWT token字符串
+        :return: (是否有效, 设备ID)
+        """
+        try:
+            # 解码token
+            payload = jwt.decode(token, self.secret_key, algorithms=["HS256"])
+
+            # 检查是否过期
+            if payload["exp"] < time.time():
+                return False, None
+
+            return True, payload["device_id"]
+        except jwt.InvalidTokenError:
+            return False, None
@@ -31,3 +31,4 @@ chardet==5.2.0
 aioconsole==0.8.1
 markitdown==0.1.1
 mcp-proxy==0.6.0
+PyJWT==2.8.0