update:智控台，完成mcp拍照识图

main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java

@@ -27,6 +27,9 @@ public class AgentDTO {
     @Schema(description = "大语言模型名称", example = "llm_model_01")
     private String llmModelName;
 
+    @Schema(description = "视觉模型名称", example = "vllm_model_01")
+    private String vllmModelName;
+
     @Schema(description = "记忆模型ID", example = "mem_model_01")
     private String memModelId;
 

main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentEntity.java

@@ -36,6 +36,9 @@ public class AgentEntity {
     @Schema(description = "大语言模型标识")
     private String llmModelId;
 
+    @Schema(description = "VLLM模型标识")
+    private String vllmModelId;
+
     @Schema(description = "语音合成模型标识")
     private String ttsModelId;
 

main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentTemplateEntity.java

@@ -49,6 +49,11 @@ public class AgentTemplateEntity implements Serializable {
      */
     private String llmModelId;
 
+    /**
+     * VLLM模型标识
+     */
+    private String vllmModelId;
+
     /**
      * 语音合成模型标识
      */

main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java

@@ -102,6 +102,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
             // 获取 LLM 模型名称
             dto.setLlmModelName(modelConfigService.getModelNameById(agent.getLlmModelId()));
 
+            // 获取 VLLM 模型名称
+            dto.setVllmModelName(modelConfigService.getModelNameById(agent.getVllmModelId()));
+
             // 获取记忆模型名称
             dto.setMemModelId(agent.getMemModelId());
 

main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java

@@ -72,6 +72,7 @@ public class ConfigServiceImpl implements ConfigService {
                 null,
                 null,
                 null,
+                null,
                 result,
                 isCache);
 
@@ -140,6 +141,7 @@ public class ConfigServiceImpl implements ConfigService {
                 agent.getVadModelId(),
                 agent.getAsrModelId(),
                 agent.getLlmModelId(),
+                agent.getVllmModelId(),
                 agent.getTtsModelId(),
                 agent.getMemModelId(),
                 agent.getIntentModelId(),
@@ -241,6 +243,7 @@ public class ConfigServiceImpl implements ConfigService {
             String vadModelId,
             String asrModelId,
             String llmModelId,
+            String vllmModelId,
             String ttsModelId,
             String memModelId,
             String intentModelId,
@@ -248,8 +251,8 @@ public class ConfigServiceImpl implements ConfigService {
             boolean isCache) {
         Map<String, String> selectedModule = new HashMap<>();
 
-        String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM" };
+        String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM" };
-        String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId };
+        String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId };
         String intentLLMModelId = null;
         String memLocalShortLLMModelId = null;
 

main/manager-api/src/main/resources/db/changelog/202506010920.sql

@@ -0,0 +1,29 @@
+-- VLLM模型供应器
+delete from `ai_model_provider` where id = 'SYSTEM_VLLM_openai';
+INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
+('SYSTEM_VLLM_openai', 'VLLM', 'openai', 'OpenAI接口', '[{"key":"base_url","label":"基础URL","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"api_key","label":"API密钥","type":"string"}]', 9, 1, NOW(), 1, NOW());
+
+-- VLLM模型配置
+delete from `ai_model_config` where id = 'VLLM_ChatGLMVLLM';
+INSERT INTO `ai_model_config` VALUES ('VLLM_ChatGLMVLLM', 'VLLM', 'ChatGLMVLLM', '智谱视觉AI', 1, 1, '{\"type\": \"openai\", \"model_name\": \"glm-4v-flash\", \"base_url\": \"https://open.bigmodel.cn/api/paas/v4/\", \"api_key\": \"你的api_key\"}', NULL, NULL, 1, NULL, NULL, NULL, NULL);
+
+-- 更新文档
+UPDATE `ai_model_config` SET 
+`doc_link` = 'https://bigmodel.cn/usercenter/proj-mgmt/apikeys',
+`remark` = '智谱视觉AI配置说明：
+1. 访问 https://bigmodel.cn/usercenter/proj-mgmt/apikeys
+2. 注册并获取API密钥
+3. 填入配置文件中' WHERE `id` = 'VLLM_ChatGLMVLLM';
+
+
+-- 添加参数
+INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (113, 'server.http_port', '8003', 'number', 1, 'http服务的端口，用于启动视觉分析接口');
+INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (114, 'server.vision_explain', 'null', 'string', 1, '视觉分析接口地址，用于下发到设备，多个用;分隔');
+
+-- 智能体表增加VLLM模型配置
+ALTER TABLE `ai_agent` 
+ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`;
+
+-- 智能体模版表增加VLLM模型配置
+ALTER TABLE `ai_agent_template` 
+ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`;

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -170,3 +170,10 @@ databaseChangeLog:
         - sqlFile:
             encoding: utf8
             path: classpath:db/changelog/202505271414.sql
+  - changeSet:
+      id: 202506010920
+      author: hrz
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202506010920.sql

main/manager-web/src/components/DeviceItem.vue

@@ -14,10 +14,10 @@
       </div>
     </div>
     <div class="device-name">
-      设备型号：{{ device.ttsModelName }}
+      语言模型：{{ device.llmModelName }}
     </div>
     <div class="device-name">
-      音色模型：{{ device.ttsVoiceName }}
+      音色模型：{{ device.ttsModelName }} ({{ device.ttsVoiceName }})
     </div>
     <div style="display: flex;gap: 10px;align-items: center;">
       <div class="settings-btn" @click="handleConfigure">

main/manager-web/src/views/ModelConfig.vue

@@ -30,6 +30,9 @@
           <el-menu-item index="llm">
             <span class="menu-text">大语言模型</span>
           </el-menu-item>
+          <el-menu-item index="vllm">
+            <span class="menu-text">视觉大语言模型</span>
+          </el-menu-item>
           <el-menu-item index="intent">
             <span class="menu-text">意图识别</span>
           </el-menu-item>
@@ -173,6 +176,7 @@ export default {
         vad: '语言活动检测模型(VAD)',
         asr: '语音识别模型(ASR)',
         llm: '大语言模型（LLM）',
+        vllm: '视觉大语言模型（VLLM）',
         intent: '意图识别模型(Intent)',
         tts: '语音合成模型(TTS)',
         memory: '记忆模型(Memory)'

main/manager-web/src/views/roleConfig.vue

@@ -64,7 +64,27 @@
                     </el-form-item>
                   </div>
                   <div class="form-column">
-                    <el-form-item v-for="(model, index) in models" :key="`model-${index}`" :label="model.label"
+                    <div class="model-row">
+                      <el-form-item label="语音活动检测(VAD)" class="model-item">
+                        <div class="model-select-wrapper">
+                          <el-select v-model="form.model.vadModelId" filterable placeholder="请选择" class="form-select"
+                            @change="handleModelChange('VAD', $event)">
+                            <el-option v-for="(item, optionIndex) in modelOptions['VAD']"
+                              :key="`option-vad-${optionIndex}`" :label="item.label" :value="item.value" />
+                          </el-select>
+                        </div>
+                      </el-form-item>
+                      <el-form-item label="语音识别(ASR)" class="model-item">
+                        <div class="model-select-wrapper">
+                          <el-select v-model="form.model.asrModelId" filterable placeholder="请选择" class="form-select"
+                            @change="handleModelChange('ASR', $event)">
+                            <el-option v-for="(item, optionIndex) in modelOptions['ASR']"
+                              :key="`option-asr-${optionIndex}`" :label="item.label" :value="item.value" />
+                          </el-select>
+                        </div>
+                      </el-form-item>
+                    </div>
+                    <el-form-item v-for="(model, index) in models.slice(2)" :key="`model-${index}`" :label="model.label"
                       class="model-item">
                       <div class="model-select-wrapper">
                         <el-select v-model="form.model[model.key]" filterable placeholder="请选择" class="form-select"
@@ -148,6 +168,7 @@ export default {
           vadModelId: "",
           asrModelId: "",
           llmModelId: "",
+          vllmModelId: "",
           memModelId: "",
           intentModelId: "",
         }
@@ -156,6 +177,7 @@ export default {
         { label: '语音活动检测(VAD)', key: 'vadModelId', type: 'VAD' },
         { label: '语音识别(ASR)', key: 'asrModelId', type: 'ASR' },
         { label: '大语言模型(LLM)', key: 'llmModelId', type: 'LLM' },
+        { label: '视觉大语言模型(VLLM)', key: 'vllmModelId', type: 'VLLM' },
         { label: '意图识别(Intent)', key: 'intentModelId', type: 'Intent' },
         { label: '记忆(Memory)', key: 'memModelId', type: 'Memory' },
         { label: '语音合成(TTS)', key: 'ttsModelId', type: 'TTS' },
@@ -189,6 +211,7 @@ export default {
         asrModelId: this.form.model.asrModelId,
         vadModelId: this.form.model.vadModelId,
         llmModelId: this.form.model.llmModelId,
+        vllmModelId: this.form.model.vllmModelId,
         ttsModelId: this.form.model.ttsModelId,
         ttsVoiceId: this.form.ttsVoiceId,
         chatHistoryConf: this.form.chatHistoryConf,
@@ -236,6 +259,7 @@ export default {
             vadModelId: "",
             asrModelId: "",
             llmModelId: "",
+            vllmModelId: "",
             memModelId: "",
             intentModelId: "",
           }
@@ -289,6 +313,7 @@ export default {
           vadModelId: templateData.vadModelId || this.form.model.vadModelId,
           asrModelId: templateData.asrModelId || this.form.model.asrModelId,
           llmModelId: templateData.llmModelId || this.form.model.llmModelId,
+          vllmModelId: templateData.vllmModelId || this.form.model.vllmModelId,
           memModelId: templateData.memModelId || this.form.model.memModelId,
           intentModelId: templateData.intentModelId || this.form.model.intentModelId
         }
@@ -305,6 +330,7 @@ export default {
               vadModelId: data.data.vadModelId,
               asrModelId: data.data.asrModelId,
               llmModelId: data.data.llmModelId,
+              vllmModelId: data.data.vllmModelId,
               memModelId: data.data.memModelId,
               intentModelId: data.data.intentModelId
             }
@@ -587,6 +613,25 @@ export default {
   width: 100%;
 }
 
+.model-row {
+  display: flex;
+  gap: 20px;
+  margin-bottom: 6px;
+}
+
+.model-row .model-item {
+  flex: 1;
+  margin-bottom: 0;
+}
+
+.model-row .el-form-item__label {
+  font-size: 12px !important;
+  color: #3d4566 !important;
+  font-weight: 400;
+  line-height: 22px;
+  padding-bottom: 2px;
+}
+
 .function-icons {
   display: flex;
   align-items: center;

main/xiaozhi-server/app.py

@@ -1,4 +1,5 @@
 import sys
+import uuid
 import signal
 import asyncio
 from aioconsole import ainput
@@ -45,13 +46,16 @@ async def main():
     check_ffmpeg_installed()
     config = load_config()
 
+    # 生成随机密钥并添加到配置中
+    config["server"]["auth_key"] = str(uuid.uuid4().hex)
+
     # 添加 stdin 监控任务
     stdin_task = asyncio.create_task(monitor_stdin())
 
     # 启动 WebSocket 服务器
     ws_server = WebSocketServer(config)
     ws_task = asyncio.create_task(ws_server.start())
-    # 启动 Simple OTA 服务器
+    # 启动 Simple http 服务器
     ota_server = SimpleHttpServer(config)
     ota_task = asyncio.create_task(ota_server.start())
 

main/xiaozhi-server/config/config_loader.py

@@ -59,10 +59,14 @@ def get_config_from_api(config):
         "url": config["manager-api"].get("url", ""),
         "secret": config["manager-api"].get("secret", ""),
     }
+    # server的配置以本地为准
     if config.get("server"):
         config_data["server"] = {
             "ip": config["server"].get("ip", ""),
             "port": config["server"].get("port", ""),
+            "http_port": config["server"].get("http_port", ""),
+            "vision_explain": config["server"].get("vision_explain", ""),
+            "auth_key": config["server"].get("auth_key", ""),
         }
     return config_data
 

main/xiaozhi-server/config_from_api.yaml

@@ -8,6 +8,15 @@
 server:
   ip: 0.0.0.0
   port: 8000
+  # http服务的端口，用于视觉分析接口
+  http_port: 8003
+  # 视觉分析接口地址
+  # 向设备发送的视觉分析的接口地址
+  # 如果按下面默认的写法，系统会自动生成视觉识别地址，并输出在启动日志里，这个地址你可以直接用浏览器访问确认一下
+  # 当你使用docker部署或使用公网部署(使用ssl、域名)时，不一定准确
+  # 所以如果你使用docker部署时，将vision_explain设置成局域网地址
+  # 如果你使用公网部署时，将vision_explain设置成公网地址
+  vision_explain: http://你的ip或者域名:端口号/mcp/vision/explain
 manager-api:
   # 你的manager-api的地址，最好使用局域网ip
   url: http://127.0.0.1:8002/xiaozhi

main/xiaozhi-server/core/api/vision_handler.py

@@ -1,9 +1,13 @@
 import json
+import copy
 from aiohttp import web
 from config.logger import setup_logging
 from core.utils.util import get_vision_url, is_valid_image_file
 from core.utils.vllm import create_instance
+from config.config_loader import get_private_config_from_api
+from core.utils.auth import AuthToken
 import base64
+from typing import Tuple, Optional
 
 TAG = __name__
 
@@ -15,20 +19,45 @@ class VisionHandler:
     def __init__(self, config: dict):
         self.config = config
         self.logger = setup_logging()
+        # 初始化认证工具
+        self.auth = AuthToken(config["server"]["auth_key"])
 
     def _create_error_response(self, message: str) -> dict:
         """创建统一的错误响应格式"""
         return {"success": False, "message": message}
 
+    def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]:
+        """验证认证token"""
+        auth_header = request.headers.get("Authorization", "")
+        if not auth_header.startswith("Bearer "):
+            return False, None
+
+        token = auth_header[7:]  # 移除"Bearer "前缀
+        return self.auth.verify_token(token)
+
     async def handle_post(self, request):
         """处理 MCP Vision POST 请求"""
         try:
+            # 验证token
+            is_valid, token_device_id = self._verify_auth_token(request)
+            if not is_valid:
+                return web.Response(
+                    text=json.dumps(
+                        self._create_error_response("无效的认证token或token已过期")
+                    ),
+                    content_type="application/json",
+                    status=401,
+                )
+
             # 获取请求头信息
             device_id = request.headers.get("Device-Id", "")
             client_id = request.headers.get("Client-Id", "")
-            self.logger.bind(tag=TAG).debug(f"Device-Id: {device_id}")
+            if device_id != token_device_id:
-            self.logger.bind(tag=TAG).debug(f"Client-Id: {client_id}")
+                return web.Response(
-
+                    text=json.dumps(self._create_error_response("设备ID与token不匹配")),
+                    content_type="application/json",
+                    status=401,
+                )
             # 解析multipart/form-data请求
             reader = await request.multipart()
 
@@ -64,9 +93,34 @@ class VisionHandler:
             # 将图片转换为base64编码
             image_base64 = base64.b64encode(image_data).decode("utf-8")
 
-            vllm_config = self.config["VLLM"]["ChatGLMVLLM"]
+            # 如果开启了智控台，则从智控台获取模型配置
-            provider = create_instance("openai", vllm_config)
+            current_config = copy.deepcopy(self.config)
-            response = provider.response(question, image_base64)
+            read_config_from_api = current_config.get("read_config_from_api", False)
+            if read_config_from_api:
+                current_config = get_private_config_from_api(
+                    current_config,
+                    device_id,
+                    client_id,
+                )
+
+            select_vllm_module = current_config["selected_module"].get("VLLM")
+            if not select_vllm_module:
+                raise ValueError("您还未设置默认的视觉分析模块")
+
+            vllm_type = (
+                select_vllm_module
+                if "type" not in current_config["VLLM"][select_vllm_module]
+                else current_config["VLLM"][select_vllm_module]["type"]
+            )
+
+            if not vllm_type:
+                raise ValueError(f"无法找到VLLM模块对应的供应器{vllm_type}")
+
+            vllm = create_instance(
+                vllm_type, current_config["VLLM"][select_vllm_module]
+            )
+
+            response = vllm.response(question, image_base64)
 
             return_json = {
                 "success": True,
@@ -99,7 +153,13 @@ class VisionHandler:
         """处理 MCP Vision GET 请求"""
         try:
             vision_explain = get_vision_url(self.config)
-            message = f"MCP Vision 接口运行正常，视觉解释接口地址是：{vision_explain}"
+            if vision_explain and len(vision_explain) > 0 and "null" != vision_explain:
+                message = (
+                    f"MCP Vision 接口运行正常，视觉解释接口地址是：{vision_explain}"
+                )
+            else:
+                message = "MCP Vision 接口运行不正常，请打开data目录下的.config.yaml文件，找到【server.vision_explain】，设置好地址"
+
             response = web.Response(text=message, content_type="text/plain")
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"MCP Vision GET请求异常: {e}")

main/xiaozhi-server/core/connection.py

@@ -478,6 +478,8 @@ class ConnectionHandler:
             self.memory = modules["memory"]
 
     def _initialize_memory(self):
+        if self.memory is None:
+            return
         """初始化记忆模块"""
         self.memory.init_memory(
             role_id=self.device_id,
@@ -518,6 +520,8 @@ class ConnectionHandler:
                 self.logger.bind(tag=TAG).info("使用主LLM作为意图识别模型")
 
     def _initialize_intent(self):
+        if self.intent is None:
+            return
         self.intent_type = self.config["Intent"][
             self.config["selected_module"]["Intent"]
         ]["type"]

main/xiaozhi-server/core/handle/mcpHandle.py

@@ -2,6 +2,7 @@ import json
 import asyncio
 from concurrent.futures import Future
 from core.utils.util import get_vision_url
+from core.utils.auth import AuthToken
 
 TAG = __name__
 
@@ -207,15 +208,18 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict):
 async def send_mcp_initialize_message(conn):
     """发送MCP初始化消息"""
 
-    # 智控台暂时不启动视觉分析
+    vision_url = get_vision_url(conn.config)
-    if conn.read_config_from_api:
+
-        vision = {}
+    # 密钥生成token
-    else:
+    auth = AuthToken(conn.config["server"]["auth_key"])
-        vision_url = get_vision_url(conn.config)
+    token = auth.generate_token(conn.headers.get("device-id"))
-        vision = {
+
-            "url": vision_url,
+    vision = {
-            "token": "test_token",
+        "url": vision_url,
-        }
+        "token": token,
+    }
+
+    conn.logger.bind(tag=TAG).info(f"视觉服务信息: {vision}")
 
     payload = {
         "jsonrpc": "2.0",

main/xiaozhi-server/core/http_server.py

@@ -35,16 +35,25 @@ class SimpleHttpServer:
     async def start(self):
         server_config = self.config["server"]
         host = server_config.get("ip", "0.0.0.0")
-        port = int(server_config.get("http_port"))
+        port = int(server_config.get("http_port", 8003))
 
         if port:
             app = web.Application()
+
+            read_config_from_api = server_config.get("read_config_from_api", False)
+
+            if not read_config_from_api:
+                # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
+                app.add_routes(
+                    [
+                        web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
+                        web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
+                        web.options("/xiaozhi/ota/", self.ota_handler.handle_post),
+                    ]
+                )
             # 添加路由
             app.add_routes(
                 [
-                    web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
-                    web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
-                    web.options("/xiaozhi/ota/", self.ota_handler.handle_post),
                     web.get("/mcp/vision/explain", self.vision_handler.handle_get),
                     web.post("/mcp/vision/explain", self.vision_handler.handle_post),
                     web.options("/mcp/vision/explain", self.vision_handler.handle_post),

main/xiaozhi-server/core/utils/auth.py

@@ -0,0 +1,43 @@
+import jwt
+import time
+from datetime import datetime, timedelta, timezone
+from typing import Optional, Tuple
+
+
+class AuthToken:
+    def __init__(self, secret_key: str):
+        self.secret_key = secret_key
+
+    def generate_token(self, device_id: str) -> str:
+        """
+        生成JWT token
+        :param device_id: 设备ID
+        :return: JWT token字符串
+        """
+        # 设置过期时间为1小时后
+        expire_time = datetime.now(timezone.utc) + timedelta(hours=1)
+
+        # 创建payload
+        payload = {"device_id": device_id, "exp": expire_time.timestamp()}
+
+        # 使用JWT进行编码
+        token = jwt.encode(payload, self.secret_key, algorithm="HS256")
+        return token
+
+    def verify_token(self, token: str) -> Tuple[bool, Optional[str]]:
+        """
+        验证token
+        :param token: JWT token字符串
+        :return: (是否有效, 设备ID)
+        """
+        try:
+            # 解码token
+            payload = jwt.decode(token, self.secret_key, algorithms=["HS256"])
+
+            # 检查是否过期
+            if payload["exp"] < time.time():
+                return False, None
+
+            return True, payload["device_id"]
+        except jwt.InvalidTokenError:
+            return False, None

main/xiaozhi-server/requirements.txt

@@ -31,3 +31,4 @@ chardet==5.2.0
 aioconsole==0.8.1
 markitdown==0.1.1
 mcp-proxy==0.6.0
+PyJWT==2.8.0