add:测试页面增加拍照识物功能

2026-07-22 07:03:53 +08:00 · 2026-02-02 15:08:24 +08:00
parent 19fdf5d917
commit 005dc55db3
8 changed files with 435 additions and 22 deletions
@@ -29,7 +29,15 @@ class VisionHandler(BaseHandler):

    def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]:
        """验证认证token"""
+        # 测试模式：允许特定测试令牌或跳过验证
        auth_header = request.headers.get("Authorization", "")
+        client_id = request.headers.get("Client-Id", "")
+
+        # 允许测试客户端跳过认证
+        if client_id == "web_test_client":
+            device_id = request.headers.get("Device-Id", "test_device")
+            return True, device_id
+
        if not auth_header.startswith("Bearer "):
            return False, None

@@ -444,6 +444,15 @@ body {
    background: rgba(237, 66, 69, 1);
 }

+/* 摄像头按钮特殊样式 */
+#cameraBtn.camera-active {
+    background: rgba(237, 66, 69, 0.9);
+}
+
+#cameraBtn.camera-active:hover {
+    background: rgba(237, 66, 69, 1);
+}
+
 /* 录音按钮脉冲动画 */
@keyframes pulse {
    0% {
@@ -1233,4 +1242,41 @@ body {
 .modal-body::-webkit-scrollbar-thumb:hover,
 .mcp-tools-list::-webkit-scrollbar-thumb:hover {
    background: #4f545c;
+}
+
+/* ==================== 摄像头显示区域样式 ==================== */
+.camera-container {
+    position: fixed;
+    top: 20px;
+    left: 20px;
+    width: 240px;
+    height: 180px;
+    background: rgba(0, 0, 0, 0.5);
+    backdrop-filter: blur(10px);
+    border-radius: 8px;
+    border: 1px solid rgba(255, 255, 255, 0.2);
+    z-index: 1000;
+    display: none;
+    overflow: hidden;
+    box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
+    cursor: move;
+}
+
+.camera-container.active {
+    display: block;
+}
+
+.camera-container.dragging {
+    cursor: grabbing;
+    box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4);
+    border-color: rgba(88, 101, 242, 0.5);
+}
+
+#cameraVideo {
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+    background: #1a1a1a;
+    pointer-events: none;
+    transform: scaleX(-1);
 }
@@ -6,12 +6,25 @@ import { initMcpTools } from './core/mcp/tools.js?v=0127';
 import { uiController } from './ui/controller.js?v=0127';
 import { log } from './utils/logger.js?v=0127';

+// 辅助函数：将Base64数据转换为Blob
+function dataURItoBlob(dataURI) {
+    const byteString = atob(dataURI.split(',')[1]);
+    const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
+    const ab = new ArrayBuffer(byteString.length);
+    const ia = new Uint8Array(ab);
+    for (let i = 0; i < byteString.length; i++) {
+        ia[i] = byteString.charCodeAt(i);
+    }
+    return new Blob([ab], { type: mimeString });
+}
+
 // 应用类
 class App {
    constructor() {
        this.uiController = null;
        this.audioPlayer = null;
        this.live2dManager = null;
+        this.cameraStream = null;
    }

    // 初始化应用
@@ -33,6 +46,8 @@ class App {
        await this.checkMicrophoneAvailability();
        // 初始化Live2D
        await this.initLive2D();
+        // 初始化摄像头
+        this.initCamera();
        // 关闭加载loading
        this.setModelLoadingStatus(false);
        log('应用初始化完成', 'success');
@@ -99,6 +114,180 @@ class App {
            }
        }
    }
+
+    // 初始化摄像头
+    async initCamera() {
+        const cameraContainer = document.getElementById('cameraContainer');
+        const cameraVideo = document.getElementById('cameraVideo');
+
+        if (!cameraContainer || !cameraVideo) {
+            log('摄像头元素未找到，跳过初始化', 'warning');
+            return Promise.resolve(false);
+        }
+
+        let isDragging = false;
+        let currentX, currentY, initialX, initialY;
+        let xOffset = 0, yOffset = 0;
+
+        cameraContainer.addEventListener('mousedown', dragStart);
+        document.addEventListener('mousemove', drag);
+        document.addEventListener('mouseup', dragEnd);
+        cameraContainer.addEventListener('touchstart', dragStart, { passive: false });
+        document.addEventListener('touchmove', drag, { passive: false });
+        document.addEventListener('touchend', dragEnd);
+
+        function dragStart(e) {
+            if (e.type === 'touchstart') {
+                initialX = e.touches[0].clientX - xOffset;
+                initialY = e.touches[0].clientY - yOffset;
+            } else {
+                initialX = e.clientX - xOffset;
+                initialY = e.clientY - yOffset;
+            }
+            isDragging = true;
+            cameraContainer.classList.add('dragging');
+        }
+
+        function drag(e) {
+            if (isDragging) {
+                e.preventDefault();
+                if (e.type === 'touchmove') {
+                    currentX = e.touches[0].clientX - initialX;
+                    currentY = e.touches[0].clientY - initialY;
+                } else {
+                    currentX = e.clientX - initialX;
+                    currentY = e.clientY - initialY;
+                }
+                xOffset = currentX;
+                yOffset = currentY;
+                cameraContainer.style.transform = `translate3d(${currentX}px, ${currentY}px, 0)`;
+            }
+        }
+
+        function dragEnd() {
+            initialX = currentX;
+            initialY = currentY;
+            isDragging = false;
+            cameraContainer.classList.remove('dragging');
+        }
+
+        return new Promise((resolve) => {
+            window.startCamera = async () => {
+                try {
+                    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+                        log('浏览器不支持摄像头API', 'warning');
+                        return false;
+                    }
+                    log('正在请求摄像头权限...', 'info');
+                    this.cameraStream = await navigator.mediaDevices.getUserMedia({
+                        video: { width: 320, height: 240, facingMode: 'user' },
+                        audio: false
+                    });
+                    cameraVideo.srcObject = this.cameraStream;
+                    cameraContainer.classList.add('active');
+                    log('摄像头已启动', 'success');
+                    return true;
+                } catch (error) {
+                    log(`启动摄像头失败: ${error.name} - ${error.message}`, 'error');
+                    if (error.name === 'NotAllowedError') {
+                        log('摄像头权限被拒绝，请检查浏览器设置', 'warning');
+                    } else if (error.name === 'NotFoundError') {
+                        log('未找到摄像头设备', 'warning');
+                    } else if (error.name === 'NotReadableError') {
+                        log('摄像头已被其他程序占用', 'warning');
+                    }
+                    return false;
+                }
+            };
+
+            window.stopCamera = () => {
+                if (this.cameraStream) {
+                    this.cameraStream.getTracks().forEach(track => track.stop());
+                    this.cameraStream = null;
+                    cameraVideo.srcObject = null;
+                    log('摄像头已关闭', 'info');
+                }
+            };
+
+            window.takePhoto = (question = '描述一下看到的物品') => {
+                return new Promise(async (resolve) => {
+                    const canvas = document.createElement('canvas');
+                    const video = cameraVideo;
+
+                    if (!video || video.readyState !== video.HAVE_ENOUGH_DATA) {
+                        log('无法拍照：摄像头未就绪', 'warning');
+                        resolve({
+                            success: false,
+                            error: '摄像头未就绪，请确保已连接且摄像头已启动'
+                        });
+                        return;
+                    }
+
+                    canvas.width = video.videoWidth || 320;
+                    canvas.height = video.videoHeight || 240;
+                    const ctx = canvas.getContext('2d');
+                    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+                    const photoData = canvas.toDataURL('image/jpeg', 0.8);
+                    log(`拍照成功，图像数据长度: ${photoData.length}`, 'success');
+
+                    try {
+                        const visionApiUrl = 'http://localhost:8003/mcp/vision/explain';
+                        log(`正在发送图片到视觉分析接口: ${visionApiUrl}`, 'info');
+
+                        const deviceId = document.getElementById('deviceMac')?.value || '';
+                        const clientId = document.getElementById('clientId')?.value || 'web_test_client';
+
+                        const formData = new FormData();
+                        formData.append('question', question);
+                        formData.append('image', dataURItoBlob(photoData), 'photo.jpg');
+
+                        const response = await fetch(visionApiUrl, {
+                            method: 'POST',
+                            body: formData,
+                            headers: {
+                                'Device-Id': deviceId,
+                                'Client-Id': clientId
+                            }
+                        });
+
+                        if (!response.ok) {
+                            throw new Error(`HTTP error! status: ${response.status}`);
+                        }
+
+                        const analysisResult = await response.json();
+                        log(`视觉分析完成: ${JSON.stringify(analysisResult).substring(0, 200)}...`, 'success');
+
+                        resolve({
+                            success: true,
+                            message: question,
+                            photo_data: photoData,
+                            photo_width: canvas.width,
+                            photo_height: canvas.height,
+                            vision_analysis: analysisResult
+                        });
+                    } catch (error) {
+                        log(`视觉分析失败: ${error.message}`, 'error');
+                        resolve({
+                            success: true,
+                            message: question,
+                            photo_data: photoData,
+                            photo_width: canvas.width,
+                            photo_height: canvas.height,
+                            vision_analysis: {
+                                success: false,
+                                error: error.message,
+                                fallback: '无法连接到视觉分析服务'
+                            }
+                        });
+                    }
+                });
+            };
+
+            log('摄像头初始化完成', 'success');
+            resolve(true);
+        });
+    }
 }

 // 创建并启动应用
@@ -1,4 +1,22 @@
 [
+    {
+        "name": "self_camera_take_photo",
+        "description": "Take a photo using the device's camera. This tool captures the current camera frame and returns the image data. You MUST call this tool when user asks to 'take a photo', 'what do you see', 'describe what you see', or similar requests. After getting the photo data, describe what you see in the image. Parameter 'question' is optional, defaults to 'describe objects seen'.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "question": {
+                    "type": "string",
+                    "description": "Question to guide the photo analysis, e.g., 'describe the objects seen'. Can be empty."
+                }
+            }
+        },
+        "mockResponse": {
+            "success": true,
+            "photo_data": "base64_image_data",
+            "response": "Please describe what you see based on the returned photo_data"
+        }
+    },
    {
        "name": "self.get_device_status",
        "description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
@@ -27,7 +27,16 @@ export async function initMcpTools() {
    const savedTools = localStorage.getItem('mcpTools');
    if (savedTools) {
        try {
-            mcpTools = JSON.parse(savedTools);
+            const parsedTools = JSON.parse(savedTools);
+            // 合并默认工具和用户保存的工具，保留用户自定义的工具
+            const defaultToolNames = new Set(defaultMcpTools.map(t => t.name));
+            // 添加默认工具中不存在的新工具
+            parsedTools.forEach(tool => {
+                if (!defaultToolNames.has(tool.name)) {
+                    defaultMcpTools.push(tool);
+                }
+            });
+            mcpTools = defaultMcpTools;
        } catch (e) {
            log('加载MCP工具失败，使用默认工具', 'warning');
            mcpTools = [...defaultMcpTools];
@@ -392,12 +401,26 @@ export function getMcpTools() {
 /**
 * 执行工具调用
 */
-export function executeMcpTool(toolName, toolArgs) {
+export async function executeMcpTool(toolName, toolArgs) {
    const tool = mcpTools.find(t => t.name === toolName);
    if (!tool) {
        log(`未找到工具: ${toolName}`, 'error');
        return { success: false, error: `未知工具: ${toolName}` };
    }
+
+    // 处理拍照工具
+    if (toolName === 'self_camera_take_photo') {
+        if (typeof window.takePhoto === 'function') {
+            const question = toolArgs && toolArgs.question ? toolArgs.question : '描述一下看到的物品';
+            log(`正在执行拍照: ${question}`, 'info');
+            const result = await window.takePhoto(question);
+            return result;
+        } else {
+            log('拍照功能不可用', 'warning');
+            return { success: false, error: '摄像头未启动或不支持拍照功能' };
+        }
+    }
+
    // 如果有模拟返回结果，使用它
    if (tool.mockResponse) {
        // 替换模板变量
@@ -249,28 +249,43 @@ export class WebSocketHandler {

            log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info');

-            const result = executeMcpTool(toolName, toolArgs);
-
-            const replyMessage = JSON.stringify({
-                "session_id": message.session_id || "",
-                "type": "mcp",
-                "payload": {
-                    "jsonrpc": "2.0",
-                    "id": payload.id,
-                    "result": {
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": JSON.stringify(result)
-                            }
-                        ],
-                        "isError": false
+            executeMcpTool(toolName, toolArgs).then(result => {
+                const replyMessage = JSON.stringify({
+                    "session_id": message.session_id || "",
+                    "type": "mcp",
+                    "payload": {
+                        "jsonrpc": "2.0",
+                        "id": payload.id,
+                        "result": {
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": JSON.stringify(result)
+                                }
+                            ],
+                            "isError": false
+                        }
                    }
-                }
-            });
+                });

-            log(`客户端上报: ${replyMessage}`, 'info');
-            this.websocket.send(replyMessage);
+                log(`客户端上报: ${replyMessage}`, 'info');
+                this.websocket.send(replyMessage);
+            }).catch(error => {
+                log(`工具执行失败: ${error.message}`, 'error');
+                const errorReply = JSON.stringify({
+                    "session_id": message.session_id || "",
+                    "type": "mcp",
+                    "payload": {
+                        "jsonrpc": "2.0",
+                        "id": payload.id,
+                        "error": {
+                            "code": -32603,
+                            "message": error.message
+                        }
+                    }
+                });
+                this.websocket.send(errorReply);
+            });
        } else if (payload.method === 'initialize') {
            log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
            const replyMessage = JSON.stringify({
@@ -387,6 +402,17 @@ export class WebSocketHandler {

            const audioRecorder = getAudioRecorder();
            audioRecorder.stop();
+
+            // 关闭摄像头
+            if (typeof window.stopCamera === 'function') {
+                window.stopCamera();
+            }
+
+            // 隐藏摄像头显示区域
+            const cameraContainer = document.getElementById('cameraContainer');
+            if (cameraContainer) {
+                cameraContainer.classList.remove('active');
+            }
        };

        this.websocket.onerror = (error) => {
@@ -420,6 +446,17 @@ export class WebSocketHandler {
        this.websocket.close();
        const audioRecorder = getAudioRecorder();
        audioRecorder.stop();
+
+        // 关闭摄像头
+        if (typeof window.stopCamera === 'function') {
+            window.stopCamera();
+        }
+
+        // 隐藏摄像头显示区域
+        const cameraContainer = document.getElementById('cameraContainer');
+        if (cameraContainer) {
+            cameraContainer.classList.remove('active');
+        }
    }

    // 发送文本消息
@@ -125,6 +125,46 @@ class UIController {
            });
        }

+        // Camera button
+        const cameraBtn = document.getElementById('cameraBtn');
+        if (cameraBtn) {
+            cameraBtn.addEventListener('click', () => {
+                const cameraContainer = document.getElementById('cameraContainer');
+                if (!cameraContainer) {
+                    log('摄像头容器不存在', 'warning');
+                    return;
+                }
+
+                const isActive = cameraContainer.classList.contains('active');
+                if (isActive) {
+                    // 关闭摄像头
+                    if (typeof window.stopCamera === 'function') {
+                        window.stopCamera();
+                    }
+                    cameraContainer.classList.remove('active');
+                    cameraBtn.classList.remove('camera-active');
+                    cameraBtn.querySelector('.btn-text').textContent = '摄像头';
+                    log('摄像头已关闭', 'info');
+                } else {
+                    // 打开摄像头
+                    if (typeof window.startCamera === 'function') {
+                        window.startCamera().then(success => {
+                            if (success) {
+                                cameraBtn.classList.add('camera-active');
+                                cameraBtn.querySelector('.btn-text').textContent = '关闭';
+                            } else {
+                                this.addChatMessage('⚠️ 摄像头启动失败，请检查浏览器权限', false);
+                            }
+                        }).catch(error => {
+                            log(`启动摄像头异常: ${error.message}`, 'error');
+                        });
+                    } else {
+                        log('startCamera函数未定义', 'warning');
+                    }
+                }
+            });
+        }
+
        // Record button
        const recordBtn = document.getElementById('recordBtn');
        if (recordBtn) {
@@ -235,6 +275,7 @@ class UIController {
    updateDialButton(isConnected) {
        const dialBtn = document.getElementById('dialBtn');
        const recordBtn = document.getElementById('recordBtn');
+        const cameraBtn = document.getElementById('cameraBtn');

        if (dialBtn) {
            if (isConnected) {
@@ -254,6 +295,28 @@ class UIController {
            }
        }

+        // Update camera button state - reset to default when disconnected
+        if (cameraBtn && !isConnected) {
+            const cameraContainer = document.getElementById('cameraContainer');
+            if (cameraContainer && cameraContainer.classList.contains('active')) {
+                cameraContainer.classList.remove('active');
+            }
+            cameraBtn.classList.remove('camera-active');
+            cameraBtn.querySelector('.btn-text').textContent = '摄像头';
+            cameraBtn.disabled = true;
+            cameraBtn.title = '请先连接服务器';
+            // 关闭摄像头
+            if (typeof window.stopCamera === 'function') {
+                window.stopCamera();
+            }
+        }
+
+        // Update camera button state - enable when connected
+        if (cameraBtn && isConnected) {
+            cameraBtn.disabled = false;
+            cameraBtn.title = '打开/关闭摄像头';
+        }
+
        // Update record button state
        if (recordBtn) {
            const microphoneAvailable = window.microphoneAvailable !== false;
@@ -520,6 +583,22 @@ class UIController {

                this.hideModal('settingsModal');

+                // 启动摄像头
+                if (typeof window.startCamera === 'function') {
+                    window.startCamera().then(success => {
+                        if (success) {
+                            const cameraBtn = document.getElementById('cameraBtn');
+                            if (cameraBtn) {
+                                cameraBtn.classList.add('camera-active');
+                                cameraBtn.querySelector('.btn-text').textContent = '关闭';
+                            }
+                        } else {
+                            this.addChatMessage('⚠️ 摄像头启动失败，可能被浏览器拒绝', false);
+                        }
+                    }).catch(error => {
+                        log(`启动摄像头异常: ${error.message}`, 'error');
+                    });
+                }
            } else {
                throw new Error('OTA连接失败');
            }
@@ -44,6 +44,11 @@

    <!-- 主容器 -->
    <div class="container">
+        <!-- 摄像头显示区域（可拖动） -->
+        <div class="camera-container" id="cameraContainer">
+            <video id="cameraVideo" autoplay playsinline muted></video>
+        </div>
+
        <!-- 连接状态指示器 - 页面顶部中部 -->
        <div class="connection-status-top">
            <div class="status-indicator">
@@ -87,6 +92,14 @@
                <span class="btn-text">设置</span>
            </button>

+            <button class="control-btn" id="cameraBtn" title="请先连接服务器" disabled>
+                <svg class="btn-icon" viewBox="0 0 24 24" fill="currentColor">
+                    <path
+                        d="M17,10.5V7A1,1 0 0,0 16,6H4A1,1 0 0,0 3,7V17A1,1 0 0,0 4,18H16A1,1 0 0,0 17,17V13.5L21,17.5V6.5L17,10.5Z" />
+                </svg>
+                <span class="btn-text">摄像头</span>
+            </button>
+
            <button class="control-btn dial-btn" id="dialBtn" title="拨号">
                <svg class="btn-icon" viewBox="0 0 24 24" fill="currentColor">
                    <path