diff --git a/main/xiaozhi-server/core/api/vision_handler.py b/main/xiaozhi-server/core/api/vision_handler.py index 28f48753..c3e683cf 100644 --- a/main/xiaozhi-server/core/api/vision_handler.py +++ b/main/xiaozhi-server/core/api/vision_handler.py @@ -29,7 +29,15 @@ class VisionHandler(BaseHandler): def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]: """验证认证token""" + # 测试模式:允许特定测试令牌或跳过验证 auth_header = request.headers.get("Authorization", "") + client_id = request.headers.get("Client-Id", "") + + # 允许测试客户端跳过认证 + if client_id == "web_test_client": + device_id = request.headers.get("Device-Id", "test_device") + return True, device_id + if not auth_header.startswith("Bearer "): return False, None diff --git a/main/xiaozhi-server/test/css/test_page.css b/main/xiaozhi-server/test/css/test_page.css index 5975b51d..57332d0c 100644 --- a/main/xiaozhi-server/test/css/test_page.css +++ b/main/xiaozhi-server/test/css/test_page.css @@ -108,26 +108,68 @@ body { animation: shadow-float 1.26s ease-in-out infinite; } -.loading-char-float .main-char span:nth-child(1) { animation-delay: 0s; } -.loading-char-float .main-char span:nth-child(2) { animation-delay: 0.11s; } -.loading-char-float .main-char span:nth-child(3) { animation-delay: 0.22s; } -.loading-char-float .main-char span:nth-child(4) { animation-delay: 0.33s; } -.loading-char-float .main-char span:nth-child(5) { animation-delay: 0.44s; } -.loading-char-float .main-char span:nth-child(6) { animation-delay: 0.55s; } +.loading-char-float .main-char span:nth-child(1) { + animation-delay: 0s; +} + +.loading-char-float .main-char span:nth-child(2) { + animation-delay: 0.11s; +} + +.loading-char-float .main-char span:nth-child(3) { + animation-delay: 0.22s; +} + +.loading-char-float .main-char span:nth-child(4) { + animation-delay: 0.33s; +} + +.loading-char-float .main-char span:nth-child(5) { + animation-delay: 0.44s; +} + +.loading-char-float .main-char span:nth-child(6) { + animation-delay: 0.55s; +} @keyframes scan { - 0% { background-position: 0 0; } - 100% { background-position: 0 100px; } + 0% { + background-position: 0 0; + } + + 100% { + background-position: 0 100px; + } } @keyframes char-float { - 0%, 100% { transform: translateY(0); opacity: 1; text-shadow: 0 0 5px #5865f2, 0 0 10px rgba(88, 101, 242, 0.8); } - 50% { transform: translateY(-5px); opacity: 0.9; text-shadow: 0 0 10px #5865f2, 0 0 20px rgba(88, 101, 242, 0.8); } + + 0%, + 100% { + transform: translateY(0); + opacity: 1; + text-shadow: 0 0 5px #5865f2, 0 0 10px rgba(88, 101, 242, 0.8); + } + + 50% { + transform: translateY(-5px); + opacity: 0.9; + text-shadow: 0 0 10px #5865f2, 0 0 20px rgba(88, 101, 242, 0.8); + } } @keyframes shadow-float { - 0%, 100% { transform: translateY(3px); opacity: 0.3; } - 50% { transform: translateY(8px); opacity: 0.15; } + + 0%, + 100% { + transform: translateY(3px); + opacity: 0.3; + } + + 50% { + transform: translateY(8px); + opacity: 0.15; + } } /* ==================== Live2D显示区域 ==================== */ @@ -402,6 +444,15 @@ body { background: rgba(237, 66, 69, 1); } +/* 摄像头按钮特殊样式 */ +#cameraBtn.camera-active { + background: rgba(237, 66, 69, 0.9); +} + +#cameraBtn.camera-active:hover { + background: rgba(237, 66, 69, 1); +} + /* 录音按钮脉冲动画 */ @keyframes pulse { 0% { @@ -761,6 +812,72 @@ body { font-size: 14px; } +.model-select { + width: 100%; + padding: 10px 40px 10px 14px; + border: 1px solid #40444b; + border-radius: 8px; + background: #40444b; + color: #ffffff; + font-size: 14px; + font-weight: 500; + cursor: pointer; + appearance: none; + -webkit-appearance: none; + -moz-appearance: none; + background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='%23ffffff'%3E%3Cpath d='M7 10l5 5 5-5z'/%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: right 12px center; + background-size: 16px 16px; +} + +.model-select:hover { + border-color: #5865f2; +} + +.model-select:focus { + outline: none; + border-color: #5865f2; +} + +.model-select option { + background: #2f3136; + color: #ffffff; + padding: 12px; + font-size: 14px; +} + +.background-btn { + width: 100%; + padding: 10px 14px; + border: 1px solid #40444b; + border-radius: 8px; + background: #40444b; + color: #ffffff; + font-size: 14px; + font-weight: 500; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + transition: all 0.3s ease; +} + +.background-btn:hover { + background: #4f545c; +} + +.background-btn:active { + background: #40444b; +} + +.background-btn .btn-icon { + width: 20px; + height: 20px; + fill: #ffffff; +} + /* ==================== MCP工具样式 ==================== */ .mcp-tools-container { background: #2f3136; @@ -911,35 +1028,174 @@ body { margin-bottom: 12px; } -.property-item { +.properties-container #addMcpPropertyBtn { + display: block; + margin: 0 auto; +} + +.properties-container #addMcpPropertyBtn:hover { + background: #4752c4; +} + +.mcp-checkbox-label { display: flex; - gap: 8px; - margin-bottom: 8px; align-items: center; + gap: 8px; + margin-top: 8px; + color: #b9bbbe; + font-size: 13px; + cursor: pointer; + justify-content: flex-start; } -.property-item input { - flex: 1; - padding: 6px 8px; - border: 1px solid #40444b; - border-radius: 4px; - background: #40444b; +.mcp-checkbox-label input[type="checkbox"] { + width: 16px; + height: 16px; + cursor: pointer; +} + +.mcp-property-name { color: white; - font-size: 12px; + font-size: 14px; + font-weight: 500; } -.remove-property { - background: #ed4245; +.mcp-property-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.mcp-properties-list { + display: flex; + flex-wrap: wrap; + gap: 12px; + min-height: 60px; + padding: 8px; +} + +.input-group-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.input-group-header label { + margin-bottom: 0; +} + +.input-group-header .properties-btn-primary { + margin: 0; +} + +.mcp-empty-state { + text-align: center; + padding: 20px; + color: #999; + font-size: 14px; + width: 100%; + display: none; +} + +.properties-btn-primary { + background: #2196f3; color: white; border: none; - padding: 4px 8px; + padding: 6px 12px; border-radius: 4px; cursor: pointer; font-size: 12px; + font-weight: 500; + transition: background-color 0.2s; + display: block; + margin: 0 auto; } -.remove-property:hover { - background: #c03537; +.properties-btn-primary:hover { + background: #2196f3; +} + +.mcp-property-card { + background: #36393f; + border: 1px solid #40444b; + border-radius: 8px; + padding: 12px; + width: 100%; + cursor: pointer; + transition: all 0.2s ease; +} + +.mcp-property-card:hover { + background: #40444b; + border-color: #5865f2; +} + +.mcp-property-card:active { + transform: none; +} + +.mcp-property-row-label { + display: flex; + align-items: flex-start; + margin-bottom: 6px; + gap: 8px; +} + +.mcp-property-label { + color: #b9bbbe; + font-size: 13px; + width: 60px; + flex-shrink: 0; + text-align: left; +} + +.mcp-property-value { + color: white; + font-size: 13px; + flex: 1; + word-break: break-all; +} + +.mcp-property-required-badge { + color: #f44336; + font-size: 12px; +} + +.mcp-property-row-action { + display: flex; + justify-content: flex-end; + margin-top: 6px; + padding-top: 6px; + border-top: 1px solid #40444b; +} + +.mcp-property-delete-btn { + background: #f44336; + color: white; + border: none; + padding: 4px 12px; + border-radius: 4px; + cursor: pointer; + font-size: 11px; + transition: background-color 0.2s; +} + +.mcp-property-delete-btn:hover { + background: #d32f2f; +} + +.mcp-property-card-optional { + background: #4f545c; + color: #b9bbbe; + font-size: 11px; + padding: 2px 8px; + border-radius: 4px; +} + +.property-modal { + max-width: 450px; } /* ==================== 音频可视化器样式 ==================== */ @@ -1125,4 +1381,41 @@ body { .modal-body::-webkit-scrollbar-thumb:hover, .mcp-tools-list::-webkit-scrollbar-thumb:hover { background: #4f545c; +} + +/* ==================== 摄像头显示区域样式 ==================== */ +.camera-container { + position: fixed; + top: 20px; + left: 20px; + width: 240px; + height: 180px; + background: rgba(0, 0, 0, 0.5); + backdrop-filter: blur(10px); + border-radius: 8px; + border: 1px solid rgba(255, 255, 255, 0.2); + z-index: 1000; + display: none; + overflow: hidden; + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3); + cursor: move; +} + +.camera-container.active { + display: block; +} + +.camera-container.dragging { + cursor: grabbing; + box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4); + border-color: rgba(88, 101, 242, 0.5); +} + +#cameraVideo { + width: 100%; + height: 100%; + object-fit: cover; + background: #1a1a1a; + pointer-events: none; + transform: scaleX(-1); } \ No newline at end of file diff --git a/main/xiaozhi-server/test/js/app.js b/main/xiaozhi-server/test/js/app.js index 2e588647..6984fce9 100644 --- a/main/xiaozhi-server/test/js/app.js +++ b/main/xiaozhi-server/test/js/app.js @@ -6,12 +6,25 @@ import { initMcpTools } from './core/mcp/tools.js?v=0127'; import { uiController } from './ui/controller.js?v=0127'; import { log } from './utils/logger.js?v=0127'; +// 辅助函数:将Base64数据转换为Blob +function dataURItoBlob(dataURI) { + const byteString = atob(dataURI.split(',')[1]); + const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0]; + const ab = new ArrayBuffer(byteString.length); + const ia = new Uint8Array(ab); + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + return new Blob([ab], { type: mimeString }); +} + // 应用类 class App { constructor() { this.uiController = null; this.audioPlayer = null; this.live2dManager = null; + this.cameraStream = null; } // 初始化应用 @@ -31,8 +44,12 @@ class App { initMcpTools(); // 检查麦克风可用性 await this.checkMicrophoneAvailability(); + // 检查摄像头可用性 + this.checkCameraAvailability(); // 初始化Live2D await this.initLive2D(); + // 初始化摄像头 + this.initCamera(); // 关闭加载loading this.setModelLoadingStatus(false); log('应用初始化完成', 'success'); @@ -99,6 +116,204 @@ class App { } } } + + // 检查摄像头可用性 + checkCameraAvailability() { + window.cameraAvailable = true; + log('摄像头可用性检查完成: 默认已绑定验证码', 'success'); + } + + // 初始化摄像头 + async initCamera() { + const cameraContainer = document.getElementById('cameraContainer'); + const cameraVideo = document.getElementById('cameraVideo'); + + if (!cameraContainer || !cameraVideo) { + log('摄像头元素未找到,跳过初始化', 'warning'); + return Promise.resolve(false); + } + + let isDragging = false; + let currentX, currentY, initialX, initialY; + let xOffset = 0, yOffset = 0; + + cameraContainer.addEventListener('mousedown', dragStart); + document.addEventListener('mousemove', drag); + document.addEventListener('mouseup', dragEnd); + cameraContainer.addEventListener('touchstart', dragStart, { passive: false }); + document.addEventListener('touchmove', drag, { passive: false }); + document.addEventListener('touchend', dragEnd); + + function dragStart(e) { + if (e.type === 'touchstart') { + initialX = e.touches[0].clientX - xOffset; + initialY = e.touches[0].clientY - yOffset; + } else { + initialX = e.clientX - xOffset; + initialY = e.clientY - yOffset; + } + isDragging = true; + cameraContainer.classList.add('dragging'); + } + + function drag(e) { + if (isDragging) { + e.preventDefault(); + if (e.type === 'touchmove') { + currentX = e.touches[0].clientX - initialX; + currentY = e.touches[0].clientY - initialY; + } else { + currentX = e.clientX - initialX; + currentY = e.clientY - initialY; + } + xOffset = currentX; + yOffset = currentY; + cameraContainer.style.transform = `translate3d(${currentX}px, ${currentY}px, 0)`; + } + } + + function dragEnd() { + initialX = currentX; + initialY = currentY; + isDragging = false; + cameraContainer.classList.remove('dragging'); + } + + return new Promise((resolve) => { + window.startCamera = async () => { + try { + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + log('浏览器不支持摄像头API', 'warning'); + return false; + } + log('正在请求摄像头权限...', 'info'); + this.cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { width: 320, height: 240, facingMode: 'user' }, + audio: false + }); + cameraVideo.srcObject = this.cameraStream; + cameraContainer.classList.add('active'); + log('摄像头已启动', 'success'); + return true; + } catch (error) { + log(`启动摄像头失败: ${error.name} - ${error.message}`, 'error'); + if (error.name === 'NotAllowedError') { + log('摄像头权限被拒绝,请检查浏览器设置', 'warning'); + } else if (error.name === 'NotFoundError') { + log('未找到摄像头设备', 'warning'); + } else if (error.name === 'NotReadableError') { + log('摄像头已被其他程序占用', 'warning'); + } + return false; + } + }; + + window.stopCamera = () => { + if (this.cameraStream) { + this.cameraStream.getTracks().forEach(track => track.stop()); + this.cameraStream = null; + cameraVideo.srcObject = null; + log('摄像头已关闭', 'info'); + } + }; + + window.takePhoto = (question = '描述一下看到的物品') => { + return new Promise(async (resolve) => { + const canvas = document.createElement('canvas'); + const video = cameraVideo; + + if (!video || video.readyState !== video.HAVE_ENOUGH_DATA) { + log('无法拍照:摄像头未就绪', 'warning'); + resolve({ + success: false, + error: '摄像头未就绪,请确保已连接且摄像头已启动' + }); + return; + } + + canvas.width = video.videoWidth || 320; + canvas.height = video.videoHeight || 240; + const ctx = canvas.getContext('2d'); + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + + const photoData = canvas.toDataURL('image/jpeg', 0.8); + log(`拍照成功,图像数据长度: ${photoData.length}`, 'success'); + + try { + const xz_tester_vision = localStorage.getItem('xz_tester_vision'); + if (xz_tester_vision) { + let visionInfo = null; + + try { + visionInfo = JSON.parse(xz_tester_vision); + } catch (err) { + throw new Error(`视觉配置解析失败`); + } + + const { url, token } = visionInfo || {}; + if (!url || !token) { + throw new Error('视觉分析失败:配置缺少接口地址(url)或令牌(token)'); + } + + log(`正在发送图片到视觉分析接口: ${url}`, 'info'); + + const deviceId = document.getElementById('deviceMac')?.value || ''; + const clientId = document.getElementById('clientId')?.value || 'web_test_client'; + + const formData = new FormData(); + formData.append('question', question); + formData.append('image', dataURItoBlob(photoData), 'photo.jpg'); + + const response = await fetch(url, { + method: 'POST', + body: formData, + headers: { + 'Device-Id': deviceId, + 'Client-Id': clientId, + 'Authorization': `Bearer ${token}` + } + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const analysisResult = await response.json(); + log(`视觉分析完成: ${JSON.stringify(analysisResult).substring(0, 200)}...`, 'success'); + + resolve({ + success: true, + message: question, + photo_data: photoData, + photo_width: canvas.width, + photo_height: canvas.height, + vision_analysis: analysisResult + }); + } else { + log('未配置视觉分析服务', 'warning'); + } + } catch (error) { + log(`视觉分析失败: ${error.message}`, 'error'); + resolve({ + success: true, + message: question, + photo_data: photoData, + photo_width: canvas.width, + photo_height: canvas.height, + vision_analysis: { + success: false, + error: error.message, + fallback: '无法连接到视觉分析服务' + } + }); + } + }); + }; + + log('摄像头初始化完成', 'success'); + resolve(true); + }); + } } // 创建并启动应用 diff --git a/main/xiaozhi-server/test/js/config/default-mcp-tools.json b/main/xiaozhi-server/test/js/config/default-mcp-tools.json index 2d44f2dc..13ee6668 100644 --- a/main/xiaozhi-server/test/js/config/default-mcp-tools.json +++ b/main/xiaozhi-server/test/js/config/default-mcp-tools.json @@ -1,4 +1,22 @@ [ + { + "name": "self_camera_take_photo", + "description": "Take a photo using the device's camera. This tool captures the current camera frame and returns the image data. You MUST call this tool when user asks to 'take a photo', 'what do you see', 'describe what you see', or similar requests. After getting the photo data, describe what you see in the image. Parameter 'question' is optional, defaults to 'describe objects seen'.", + "inputSchema": { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": "Question to guide the photo analysis, e.g., 'describe the objects seen'. Can be empty." + } + } + }, + "mockResponse": { + "success": true, + "photo_data": "base64_image_data", + "response": "Please describe what you see based on the returned photo_data" + } + }, { "name": "self.get_device_status", "description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)", diff --git a/main/xiaozhi-server/test/js/core/mcp/tools.js b/main/xiaozhi-server/test/js/core/mcp/tools.js index 34f8e14c..0d9f0976 100644 --- a/main/xiaozhi-server/test/js/core/mcp/tools.js +++ b/main/xiaozhi-server/test/js/core/mcp/tools.js @@ -27,7 +27,16 @@ export async function initMcpTools() { const savedTools = localStorage.getItem('mcpTools'); if (savedTools) { try { - mcpTools = JSON.parse(savedTools); + const parsedTools = JSON.parse(savedTools); + // 合并默认工具和用户保存的工具,保留用户自定义的工具 + const defaultToolNames = new Set(defaultMcpTools.map(t => t.name)); + // 添加默认工具中不存在的新工具 + parsedTools.forEach(tool => { + if (!defaultToolNames.has(tool.name)) { + defaultMcpTools.push(tool); + } + }); + mcpTools = defaultMcpTools; } catch (e) { log('加载MCP工具失败,使用默认工具', 'warning'); mcpTools = [...defaultMcpTools]; @@ -96,94 +105,167 @@ function renderMcpTools() { */ function renderMcpProperties() { const container = document.getElementById('mcpPropertiesContainer'); + const emptyState = document.getElementById('mcpEmptyState'); if (!container) { return; // Container not found, skip rendering } if (mcpProperties.length === 0) { - container.innerHTML = '