diff --git a/main/xiaozhi-server/.claude/settings.local.json b/main/xiaozhi-server/.claude/settings.local.json new file mode 100644 index 00000000..b9a1f5de --- /dev/null +++ b/main/xiaozhi-server/.claude/settings.local.json @@ -0,0 +1,10 @@ +{ + "permissions": { + "allow": [ + "Bash(tree:*)", + "Bash(find:*)" + ], + "deny": [], + "ask": [] + } +} diff --git a/main/xiaozhi-server/test/css/bg.png b/main/xiaozhi-server/test/css/bg.png new file mode 100644 index 00000000..1e63f4db Binary files /dev/null and b/main/xiaozhi-server/test/css/bg.png differ diff --git a/main/xiaozhi-server/test/test_page.css b/main/xiaozhi-server/test/css/test_page.css similarity index 56% rename from main/xiaozhi-server/test/test_page.css rename to main/xiaozhi-server/test/css/test_page.css index 17c6411c..1642ae2d 100644 --- a/main/xiaozhi-server/test/test_page.css +++ b/main/xiaozhi-server/test/css/test_page.css @@ -1,17 +1,40 @@ +/* ==================== 全局样式 ==================== */ +* { + box-sizing: border-box; +} + body { font-family: 'PingFang SC', 'Microsoft YaHei', sans-serif; margin: 0; padding: 0px; - background-color: #f7f8fb; + background-image: url('bg.png'); } +/* ==================== 容器布局 ==================== */ .container { - max-width: 80%; + max-width: 1400px; margin: 0 auto; border-radius: 10px; padding: 0px; } +/* 两栏布局 */ +.two-column-layout { + display: flex; + gap: 20px; + align-items: stretch; + margin-top: 10px; + margin-bottom: 10px; +} + +.two-column-layout>.section { + flex: 1; + margin-top: 0; + margin-bottom: 0; + min-width: 0; +} + +/* ==================== 通用区块样式 ==================== */ .section { margin-top: 10px; margin-bottom: 10px; @@ -23,7 +46,7 @@ body { .section h2 { margin: 0; - color: #444; + color: #000; font-size: 14px; font-weight: 600; letter-spacing: 0.02em; @@ -32,6 +55,23 @@ body { gap: 10px; } +/* ==================== 通用按钮样式 ==================== */ +button { + padding: 8px 15px; + border: none; + border-radius: 5px; + background-color: #007aff; + color: white; + cursor: pointer; + transition: background-color 0.2s; +} + +button:disabled { + background-color: #cccccc; + cursor: not-allowed; +} + +/* 切换按钮和连接按钮 */ .toggle-button, .connect-button { margin-left: auto; @@ -53,12 +93,13 @@ body { box-shadow: 0px 0px 10px 2px rgba(0, 122, 255, 0.28); } +/* ==================== 设备配置面板 ==================== */ .config-panel { display: block; transition: all 0.3s ease; } -/* 只为MCP工具面板设置默认隐藏 */ +/* MCP工具面板默认隐藏 */ #mcpToolsPanel { display: none; } @@ -74,49 +115,213 @@ body { margin-top: 10px; } +/* 配置行 */ .config-row { display: flex; gap: 10px; width: 100%; } +/* 配置项 */ .config-item { flex: 1; display: flex; align-items: center; gap: 10px; min-width: 0; + margin-bottom: 8px; + width: 100%; +} + +.config-item label { + white-space: nowrap; + min-width: 70px; + text-align: left; + color: #666; + font-size: 12px; } .config-item input { flex: 1; width: 0; - /* 确保input能自动填充剩余空间 */ -} - -button { - padding: 8px 15px; - border: none; + flex-grow: 1; + padding: 6px; + font-size: 12px; + border: 1px solid #b1c9f8; border-radius: 5px; - background-color: #007aff; - color: white; - cursor: pointer; - transition: background-color 0.2s; } -button:disabled { - background-color: #cccccc; - cursor: not-allowed; +.config-item input:disabled { + background-color: #fff; + border-color: #fff; + cursor: default; +} + +/* ==================== 连接信息面板 ==================== */ +.connection-controls { + display: flex; + flex-direction: column; + gap: 10px; + width: 100%; + margin-top: 10px; +} + +.connection-controls input { + width: 100%; + padding: 8px; + border: 1px solid #ddd; + border-radius: 5px; + box-sizing: border-box; +} + +.connection-controls button { + white-space: nowrap; + padding: 8px 15px; } #serverUrl, #otaUrl { flex-grow: 1; padding: 8px; - border: 1px solid #ddd; border-radius: 5px; } +#otaUrl { + border: 1px solid #b1c9f8; +} + +#serverUrl { + border: 1px solid #fafafa; + background-color: #fafafa; +} + +/* 连接状态 */ +.connection-status { + display: flex; + align-items: center; + gap: 20px; + margin-left: 0px; + padding: 0 15px; + height: 28px; + line-height: 28px; +} + +.connection-status span { + font-size: 13px; +} + +.connection-status .status { + background-color: #fef2f2; + color: #b91c1c; + font-size: 12px; + padding: 0px 8px; + border-radius: 20px; + transition: all 0.3s ease; +} + +/* 已连接状态 */ +.connection-status .status.connected { + background-color: #ecfdf3; + color: #15803d; +} + +/* 会话状态 - 离线中 */ +.connection-status .status.offline { + background-color: #fef2f2; + color: #b91c1c; +} + +/* 会话状态 - 聆听中 */ +.connection-status .status.listening { + background-color: #ecfdf3; + color: #15803d; +} + +/* 会话状态 - 说话中 */ +.connection-status .status.speaking { + background-color: #fff7ed; + color: #c2410c; + animation: pulse-speaking 1.5s infinite; +} + +@keyframes pulse-speaking { + 0%, + 100% { + opacity: 1; + } + + 50% { + opacity: 0.7; + } +} + +/* LLM状态emoji样式 */ +span.connection-status.llm-emoji { + display: block !important; + text-align: center !important; + width: 100% !important; + margin: 0 auto !important; +} + +.llm-emoji .status { + font-size: 14px !important; + padding: 8px 20px !important; +} + +.emoji-large { + font-size: 2em !important; + vertical-align: middle; + display: inline-block; + line-height: 1; +} + +/* ==================== 标签页样式 ==================== */ +.tabs { + display: flex; + margin-bottom: 20px; + border-bottom: 1px solid #e0e0e0; + align-items: center; +} + +.tab { + padding: 10px 20px; + cursor: pointer; + border: none; + background: none; + font-size: 14px; + color: #666; + position: relative; + transition: all 0.3s ease; +} + +.tab:hover { + color: #007aff; +} + +.tab.active { + color: #007aff; + font-weight: bold; +} + +.tab.active::after { + content: ''; + position: absolute; + bottom: -1px; + left: 0; + width: 100%; + height: 1px; + background-color: #007aff; +} + +.tab-content { + display: none; +} + +.tab-content.active { + display: block; +} + +/* ==================== 文本消息输入 ==================== */ .message-input { display: flex; gap: 10px; @@ -125,49 +330,22 @@ button:disabled { #messageInput { flex-grow: 1; - padding: 8px; - border: 1px solid #ddd; + padding: 8px 20px 8px 20px; + border: 1px solid #b1c9f8; border-radius: 10px; } +#messageInput:disabled { + border: 1px solid #fafafa; + background-color: #fafafa; +} + #sendTextButton, #recordButton { border-radius: 20px; } -.conversation { - max-height: 300px; - overflow-y: auto; - border: 1px solid #ddd; - border-right: none; - border-radius: 5px 0px 0px 5px; - padding: 10px; - flex: 1; -} - -.message { - margin-bottom: 10px; - padding: 8px 12px; - border-radius: 8px; - width: fit-content; - max-width: 80%; -} - -.user { - background-color: #95ec69; - margin-left: auto; - margin-right: 10px; - text-align: right; - box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1); -} - -.server { - background-color: #fff; - margin-right: auto; - margin-left: 10px; - box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1); -} - +/* ==================== 语音消息控制 ==================== */ .audio-controls { display: flex; justify-content: center; @@ -210,6 +388,49 @@ button:disabled { } } +/* ==================== 会话记录和日志 ==================== */ +.flex-container { + display: flex; + margin-top: 10px; + background-color: #f9fafb; +} + +/* 会话记录容器 */ +.conversation { + max-height: 300px; + overflow-y: auto; + border: 1px solid #ddd; + border-right: none; + border-radius: 5px 0px 0px 5px; + padding: 10px; + flex: 1; +} + +/* 消息气泡 */ +.message { + margin-bottom: 10px; + padding: 8px 12px; + border-radius: 8px; + width: fit-content; + max-width: 80%; +} + +.user { + background-color: #fff; + margin-left: auto; + margin-right: 10px; + text-align: right; + box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1); +} + +.server { + background-color: #95ec69; + margin-right: auto; + margin-left: 10px; + box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1); +} + +/* 日志容器 */ #logContainer { margin-top: 0; padding: 10px; @@ -238,253 +459,7 @@ button:disabled { color: #0f9d58; } -.script-status { - display: inline-block; - width: 10px; - height: 10px; - border-radius: 50%; - margin-right: 5px; -} - -.script-loaded { - background-color: #0f9d58; -} - -.script-loading { - background-color: #f4b400; -} - -.script-error { - background-color: #db4437; -} - -.script-list { - margin: 10px 0; - padding: 10px; - background-color: #f9f9f9; - border-radius: 5px; - font-family: monospace; - font-size: 11px; -} - -#scriptStatus.success { - background-color: #e6f4ea; - color: #0f9d58; - border-left: 4px solid #0f9d58; -} - -#scriptStatus.error { - background-color: #fce8e6; - color: #db4437; - border-left: 4px solid #db4437; -} - -#scriptStatus.warning { - background-color: #fef7e0; - color: #f4b400; - border-left: 4px solid #f4b400; -} - -/* 标签页样式 */ -.tabs { - display: flex; - margin-bottom: 20px; - border-bottom: 2px solid #e0e0e0; -} - -.tab { - padding: 10px 20px; - cursor: pointer; - border: none; - background: none; - font-size: 16px; - color: #666; - position: relative; - transition: all 0.3s ease; -} - -.tab:hover { - color: #007aff; -} - -.tab.active { - color: #007aff; - font-weight: bold; -} - -.tab.active::after { - content: ''; - position: absolute; - bottom: -2px; - left: 0; - width: 100%; - height: 2px; - background-color: #007aff; -} - -.tab-content { - display: none; -} - -.tab-content.active { - display: block; -} - -.flex-container { - display: flex; - margin-top: 10px; - background-color: #f9fafb; -} - -.config-item { - display: flex; - align-items: center; - margin-bottom: 8px; - width: 100%; -} - -.config-item label { - white-space: nowrap; - min-width: 70px; - text-align: left; - color: #666; - font-size: 14px; -} - -.config-item input { - flex-grow: 1; - padding: 6px; - border: 1px solid #ddd; - border-radius: 5px; -} - -.config-item input:disabled { - background-color: #f9f9f9; - border-color: #e0e0e0; - cursor: default; -} - -/* 两栏布局样式 */ -.two-column-layout { - display: flex; - gap: 15px; - align-items: stretch; -} - -.two-column-layout>.section { - flex: 1; - margin-bottom: 5px; -} - -.control-panel { - display: flex; - flex-direction: column; - gap: 10px; - margin-top: 10px; -} - -.connection-controls { - display: flex; - flex-direction: column; - gap: 10px; - width: 100%; - margin-top: 10px; -} - -.connection-controls input { - width: 100%; - padding: 8px; - border: 1px solid #ddd; - border-radius: 5px; - box-sizing: border-box; -} - -.connection-controls button { - white-space: nowrap; - padding: 8px 15px; -} - -.connection-status { - display: flex; - align-items: center; - gap: 20px; - margin-left: 20px; - padding: 0 15px; - height: 28px; - line-height: 28px; -} - -.connection-status span { - font-size: 13px; -} - -.connection-status .status { - /* 默认未连接状态 */ - background-color: #fef2f2; - color: #b91c1c; - font-size: 12px; - padding: 0px 8px; - border-radius: 20px; - transition: all 0.3s ease; -} - -/* 已连接状态样式 */ -.connection-status .status.connected { - background-color: #ecfdf3; - color: #15803d; -} - -#fileProtocolWarning { - position: fixed; - top: 0; - left: 0; - width: 100%; - height: 100%; - background-color: rgba(0, 0, 0, 0.8); - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - z-index: 9999; - color: white; - padding: 20px; - box-sizing: border-box; -} - -#fileProtocolWarning h2 { - color: #ff4d4d; - margin-bottom: 20px; -} - -#fileProtocolWarning pre { - background-color: green; - font-size: 18px; - padding: 15px; - border-radius: 5px; - font-family: monospace; - overflow-x: auto; - margin: 15px 0; -} - -#fileProtocolWarning button { - background-color: #4CAF50; - color: white; - border: none; - padding: 10px 20px; - text-align: center; - text-decoration: none; - display: inline-block; - font-size: 16px; - margin: 10px 2px; - cursor: pointer; - border-radius: 4px; -} - -#fileProtocolWarning button:hover { - background-color: #45a049; -} - -/* MCP 工具管理样式 */ +/* ==================== MCP 工具管理 ==================== */ .mcp-tools-container { display: grid; gap: 12px; @@ -558,6 +533,7 @@ button:disabled { font-family: 'Courier New', monospace; } +/* MCP 属性项 */ .mcp-property-item { background-color: white; border: 1px solid #e0e0e0; @@ -639,4 +615,382 @@ button:disabled { .mcp-badge-optional { background-color: #e3f2fd; color: #1976d2; -} \ No newline at end of file +} + +/* ==================== 脚本加载状态 ==================== */ +.script-status { + display: inline-block; + width: 10px; + height: 10px; + border-radius: 50%; + margin-right: 5px; +} + +.script-loaded { + background-color: #0f9d58; +} + +.script-loading { + background-color: #f4b400; +} + +.script-error { + background-color: #db4437; +} + +.script-list { + margin: 10px 0; + padding: 10px; + background-color: #f9f9f9; + border-radius: 5px; + font-family: monospace; + font-size: 11px; +} + +#scriptStatus.success { + background-color: #e6f4ea; + color: #0f9d58; + border-left: 4px solid #0f9d58; +} + +#scriptStatus.error { + background-color: #fce8e6; + color: #db4437; + border-left: 4px solid #db4437; +} + +#scriptStatus.warning { + background-color: #fef7e0; + color: #f4b400; + border-left: 4px solid #f4b400; +} + +/* ==================== File协议警告 ==================== */ +#fileProtocolWarning { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.8); + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + z-index: 9999; + color: white; + padding: 20px; + box-sizing: border-box; +} + +#fileProtocolWarning h2 { + color: #ff4d4d; + margin-bottom: 20px; +} + +#fileProtocolWarning pre { + background-color: green; + font-size: 18px; + padding: 15px; + border-radius: 5px; + font-family: monospace; + overflow-x: auto; + margin: 15px 0; +} + +#fileProtocolWarning button { + background-color: #4CAF50; + color: white; + border: none; + padding: 10px 20px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 16px; + margin: 10px 2px; + cursor: pointer; + border-radius: 4px; +} + +#fileProtocolWarning button:hover { + background-color: #45a049; +} + +/* ==================== 响应式布局 ==================== */ + +/* 大屏幕 (宽度 > 1200px) */ +@media (min-width: 1201px) { + .container { + max-width: 80%; + } +} + +/* 中等屏幕 (宽度 <= 1200px) */ +@media (max-width: 1200px) { + .container { + max-width: 95%; + padding: 10px; + } +} + +/* 平板设备 (宽度 <= 1060px) */ +@media (max-width: 1060px) { + + /* 两栏布局改为单栏 */ + .two-column-layout { + flex-direction: column; + gap: 0; + } + + .two-column-layout>.section { + width: 100%; + margin-top: 10px; + margin-bottom: 10px; + } + + /* 连接状态标签调整 */ + .connection-status { + flex-wrap: wrap; + gap: 8px; + margin-left: 0px; + padding: 0 8px; + } + + /* 会话记录和日志容器改为上下布局 */ + .flex-container { + flex-direction: column; + } + + .conversation, + #logContainer { + border-radius: 5px; + border: 1px solid #ddd; + max-height: 200px; + } + + .conversation { + margin-bottom: 10px; + } + + /* MCP 属性行改为单列 */ + .mcp-property-row { + grid-template-columns: 1fr; + } +} + +/* 移动设备 (宽度 <= 768px) */ +@media (max-width: 768px) { + body { + padding: 0; + } + + .container { + max-width: 100%; + padding: 5px; + } + + /* 调整区块内边距 */ + .section { + padding: 8px; + margin-top: 8px; + margin-bottom: 8px; + border-radius: 10px; + } + + /* 调整标题字体 */ + .section h2 { + font-size: 13px; + flex-wrap: wrap; + gap: 5px; + } + + /* 配置行改为单列 */ + .config-row { + flex-direction: column; + } + + /* 按钮调整 */ + .toggle-button, + .connect-button { + padding: 3px 10px; + font-size: 12px; + height: 24px; + line-height: 18px; + } + + button { + padding: 6px 12px; + font-size: 13px; + } + + /* 输入框 */ + #serverUrl, + #otaUrl, + #messageInput { + font-size: 14px; + padding: 6px; + } + + /* 配置项标签 */ + .config-item label { + min-width: 60px; + font-size: 12px; + } + + .config-item input { + font-size: 13px; + padding: 5px; + } + + /* 标签页 */ + .tab { + padding: 8px 12px; + font-size: 14px; + } + + /* 消息气泡 */ + .message { + max-width: 90%; + font-size: 14px; + padding: 6px 10px; + } + + /* 日志容器 */ + #logContainer { + font-size: 11px; + padding: 8px; + } + + .log-entry { + font-size: 11px; + } + + /* 连接状态 */ + .connection-status { + margin-left: 0; + padding: 5px 0; + height: auto; + line-height: normal; + } + + .connection-status .status { + font-size: 11px; + padding: 0px 6px; + } + + /* 音频可视化 */ + .audio-visualizer { + height: 50px; + } + + /* MCP 工具卡片 */ + .mcp-tool-card { + padding: 10px; + } + + .mcp-tool-name { + font-size: 13px; + } + + .mcp-tool-description { + font-size: 12px; + } + + .mcp-tool-info { + font-size: 11px; + padding: 6px; + } + + .mcp-tool-info-row { + flex-direction: column; + gap: 2px; + } + + .mcp-tool-info-label { + min-width: auto; + } + + /* 消息输入框 */ + .message-input { + gap: 5px; + } + + /* 会话记录高度调整 */ + .conversation, + #logContainer { + max-height: 180px; + } +} + +/* 小型移动设备 (宽度 <= 480px) */ +@media (max-width: 480px) { + .container { + padding: 3px; + } + + .section { + padding: 6px; + margin-top: 5px; + margin-bottom: 5px; + } + + .section h2 { + font-size: 12px; + } + + /* 更紧凑的按钮 */ + .toggle-button, + .connect-button { + padding: 2px 8px; + font-size: 11px; + height: 22px; + line-height: 16px; + } + + button { + padding: 5px 10px; + font-size: 12px; + } + + .config-item label { + min-width: 50px; + font-size: 11px; + } + + .config-item input { + font-size: 12px; + } + + /* 标签页 */ + .tab { + padding: 6px 10px; + font-size: 13px; + } + + /* 消息气泡 */ + .message { + max-width: 95%; + font-size: 13px; + } + + /* 会话记录 */ + .conversation, + #logContainer { + max-height: 150px; + padding: 8px; + } + + /* 连接状态文字更小 */ + .connection-status .status { + font-size: 10px; + } + + /* MCP 工具 */ + .mcp-tool-name { + font-size: 12px; + } + + .mcp-tool-description { + font-size: 11px; + } +} diff --git a/main/xiaozhi-server/test/js/app.js b/main/xiaozhi-server/test/js/app.js new file mode 100644 index 00000000..03930f6a --- /dev/null +++ b/main/xiaozhi-server/test/js/app.js @@ -0,0 +1,50 @@ +// 主应用入口 +import { log } from './utils/logger.js'; +import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js'; +import { getUIController } from './ui/controller.js'; +import { getAudioPlayer } from './core/audio/player.js'; +import { initMcpTools } from './core/mcp/tools.js'; + +// 应用类 +class App { + constructor() { + this.uiController = null; + this.audioPlayer = null; + } + + // 初始化应用 + async init() { + log('正在初始化应用...', 'info'); + + // 初始化UI控制器 + this.uiController = getUIController(); + this.uiController.init(); + + // 检查Opus库 + checkOpusLoaded(); + + // 初始化Opus编码器 + initOpusEncoder(); + + // 初始化音频播放器 + this.audioPlayer = getAudioPlayer(); + await this.audioPlayer.start(); + + // 初始化MCP工具 + initMcpTools(); + + log('应用初始化完成', 'success'); + } +} + +// 创建并启动应用 +const app = new App(); + +// DOM加载完成后初始化 +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => app.init()); +} else { + app.init(); +} + +export default app; diff --git a/main/xiaozhi-server/test/js/default-mcp-tools.json b/main/xiaozhi-server/test/js/config/default-mcp-tools.json similarity index 100% rename from main/xiaozhi-server/test/js/default-mcp-tools.json rename to main/xiaozhi-server/test/js/config/default-mcp-tools.json diff --git a/main/xiaozhi-server/test/js/config/manager.js b/main/xiaozhi-server/test/js/config/manager.js new file mode 100644 index 00000000..95eac7b9 --- /dev/null +++ b/main/xiaozhi-server/test/js/config/manager.js @@ -0,0 +1,86 @@ +// 配置管理模块 +import { log } from '../utils/logger.js'; + +// 生成随机MAC地址 +function generateRandomMac() { + const hexDigits = '0123456789ABCDEF'; + let mac = ''; + for (let i = 0; i < 6; i++) { + if (i > 0) mac += ':'; + for (let j = 0; j < 2; j++) { + mac += hexDigits.charAt(Math.floor(Math.random() * 16)); + } + } + return mac; +} + +// 加载配置 +export function loadConfig() { + const deviceMacInput = document.getElementById('deviceMac'); + const deviceNameInput = document.getElementById('deviceName'); + const clientIdInput = document.getElementById('clientId'); + const tokenInput = document.getElementById('token'); + const otaUrlInput = document.getElementById('otaUrl'); + + // 从localStorage加载MAC地址,如果没有则生成新的 + let savedMac = localStorage.getItem('deviceMac'); + if (!savedMac) { + savedMac = generateRandomMac(); + localStorage.setItem('deviceMac', savedMac); + } + deviceMacInput.value = savedMac; + + // 从localStorage加载其他配置 + const savedDeviceName = localStorage.getItem('deviceName'); + if (savedDeviceName) { + deviceNameInput.value = savedDeviceName; + } + + const savedClientId = localStorage.getItem('clientId'); + if (savedClientId) { + clientIdInput.value = savedClientId; + } + + const savedToken = localStorage.getItem('token'); + if (savedToken) { + tokenInput.value = savedToken; + } + + const savedOtaUrl = localStorage.getItem('otaUrl'); + if (savedOtaUrl) { + otaUrlInput.value = savedOtaUrl; + } +} + +// 保存配置 +export function saveConfig() { + const deviceMacInput = document.getElementById('deviceMac'); + const deviceNameInput = document.getElementById('deviceName'); + const clientIdInput = document.getElementById('clientId'); + const tokenInput = document.getElementById('token'); + + localStorage.setItem('deviceMac', deviceMacInput.value); + localStorage.setItem('deviceName', deviceNameInput.value); + localStorage.setItem('clientId', clientIdInput.value); + localStorage.setItem('token', tokenInput.value); +} + +// 获取配置值 +export function getConfig() { + const deviceMac = document.getElementById('deviceMac').value.trim(); + return { + deviceId: deviceMac, // 使用MAC地址作为deviceId + deviceName: document.getElementById('deviceName').value.trim(), + deviceMac: deviceMac, + clientId: document.getElementById('clientId').value.trim(), + token: document.getElementById('token').value.trim() + }; +} + +// 保存连接URL +export function saveConnectionUrls() { + const otaUrl = document.getElementById('otaUrl').value.trim(); + const wsUrl = document.getElementById('serverUrl').value.trim(); + localStorage.setItem('otaUrl', otaUrl); + localStorage.setItem('wsUrl', wsUrl); +} diff --git a/main/xiaozhi-server/test/js/opus.js b/main/xiaozhi-server/test/js/core/audio/opus-codec.js similarity index 98% rename from main/xiaozhi-server/test/js/opus.js rename to main/xiaozhi-server/test/js/core/audio/opus-codec.js index 67e7ec30..8fb856f2 100644 --- a/main/xiaozhi-server/test/js/opus.js +++ b/main/xiaozhi-server/test/js/core/audio/opus-codec.js @@ -1,5 +1,5 @@ -import { log } from './utils/logger.js'; -import { updateScriptStatus } from './document.js' +import { log } from '../../utils/logger.js'; +import { updateScriptStatus } from '../../ui/dom-helper.js' // 检查Opus库是否已加载 diff --git a/main/xiaozhi-server/test/js/core/audio/player.js b/main/xiaozhi-server/test/js/core/audio/player.js new file mode 100644 index 00000000..459fbb54 --- /dev/null +++ b/main/xiaozhi-server/test/js/core/audio/player.js @@ -0,0 +1,262 @@ +// 音频播放模块 +import { log } from '../../utils/logger.js'; +import BlockingQueue from '../../utils/blocking-queue.js'; +import { createStreamingContext } from './stream-context.js'; + +// 音频播放器类 +export class AudioPlayer { + constructor() { + // 音频参数 + this.SAMPLE_RATE = 16000; + this.CHANNELS = 1; + this.FRAME_SIZE = 960; + this.MIN_AUDIO_DURATION = 0.12; + + // 状态 + this.audioContext = null; + this.opusDecoder = null; + this.streamingContext = null; + this.queue = new BlockingQueue(); + this.isPlaying = false; + } + + // 获取或创建AudioContext + getAudioContext() { + if (!this.audioContext) { + this.audioContext = new (window.AudioContext || window.webkitAudioContext)({ + sampleRate: this.SAMPLE_RATE, + latencyHint: 'interactive' + }); + log('创建音频上下文,采样率: ' + this.SAMPLE_RATE + 'Hz', 'debug'); + } + return this.audioContext; + } + + // 初始化Opus解码器 + async initOpusDecoder() { + if (this.opusDecoder) return this.opusDecoder; + + try { + if (typeof window.ModuleInstance === 'undefined') { + if (typeof Module !== 'undefined') { + window.ModuleInstance = Module; + log('使用全局Module作为ModuleInstance', 'info'); + } else { + throw new Error('Opus库未加载,ModuleInstance和Module对象都不存在'); + } + } + + const mod = window.ModuleInstance; + + this.opusDecoder = { + channels: this.CHANNELS, + rate: this.SAMPLE_RATE, + frameSize: this.FRAME_SIZE, + module: mod, + decoderPtr: null, + + init: function () { + if (this.decoderPtr) return true; + + const decoderSize = mod._opus_decoder_get_size(this.channels); + log(`Opus解码器大小: ${decoderSize}字节`, 'debug'); + + this.decoderPtr = mod._malloc(decoderSize); + if (!this.decoderPtr) { + throw new Error("无法分配解码器内存"); + } + + const err = mod._opus_decoder_init( + this.decoderPtr, + this.rate, + this.channels + ); + + if (err < 0) { + this.destroy(); + throw new Error(`Opus解码器初始化失败: ${err}`); + } + + log("Opus解码器初始化成功", 'success'); + return true; + }, + + decode: function (opusData) { + if (!this.decoderPtr) { + if (!this.init()) { + throw new Error("解码器未初始化且无法初始化"); + } + } + + try { + const mod = this.module; + + const opusPtr = mod._malloc(opusData.length); + mod.HEAPU8.set(opusData, opusPtr); + + const pcmPtr = mod._malloc(this.frameSize * 2); + + const decodedSamples = mod._opus_decode( + this.decoderPtr, + opusPtr, + opusData.length, + pcmPtr, + this.frameSize, + 0 + ); + + if (decodedSamples < 0) { + mod._free(opusPtr); + mod._free(pcmPtr); + throw new Error(`Opus解码失败: ${decodedSamples}`); + } + + const decodedData = new Int16Array(decodedSamples); + for (let i = 0; i < decodedSamples; i++) { + decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i]; + } + + mod._free(opusPtr); + mod._free(pcmPtr); + + return decodedData; + } catch (error) { + log(`Opus解码错误: ${error.message}`, 'error'); + return new Int16Array(0); + } + }, + + destroy: function () { + if (this.decoderPtr) { + this.module._free(this.decoderPtr); + this.decoderPtr = null; + } + } + }; + + if (!this.opusDecoder.init()) { + throw new Error("Opus解码器初始化失败"); + } + + return this.opusDecoder; + + } catch (error) { + log(`Opus解码器初始化失败: ${error.message}`, 'error'); + this.opusDecoder = null; + throw error; + } + } + + // 启动音频缓冲 + async startAudioBuffering() { + log("开始音频缓冲...", 'info'); + + this.initOpusDecoder().catch(error => { + log(`预初始化Opus解码器失败: ${error.message}`, 'warning'); + }); + + const timeout = 400; + while (true) { + const packets = await this.queue.dequeue( + 6, + timeout, + (count) => { + log(`缓冲超时,当前缓冲包数: ${count},开始播放`, 'info'); + } + ); + if (packets.length) { + log(`已缓冲 ${packets.length} 个音频包,开始播放`, 'info'); + this.streamingContext.pushAudioBuffer(packets); + } + + while (true) { + const data = await this.queue.dequeue(99, 30); + if (data.length) { + this.streamingContext.pushAudioBuffer(data); + } else { + break; + } + } + } + } + + // 播放已缓冲的音频 + async playBufferedAudio() { + try { + this.audioContext = this.getAudioContext(); + + if (!this.opusDecoder) { + log('初始化Opus解码器...', 'info'); + try { + this.opusDecoder = await this.initOpusDecoder(); + if (!this.opusDecoder) { + throw new Error('解码器初始化失败'); + } + log('Opus解码器初始化成功', 'success'); + } catch (error) { + log('Opus解码器初始化失败: ' + error.message, 'error'); + this.isPlaying = false; + return; + } + } + + if (!this.streamingContext) { + this.streamingContext = createStreamingContext( + this.opusDecoder, + this.audioContext, + this.SAMPLE_RATE, + this.CHANNELS, + this.MIN_AUDIO_DURATION + ); + } + + this.streamingContext.decodeOpusFrames(); + this.streamingContext.startPlaying(); + + } catch (error) { + log(`播放已缓冲的音频出错: ${error.message}`, 'error'); + this.isPlaying = false; + this.streamingContext = null; + } + } + + // 添加音频数据到队列 + enqueueAudioData(opusData) { + if (opusData.length > 0) { + this.queue.enqueue(opusData); + } else { + log('收到空音频数据帧,可能是结束标志', 'warning'); + if (this.isPlaying && this.streamingContext) { + this.streamingContext.endOfStream = true; + } + } + } + + // 预加载解码器 + async preload() { + log('预加载Opus解码器...', 'info'); + try { + await this.initOpusDecoder(); + log('Opus解码器预加载成功', 'success'); + } catch (error) { + log(`Opus解码器预加载失败: ${error.message},将在需要时重试`, 'warning'); + } + } + + // 启动播放系统 + async start() { + await this.preload(); + this.playBufferedAudio(); + this.startAudioBuffering(); + } +} + +// 创建单例 +let audioPlayerInstance = null; + +export function getAudioPlayer() { + if (!audioPlayerInstance) { + audioPlayerInstance = new AudioPlayer(); + } + return audioPlayerInstance; +} diff --git a/main/xiaozhi-server/test/js/core/audio/recorder.js b/main/xiaozhi-server/test/js/core/audio/recorder.js new file mode 100644 index 00000000..9f94f895 --- /dev/null +++ b/main/xiaozhi-server/test/js/core/audio/recorder.js @@ -0,0 +1,420 @@ +// 音频录制模块 +import { log } from '../../utils/logger.js'; +import { initOpusEncoder } from './opus-codec.js'; +import { getAudioPlayer } from './player.js'; + +// 音频录制器类 +export class AudioRecorder { + constructor() { + this.isRecording = false; + this.audioContext = null; + this.analyser = null; + this.audioProcessor = null; + this.audioProcessorType = null; + this.audioSource = null; + this.opusEncoder = null; + this.pcmDataBuffer = new Int16Array(); + this.audioBuffers = []; + this.totalAudioSize = 0; + this.visualizationRequest = null; + this.recordingTimer = null; + this.websocket = null; + + // 回调函数 + this.onRecordingStart = null; + this.onRecordingStop = null; + this.onVisualizerUpdate = null; + } + + // 设置WebSocket实例 + setWebSocket(ws) { + this.websocket = ws; + } + + // 获取AudioContext实例 + getAudioContext() { + const audioPlayer = getAudioPlayer(); + return audioPlayer.getAudioContext(); + } + + // 初始化编码器 + initEncoder() { + if (!this.opusEncoder) { + this.opusEncoder = initOpusEncoder(); + } + return this.opusEncoder; + } + + // PCM处理器代码 + getAudioProcessorCode() { + return ` + class AudioRecorderProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.buffers = []; + this.frameSize = 960; + this.buffer = new Int16Array(this.frameSize); + this.bufferIndex = 0; + this.isRecording = false; + + this.port.onmessage = (event) => { + if (event.data.command === 'start') { + this.isRecording = true; + this.port.postMessage({ type: 'status', status: 'started' }); + } else if (event.data.command === 'stop') { + this.isRecording = false; + + if (this.bufferIndex > 0) { + const finalBuffer = this.buffer.slice(0, this.bufferIndex); + this.port.postMessage({ + type: 'buffer', + buffer: finalBuffer + }); + this.bufferIndex = 0; + } + + this.port.postMessage({ type: 'status', status: 'stopped' }); + } + }; + } + + process(inputs, outputs, parameters) { + if (!this.isRecording) return true; + + const input = inputs[0][0]; + if (!input) return true; + + for (let i = 0; i < input.length; i++) { + if (this.bufferIndex >= this.frameSize) { + this.port.postMessage({ + type: 'buffer', + buffer: this.buffer.slice(0) + }); + this.bufferIndex = 0; + } + + this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767))); + } + + return true; + } + } + + registerProcessor('audio-recorder-processor', AudioRecorderProcessor); + `; + } + + // 创建音频处理器 + async createAudioProcessor() { + this.audioContext = this.getAudioContext(); + + try { + if (this.audioContext.audioWorklet) { + const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' }); + const url = URL.createObjectURL(blob); + await this.audioContext.audioWorklet.addModule(url); + URL.revokeObjectURL(url); + + const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor'); + + audioProcessor.port.onmessage = (event) => { + if (event.data.type === 'buffer') { + this.processPCMBuffer(event.data.buffer); + } + }; + + log('使用AudioWorklet处理音频', 'success'); + + const silent = this.audioContext.createGain(); + silent.gain.value = 0; + audioProcessor.connect(silent); + silent.connect(this.audioContext.destination); + return { node: audioProcessor, type: 'worklet' }; + } else { + log('AudioWorklet不可用,使用ScriptProcessorNode作为回退方案', 'warning'); + return this.createScriptProcessor(); + } + } catch (error) { + log(`创建音频处理器失败: ${error.message},尝试回退方案`, 'error'); + return this.createScriptProcessor(); + } + } + + // 创建ScriptProcessor作为回退 + createScriptProcessor() { + try { + const frameSize = 4096; + const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1); + + scriptProcessor.onaudioprocess = (event) => { + if (!this.isRecording) return; + + const input = event.inputBuffer.getChannelData(0); + const buffer = new Int16Array(input.length); + + for (let i = 0; i < input.length; i++) { + buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767))); + } + + this.processPCMBuffer(buffer); + }; + + const silent = this.audioContext.createGain(); + silent.gain.value = 0; + scriptProcessor.connect(silent); + silent.connect(this.audioContext.destination); + + log('使用ScriptProcessorNode作为回退方案成功', 'warning'); + return { node: scriptProcessor, type: 'processor' }; + } catch (fallbackError) { + log(`回退方案也失败: ${fallbackError.message}`, 'error'); + return null; + } + } + + // 处理PCM缓冲数据 + processPCMBuffer(buffer) { + if (!this.isRecording) return; + + const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length); + newBuffer.set(this.pcmDataBuffer); + newBuffer.set(buffer, this.pcmDataBuffer.length); + this.pcmDataBuffer = newBuffer; + + const samplesPerFrame = 960; + + while (this.pcmDataBuffer.length >= samplesPerFrame) { + const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame); + this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame); + + this.encodeAndSendOpus(frameData); + } + } + + // 编码并发送Opus数据 + encodeAndSendOpus(pcmData = null) { + if (!this.opusEncoder) { + log('Opus编码器未初始化', 'error'); + return; + } + + try { + if (pcmData) { + const opusData = this.opusEncoder.encode(pcmData); + + if (opusData && opusData.length > 0) { + this.audioBuffers.push(opusData.buffer); + this.totalAudioSize += opusData.length; + + if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { + try { + this.websocket.send(opusData.buffer); + log(`发送Opus帧,大小:${opusData.length}字节`, 'debug'); + } catch (error) { + log(`WebSocket发送错误: ${error.message}`, 'error'); + } + } + } else { + log('Opus编码失败,无有效数据返回', 'error'); + } + } else { + if (this.pcmDataBuffer.length > 0) { + const samplesPerFrame = 960; + if (this.pcmDataBuffer.length < samplesPerFrame) { + const paddedBuffer = new Int16Array(samplesPerFrame); + paddedBuffer.set(this.pcmDataBuffer); + this.encodeAndSendOpus(paddedBuffer); + } else { + this.encodeAndSendOpus(this.pcmDataBuffer.slice(0, samplesPerFrame)); + } + this.pcmDataBuffer = new Int16Array(0); + } + } + } catch (error) { + log(`Opus编码错误: ${error.message}`, 'error'); + } + } + + // 开始录音 + async start() { + if (this.isRecording) return false; + + try { + if (!this.initEncoder()) { + log('无法启动录音: Opus编码器初始化失败', 'error'); + return false; + } + + log('请至少录制1-2秒钟的音频,确保采集到足够数据', 'info'); + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + sampleRate: 16000, + channelCount: 1 + } + }); + + this.audioContext = this.getAudioContext(); + + if (this.audioContext.state === 'suspended') { + await this.audioContext.resume(); + } + + const processorResult = await this.createAudioProcessor(); + if (!processorResult) { + log('无法创建音频处理器', 'error'); + return false; + } + + this.audioProcessor = processorResult.node; + this.audioProcessorType = processorResult.type; + + this.audioSource = this.audioContext.createMediaStreamSource(stream); + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 2048; + + this.audioSource.connect(this.analyser); + this.audioSource.connect(this.audioProcessor); + + this.pcmDataBuffer = new Int16Array(); + this.audioBuffers = []; + this.totalAudioSize = 0; + this.isRecording = true; + + if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) { + this.audioProcessor.port.postMessage({ command: 'start' }); + } + + // 发送监听开始消息 + if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { + const listenMessage = { + type: 'listen', + mode: 'manual', + state: 'start' + }; + + log(`发送录音开始消息: ${JSON.stringify(listenMessage)}`, 'info'); + this.websocket.send(JSON.stringify(listenMessage)); + } else { + log('WebSocket未连接,无法发送开始消息', 'error'); + return false; + } + + // 开始可视化 + if (this.onVisualizerUpdate) { + const dataArray = new Uint8Array(this.analyser.frequencyBinCount); + this.startVisualization(dataArray); + } + + // 启动录音计时器 + let recordingSeconds = 0; + this.recordingTimer = setInterval(() => { + recordingSeconds += 0.1; + if (this.onRecordingStart) { + this.onRecordingStart(recordingSeconds); + } + }, 100); + + log('开始PCM直接录音', 'success'); + return true; + } catch (error) { + log(`直接录音启动错误: ${error.message}`, 'error'); + this.isRecording = false; + return false; + } + } + + // 开始可视化 + startVisualization(dataArray) { + const draw = () => { + this.visualizationRequest = requestAnimationFrame(() => draw()); + + if (!this.isRecording) return; + + this.analyser.getByteFrequencyData(dataArray); + + if (this.onVisualizerUpdate) { + this.onVisualizerUpdate(dataArray); + } + }; + draw(); + } + + // 停止录音 + stop() { + if (!this.isRecording) return false; + + try { + this.isRecording = false; + + if (this.audioProcessor) { + if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) { + this.audioProcessor.port.postMessage({ command: 'stop' }); + } + + this.audioProcessor.disconnect(); + this.audioProcessor = null; + } + + if (this.audioSource) { + this.audioSource.disconnect(); + this.audioSource = null; + } + + if (this.visualizationRequest) { + cancelAnimationFrame(this.visualizationRequest); + this.visualizationRequest = null; + } + + if (this.recordingTimer) { + clearInterval(this.recordingTimer); + this.recordingTimer = null; + } + + // 编码并发送剩余的数据 + this.encodeAndSendOpus(); + + // 发送结束信号 + if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { + const emptyOpusFrame = new Uint8Array(0); + this.websocket.send(emptyOpusFrame); + + const stopMessage = { + type: 'listen', + mode: 'manual', + state: 'stop' + }; + + this.websocket.send(JSON.stringify(stopMessage)); + log('已发送录音停止信号', 'info'); + } + + if (this.onRecordingStop) { + this.onRecordingStop(); + } + + log('停止PCM直接录音', 'success'); + return true; + } catch (error) { + log(`直接录音停止错误: ${error.message}`, 'error'); + return false; + } + } + + // 获取分析器 + getAnalyser() { + return this.analyser; + } +} + +// 创建单例 +let audioRecorderInstance = null; + +export function getAudioRecorder() { + if (!audioRecorderInstance) { + audioRecorderInstance = new AudioRecorder(); + } + return audioRecorderInstance; +} diff --git a/main/xiaozhi-server/test/js/streaming-context.js b/main/xiaozhi-server/test/js/core/audio/stream-context.js similarity index 98% rename from main/xiaozhi-server/test/js/streaming-context.js rename to main/xiaozhi-server/test/js/core/audio/stream-context.js index 198269e9..9c722505 100644 --- a/main/xiaozhi-server/test/js/streaming-context.js +++ b/main/xiaozhi-server/test/js/core/audio/stream-context.js @@ -1,5 +1,5 @@ -import BlockingQueue from './utils/blocking-queue.js'; -import { log } from './utils/logger.js'; +import BlockingQueue from '../../utils/blocking-queue.js'; +import { log } from '../../utils/logger.js'; // 音频流播放上下文类 export class StreamingContext { diff --git a/main/xiaozhi-server/test/js/mcp-tools.js b/main/xiaozhi-server/test/js/core/mcp/tools.js similarity index 99% rename from main/xiaozhi-server/test/js/mcp-tools.js rename to main/xiaozhi-server/test/js/core/mcp/tools.js index da11d815..d4579be1 100644 --- a/main/xiaozhi-server/test/js/mcp-tools.js +++ b/main/xiaozhi-server/test/js/core/mcp/tools.js @@ -1,4 +1,4 @@ -import { log } from './utils/logger.js'; +import { log } from '../../utils/logger.js'; // ========================================== // MCP 工具管理逻辑 @@ -23,7 +23,7 @@ export function setWebSocket(ws) { */ export async function initMcpTools() { // 加载默认工具数据 - const defaultMcpTools = await fetch("js/default-mcp-tools.json").then(res => res.json()); + const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json()); const savedTools = localStorage.getItem('mcpTools'); if (savedTools) { diff --git a/main/xiaozhi-server/test/js/xiaozhi-connect.js b/main/xiaozhi-server/test/js/core/network/ota-connector.js similarity index 97% rename from main/xiaozhi-server/test/js/xiaozhi-connect.js rename to main/xiaozhi-server/test/js/core/network/ota-connector.js index d8fbe7a3..78096959 100644 --- a/main/xiaozhi-server/test/js/xiaozhi-connect.js +++ b/main/xiaozhi-server/test/js/core/network/ota-connector.js @@ -1,5 +1,5 @@ -import { otaStatusStyle } from './document.js'; -import { log } from './utils/logger.js'; +import { otaStatusStyle } from '../../ui/dom-helper.js'; +import { log } from '../../utils/logger.js'; // WebSocket 连接 export async function webSocketConnect(otaUrl, config) { diff --git a/main/xiaozhi-server/test/js/core/network/websocket.js b/main/xiaozhi-server/test/js/core/network/websocket.js new file mode 100644 index 00000000..b62c710b --- /dev/null +++ b/main/xiaozhi-server/test/js/core/network/websocket.js @@ -0,0 +1,372 @@ +// WebSocket消息处理模块 +import { log } from '../../utils/logger.js'; +import { addMessage } from '../../ui/dom-helper.js'; +import { webSocketConnect } from './ota-connector.js'; +import { getConfig, saveConnectionUrls } from '../../config/manager.js'; +import { getAudioPlayer } from '../audio/player.js'; +import { getAudioRecorder } from '../audio/recorder.js'; +import { getMcpTools, executeMcpTool, setWebSocket as setMcpWebSocket } from '../mcp/tools.js'; + +// WebSocket处理器类 +export class WebSocketHandler { + constructor() { + this.websocket = null; + this.onConnectionStateChange = null; + this.onRecordButtonStateChange = null; + this.onSessionStateChange = null; + this.onSessionEmotionChange = null; + this.currentSessionId = null; + this.isRemoteSpeaking = false; + } + + // 发送hello握手消息 + async sendHelloMessage() { + if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) return false; + + try { + const config = getConfig(); + + const helloMessage = { + type: 'hello', + device_id: config.deviceId, + device_name: config.deviceName, + device_mac: config.deviceMac, + token: config.token, + features: { + mcp: true + } + }; + + log('发送hello握手消息', 'info'); + this.websocket.send(JSON.stringify(helloMessage)); + + return new Promise(resolve => { + const timeout = setTimeout(() => { + log('等待hello响应超时', 'error'); + log('提示: 请尝试点击"测试认证"按钮进行连接排查', 'info'); + resolve(false); + }, 5000); + + const onMessageHandler = (event) => { + try { + const response = JSON.parse(event.data); + if (response.type === 'hello' && response.session_id) { + log(`服务器握手成功,会话ID: ${response.session_id}`, 'success'); + clearTimeout(timeout); + this.websocket.removeEventListener('message', onMessageHandler); + resolve(true); + } + } catch (e) { + // 忽略非JSON消息 + } + }; + + this.websocket.addEventListener('message', onMessageHandler); + }); + } catch (error) { + log(`发送hello消息错误: ${error.message}`, 'error'); + return false; + } + } + + // 处理文本消息 + handleTextMessage(message) { + if (message.type === 'hello') { + log(`服务器回应:${JSON.stringify(message, null, 2)}`, 'success'); + } else if (message.type === 'tts') { + this.handleTTSMessage(message); + } else if (message.type === 'audio') { + log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info'); + } else if (message.type === 'stt') { + log(`识别结果: ${message.text}`, 'info'); + addMessage(`${message.text}`, true); + } else if (message.type === 'llm') { + log(`大模型回复: ${message.text}`, 'info'); + + // 如果包含表情,更新sessionStatus表情 + if (message.text && /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u.test(message.text)) { + // 提取表情符号 + const emojiMatch = message.text.match(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u); + if (emojiMatch && this.onSessionEmotionChange) { + this.onSessionEmotionChange(emojiMatch[0]); + } + } + + // 只有当文本不仅仅是表情时,才添加到对话中 + // 移除文本中的表情后检查是否还有内容 + const textWithoutEmoji = message.text ? message.text.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim() : ''; + if (textWithoutEmoji) { + addMessage(message.text); + } + } else if (message.type === 'mcp') { + this.handleMCPMessage(message); + } else { + log(`未知消息类型: ${message.type}`, 'info'); + addMessage(JSON.stringify(message, null, 2)); + } + } + + // 处理TTS消息 + handleTTSMessage(message) { + if (message.state === 'start') { + log('服务器开始发送语音', 'info'); + this.currentSessionId = message.session_id; + this.isRemoteSpeaking = true; + if (this.onSessionStateChange) { + this.onSessionStateChange(true); + } + } else if (message.state === 'sentence_start') { + log(`服务器发送语音段: ${message.text}`, 'info'); + if (message.text) { + addMessage(message.text); + } + } else if (message.state === 'sentence_end') { + log(`语音段结束: ${message.text}`, 'info'); + } else if (message.state === 'stop') { + log('服务器语音传输结束', 'info'); + this.isRemoteSpeaking = false; + if (this.onRecordButtonStateChange) { + this.onRecordButtonStateChange(false); + } + if (this.onSessionStateChange) { + this.onSessionStateChange(false); + } + } + } + + // 处理MCP消息 + handleMCPMessage(message) { + const payload = message.payload || {}; + log(`服务器下发: ${JSON.stringify(message)}`, 'info'); + + if (payload.method === 'tools/list') { + const tools = getMcpTools(); + + const replyMessage = JSON.stringify({ + "session_id": message.session_id || "", + "type": "mcp", + "payload": { + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "tools": tools + } + } + }); + log(`客户端上报: ${replyMessage}`, 'info'); + this.websocket.send(replyMessage); + log(`回复MCP工具列表: ${tools.length} 个工具`, 'info'); + + } else if (payload.method === 'tools/call') { + const toolName = payload.params?.name; + const toolArgs = payload.params?.arguments; + + log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info'); + + const result = executeMcpTool(toolName, toolArgs); + + const replyMessage = JSON.stringify({ + "session_id": message.session_id || "", + "type": "mcp", + "payload": { + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "content": [ + { + "type": "text", + "text": JSON.stringify(result) + } + ], + "isError": false + } + } + }); + + log(`客户端上报: ${replyMessage}`, 'info'); + this.websocket.send(replyMessage); + } else if (payload.method === 'initialize') { + log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info'); + } else { + log(`未知的MCP方法: ${payload.method}`, 'warning'); + } + } + + // 处理二进制消息 + async handleBinaryMessage(data) { + try { + let arrayBuffer; + if (data instanceof ArrayBuffer) { + arrayBuffer = data; + log(`收到ArrayBuffer音频数据,大小: ${data.byteLength}字节`, 'debug'); + } else if (data instanceof Blob) { + arrayBuffer = await data.arrayBuffer(); + log(`收到Blob音频数据,大小: ${arrayBuffer.byteLength}字节`, 'debug'); + } else { + log(`收到未知类型的二进制数据: ${typeof data}`, 'warning'); + return; + } + + const opusData = new Uint8Array(arrayBuffer); + const audioPlayer = getAudioPlayer(); + audioPlayer.enqueueAudioData(opusData); + } catch (error) { + log(`处理二进制消息出错: ${error.message}`, 'error'); + } + } + + // 连接WebSocket服务器 + async connect() { + const config = getConfig(); + log('正在检查OTA状态...', 'info'); + saveConnectionUrls(); + + try { + const otaUrl = document.getElementById('otaUrl').value.trim(); + const ws = await webSocketConnect(otaUrl, config); + if (ws === undefined) { + return false; + } + this.websocket = ws; + + // 设置接收二进制数据的类型为ArrayBuffer + this.websocket.binaryType = 'arraybuffer'; + + // 设置 MCP 模块的 WebSocket 实例 + setMcpWebSocket(this.websocket); + + // 设置录音器的WebSocket + const audioRecorder = getAudioRecorder(); + audioRecorder.setWebSocket(this.websocket); + + this.setupEventHandlers(); + + return true; + } catch (error) { + log(`连接错误: ${error.message}`, 'error'); + if (this.onConnectionStateChange) { + this.onConnectionStateChange(false); + } + return false; + } + } + + // 设置事件处理器 + setupEventHandlers() { + this.websocket.onopen = async () => { + const url = document.getElementById('serverUrl').value; + log(`已连接到服务器: ${url}`, 'success'); + + if (this.onConnectionStateChange) { + this.onConnectionStateChange(true); + } + + // 连接成功后,默认状态为聆听中 + this.isRemoteSpeaking = false; + if (this.onSessionStateChange) { + this.onSessionStateChange(false); + } + + await this.sendHelloMessage(); + }; + + this.websocket.onclose = () => { + log('已断开连接', 'info'); + + if (this.onConnectionStateChange) { + this.onConnectionStateChange(false); + } + + const audioRecorder = getAudioRecorder(); + audioRecorder.stop(); + }; + + this.websocket.onerror = (error) => { + log(`WebSocket错误: ${error.message || '未知错误'}`, 'error'); + + if (this.onConnectionStateChange) { + this.onConnectionStateChange(false); + } + }; + + this.websocket.onmessage = (event) => { + try { + if (typeof event.data === 'string') { + const message = JSON.parse(event.data); + this.handleTextMessage(message); + } else { + this.handleBinaryMessage(event.data); + } + } catch (error) { + log(`WebSocket消息处理错误: ${error.message}`, 'error'); + if (typeof event.data === 'string') { + addMessage(event.data); + } + } + }; + } + + // 断开连接 + disconnect() { + if (!this.websocket) return; + + this.websocket.close(); + const audioRecorder = getAudioRecorder(); + audioRecorder.stop(); + } + + // 发送文本消息 + sendTextMessage(text) { + if (text === '' || !this.websocket || this.websocket.readyState !== WebSocket.OPEN) { + return false; + } + + try { + // 如果对方正在说话,先发送打断消息 + if (this.isRemoteSpeaking && this.currentSessionId) { + const abortMessage = { + session_id: this.currentSessionId, + type: 'abort', + reason: 'wake_word_detected' + }; + this.websocket.send(JSON.stringify(abortMessage)); + log('发送打断消息', 'info'); + } + + const listenMessage = { + type: 'listen', + mode: 'manual', + state: 'detect', + text: text + }; + + this.websocket.send(JSON.stringify(listenMessage)); + log(`发送文本消息: ${text}`, 'info'); + + return true; + } catch (error) { + log(`发送消息错误: ${error.message}`, 'error'); + return false; + } + } + + // 获取WebSocket实例 + getWebSocket() { + return this.websocket; + } + + // 检查是否已连接 + isConnected() { + return this.websocket && this.websocket.readyState === WebSocket.OPEN; + } +} + +// 创建单例 +let wsHandlerInstance = null; + +export function getWebSocketHandler() { + if (!wsHandlerInstance) { + wsHandlerInstance = new WebSocketHandler(); + } + return wsHandlerInstance; +} diff --git a/main/xiaozhi-server/test/js/ui/controller.js b/main/xiaozhi-server/test/js/ui/controller.js new file mode 100644 index 00000000..dd58555b --- /dev/null +++ b/main/xiaozhi-server/test/js/ui/controller.js @@ -0,0 +1,269 @@ +// UI控制模块 +import { loadConfig, saveConfig } from '../config/manager.js'; +import { getAudioRecorder } from '../core/audio/recorder.js'; +import { getWebSocketHandler } from '../core/network/websocket.js'; + +// UI控制器类 +export class UIController { + constructor() { + this.isEditing = false; + this.visualizerCanvas = null; + this.visualizerContext = null; + } + + // 初始化 + init() { + this.visualizerCanvas = document.getElementById('audioVisualizer'); + this.visualizerContext = this.visualizerCanvas.getContext('2d'); + + this.initVisualizer(); + this.initEventListeners(); + loadConfig(); + } + + // 初始化可视化器 + initVisualizer() { + this.visualizerCanvas.width = this.visualizerCanvas.clientWidth; + this.visualizerCanvas.height = this.visualizerCanvas.clientHeight; + this.visualizerContext.fillStyle = '#fafafa'; + this.visualizerContext.fillRect(0, 0, this.visualizerCanvas.width, this.visualizerCanvas.height); + } + + // 更新状态显示 + updateStatusDisplay(element, text) { + element.textContent = text; + element.removeAttribute('style'); + element.classList.remove('connected'); + if (text.includes('已连接')) { + element.classList.add('connected'); + } + console.log('更新状态:', text, '类列表:', element.className, '样式属性:', element.getAttribute('style')); + } + + // 更新连接状态UI + updateConnectionUI(isConnected) { + const connectionStatus = document.getElementById('connectionStatus'); + const otaStatus = document.getElementById('otaStatus'); + const connectButton = document.getElementById('connectButton'); + const messageInput = document.getElementById('messageInput'); + const sendTextButton = document.getElementById('sendTextButton'); + const recordButton = document.getElementById('recordButton'); + + if (isConnected) { + this.updateStatusDisplay(connectionStatus, '● WS已连接'); + this.updateStatusDisplay(otaStatus, '● OTA已连接'); + connectButton.textContent = '断开'; + messageInput.disabled = false; + sendTextButton.disabled = false; + recordButton.disabled = false; + } else { + this.updateStatusDisplay(connectionStatus, '● WS未连接'); + this.updateStatusDisplay(otaStatus, '● OTA未连接'); + connectButton.textContent = '连接'; + messageInput.disabled = true; + sendTextButton.disabled = true; + recordButton.disabled = true; + // 断开连接时,会话状态变为离线 + this.updateSessionStatus(null); + } + } + + // 更新录音按钮状态 + updateRecordButtonState(isRecording, seconds = 0) { + const recordButton = document.getElementById('recordButton'); + if (isRecording) { + recordButton.textContent = `停止录音 ${seconds.toFixed(1)}秒`; + recordButton.classList.add('recording'); + } else { + recordButton.textContent = '开始录音'; + recordButton.classList.remove('recording'); + } + recordButton.disabled = false; + } + + // 更新会话状态UI + updateSessionStatus(isSpeaking) { + const sessionStatus = document.getElementById('sessionStatus'); + if (!sessionStatus) return; + + if (isSpeaking === null) { + // 离线状态 + sessionStatus.innerHTML = '😶 小智离线中'; + sessionStatus.className = 'status offline'; + } else if (isSpeaking) { + // 说话中 + sessionStatus.innerHTML = '😶 小智说话中'; + sessionStatus.className = 'status speaking'; + } else { + // 聆听中 + sessionStatus.innerHTML = '😶 小智聆听中'; + sessionStatus.className = 'status listening'; + } + } + + // 更新会话表情 + updateSessionEmotion(emoji) { + const sessionStatus = document.getElementById('sessionStatus'); + if (!sessionStatus) return; + + // 获取当前文本内容,提取非表情部分 + let currentText = sessionStatus.textContent; + // 移除现有的表情符号 + currentText = currentText.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim(); + // 使用 innerHTML 添加带样式的表情 + sessionStatus.innerHTML = `${emoji} ${currentText}`; + } + + // 绘制音频可视化效果 + drawVisualizer(dataArray) { + this.visualizerContext.fillStyle = '#fafafa'; + this.visualizerContext.fillRect(0, 0, this.visualizerCanvas.width, this.visualizerCanvas.height); + + const barWidth = (this.visualizerCanvas.width / dataArray.length) * 2.5; + let barHeight; + let x = 0; + + for (let i = 0; i < dataArray.length; i++) { + barHeight = dataArray[i] / 2; + + this.visualizerContext.fillStyle = `rgb(${barHeight + 100}, 50, 50)`; + this.visualizerContext.fillRect(x, this.visualizerCanvas.height - barHeight, barWidth, barHeight); + + x += barWidth + 1; + } + } + + // 初始化事件监听器 + initEventListeners() { + const wsHandler = getWebSocketHandler(); + const audioRecorder = getAudioRecorder(); + + // 设置WebSocket回调 + wsHandler.onConnectionStateChange = (isConnected) => { + this.updateConnectionUI(isConnected); + }; + + wsHandler.onRecordButtonStateChange = (isRecording) => { + this.updateRecordButtonState(isRecording); + }; + + wsHandler.onSessionStateChange = (isSpeaking) => { + this.updateSessionStatus(isSpeaking); + }; + + wsHandler.onSessionEmotionChange = (emoji) => { + this.updateSessionEmotion(emoji); + }; + + // 设置录音器回调 + audioRecorder.onRecordingStart = (seconds) => { + this.updateRecordButtonState(true, seconds); + }; + + audioRecorder.onRecordingStop = () => { + this.updateRecordButtonState(false); + }; + + audioRecorder.onVisualizerUpdate = (dataArray) => { + this.drawVisualizer(dataArray); + }; + + // 连接按钮 + const connectButton = document.getElementById('connectButton'); + let isConnecting = false; + + const handleConnect = async () => { + if (isConnecting) return; + + if (wsHandler.isConnected()) { + wsHandler.disconnect(); + } else { + isConnecting = true; + await wsHandler.connect(); + isConnecting = false; + } + }; + + connectButton.addEventListener('click', handleConnect); + + // 设备配置面板编辑/确定切换 + const toggleButton = document.getElementById('toggleConfig'); + const deviceMacInput = document.getElementById('deviceMac'); + const deviceNameInput = document.getElementById('deviceName'); + const clientIdInput = document.getElementById('clientId'); + const tokenInput = document.getElementById('token'); + + toggleButton.addEventListener('click', () => { + this.isEditing = !this.isEditing; + + deviceMacInput.disabled = !this.isEditing; + deviceNameInput.disabled = !this.isEditing; + clientIdInput.disabled = !this.isEditing; + tokenInput.disabled = !this.isEditing; + + toggleButton.textContent = this.isEditing ? '确定' : '编辑'; + + if (!this.isEditing) { + saveConfig(); + } + }); + + // 标签页切换 + const tabs = document.querySelectorAll('.tab'); + tabs.forEach(tab => { + tab.addEventListener('click', () => { + tabs.forEach(t => t.classList.remove('active')); + document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active')); + + tab.classList.add('active'); + const tabContent = document.getElementById(`${tab.dataset.tab}Tab`); + tabContent.classList.add('active'); + + if (tab.dataset.tab === 'voice') { + setTimeout(() => { + this.initVisualizer(); + }, 50); + } + }); + }); + + // 发送文本消息 + const messageInput = document.getElementById('messageInput'); + const sendTextButton = document.getElementById('sendTextButton'); + + const sendMessage = () => { + const message = messageInput.value.trim(); + if (message && wsHandler.sendTextMessage(message)) { + messageInput.value = ''; + } + }; + + sendTextButton.addEventListener('click', sendMessage); + messageInput.addEventListener('keypress', (e) => { + if (e.key === 'Enter') sendMessage(); + }); + + // 录音按钮 + const recordButton = document.getElementById('recordButton'); + recordButton.addEventListener('click', () => { + if (audioRecorder.isRecording) { + audioRecorder.stop(); + } else { + audioRecorder.start(); + } + }); + + // 窗口大小变化 + window.addEventListener('resize', () => this.initVisualizer()); + } +} + +// 创建单例 +let uiControllerInstance = null; + +export function getUIController() { + if (!uiControllerInstance) { + uiControllerInstance = new UIController(); + } + return uiControllerInstance; +} diff --git a/main/xiaozhi-server/test/js/document.js b/main/xiaozhi-server/test/js/ui/dom-helper.js similarity index 100% rename from main/xiaozhi-server/test/js/document.js rename to main/xiaozhi-server/test/js/ui/dom-helper.js diff --git a/main/xiaozhi-server/test/js/utils/logger.js b/main/xiaozhi-server/test/js/utils/logger.js index 913c2c9a..29f79444 100644 --- a/main/xiaozhi-server/test/js/utils/logger.js +++ b/main/xiaozhi-server/test/js/utils/logger.js @@ -1,4 +1,4 @@ -import { getLogContainer } from '../document.js' +import { getLogContainer } from '../ui/dom-helper.js' const logContainer = getLogContainer(); // 日志记录函数 diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index 96f72e84..97af77ac 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -5,7 +5,7 @@