mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-27 09:33:55 +08:00
update:test迁移重命名为digital-human
This commit is contained in:
3030332422
@@ -0,0 +1,373 @@
|
||||
// 主应用入口
|
||||
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js?v=0205';
|
||||
import { getAudioPlayer } from './core/audio/player.js?v=0205';
|
||||
import { checkMicrophoneAvailability, isHttpNonLocalhost } from './core/audio/recorder.js?v=0205';
|
||||
import { initMcpTools } from './core/mcp/tools.js?v=0205';
|
||||
import { startWakewordBridgeListener } from './core/network/wakeword-bridge.js?v=0205';
|
||||
import { uiController } from './ui/controller.js?v=0205';
|
||||
import { log } from './utils/logger.js?v=0205';
|
||||
|
||||
// 辅助函数：将Base64数据转换为Blob
|
||||
function dataURItoBlob(dataURI) {
|
||||
const byteString = atob(dataURI.split(',')[1]);
|
||||
const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
|
||||
const ab = new ArrayBuffer(byteString.length);
|
||||
const ia = new Uint8Array(ab);
|
||||
for (let i = 0; i < byteString.length; i++) {
|
||||
ia[i] = byteString.charCodeAt(i);
|
||||
}
|
||||
return new Blob([ab], { type: mimeString });
|
||||
}
|
||||
|
||||
// 应用类
|
||||
class App {
|
||||
constructor() {
|
||||
this.uiController = null;
|
||||
this.audioPlayer = null;
|
||||
this.live2dManager = null;
|
||||
this.cameraStream = null;
|
||||
this.currentFacingMode = 'user';
|
||||
}
|
||||
|
||||
// 初始化应用
|
||||
async init() {
|
||||
log('正在初始化应用...', 'info');
|
||||
// 初始化UI控制器
|
||||
this.uiController = uiController;
|
||||
this.uiController.init();
|
||||
// 检查Opus库
|
||||
checkOpusLoaded();
|
||||
// 初始化Opus编码器
|
||||
initOpusEncoder();
|
||||
// 初始化音频播放器
|
||||
this.audioPlayer = getAudioPlayer();
|
||||
await this.audioPlayer.start();
|
||||
// 初始化MCP工具
|
||||
initMcpTools();
|
||||
// 初始化本地唤醒事件监听
|
||||
startWakewordBridgeListener();
|
||||
// 检查麦克风可用性
|
||||
await this.checkMicrophoneAvailability();
|
||||
// 检查摄像头可用性
|
||||
this.checkCameraAvailability();
|
||||
// 初始化Live2D
|
||||
await this.initLive2D();
|
||||
// 初始化摄像头
|
||||
this.initCamera();
|
||||
// 关闭加载loading
|
||||
this.setModelLoadingStatus(false);
|
||||
log('应用初始化完成', 'success');
|
||||
}
|
||||
|
||||
// 初始化Live2D
|
||||
async initLive2D() {
|
||||
try {
|
||||
// 检查Live2DManager是否已加载
|
||||
if (typeof window.Live2DManager === 'undefined') {
|
||||
throw new Error('Live2DManager未加载，请检查脚本引入顺序');
|
||||
}
|
||||
this.live2dManager = new window.Live2DManager();
|
||||
await this.live2dManager.initializeLive2D();
|
||||
// 更新UI状态
|
||||
const live2dStatus = document.getElementById('live2dStatus');
|
||||
if (live2dStatus) {
|
||||
live2dStatus.textContent = '● 已加载';
|
||||
live2dStatus.className = 'status loaded';
|
||||
}
|
||||
log('Live2D初始化完成', 'success');
|
||||
} catch (error) {
|
||||
log(`Live2D初始化失败: ${error.message}`, 'error');
|
||||
// 更新UI状态
|
||||
const live2dStatus = document.getElementById('live2dStatus');
|
||||
if (live2dStatus) {
|
||||
live2dStatus.textContent = '● 加载失败';
|
||||
live2dStatus.className = 'status error';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 设置model加载状态
|
||||
setModelLoadingStatus(isLoading) {
|
||||
const modelLoading = document.getElementById('modelLoading');
|
||||
if (modelLoading) {
|
||||
modelLoading.style.display = isLoading ? 'flex' : 'none';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查麦克风可用性
|
||||
* 在应用初始化时调用，检查麦克风是否可用并更新UI状态
|
||||
*/
|
||||
async checkMicrophoneAvailability() {
|
||||
try {
|
||||
const isAvailable = await checkMicrophoneAvailability();
|
||||
const isHttp = isHttpNonLocalhost();
|
||||
// 保存可用性状态到全局变量
|
||||
window.microphoneAvailable = isAvailable;
|
||||
window.isHttpNonLocalhost = isHttp;
|
||||
// 更新UI
|
||||
if (this.uiController) {
|
||||
this.uiController.updateMicrophoneAvailability(isAvailable, isHttp);
|
||||
}
|
||||
log(`麦克风可用性检查完成: ${isAvailable ? '可用' : '不可用'}`, isAvailable ? 'success' : 'warning');
|
||||
} catch (error) {
|
||||
log(`检查麦克风可用性失败: ${error.message}`, 'error');
|
||||
// 默认设置为不可用
|
||||
window.microphoneAvailable = false;
|
||||
window.isHttpNonLocalhost = isHttpNonLocalhost();
|
||||
if (this.uiController) {
|
||||
this.uiController.updateMicrophoneAvailability(false, window.isHttpNonLocalhost);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 检查摄像头可用性
|
||||
checkCameraAvailability() {
|
||||
window.cameraAvailable = true;
|
||||
log('摄像头可用性检查完成: 默认已绑定验证码', 'success');
|
||||
}
|
||||
|
||||
// 初始化摄像头
|
||||
async initCamera() {
|
||||
const cameraContainer = document.getElementById('cameraContainer');
|
||||
const cameraVideo = document.getElementById('cameraVideo');
|
||||
const cameraSwitch = document.getElementById('cameraSwitch');
|
||||
const cameraSwitchMask = document.getElementById('cameraSwitchMask');
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
|
||||
if (!cameraContainer || !cameraVideo) {
|
||||
log('摄像头元素未找到，跳过初始化', 'warning');
|
||||
return Promise.resolve(false);
|
||||
}
|
||||
|
||||
let isDragging = false;
|
||||
let currentX, currentY, initialX, initialY;
|
||||
let xOffset = 0, yOffset = 0;
|
||||
|
||||
cameraContainer.addEventListener('mousedown', dragStart);
|
||||
document.addEventListener('mousemove', drag);
|
||||
document.addEventListener('mouseup', dragEnd);
|
||||
cameraContainer.addEventListener('touchstart', dragStart, { passive: false });
|
||||
document.addEventListener('touchmove', drag, { passive: false });
|
||||
document.addEventListener('touchend', dragEnd);
|
||||
|
||||
function dragStart(e) {
|
||||
if (e.type === 'touchstart') {
|
||||
initialX = e.touches[0].clientX - xOffset;
|
||||
initialY = e.touches[0].clientY - yOffset;
|
||||
} else {
|
||||
initialX = e.clientX - xOffset;
|
||||
initialY = e.clientY - yOffset;
|
||||
}
|
||||
isDragging = true;
|
||||
cameraContainer.classList.add('dragging');
|
||||
}
|
||||
|
||||
function drag(e) {
|
||||
if (isDragging) {
|
||||
e.preventDefault();
|
||||
if (e.type === 'touchmove') {
|
||||
currentX = e.touches[0].clientX - initialX;
|
||||
currentY = e.touches[0].clientY - initialY;
|
||||
} else {
|
||||
currentX = e.clientX - initialX;
|
||||
currentY = e.clientY - initialY;
|
||||
}
|
||||
xOffset = currentX;
|
||||
yOffset = currentY;
|
||||
cameraContainer.style.transform = `translate3d(${currentX}px, ${currentY}px, 0)`;
|
||||
}
|
||||
}
|
||||
|
||||
function dragEnd() {
|
||||
initialX = currentX;
|
||||
initialY = currentY;
|
||||
isDragging = false;
|
||||
cameraContainer.classList.remove('dragging');
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
window.startCamera = async () => {
|
||||
try {
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log('浏览器不支持摄像头API', 'warning');
|
||||
return false;
|
||||
}
|
||||
log('正在请求摄像头权限...', 'info');
|
||||
this.cameraStream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: 180, height: 240, facingMode: this.currentFacingMode },
|
||||
audio: false
|
||||
});
|
||||
cameraVideo.srcObject = this.cameraStream;
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
const videoDevices = devices.filter(device => device.kind === 'videoinput');
|
||||
if (videoDevices.length > 1) {
|
||||
if (cameraSwitch) cameraSwitch.classList.add('active');
|
||||
}
|
||||
cameraContainer.classList.add('active');
|
||||
|
||||
// 切换时挂断情况
|
||||
const hasActive = dialBtn.classList.contains('dial-active');
|
||||
if (!hasActive) {
|
||||
cameraContainer.classList.remove('active');
|
||||
cameraSwitch.classList.remove('active');
|
||||
window.stopCamera();
|
||||
}
|
||||
log('摄像头已启动', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`启动摄像头失败: ${error.name} - ${error.message}`, 'error');
|
||||
if (error.name === 'NotAllowedError') {
|
||||
log('摄像头权限被拒绝，请检查浏览器设置', 'warning');
|
||||
} else if (error.name === 'NotFoundError') {
|
||||
log('未找到摄像头设备', 'warning');
|
||||
} else if (error.name === 'NotReadableError') {
|
||||
log('摄像头已被其他程序占用', 'warning');
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
window.stopCamera = () => {
|
||||
if (this.cameraStream) {
|
||||
this.cameraStream.getTracks().forEach(track => track.stop());
|
||||
this.cameraStream = null;
|
||||
cameraVideo.srcObject = null;
|
||||
log('摄像头已关闭', 'info');
|
||||
}
|
||||
};
|
||||
|
||||
window.switchCamera = async() => {
|
||||
if (window.switchCameraTimer) return;
|
||||
if (this.cameraStream) {
|
||||
const currentTransform = window.getComputedStyle(cameraContainer).transform;
|
||||
const originalTransform = currentTransform === 'none' ? 'translate(0px, 0px)' : currentTransform;
|
||||
cameraContainer.style.setProperty('--original-transform', originalTransform);
|
||||
cameraContainer.classList.add('flip');
|
||||
if (cameraSwitchMask) cameraSwitchMask.style.opacity = 0;
|
||||
this.currentFacingMode = this.currentFacingMode === 'user' ? 'environment' : 'user';
|
||||
window.stopCamera();
|
||||
window.startCamera();
|
||||
|
||||
window.switchCameraTimer = setTimeout(() => {
|
||||
if (this.currentFacingMode === 'user') {
|
||||
cameraVideo.style.transform = 'scaleX(-1)';
|
||||
} else {
|
||||
cameraVideo.style.transform = 'scaleX(1)';
|
||||
}
|
||||
window.switchCameraTimer = null;
|
||||
cameraContainer.classList.remove('flip');
|
||||
cameraContainer.style.removeProperty('--original-transform');
|
||||
if (cameraSwitchMask) cameraSwitchMask.style.opacity = 1;
|
||||
}, 500);
|
||||
}
|
||||
};
|
||||
|
||||
window.takePhoto = (question = '描述一下看到的物品') => {
|
||||
return new Promise(async (resolve) => {
|
||||
const canvas = document.createElement('canvas');
|
||||
const video = cameraVideo;
|
||||
|
||||
if (!video || video.readyState !== video.HAVE_ENOUGH_DATA) {
|
||||
log('无法拍照：摄像头未就绪', 'warning');
|
||||
resolve({
|
||||
success: false,
|
||||
error: '摄像头未就绪，请确保已连接且摄像头已启动'
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
canvas.width = video.videoWidth || 180;
|
||||
canvas.height = video.videoHeight || 240;
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
|
||||
const photoData = canvas.toDataURL('image/jpeg', 0.8);
|
||||
log(`拍照成功，图像数据长度: ${photoData.length}`, 'success');
|
||||
|
||||
try {
|
||||
const xz_tester_vision = localStorage.getItem('xz_tester_vision');
|
||||
if (xz_tester_vision) {
|
||||
let visionInfo = null;
|
||||
|
||||
try {
|
||||
visionInfo = JSON.parse(xz_tester_vision);
|
||||
} catch (err) {
|
||||
throw new Error(`视觉配置解析失败`);
|
||||
}
|
||||
|
||||
const { url, token } = visionInfo || {};
|
||||
if (!url || !token) {
|
||||
throw new Error('视觉分析失败：配置缺少接口地址(url)或令牌(token)');
|
||||
}
|
||||
|
||||
log(`正在发送图片到视觉分析接口: ${url}`, 'info');
|
||||
|
||||
const deviceId = document.getElementById('deviceMac')?.value || '';
|
||||
const clientId = document.getElementById('clientId')?.value || 'web_test_client';
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('question', question);
|
||||
formData.append('image', dataURItoBlob(photoData), 'photo.jpg');
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
headers: {
|
||||
'Device-Id': deviceId,
|
||||
'Client-Id': clientId,
|
||||
'Authorization': `Bearer ${token}`
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const analysisResult = await response.json();
|
||||
log(`视觉分析完成: ${JSON.stringify(analysisResult).substring(0, 200)}...`, 'success');
|
||||
|
||||
resolve({
|
||||
success: true,
|
||||
message: question,
|
||||
photo_data: photoData,
|
||||
photo_width: canvas.width,
|
||||
photo_height: canvas.height,
|
||||
vision_analysis: analysisResult
|
||||
});
|
||||
} else {
|
||||
log('未配置视觉分析服务', 'warning');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`视觉分析失败: ${error.message}`, 'error');
|
||||
resolve({
|
||||
success: true,
|
||||
message: question,
|
||||
photo_data: photoData,
|
||||
photo_width: canvas.width,
|
||||
photo_height: canvas.height,
|
||||
vision_analysis: {
|
||||
success: false,
|
||||
error: error.message,
|
||||
fallback: '无法连接到视觉分析服务'
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
log('摄像头初始化完成', 'success');
|
||||
resolve(true);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 创建并启动应用
|
||||
const app = new App();
|
||||
// 将应用实例暴露到全局，供其他模块访问
|
||||
window.chatApp = app;
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
// 初始化应用
|
||||
app.init();
|
||||
});
|
||||
export default app;
|
||||
@@ -0,0 +1,90 @@
|
||||
[
|
||||
{
|
||||
"name": "self_camera_take_photo",
|
||||
"description": "Take a photo using the device's camera. This tool captures the current camera frame and returns the image data. You MUST call this tool when user asks to 'take a photo', 'what do you see', 'describe what you see', or similar requests. After getting the photo data, describe what you see in the image. Parameter 'question' is optional, defaults to 'describe objects seen'.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"question": {
|
||||
"type": "string",
|
||||
"description": "Question to guide the photo analysis, e.g., 'describe the objects seen'. Can be empty."
|
||||
}
|
||||
}
|
||||
},
|
||||
"mockResponse": {
|
||||
"success": true,
|
||||
"photo_data": "base64_image_data",
|
||||
"response": "Please describe what you see based on the returned photo_data"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "self.get_device_status",
|
||||
"description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
},
|
||||
"mockResponse": {
|
||||
"audio_speaker": {
|
||||
"volume": 50,
|
||||
"muted": false
|
||||
},
|
||||
"screen": {
|
||||
"brightness": 80,
|
||||
"theme": "light"
|
||||
},
|
||||
"battery": {
|
||||
"level": 85,
|
||||
"charging": false
|
||||
},
|
||||
"network": {
|
||||
"connected": true,
|
||||
"type": "wifi"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "self.audio_speaker.set_volume",
|
||||
"description": "Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"volume": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 100
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"volume"
|
||||
]
|
||||
},
|
||||
"mockResponse": {
|
||||
"success": true,
|
||||
"volume": "${volume}",
|
||||
"message": "音量已设置为 ${volume}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "self.screen.set_brightness",
|
||||
"description": "Set the brightness of the screen.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"brightness": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 100
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"brightness"
|
||||
]
|
||||
},
|
||||
"mockResponse": {
|
||||
"success": true,
|
||||
"brightness": "${brightness}",
|
||||
"message": "亮度已设置为 ${brightness}"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,127 @@
|
||||
// 配置管理模块
|
||||
|
||||
// 默认唤醒词列表
|
||||
export const DEFAULT_WAKE_WORDS = '你好小智\n你好小志\n小爱同学\n你好小鑫\n你好小新\n小美同学\n小龙小龙\n喵喵同学\n小滨小滨\n小冰小冰\n嘿你好呀';
|
||||
|
||||
// 生成随机MAC地址
|
||||
function generateRandomMac() {
|
||||
const hexDigits = '0123456789ABCDEF';
|
||||
let mac = '';
|
||||
for (let i = 0; i < 6; i++) {
|
||||
if (i > 0) mac += ':';
|
||||
for (let j = 0; j < 2; j++) {
|
||||
mac += hexDigits.charAt(Math.floor(Math.random() * 16));
|
||||
}
|
||||
}
|
||||
return mac;
|
||||
}
|
||||
|
||||
// 加载配置
|
||||
export function loadConfig() {
|
||||
const deviceMacInput = document.getElementById('deviceMac');
|
||||
const deviceNameInput = document.getElementById('deviceName');
|
||||
const clientIdInput = document.getElementById('clientId');
|
||||
const otaUrlInput = document.getElementById('otaUrl');
|
||||
const wakewordWsUrlInput = document.getElementById('wakewordWsUrl');
|
||||
const wakewordEnabledInput = document.getElementById('wakewordEnabled');
|
||||
const wakewordListInput = document.getElementById('wakewordList');
|
||||
|
||||
// 从localStorage加载MAC地址，如果没有则生成新的
|
||||
let savedMac = localStorage.getItem('xz_tester_deviceMac');
|
||||
if (!savedMac) {
|
||||
savedMac = generateRandomMac();
|
||||
localStorage.setItem('xz_tester_deviceMac', savedMac);
|
||||
}
|
||||
deviceMacInput.value = savedMac;
|
||||
|
||||
// 从localStorage加载其他配置
|
||||
const savedDeviceName = localStorage.getItem('xz_tester_deviceName');
|
||||
if (savedDeviceName) {
|
||||
deviceNameInput.value = savedDeviceName;
|
||||
}
|
||||
|
||||
const savedClientId = localStorage.getItem('xz_tester_clientId');
|
||||
if (savedClientId) {
|
||||
clientIdInput.value = savedClientId;
|
||||
}
|
||||
|
||||
const savedOtaUrl = localStorage.getItem('xz_tester_otaUrl');
|
||||
if (savedOtaUrl) {
|
||||
otaUrlInput.value = savedOtaUrl;
|
||||
}
|
||||
|
||||
const savedWakewordWsUrl = localStorage.getItem('xz_tester_wakewordWsUrl');
|
||||
if (savedWakewordWsUrl !== null && wakewordWsUrlInput) {
|
||||
wakewordWsUrlInput.value = savedWakewordWsUrl;
|
||||
}
|
||||
|
||||
const savedWakewordEnabled = localStorage.getItem('xz_tester_wakewordEnabled');
|
||||
if (savedWakewordEnabled !== null && wakewordEnabledInput) {
|
||||
wakewordEnabledInput.value = savedWakewordEnabled;
|
||||
}
|
||||
|
||||
const savedWakewordList = localStorage.getItem('xz_tester_wakewordList');
|
||||
if (savedWakewordList !== null && wakewordListInput) {
|
||||
wakewordListInput.value = savedWakewordList;
|
||||
} else if (wakewordListInput) {
|
||||
wakewordListInput.value = DEFAULT_WAKE_WORDS;
|
||||
}
|
||||
|
||||
const emojiEnabledInput = document.getElementById('emojiEnabled');
|
||||
const savedEmojiEnabled = localStorage.getItem('xz_tester_emojiEnabled');
|
||||
if (savedEmojiEnabled !== null && emojiEnabledInput) {
|
||||
emojiEnabledInput.value = savedEmojiEnabled;
|
||||
}
|
||||
}
|
||||
|
||||
// 保存配置
|
||||
export function saveConfig() {
|
||||
const deviceMacInput = document.getElementById('deviceMac');
|
||||
const deviceNameInput = document.getElementById('deviceName');
|
||||
const clientIdInput = document.getElementById('clientId');
|
||||
const wakewordWsUrlInput = document.getElementById('wakewordWsUrl');
|
||||
const wakewordEnabledInput = document.getElementById('wakewordEnabled');
|
||||
const wakewordListInput = document.getElementById('wakewordList');
|
||||
|
||||
localStorage.setItem('xz_tester_deviceMac', deviceMacInput.value);
|
||||
localStorage.setItem('xz_tester_deviceName', deviceNameInput.value);
|
||||
localStorage.setItem('xz_tester_clientId', clientIdInput.value);
|
||||
const emojiEnabledInput = document.getElementById('emojiEnabled');
|
||||
if (emojiEnabledInput) {
|
||||
localStorage.setItem('xz_tester_emojiEnabled', emojiEnabledInput.value);
|
||||
}
|
||||
if (wakewordEnabledInput) {
|
||||
localStorage.setItem('xz_tester_wakewordEnabled', wakewordEnabledInput.value);
|
||||
}
|
||||
if (wakewordListInput) {
|
||||
localStorage.setItem('xz_tester_wakewordList', wakewordListInput.value);
|
||||
}
|
||||
if (wakewordWsUrlInput && wakewordWsUrlInput.value.trim()) {
|
||||
localStorage.setItem('xz_tester_wakewordWsUrl', wakewordWsUrlInput.value.trim());
|
||||
}
|
||||
}
|
||||
|
||||
// 获取配置值
|
||||
export function getConfig() {
|
||||
// 从DOM获取值
|
||||
const deviceMac = document.getElementById('deviceMac')?.value.trim() || '';
|
||||
const deviceName = document.getElementById('deviceName')?.value.trim() || '';
|
||||
const clientId = document.getElementById('clientId')?.value.trim() || '';
|
||||
const emojiEnabled = document.getElementById('emojiEnabled')?.value !== 'false';
|
||||
|
||||
return {
|
||||
deviceId: deviceMac, // 使用MAC地址作为deviceId
|
||||
deviceName,
|
||||
deviceMac,
|
||||
clientId,
|
||||
emojiEnabled
|
||||
};
|
||||
}
|
||||
|
||||
// 保存连接URL
|
||||
export function saveConnectionUrls() {
|
||||
const otaUrl = document.getElementById('otaUrl').value.trim();
|
||||
const wsUrl = document.getElementById('serverUrl').value.trim();
|
||||
localStorage.setItem('xz_tester_otaUrl', otaUrl);
|
||||
localStorage.setItem('xz_tester_wsUrl', wsUrl);
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
export function checkOpusLoaded() {
|
||||
try {
|
||||
// 检查Module是否存在（本地库导出的全局变量）
|
||||
if (typeof Module === 'undefined') {
|
||||
throw new Error('Opus库未加载，Module对象不存在');
|
||||
}
|
||||
|
||||
// 尝试先使用Module.instance（libopus.js最后一行导出方式）
|
||||
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
|
||||
// 使用Module.instance对象替换全局Module对象
|
||||
window.ModuleInstance = Module.instance;
|
||||
log('Opus库加载成功（使用Module.instance）', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
// 如果没有Module.instance，检查全局Module函数
|
||||
if (typeof Module._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module;
|
||||
log('Opus库加载成功（使用全局Module）', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error('Opus解码函数未找到，可能Module结构不正确');
|
||||
} catch (err) {
|
||||
log(`Opus库加载失败，请检查libopus.js文件是否存在且正确: ${err.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 创建一个Opus编码器
|
||||
let opusEncoder = null;
|
||||
export function initOpusEncoder() {
|
||||
try {
|
||||
if (opusEncoder) {
|
||||
return opusEncoder; // 已经初始化过
|
||||
}
|
||||
|
||||
if (!window.ModuleInstance) {
|
||||
log('无法创建Opus编码器：ModuleInstance不可用', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 初始化一个Opus编码器
|
||||
const mod = window.ModuleInstance;
|
||||
const sampleRate = 16000; // 16kHz采样率
|
||||
const channels = 1; // 单声道
|
||||
const application = 2048; // OPUS_APPLICATION_VOIP = 2048
|
||||
|
||||
// 创建编码器
|
||||
opusEncoder = {
|
||||
channels: channels,
|
||||
sampleRate: sampleRate,
|
||||
frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
|
||||
maxPacketSize: 4000, // 最大包大小
|
||||
module: mod,
|
||||
|
||||
// 初始化编码器
|
||||
init: function () {
|
||||
try {
|
||||
// 获取编码器大小
|
||||
const encoderSize = mod._opus_encoder_get_size(this.channels);
|
||||
log(`Opus编码器大小: ${encoderSize}字节`, 'info');
|
||||
|
||||
// 分配内存
|
||||
this.encoderPtr = mod._malloc(encoderSize);
|
||||
if (!this.encoderPtr) {
|
||||
throw new Error("无法分配编码器内存");
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
const err = mod._opus_encoder_init(
|
||||
this.encoderPtr,
|
||||
this.sampleRate,
|
||||
this.channels,
|
||||
application
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
throw new Error(`Opus编码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
// 设置位率 (16kbps)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4002, 16000); // OPUS_SET_BITRATE
|
||||
|
||||
// 设置复杂度 (0-10, 越高质量越好但CPU使用越多)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4010, 5); // OPUS_SET_COMPLEXITY
|
||||
|
||||
// 设置使用DTX (不传输静音帧)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4016, 1); // OPUS_SET_DTX
|
||||
|
||||
log("Opus编码器初始化成功", 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (this.encoderPtr) {
|
||||
mod._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
log(`Opus编码器初始化失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
// 编码PCM数据为Opus
|
||||
encode: function (pcmData) {
|
||||
if (!this.encoderPtr) {
|
||||
if (!this.init()) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
// 为PCM数据分配内存
|
||||
const pcmPtr = mod._malloc(pcmData.length * 2); // 2字节/int16
|
||||
|
||||
// 将PCM数据复制到HEAP
|
||||
for (let i = 0; i < pcmData.length; i++) {
|
||||
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
|
||||
}
|
||||
|
||||
// 为输出分配内存
|
||||
const outPtr = mod._malloc(this.maxPacketSize);
|
||||
|
||||
// 进行编码
|
||||
const encodedLen = mod._opus_encode(
|
||||
this.encoderPtr,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
outPtr,
|
||||
this.maxPacketSize
|
||||
);
|
||||
|
||||
if (encodedLen < 0) {
|
||||
throw new Error(`Opus编码失败: ${encodedLen}`);
|
||||
}
|
||||
|
||||
// 复制编码后的数据
|
||||
const opusData = new Uint8Array(encodedLen);
|
||||
for (let i = 0; i < encodedLen; i++) {
|
||||
opusData[i] = mod.HEAPU8[outPtr + i];
|
||||
}
|
||||
|
||||
// 释放内存
|
||||
mod._free(pcmPtr);
|
||||
mod._free(outPtr);
|
||||
|
||||
return opusData;
|
||||
} catch (error) {
|
||||
log(`Opus编码出错: ${error.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
},
|
||||
|
||||
// 销毁编码器
|
||||
destroy: function () {
|
||||
if (this.encoderPtr) {
|
||||
this.module._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
opusEncoder.init();
|
||||
return opusEncoder;
|
||||
} catch (error) {
|
||||
log(`创建Opus编码器失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
// 音频播放模块
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { createStreamingContext } from './stream-context.js?v=0205';
|
||||
|
||||
// 音频播放器类
|
||||
export class AudioPlayer {
|
||||
constructor() {
|
||||
// 音频参数
|
||||
this.SAMPLE_RATE = 16000;
|
||||
this.CHANNELS = 1;
|
||||
this.FRAME_SIZE = 960;
|
||||
this.MIN_AUDIO_DURATION = 0.12;
|
||||
|
||||
// 状态
|
||||
this.audioContext = null;
|
||||
this.opusDecoder = null;
|
||||
this.streamingContext = null;
|
||||
this.queue = new BlockingQueue();
|
||||
this.isPlaying = false;
|
||||
}
|
||||
|
||||
// 获取或创建AudioContext
|
||||
getAudioContext() {
|
||||
if (!this.audioContext) {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: this.SAMPLE_RATE,
|
||||
latencyHint: 'interactive'
|
||||
});
|
||||
log('创建音频上下文，采样率: ' + this.SAMPLE_RATE + 'Hz', 'debug');
|
||||
}
|
||||
return this.audioContext;
|
||||
}
|
||||
|
||||
// 初始化Opus解码器
|
||||
async initOpusDecoder() {
|
||||
if (this.opusDecoder) return this.opusDecoder;
|
||||
|
||||
try {
|
||||
if (typeof window.ModuleInstance === 'undefined') {
|
||||
if (typeof Module !== 'undefined') {
|
||||
window.ModuleInstance = Module;
|
||||
log('使用全局Module作为ModuleInstance', 'info');
|
||||
} else {
|
||||
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
|
||||
}
|
||||
}
|
||||
|
||||
const mod = window.ModuleInstance;
|
||||
|
||||
this.opusDecoder = {
|
||||
channels: this.CHANNELS,
|
||||
rate: this.SAMPLE_RATE,
|
||||
frameSize: this.FRAME_SIZE,
|
||||
module: mod,
|
||||
decoderPtr: null,
|
||||
|
||||
init: function () {
|
||||
if (this.decoderPtr) return true;
|
||||
|
||||
const decoderSize = mod._opus_decoder_get_size(this.channels);
|
||||
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
|
||||
|
||||
this.decoderPtr = mod._malloc(decoderSize);
|
||||
if (!this.decoderPtr) {
|
||||
throw new Error("无法分配解码器内存");
|
||||
}
|
||||
|
||||
const err = mod._opus_decoder_init(
|
||||
this.decoderPtr,
|
||||
this.rate,
|
||||
this.channels
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
this.destroy();
|
||||
throw new Error(`Opus解码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
log("Opus解码器初始化成功", 'success');
|
||||
return true;
|
||||
},
|
||||
|
||||
decode: function (opusData) {
|
||||
if (!this.decoderPtr) {
|
||||
if (!this.init()) {
|
||||
throw new Error("解码器未初始化且无法初始化");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
const opusPtr = mod._malloc(opusData.length);
|
||||
mod.HEAPU8.set(opusData, opusPtr);
|
||||
|
||||
const pcmPtr = mod._malloc(this.frameSize * 2);
|
||||
|
||||
const decodedSamples = mod._opus_decode(
|
||||
this.decoderPtr,
|
||||
opusPtr,
|
||||
opusData.length,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
0
|
||||
);
|
||||
|
||||
if (decodedSamples < 0) {
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
throw new Error(`Opus解码失败: ${decodedSamples}`);
|
||||
}
|
||||
|
||||
const decodedData = new Int16Array(decodedSamples);
|
||||
for (let i = 0; i < decodedSamples; i++) {
|
||||
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
|
||||
}
|
||||
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
|
||||
return decodedData;
|
||||
} catch (error) {
|
||||
log(`Opus解码错误: ${error.message}`, 'error');
|
||||
return new Int16Array(0);
|
||||
}
|
||||
},
|
||||
|
||||
destroy: function () {
|
||||
if (this.decoderPtr) {
|
||||
this.module._free(this.decoderPtr);
|
||||
this.decoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (!this.opusDecoder.init()) {
|
||||
throw new Error("Opus解码器初始化失败");
|
||||
}
|
||||
|
||||
return this.opusDecoder;
|
||||
|
||||
} catch (error) {
|
||||
log(`Opus解码器初始化失败: ${error.message}`, 'error');
|
||||
this.opusDecoder = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 启动音频缓冲
|
||||
async startAudioBuffering() {
|
||||
log("开始音频缓冲...", 'info');
|
||||
|
||||
this.initOpusDecoder().catch(error => {
|
||||
log(`预初始化Opus解码器失败: ${error.message}`, 'warning');
|
||||
});
|
||||
|
||||
const timeout = 400;
|
||||
while (true) {
|
||||
const packets = await this.queue.dequeue(
|
||||
6,
|
||||
timeout,
|
||||
(count) => {
|
||||
log(`缓冲超时，当前缓冲包数: ${count}，开始播放`, 'info');
|
||||
}
|
||||
);
|
||||
if (packets.length) {
|
||||
log(`已缓冲 ${packets.length} 个音频包，开始播放`, 'info');
|
||||
this.streamingContext.pushAudioBuffer(packets);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const data = await this.queue.dequeue(99, 30);
|
||||
if (data.length) {
|
||||
this.streamingContext.pushAudioBuffer(data);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 播放已缓冲的音频
|
||||
async playBufferedAudio() {
|
||||
try {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (!this.opusDecoder) {
|
||||
log('初始化Opus解码器...', 'info');
|
||||
try {
|
||||
this.opusDecoder = await this.initOpusDecoder();
|
||||
if (!this.opusDecoder) {
|
||||
throw new Error('解码器初始化失败');
|
||||
}
|
||||
log('Opus解码器初始化成功', 'success');
|
||||
} catch (error) {
|
||||
log('Opus解码器初始化失败: ' + error.message, 'error');
|
||||
this.isPlaying = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.streamingContext) {
|
||||
this.streamingContext = createStreamingContext(
|
||||
this.opusDecoder,
|
||||
this.audioContext,
|
||||
this.SAMPLE_RATE,
|
||||
this.CHANNELS,
|
||||
this.MIN_AUDIO_DURATION
|
||||
);
|
||||
}
|
||||
|
||||
this.streamingContext.decodeOpusFrames();
|
||||
this.streamingContext.startPlaying();
|
||||
|
||||
} catch (error) {
|
||||
log(`播放已缓冲的音频出错: ${error.message}`, 'error');
|
||||
this.isPlaying = false;
|
||||
this.streamingContext = null;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加音频数据到队列
|
||||
enqueueAudioData(opusData) {
|
||||
if (opusData.length > 0) {
|
||||
this.queue.enqueue(opusData);
|
||||
} else {
|
||||
log('收到空音频数据帧，可能是结束标志', 'warning');
|
||||
if (this.isPlaying && this.streamingContext) {
|
||||
this.streamingContext.endOfStream = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 预加载解码器
|
||||
async preload() {
|
||||
log('预加载Opus解码器...', 'info');
|
||||
try {
|
||||
await this.initOpusDecoder();
|
||||
log('Opus解码器预加载成功', 'success');
|
||||
} catch (error) {
|
||||
log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 启动播放系统
|
||||
async start() {
|
||||
await this.preload();
|
||||
this.playBufferedAudio();
|
||||
this.startAudioBuffering();
|
||||
}
|
||||
|
||||
// 获取音频包统计信息
|
||||
getAudioStats() {
|
||||
if (!this.streamingContext) {
|
||||
return {
|
||||
pendingDecode: 0,
|
||||
pendingPlay: 0,
|
||||
totalPending: 0
|
||||
};
|
||||
}
|
||||
|
||||
const pendingDecode = this.streamingContext.getPendingDecodeCount();
|
||||
const pendingPlay = this.streamingContext.getPendingPlayCount();
|
||||
|
||||
return {
|
||||
pendingDecode, // 待解码包数
|
||||
pendingPlay, // 待播放包数
|
||||
totalPending: pendingDecode + pendingPlay // 总待处理包数
|
||||
};
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲并停止播放
|
||||
clearAllAudio() {
|
||||
log('AudioPlayer: 清空所有音频', 'info');
|
||||
|
||||
// 清空接收队列（使用clear方法保持对象引用）
|
||||
this.queue.clear();
|
||||
|
||||
// 清空流上下文的所有缓冲
|
||||
if (this.streamingContext) {
|
||||
this.streamingContext.clearAllBuffers();
|
||||
}
|
||||
|
||||
log('AudioPlayer: 音频已清空', 'success');
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let audioPlayerInstance = null;
|
||||
|
||||
export function getAudioPlayer() {
|
||||
if (!audioPlayerInstance) {
|
||||
audioPlayerInstance = new AudioPlayer();
|
||||
}
|
||||
return audioPlayerInstance;
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
// Audio recording module
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { initOpusEncoder } from './opus-codec.js?v=0205';
|
||||
import { getAudioPlayer } from './player.js?v=0205';
|
||||
|
||||
// Audio recorder class
|
||||
export class AudioRecorder {
|
||||
constructor() {
|
||||
this.isRecording = false;
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
this.audioProcessor = null;
|
||||
this.audioProcessorType = null;
|
||||
this.audioSource = null;
|
||||
this.opusEncoder = null;
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.visualizationRequest = null;
|
||||
this.recordingTimer = null;
|
||||
this.websocket = null;
|
||||
// Callback functions
|
||||
this.onRecordingStart = null;
|
||||
this.onRecordingStop = null;
|
||||
this.onVisualizerUpdate = null;
|
||||
}
|
||||
|
||||
// Set WebSocket instance
|
||||
setWebSocket(ws) {
|
||||
this.websocket = ws;
|
||||
}
|
||||
|
||||
// Get AudioContext instance
|
||||
getAudioContext() {
|
||||
return getAudioPlayer().getAudioContext();
|
||||
}
|
||||
|
||||
// Initialize encoder
|
||||
initEncoder() {
|
||||
if (!this.opusEncoder) {
|
||||
this.opusEncoder = initOpusEncoder();
|
||||
}
|
||||
return this.opusEncoder;
|
||||
}
|
||||
|
||||
// PCM processor code
|
||||
getAudioProcessorCode() {
|
||||
return `
|
||||
class AudioRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffers = [];
|
||||
this.frameSize = 960;
|
||||
this.buffer = new Int16Array(this.frameSize);
|
||||
this.bufferIndex = 0;
|
||||
this.isRecording = false;
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.command === 'start') {
|
||||
this.isRecording = true;
|
||||
this.port.postMessage({ type: 'status', status: 'started' });
|
||||
} else if (event.data.command === 'stop') {
|
||||
this.isRecording = false;
|
||||
if (this.bufferIndex > 0) {
|
||||
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
||||
this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
this.port.postMessage({ type: 'status', status: 'stopped' });
|
||||
}
|
||||
};
|
||||
}
|
||||
process(inputs, outputs, parameters) {
|
||||
if (!this.isRecording) return true;
|
||||
const input = inputs[0][0];
|
||||
if (!input) return true;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (this.bufferIndex >= this.frameSize) {
|
||||
this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
|
||||
`;
|
||||
}
|
||||
|
||||
// Create audio processor
|
||||
async createAudioProcessor() {
|
||||
this.audioContext = this.getAudioContext();
|
||||
try {
|
||||
if (this.audioContext.audioWorklet) {
|
||||
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
await this.audioContext.audioWorklet.addModule(url);
|
||||
URL.revokeObjectURL(url);
|
||||
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
|
||||
audioProcessor.port.onmessage = (event) => {
|
||||
if (event.data.type === 'buffer') {
|
||||
this.processPCMBuffer(event.data.buffer);
|
||||
}
|
||||
};
|
||||
log('使用AudioWorklet处理音频', 'success');
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
audioProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
return { node: audioProcessor, type: 'worklet' };
|
||||
} else {
|
||||
log('AudioWorklet不可用，使用ScriptProcessorNode作为后备方案', 'warning');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
} catch (error) {
|
||||
log(`创建音频处理器失败: ${error.message}，尝试后备方案`, 'error');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
// Create ScriptProcessor as fallback
|
||||
createScriptProcessor() {
|
||||
try {
|
||||
const frameSize = 4096;
|
||||
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
|
||||
scriptProcessor.onaudioprocess = (event) => {
|
||||
if (!this.isRecording) return;
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
const buffer = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
this.processPCMBuffer(buffer);
|
||||
};
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
scriptProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
log('使用ScriptProcessorNode作为后备方案成功', 'warning');
|
||||
return { node: scriptProcessor, type: 'processor' };
|
||||
} catch (fallbackError) {
|
||||
log(`后备方案也失败: ${fallbackError.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Process PCM buffer data
|
||||
processPCMBuffer(buffer) {
|
||||
if (!this.isRecording) return;
|
||||
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
|
||||
newBuffer.set(this.pcmDataBuffer);
|
||||
newBuffer.set(buffer, this.pcmDataBuffer.length);
|
||||
this.pcmDataBuffer = newBuffer;
|
||||
const samplesPerFrame = 960;
|
||||
while (this.pcmDataBuffer.length >= samplesPerFrame) {
|
||||
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
|
||||
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
|
||||
this.encodeAndSendOpus(frameData);
|
||||
}
|
||||
}
|
||||
|
||||
// Encode and send Opus data
|
||||
encodeAndSendOpus(pcmData = null) {
|
||||
if (!this.opusEncoder) {
|
||||
log('Opus编码器未初始化', 'error');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (pcmData) {
|
||||
const opusData = this.opusEncoder.encode(pcmData);
|
||||
if (opusData && opusData.length > 0) {
|
||||
this.audioBuffers.push(opusData.buffer);
|
||||
this.totalAudioSize += opusData.length;
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.websocket.send(opusData.buffer);
|
||||
} catch (error) {
|
||||
log(`WebSocket发送错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('Opus编码失败，未返回有效数据', 'error');
|
||||
}
|
||||
} else {
|
||||
if (this.pcmDataBuffer.length > 0) {
|
||||
const samplesPerFrame = 960;
|
||||
if (this.pcmDataBuffer.length < samplesPerFrame) {
|
||||
const paddedBuffer = new Int16Array(samplesPerFrame);
|
||||
paddedBuffer.set(this.pcmDataBuffer);
|
||||
this.encodeAndSendOpus(paddedBuffer);
|
||||
} else {
|
||||
this.encodeAndSendOpus(this.pcmDataBuffer.slice(0, samplesPerFrame));
|
||||
}
|
||||
this.pcmDataBuffer = new Int16Array(0);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log(`Opus编码错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// Start recording
|
||||
async start() {
|
||||
if (this.isRecording) return false;
|
||||
try {
|
||||
if (!this.initEncoder()) {
|
||||
log('无法开始录音: Opus编码器初始化失败', 'error');
|
||||
return false;
|
||||
}
|
||||
log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
this.audioContext = this.getAudioContext();
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
const processorResult = await this.createAudioProcessor();
|
||||
if (!processorResult) {
|
||||
log('无法创建音频处理器', 'error');
|
||||
return false;
|
||||
}
|
||||
this.audioProcessor = processorResult.node;
|
||||
this.audioProcessorType = processorResult.type;
|
||||
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 2048;
|
||||
this.audioSource.connect(this.analyser);
|
||||
this.audioSource.connect(this.audioProcessor);
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.isRecording = true;
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'start' });
|
||||
}
|
||||
// Send listening start message
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
log(`已发送录音开始消息`, 'info');
|
||||
} else {
|
||||
log('WebSocket未连接，无法发送开始消息', 'error');
|
||||
return false;
|
||||
}
|
||||
// Start visualization
|
||||
if (this.onVisualizerUpdate) {
|
||||
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
this.startVisualization(dataArray);
|
||||
}
|
||||
// Immediately notify recording start, update button state
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(0);
|
||||
}
|
||||
// Start recording timer
|
||||
let recordingSeconds = 0;
|
||||
this.recordingTimer = setInterval(() => {
|
||||
recordingSeconds += 0.1;
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(recordingSeconds);
|
||||
}
|
||||
}, 100);
|
||||
log('已开始PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音启动错误: ${error.message}`, 'error');
|
||||
this.isRecording = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Start visualization
|
||||
startVisualization(dataArray) {
|
||||
const draw = () => {
|
||||
this.visualizationRequest = requestAnimationFrame(() => draw());
|
||||
if (!this.isRecording) return;
|
||||
this.analyser.getByteFrequencyData(dataArray);
|
||||
if (this.onVisualizerUpdate) {
|
||||
this.onVisualizerUpdate(dataArray);
|
||||
}
|
||||
};
|
||||
draw();
|
||||
}
|
||||
|
||||
// Stop recording
|
||||
stop() {
|
||||
if (!this.isRecording) return false;
|
||||
try {
|
||||
this.isRecording = false;
|
||||
if (this.audioProcessor) {
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'stop' });
|
||||
}
|
||||
this.audioProcessor.disconnect();
|
||||
this.audioProcessor = null;
|
||||
}
|
||||
if (this.audioSource) {
|
||||
this.audioSource.disconnect();
|
||||
this.audioSource = null;
|
||||
}
|
||||
if (this.visualizationRequest) {
|
||||
cancelAnimationFrame(this.visualizationRequest);
|
||||
this.visualizationRequest = null;
|
||||
}
|
||||
if (this.recordingTimer) {
|
||||
clearInterval(this.recordingTimer);
|
||||
this.recordingTimer = null;
|
||||
}
|
||||
// Encode and send remaining data
|
||||
this.encodeAndSendOpus();
|
||||
// Send end signal
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const emptyOpusFrame = new Uint8Array(0);
|
||||
this.websocket.send(emptyOpusFrame);
|
||||
log('已发送录音停止信号', 'info');
|
||||
}
|
||||
if (this.onRecordingStop) {
|
||||
this.onRecordingStop();
|
||||
}
|
||||
log('已停止PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音停止错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Get analyser
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
let audioRecorderInstance = null;
|
||||
|
||||
export function getAudioRecorder() {
|
||||
if (!audioRecorderInstance) {
|
||||
audioRecorderInstance = new AudioRecorder();
|
||||
}
|
||||
return audioRecorderInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if microphone is available
|
||||
* @returns {Promise<boolean>} Returns true if available, false if not available
|
||||
*/
|
||||
export async function checkMicrophoneAvailability() {
|
||||
// Check if browser supports getUserMedia API
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log('浏览器不支持getUserMedia API', 'warning');
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
// Try to access microphone
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
// Immediately stop all tracks to release microphone
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
log('麦克风可用性检查成功', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`麦克风不可用: ${error.message}`, 'warning');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if it is HTTP non-localhost access
|
||||
* @returns {boolean} Returns true if it is HTTP non-localhost access
|
||||
*/
|
||||
export function isHttpNonLocalhost() {
|
||||
const protocol = window.location.protocol;
|
||||
const hostname = window.location.hostname;
|
||||
// Check if it is HTTP protocol
|
||||
if (protocol !== 'http:') {
|
||||
return false;
|
||||
}
|
||||
// localhost and 127.0.0.1 can use microphone
|
||||
if (hostname === 'localhost' || hostname === '127.0.0.1') {
|
||||
return false;
|
||||
}
|
||||
// Private IP addresses can also use microphone (browser allows)
|
||||
if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
|
||||
return false;
|
||||
}
|
||||
// Other HTTP access is considered non-localhost
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
// 音频流播放上下文类
|
||||
export class StreamingContext {
|
||||
constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
this.opusDecoder = opusDecoder;
|
||||
this.audioContext = audioContext;
|
||||
|
||||
// 音频参数
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
this.minAudioDuration = minAudioDuration;
|
||||
|
||||
// 初始化队列和状态
|
||||
this.queue = []; // 已解码的PCM队列。正在播放
|
||||
this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放
|
||||
this.pendingAudioBufferQueue = []; // 待处理的缓存队列
|
||||
this.audioBufferQueue = new BlockingQueue(); // 缓存队列
|
||||
this.playing = false; // 是否正在播放
|
||||
this.endOfStream = false; // 是否收到结束信号
|
||||
this.source = null; // 当前音频源
|
||||
this.totalSamples = 0; // 累积的总样本数
|
||||
this.lastPlayTime = 0; // 上次播放的时间戳
|
||||
this.scheduledEndTime = 0; // 已调度音频的结束时间
|
||||
|
||||
// 初始化分析器节点（供Live2D使用）
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 256;
|
||||
}
|
||||
|
||||
// 缓存音频数组
|
||||
pushAudioBuffer(item) {
|
||||
this.audioBufferQueue.enqueue(...item);
|
||||
}
|
||||
|
||||
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
|
||||
async getPendingAudioBufferQueue() {
|
||||
// 等待数据到达并获取
|
||||
const data = await this.audioBufferQueue.dequeue();
|
||||
// 赋值给待处理队列
|
||||
this.pendingAudioBufferQueue = data;
|
||||
}
|
||||
|
||||
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
|
||||
async getQueue(minSamples) {
|
||||
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
|
||||
|
||||
// 等待数据并获取
|
||||
const tempArray = await this.activeQueue.dequeue(num);
|
||||
this.queue.push(...tempArray);
|
||||
}
|
||||
|
||||
// 将Int16音频数据转换为Float32音频数据
|
||||
convertInt16ToFloat32(int16Data) {
|
||||
const float32Data = new Float32Array(int16Data.length);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
// 将[-32768,32767]范围转换为[-1,1]，统一使用32768.0避免不对称失真
|
||||
float32Data[i] = int16Data[i] / 32768.0;
|
||||
}
|
||||
return float32Data;
|
||||
}
|
||||
|
||||
// 获取待解码包数
|
||||
getPendingDecodeCount() {
|
||||
return this.audioBufferQueue.length + this.pendingAudioBufferQueue.length;
|
||||
}
|
||||
|
||||
// 获取待播放样本数（转换为包数，每包960样本）
|
||||
getPendingPlayCount() {
|
||||
// 计算已在队列中的样本
|
||||
const queuedSamples = this.activeQueue.length + this.queue.length;
|
||||
|
||||
// 计算已调度但未播放的样本（在Web Audio缓冲区中）
|
||||
let scheduledSamples = 0;
|
||||
if (this.playing && this.scheduledEndTime) {
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const remainingTime = Math.max(0, this.scheduledEndTime - currentTime);
|
||||
scheduledSamples = Math.floor(remainingTime * this.sampleRate);
|
||||
}
|
||||
|
||||
const totalSamples = queuedSamples + scheduledSamples;
|
||||
return Math.ceil(totalSamples / 960);
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲
|
||||
clearAllBuffers() {
|
||||
log('清空所有音频缓冲', 'info');
|
||||
|
||||
// 清空所有队列（使用clear方法保持对象引用）
|
||||
this.audioBufferQueue.clear();
|
||||
this.pendingAudioBufferQueue = [];
|
||||
this.activeQueue.clear();
|
||||
this.queue = [];
|
||||
|
||||
// 停止当前播放的音频源
|
||||
if (this.source) {
|
||||
try {
|
||||
this.source.stop();
|
||||
this.source.disconnect();
|
||||
} catch (e) {
|
||||
// 忽略已经停止的错误
|
||||
}
|
||||
this.source = null;
|
||||
}
|
||||
|
||||
// 重置状态
|
||||
this.playing = false;
|
||||
this.scheduledEndTime = this.audioContext.currentTime;
|
||||
this.totalSamples = 0;
|
||||
|
||||
log('音频缓冲已清空', 'success');
|
||||
}
|
||||
|
||||
// 获取分析器节点（供Live2D使用）
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
|
||||
// 将Opus数据解码为PCM
|
||||
async decodeOpusFrames() {
|
||||
if (!this.opusDecoder) {
|
||||
log('Opus解码器未初始化，无法解码', 'error');
|
||||
return;
|
||||
} else {
|
||||
log('Opus解码器启动', 'info');
|
||||
}
|
||||
|
||||
while (true) {
|
||||
let decodedSamples = [];
|
||||
for (const frame of this.pendingAudioBufferQueue) {
|
||||
try {
|
||||
// 使用Opus解码器解码
|
||||
const frameData = this.opusDecoder.decode(frame);
|
||||
if (frameData && frameData.length > 0) {
|
||||
// 转换为Float32
|
||||
const floatData = this.convertInt16ToFloat32(frameData);
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < floatData.length; i++) {
|
||||
decodedSamples.push(floatData[i]);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log("Opus解码失败: " + error.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedSamples.length > 0) {
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < decodedSamples.length; i++) {
|
||||
this.activeQueue.enqueue(decodedSamples[i]);
|
||||
}
|
||||
this.totalSamples += decodedSamples.length;
|
||||
} else {
|
||||
log('没有成功解码的样本', 'warning');
|
||||
}
|
||||
await this.getPendingAudioBufferQueue();
|
||||
}
|
||||
}
|
||||
|
||||
// 开始播放音频
|
||||
async startPlaying() {
|
||||
this.scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
|
||||
|
||||
while (true) {
|
||||
// 初始缓冲：等待足够的样本再开始播放
|
||||
const minSamples = this.sampleRate * this.minAudioDuration * 2;
|
||||
if (!this.playing && this.queue.length < minSamples) {
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
this.playing = true;
|
||||
|
||||
// 持续播放队列中的音频，每次播放一个小块
|
||||
while (this.playing && this.queue.length > 0) {
|
||||
// 每次播放120ms的音频（2个Opus包）
|
||||
const playDuration = 0.12;
|
||||
const targetSamples = Math.floor(this.sampleRate * playDuration);
|
||||
const actualSamples = Math.min(this.queue.length, targetSamples);
|
||||
|
||||
if (actualSamples === 0) break;
|
||||
|
||||
const currentSamples = this.queue.splice(0, actualSamples);
|
||||
const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate);
|
||||
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
|
||||
|
||||
// 创建音频源
|
||||
this.source = this.audioContext.createBufferSource();
|
||||
this.source.buffer = audioBuffer;
|
||||
|
||||
// 精确调度播放时间
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const startTime = Math.max(this.scheduledEndTime, currentTime);
|
||||
|
||||
// 连接到分析器和输出
|
||||
this.source.connect(this.analyser);
|
||||
this.source.connect(this.audioContext.destination);
|
||||
|
||||
log(`调度播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'debug');
|
||||
this.source.start(startTime);
|
||||
|
||||
// 更新下一个音频块的调度时间
|
||||
const duration = audioBuffer.duration;
|
||||
this.scheduledEndTime = startTime + duration;
|
||||
this.lastPlayTime = startTime;
|
||||
|
||||
// 如果队列中数据不足，等待新数据
|
||||
if (this.queue.length < targetSamples) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 等待新数据
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 创建streamingContext实例的工厂函数
|
||||
export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration);
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
// ==========================================
|
||||
// MCP 工具管理逻辑
|
||||
// ==========================================
|
||||
|
||||
// 全局变量
|
||||
let mcpTools = [];
|
||||
let mcpEditingIndex = null;
|
||||
let mcpProperties = [];
|
||||
let websocket = null; // 将从外部设置
|
||||
|
||||
/**
|
||||
* 设置 WebSocket 实例
|
||||
* @param {WebSocket} ws - WebSocket 连接实例
|
||||
*/
|
||||
export function setWebSocket(ws) {
|
||||
websocket = ws;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化 MCP 工具
|
||||
*/
|
||||
export async function initMcpTools() {
|
||||
// 加载默认工具数据
|
||||
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
|
||||
const savedTools = localStorage.getItem('mcpTools');
|
||||
if (savedTools) {
|
||||
try {
|
||||
const parsedTools = JSON.parse(savedTools);
|
||||
// 合并默认工具和用户保存的工具，保留用户自定义的工具
|
||||
const defaultToolNames = new Set(defaultMcpTools.map(t => t.name));
|
||||
// 添加默认工具中不存在的新工具
|
||||
parsedTools.forEach(tool => {
|
||||
if (!defaultToolNames.has(tool.name)) {
|
||||
defaultMcpTools.push(tool);
|
||||
}
|
||||
});
|
||||
mcpTools = defaultMcpTools;
|
||||
} catch (e) {
|
||||
log('加载MCP工具失败，使用默认工具', 'warning');
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
} else {
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
renderMcpTools();
|
||||
setupMcpEventListeners();
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染工具列表
|
||||
*/
|
||||
function renderMcpTools() {
|
||||
const container = document.getElementById('mcpToolsContainer');
|
||||
const countSpan = document.getElementById('mcpToolsCount');
|
||||
if (!container) {
|
||||
return; // Container not found, skip rendering
|
||||
}
|
||||
if (countSpan) {
|
||||
countSpan.textContent = `${mcpTools.length} 个工具`;
|
||||
}
|
||||
if (mcpTools.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
|
||||
return;
|
||||
}
|
||||
container.innerHTML = mcpTools.map((tool, index) => {
|
||||
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
|
||||
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
|
||||
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
|
||||
return `
|
||||
<div class="mcp-tool-card">
|
||||
<div class="mcp-tool-header">
|
||||
<div class="mcp-tool-name">${tool.name}</div>
|
||||
<div class="mcp-tool-actions">
|
||||
<button class="mcp-edit-btn" onclick="window.mcpModule.editMcpTool(${index})">
|
||||
✏️ 编辑
|
||||
</button>
|
||||
<button class="mcp-delete-btn" onclick="window.mcpModule.deleteMcpTool(${index})">
|
||||
🗑️ 删除
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mcp-tool-description">${tool.description}</div>
|
||||
<div class="mcp-tool-info">
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">参数数量:</span>
|
||||
<span class="mcp-tool-info-value">${paramCount} 个 ${requiredCount > 0 ? `(${requiredCount} 个必填)` : ''}</span>
|
||||
</div>
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">模拟返回:</span>
|
||||
<span class="mcp-tool-info-value">${hasMockResponse ? '✅ 已配置: ' + JSON.stringify(tool.mockResponse) : '⚪ 使用默认'}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染参数列表
|
||||
*/
|
||||
function renderMcpProperties() {
|
||||
const container = document.getElementById('mcpPropertiesContainer');
|
||||
const emptyState = document.getElementById('mcpEmptyState');
|
||||
if (!container) {
|
||||
return; // Container not found, skip rendering
|
||||
}
|
||||
if (mcpProperties.length === 0) {
|
||||
if (emptyState) {
|
||||
emptyState.style.display = 'block';
|
||||
}
|
||||
container.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
if (emptyState) {
|
||||
emptyState.style.display = 'none';
|
||||
}
|
||||
container.innerHTML = mcpProperties.map((prop, index) => `
|
||||
<div class="mcp-property-card" onclick="window.mcpModule.editMcpProperty(${index})">
|
||||
<div class="mcp-property-row-label">
|
||||
<span class="mcp-property-label">参数名称</span>
|
||||
<span class="mcp-property-value">${prop.name}${prop.required ? ' <span class="mcp-property-required-badge">[必填]</span>' : ''}</span>
|
||||
</div>
|
||||
<div class="mcp-property-row-label">
|
||||
<span class="mcp-property-label">数据类型</span>
|
||||
<span class="mcp-property-value">${getTypeLabel(prop.type)}</span>
|
||||
</div>
|
||||
<div class="mcp-property-row-label">
|
||||
<span class="mcp-property-label">描述</span>
|
||||
<span class="mcp-property-value">${prop.description || '-'}</span>
|
||||
</div>
|
||||
<div class="mcp-property-row-action">
|
||||
<button class="mcp-property-delete-btn" onclick="event.stopPropagation(); window.mcpModule.deleteMcpProperty(${index})">删除</button>
|
||||
</div>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取数据类型标签
|
||||
*/
|
||||
function getTypeLabel(type) {
|
||||
const typeMap = {
|
||||
'string': '字符串',
|
||||
'integer': '整数',
|
||||
'number': '数字',
|
||||
'boolean': '布尔值',
|
||||
'array': '数组',
|
||||
'object': '对象'
|
||||
};
|
||||
return typeMap[type] || type;
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加参数 - 打开参数编辑模态框
|
||||
*/
|
||||
function addMcpProperty() {
|
||||
openPropertyModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 编辑参数 - 打开参数编辑模态框
|
||||
*/
|
||||
function editMcpProperty(index) {
|
||||
openPropertyModal(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开参数编辑模态框
|
||||
*/
|
||||
function openPropertyModal(index = null) {
|
||||
const form = document.getElementById('mcpPropertyForm');
|
||||
const title = document.getElementById('mcpPropertyModalTitle');
|
||||
document.getElementById('mcpPropertyIndex').value = index !== null ? index : -1;
|
||||
|
||||
if (index !== null) {
|
||||
const prop = mcpProperties[index];
|
||||
title.textContent = '编辑参数';
|
||||
document.getElementById('mcpPropertyName').value = prop.name;
|
||||
document.getElementById('mcpPropertyType').value = prop.type || 'string';
|
||||
document.getElementById('mcpPropertyMinimum').value = prop.minimum !== undefined ? prop.minimum : '';
|
||||
document.getElementById('mcpPropertyMaximum').value = prop.maximum !== undefined ? prop.maximum : '';
|
||||
document.getElementById('mcpPropertyDescription').value = prop.description || '';
|
||||
document.getElementById('mcpPropertyRequired').checked = prop.required || false;
|
||||
} else {
|
||||
title.textContent = '添加参数';
|
||||
form.reset();
|
||||
document.getElementById('mcpPropertyName').value = `param_${mcpProperties.length + 1}`;
|
||||
document.getElementById('mcpPropertyType').value = 'string';
|
||||
document.getElementById('mcpPropertyMinimum').value = '';
|
||||
document.getElementById('mcpPropertyMaximum').value = '';
|
||||
document.getElementById('mcpPropertyDescription').value = '';
|
||||
document.getElementById('mcpPropertyRequired').checked = false;
|
||||
}
|
||||
|
||||
updatePropertyRangeVisibility();
|
||||
document.getElementById('mcpPropertyModal').style.display = 'flex';
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭参数编辑模态框
|
||||
*/
|
||||
function closePropertyModal() {
|
||||
document.getElementById('mcpPropertyModal').style.display = 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新数值范围输入框的可见性
|
||||
*/
|
||||
function updatePropertyRangeVisibility() {
|
||||
const type = document.getElementById('mcpPropertyType').value;
|
||||
const rangeGroup = document.getElementById('mcpPropertyRangeGroup');
|
||||
if (type === 'integer' || type === 'number') {
|
||||
rangeGroup.style.display = 'block';
|
||||
} else {
|
||||
rangeGroup.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理参数表单提交
|
||||
*/
|
||||
function handlePropertySubmit(e) {
|
||||
e.preventDefault();
|
||||
const index = parseInt(document.getElementById('mcpPropertyIndex').value);
|
||||
const name = document.getElementById('mcpPropertyName').value.trim();
|
||||
const type = document.getElementById('mcpPropertyType').value;
|
||||
const minimum = document.getElementById('mcpPropertyMinimum').value;
|
||||
const maximum = document.getElementById('mcpPropertyMaximum').value;
|
||||
const description = document.getElementById('mcpPropertyDescription').value.trim();
|
||||
const required = document.getElementById('mcpPropertyRequired').checked;
|
||||
|
||||
// 检查名称重复
|
||||
const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === name);
|
||||
if (isDuplicate) {
|
||||
alert('参数名称已存在，请使用不同的名称');
|
||||
return;
|
||||
}
|
||||
|
||||
const propData = {
|
||||
name,
|
||||
type,
|
||||
description,
|
||||
required
|
||||
};
|
||||
|
||||
// 数值类型添加范围限制
|
||||
if (type === 'integer' || type === 'number') {
|
||||
if (minimum !== '') {
|
||||
propData.minimum = parseFloat(minimum);
|
||||
}
|
||||
if (maximum !== '') {
|
||||
propData.maximum = parseFloat(maximum);
|
||||
}
|
||||
}
|
||||
|
||||
if (index >= 0) {
|
||||
mcpProperties[index] = propData;
|
||||
} else {
|
||||
mcpProperties.push(propData);
|
||||
}
|
||||
|
||||
renderMcpProperties();
|
||||
closePropertyModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除参数
|
||||
*/
|
||||
function deleteMcpProperty(index) {
|
||||
mcpProperties.splice(index, 1);
|
||||
renderMcpProperties();
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置事件监听
|
||||
*/
|
||||
function setupMcpEventListeners() {
|
||||
const panel = document.getElementById('mcpToolsPanel');
|
||||
const addBtn = document.getElementById('addMcpToolBtn');
|
||||
const modal = document.getElementById('mcpToolModal');
|
||||
const closeBtn = document.getElementById('closeMcpModalBtn');
|
||||
const cancelBtn = document.getElementById('cancelMcpBtn');
|
||||
const form = document.getElementById('mcpToolForm');
|
||||
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
|
||||
|
||||
// 参数编辑模态框相关元素
|
||||
const propertyModal = document.getElementById('mcpPropertyModal');
|
||||
const closePropertyBtn = document.getElementById('closeMcpPropertyModalBtn');
|
||||
const cancelPropertyBtn = document.getElementById('cancelMcpPropertyBtn');
|
||||
const propertyForm = document.getElementById('mcpPropertyForm');
|
||||
const propertyTypeSelect = document.getElementById('mcpPropertyType');
|
||||
|
||||
// Return early if required elements don't exist (e.g., in test environment)
|
||||
if (!panel || !addBtn || !modal || !closeBtn || !cancelBtn || !form || !addPropertyBtn) {
|
||||
return;
|
||||
}
|
||||
addBtn.addEventListener('click', () => openMcpModal());
|
||||
closeBtn.addEventListener('click', closeMcpModal);
|
||||
cancelBtn.addEventListener('click', closeMcpModal);
|
||||
addPropertyBtn.addEventListener('click', addMcpProperty);
|
||||
form.addEventListener('submit', handleMcpSubmit);
|
||||
|
||||
// 参数编辑模态框事件
|
||||
if (propertyModal && closePropertyBtn && cancelPropertyBtn && propertyForm && propertyTypeSelect) {
|
||||
closePropertyBtn.addEventListener('click', closePropertyModal);
|
||||
cancelPropertyBtn.addEventListener('click', closePropertyModal);
|
||||
propertyForm.addEventListener('submit', handlePropertySubmit);
|
||||
propertyTypeSelect.addEventListener('change', updatePropertyRangeVisibility);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开模态框
|
||||
*/
|
||||
function openMcpModal(index = null) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
mcpEditingIndex = index;
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
if (index !== null) {
|
||||
document.getElementById('mcpModalTitle').textContent = '编辑工具';
|
||||
const tool = mcpTools[index];
|
||||
document.getElementById('mcpToolName').value = tool.name;
|
||||
document.getElementById('mcpToolDescription').value = tool.description;
|
||||
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
|
||||
mcpProperties = [];
|
||||
const schema = tool.inputSchema;
|
||||
if (schema.properties) {
|
||||
Object.keys(schema.properties).forEach(key => {
|
||||
const prop = schema.properties[key];
|
||||
mcpProperties.push({
|
||||
name: key,
|
||||
type: prop.type || 'string',
|
||||
minimum: prop.minimum,
|
||||
maximum: prop.maximum,
|
||||
description: prop.description || '',
|
||||
required: schema.required && schema.required.includes(key)
|
||||
});
|
||||
});
|
||||
}
|
||||
} else {
|
||||
document.getElementById('mcpModalTitle').textContent = '添加工具';
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
}
|
||||
renderMcpProperties();
|
||||
document.getElementById('mcpToolModal').style.display = 'flex';
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭模态框
|
||||
*/
|
||||
function closeMcpModal() {
|
||||
document.getElementById('mcpToolModal').style.display = 'none';
|
||||
mcpEditingIndex = null;
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
document.getElementById('mcpErrorContainer').innerHTML = '';
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理表单提交
|
||||
*/
|
||||
function handleMcpSubmit(e) {
|
||||
e.preventDefault();
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
const name = document.getElementById('mcpToolName').value.trim();
|
||||
const description = document.getElementById('mcpToolDescription').value.trim();
|
||||
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
|
||||
// 检查名称重复
|
||||
const isDuplicate = mcpTools.some((tool, index) => tool.name === name && index !== mcpEditingIndex);
|
||||
if (isDuplicate) {
|
||||
showMcpError('工具名称已存在，请使用不同的名称');
|
||||
return;
|
||||
}
|
||||
// 解析模拟返回结果
|
||||
let mockResponse = null;
|
||||
if (mockResponseText) {
|
||||
try {
|
||||
mockResponse = JSON.parse(mockResponseText);
|
||||
} catch (e) {
|
||||
showMcpError('模拟返回结果不是有效的 JSON 格式: ' + e.message);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// 构建 inputSchema
|
||||
const inputSchema = { type: "object", properties: {}, required: [] };
|
||||
mcpProperties.forEach(prop => {
|
||||
const propSchema = { type: prop.type };
|
||||
if (prop.description) {
|
||||
propSchema.description = prop.description;
|
||||
}
|
||||
if ((prop.type === 'integer' || prop.type === 'number')) {
|
||||
if (prop.minimum !== undefined && prop.minimum !== '') {
|
||||
propSchema.minimum = prop.minimum;
|
||||
}
|
||||
if (prop.maximum !== undefined && prop.maximum !== '') {
|
||||
propSchema.maximum = prop.maximum;
|
||||
}
|
||||
}
|
||||
inputSchema.properties[prop.name] = propSchema;
|
||||
if (prop.required) {
|
||||
inputSchema.required.push(prop.name);
|
||||
}
|
||||
});
|
||||
if (inputSchema.required.length === 0) {
|
||||
delete inputSchema.required;
|
||||
}
|
||||
const tool = { name, description, inputSchema, mockResponse };
|
||||
if (mcpEditingIndex !== null) {
|
||||
mcpTools[mcpEditingIndex] = tool;
|
||||
log(`已更新工具: ${name}`, 'success');
|
||||
} else {
|
||||
mcpTools.push(tool);
|
||||
log(`已添加工具: ${name}`, 'success');
|
||||
}
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
closeMcpModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 显示错误
|
||||
*/
|
||||
function showMcpError(message) {
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = `<div class="mcp-error">${message}</div>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编辑工具
|
||||
*/
|
||||
function editMcpTool(index) {
|
||||
openMcpModal(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除工具
|
||||
*/
|
||||
function deleteMcpTool(index) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
if (confirm(`确定要删除工具 "${mcpTools[index].name}" 吗？`)) {
|
||||
const toolName = mcpTools[index].name;
|
||||
mcpTools.splice(index, 1);
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
log(`已删除工具: ${toolName}`, 'info');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存工具
|
||||
*/
|
||||
function saveMcpTools() {
|
||||
localStorage.setItem('mcpTools', JSON.stringify(mcpTools));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取工具列表
|
||||
*/
|
||||
export function getMcpTools() {
|
||||
return mcpTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行工具调用
|
||||
*/
|
||||
export async function executeMcpTool(toolName, toolArgs) {
|
||||
const tool = mcpTools.find(t => t.name === toolName);
|
||||
if (!tool) {
|
||||
log(`未找到工具: ${toolName}`, 'error');
|
||||
return { success: false, error: `未知工具: ${toolName}` };
|
||||
}
|
||||
|
||||
// 处理拍照工具
|
||||
if (toolName === 'self_camera_take_photo') {
|
||||
if (typeof window.takePhoto === 'function') {
|
||||
const question = toolArgs && toolArgs.question ? toolArgs.question : '描述一下看到的物品';
|
||||
log(`正在执行拍照: ${question}`, 'info');
|
||||
const result = await window.takePhoto(question);
|
||||
return result;
|
||||
} else {
|
||||
log('拍照功能不可用', 'warning');
|
||||
return { success: false, error: '摄像头未启动或不支持拍照功能' };
|
||||
}
|
||||
}
|
||||
|
||||
// 如果有模拟返回结果，使用它
|
||||
if (tool.mockResponse) {
|
||||
// 替换模板变量
|
||||
let responseStr = JSON.stringify(tool.mockResponse);
|
||||
// 替换 ${paramName} 格式的变量
|
||||
if (toolArgs) {
|
||||
Object.keys(toolArgs).forEach(key => {
|
||||
const regex = new RegExp(`\\$\\{${key}\\}`, 'g');
|
||||
responseStr = responseStr.replace(regex, toolArgs[key]);
|
||||
});
|
||||
}
|
||||
try {
|
||||
const response = JSON.parse(responseStr);
|
||||
log(`工具 ${toolName} 执行成功，返回模拟结果: ${responseStr}`, 'success');
|
||||
return response;
|
||||
} catch (e) {
|
||||
log(`解析模拟返回结果失败: ${e.message}`, 'error');
|
||||
return tool.mockResponse;
|
||||
}
|
||||
}
|
||||
// 没有模拟返回结果，返回默认成功消息
|
||||
log(`工具 ${toolName} 执行成功，返回默认结果`, 'success');
|
||||
return { success: true, message: `工具 ${toolName} 执行成功`, tool: toolName, arguments: toolArgs };
|
||||
}
|
||||
|
||||
// 暴露全局方法供 HTML 内联事件调用
|
||||
window.mcpModule = { addMcpProperty, editMcpProperty, deleteMcpProperty, editMcpTool, deleteMcpTool };
|
||||
@@ -0,0 +1,109 @@
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
// WebSocket 连接
|
||||
export async function webSocketConnect(otaUrl, config) {
|
||||
|
||||
if (!validateConfig(config)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 发送OTA请求并获取返回的websocket信息
|
||||
const otaResult = await sendOTA(otaUrl, config);
|
||||
if (!otaResult) {
|
||||
log('无法从OTA服务器获取信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 从OTA响应中提取websocket信息
|
||||
const { websocket } = otaResult;
|
||||
if (!websocket || !websocket.url) {
|
||||
log('OTA响应中缺少websocket信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 使用OTA返回的websocket URL
|
||||
let connUrl = new URL(websocket.url);
|
||||
|
||||
// 添加token参数（从OTA响应中获取）
|
||||
if (websocket.token) {
|
||||
if (websocket.token.startsWith("Bearer ")) {
|
||||
connUrl.searchParams.append('authorization', websocket.token);
|
||||
} else {
|
||||
connUrl.searchParams.append('authorization', 'Bearer ' + websocket.token);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加认证参数（保持原有逻辑）
|
||||
connUrl.searchParams.append('device-id', config.deviceId);
|
||||
connUrl.searchParams.append('client-id', config.clientId);
|
||||
|
||||
const wsurl = connUrl.toString()
|
||||
|
||||
log(`正在连接: ${wsurl}`, 'info');
|
||||
|
||||
if (wsurl) {
|
||||
document.getElementById('serverUrl').value = wsurl;
|
||||
}
|
||||
|
||||
return new WebSocket(connUrl.toString());
|
||||
}
|
||||
|
||||
// 验证配置
|
||||
function validateConfig(config) {
|
||||
if (!config.deviceMac) {
|
||||
log('设备MAC地址不能为空', 'error');
|
||||
return false;
|
||||
}
|
||||
if (!config.clientId) {
|
||||
log('客户端ID不能为空', 'error');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// OTA发送请求，验证状态，并返回响应数据
|
||||
async function sendOTA(otaUrl, config) {
|
||||
try {
|
||||
const res = await fetch(otaUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Device-Id': config.deviceId,
|
||||
'Client-Id': config.clientId
|
||||
},
|
||||
body: JSON.stringify({
|
||||
version: 0,
|
||||
uuid: '',
|
||||
application: {
|
||||
name: 'xiaozhi-web-test',
|
||||
version: '1.0.0',
|
||||
compile_time: '2025-04-16 10:00:00',
|
||||
idf_version: '4.4.3',
|
||||
elf_sha256: '1234567890abcdef1234567890abcdef1234567890abcdef'
|
||||
},
|
||||
ota: { label: 'xiaozhi-web-test' },
|
||||
board: {
|
||||
type: config.deviceName,
|
||||
ssid: 'xiaozhi-web-test',
|
||||
rssi: 0,
|
||||
channel: 0,
|
||||
ip: '192.168.1.1',
|
||||
mac: config.deviceMac
|
||||
},
|
||||
flash_size: 0,
|
||||
minimum_free_heap_size: 0,
|
||||
mac_address: config.deviceMac,
|
||||
chip_model_name: '',
|
||||
chip_info: { model: 0, cores: 0, revision: 0, features: 0 },
|
||||
partition_table: [{ label: '', type: 0, subtype: 0, address: 0, size: 0 }]
|
||||
})
|
||||
});
|
||||
|
||||
if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
|
||||
|
||||
const result = await res.json();
|
||||
return result; // 返回完整的响应数据
|
||||
} catch (err) {
|
||||
return null; // 失败返回null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { uiController } from '../../ui/controller.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
let wakewordSocket = null;
|
||||
let reconnectTimer = null;
|
||||
let reconnectAttempts = 0;
|
||||
let shouldReconnect = true;
|
||||
let wakewordRequestSeq = 0;
|
||||
let onNextBridgeConnectedCallback = null;
|
||||
|
||||
const pendingWakewordRequests = new Map();
|
||||
|
||||
export function startWakewordBridgeListener() {
|
||||
if (wakewordSocket) {
|
||||
return wakewordSocket;
|
||||
}
|
||||
|
||||
shouldReconnect = true;
|
||||
log('正在连接本地唤醒事件桥...', 'info');
|
||||
tryConnect();
|
||||
return wakewordSocket;
|
||||
}
|
||||
|
||||
function tryConnect() {
|
||||
const bridgeUrl = buildWakewordBridgeUrl();
|
||||
|
||||
try {
|
||||
wakewordSocket = new WebSocket(bridgeUrl);
|
||||
wakewordSocket.onopen = () => {
|
||||
reconnectAttempts = 0;
|
||||
log(`本地唤醒事件桥已连接: ${bridgeUrl}`, 'success');
|
||||
// 连接成功后自动保存地址，刷新后仍能记住
|
||||
localStorage.setItem('xz_tester_wakewordWsUrl', bridgeUrl);
|
||||
const urlInput = document.getElementById('wakewordWsUrl');
|
||||
if (urlInput) urlInput.value = bridgeUrl;
|
||||
};
|
||||
|
||||
wakewordSocket.onerror = () => {
|
||||
log(`本地唤醒事件桥连接失败: ${bridgeUrl}`, 'error');
|
||||
};
|
||||
|
||||
wakewordSocket.onmessage = async (event) => {
|
||||
try {
|
||||
const message = parseWakewordBridgeMessage(event.data);
|
||||
if (message.requestId && pendingWakewordRequests.has(message.requestId)) {
|
||||
settleWakewordRequest(message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.success === false) {
|
||||
log(`本地唤醒事件桥返回错误: ${message.error || '未知错误'}`, 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'bridge_connected') {
|
||||
log('本地唤醒监听已就绪', 'info');
|
||||
if (onNextBridgeConnectedCallback) {
|
||||
const cb = onNextBridgeConnectedCallback;
|
||||
onNextBridgeConnectedCallback = null;
|
||||
cb(message);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'service_ready') {
|
||||
log('本地唤醒服务已启动', 'info');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'wakeword_config') {
|
||||
uiController.applyWakewordConfig(message.payload || {});
|
||||
log('已同步本地唤醒词配置', 'info');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'service_stopping') {
|
||||
log('本地唤醒服务正在停止', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'wake_word_detected') {
|
||||
const wakeWord = message.payload?.wake_word || '唤醒词';
|
||||
log(`检测到本地唤醒事件: ${wakeWord}`, 'info');
|
||||
await uiController.triggerWakewordDial(wakeWord);
|
||||
}
|
||||
} catch (error) {
|
||||
log(`解析本地唤醒事件失败: ${error.message}`, 'error');
|
||||
}
|
||||
};
|
||||
|
||||
wakewordSocket.onclose = () => {
|
||||
if (wakewordSocket) {
|
||||
wakewordSocket = null;
|
||||
}
|
||||
|
||||
rejectAllWakewordRequests('本地唤醒事件桥已断开');
|
||||
|
||||
if (!shouldReconnect) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (reconnectTimer) {
|
||||
return;
|
||||
}
|
||||
|
||||
reconnectAttempts += 1;
|
||||
const delay = Math.min(1000 * reconnectAttempts, 5000);
|
||||
log(`本地唤醒事件桥将在 ${delay}ms 后重连: ${bridgeUrl}`, 'warning');
|
||||
reconnectTimer = window.setTimeout(() => {
|
||||
reconnectTimer = null;
|
||||
tryConnect();
|
||||
}, delay);
|
||||
};
|
||||
|
||||
return wakewordSocket;
|
||||
} catch (error) {
|
||||
log(`启动本地唤醒监听失败: ${error.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function stopWakewordBridgeListener() {
|
||||
shouldReconnect = false;
|
||||
|
||||
if (reconnectTimer) {
|
||||
window.clearTimeout(reconnectTimer);
|
||||
reconnectTimer = null;
|
||||
}
|
||||
|
||||
if (!wakewordSocket) {
|
||||
return;
|
||||
}
|
||||
|
||||
wakewordSocket.onclose = null;
|
||||
wakewordSocket.close();
|
||||
wakewordSocket = null;
|
||||
}
|
||||
|
||||
export function sendWakewordBridgeMessage(type, payload = {}, requestId = null) {
|
||||
if (!wakewordSocket || wakewordSocket.readyState !== WebSocket.OPEN) {
|
||||
log('本地唤醒事件桥未连接，无法发送消息', 'warning');
|
||||
return false;
|
||||
}
|
||||
|
||||
wakewordSocket.send(JSON.stringify({
|
||||
type,
|
||||
requestId,
|
||||
payload,
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
|
||||
export function requestWakewordBridge(type, payload = {}, timeout = 5000) {
|
||||
const requestId = `wakeword-${Date.now()}-${++wakewordRequestSeq}`;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = window.setTimeout(() => {
|
||||
pendingWakewordRequests.delete(requestId);
|
||||
reject(new Error('本地唤醒服务响应超时'));
|
||||
}, timeout);
|
||||
|
||||
pendingWakewordRequests.set(requestId, { resolve, reject, timer });
|
||||
|
||||
if (!sendWakewordBridgeMessage(type, payload, requestId)) {
|
||||
window.clearTimeout(timer);
|
||||
pendingWakewordRequests.delete(requestId);
|
||||
reject(new Error('本地唤醒事件桥未连接'));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function getWakewordBridgeUrl() {
|
||||
if (wakewordSocket && wakewordSocket.url) {
|
||||
return wakewordSocket.url;
|
||||
}
|
||||
return buildWakewordBridgeUrl();
|
||||
}
|
||||
|
||||
export function onNextBridgeConnected(callback) {
|
||||
onNextBridgeConnectedCallback = callback;
|
||||
}
|
||||
|
||||
function buildWakewordBridgeUrl() {
|
||||
const configured = localStorage.getItem('xz_tester_wakewordWsUrl');
|
||||
if (configured && configured.trim()) {
|
||||
return configured.trim();
|
||||
}
|
||||
return 'ws://127.0.0.1:8006/wakeword-ws';
|
||||
}
|
||||
|
||||
function parseWakewordBridgeMessage(rawData) {
|
||||
const message = JSON.parse(rawData);
|
||||
return {
|
||||
type: message.type || '',
|
||||
requestId: message.requestId || null,
|
||||
success: message.success !== false,
|
||||
payload: message.payload || {},
|
||||
error: message.error || null,
|
||||
};
|
||||
}
|
||||
|
||||
function settleWakewordRequest(message) {
|
||||
const pendingRequest = pendingWakewordRequests.get(message.requestId);
|
||||
if (!pendingRequest) {
|
||||
return;
|
||||
}
|
||||
|
||||
window.clearTimeout(pendingRequest.timer);
|
||||
pendingWakewordRequests.delete(message.requestId);
|
||||
|
||||
if (message.success === false) {
|
||||
pendingRequest.reject(new Error(message.error || '本地唤醒服务返回失败'));
|
||||
return;
|
||||
}
|
||||
|
||||
pendingRequest.resolve(message);
|
||||
}
|
||||
|
||||
function rejectAllWakewordRequests(errorMessage) {
|
||||
pendingWakewordRequests.forEach((pendingRequest) => {
|
||||
window.clearTimeout(pendingRequest.timer);
|
||||
pendingRequest.reject(new Error(errorMessage));
|
||||
});
|
||||
pendingWakewordRequests.clear();
|
||||
}
|
||||
@@ -0,0 +1,591 @@
|
||||
// WebSocket消息处理模块
|
||||
import { getConfig, saveConnectionUrls } from '../../config/manager.js?v=0205';
|
||||
import { uiController } from '../../ui/controller.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { getAudioPlayer } from '../audio/player.js?v=0205';
|
||||
import { getAudioRecorder } from '../audio/recorder.js?v=0205';
|
||||
import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js?v=0205';
|
||||
import { webSocketConnect } from './ota-connector.js?v=0205';
|
||||
|
||||
// WebSocket处理器类
|
||||
export class WebSocketHandler {
|
||||
constructor() {
|
||||
this.websocket = null;
|
||||
this.onConnectionStateChange = null;
|
||||
this.onRecordButtonStateChange = null;
|
||||
this.onSessionStateChange = null;
|
||||
this.onSessionEmotionChange = null;
|
||||
this.onChatMessage = null; // 新增：聊天消息回调
|
||||
this.currentSessionId = null;
|
||||
this.isRemoteSpeaking = false;
|
||||
}
|
||||
|
||||
// 发送hello握手消息
|
||||
async sendHelloMessage() {
|
||||
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) return false;
|
||||
|
||||
try {
|
||||
const config = getConfig();
|
||||
|
||||
const helloMessage = {
|
||||
type: 'hello',
|
||||
device_id: config.deviceId,
|
||||
device_name: config.deviceName,
|
||||
device_mac: config.deviceMac,
|
||||
token: config.token,
|
||||
features: {
|
||||
mcp: true,
|
||||
emoji: config.emojiEnabled
|
||||
}
|
||||
};
|
||||
|
||||
log('发送hello握手消息', 'info');
|
||||
this.websocket.send(JSON.stringify(helloMessage));
|
||||
|
||||
return new Promise(resolve => {
|
||||
const timeout = setTimeout(() => {
|
||||
log('等待hello响应超时', 'error');
|
||||
log('提示: 请尝试点击"测试认证"按钮进行连接排查', 'info');
|
||||
resolve(false);
|
||||
}, 5000);
|
||||
|
||||
const onMessageHandler = (event) => {
|
||||
try {
|
||||
const response = JSON.parse(event.data);
|
||||
if (response.type === 'hello' && response.session_id) {
|
||||
log(`服务器握手成功，会话ID: ${response.session_id}`, 'success');
|
||||
clearTimeout(timeout);
|
||||
this.websocket.removeEventListener('message', onMessageHandler);
|
||||
resolve(true);
|
||||
}
|
||||
} catch (e) {
|
||||
// 忽略非JSON消息
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.addEventListener('message', onMessageHandler);
|
||||
});
|
||||
} catch (error) {
|
||||
log(`发送hello消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
_sendWakeupMessages(sessionId) {
|
||||
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) return;
|
||||
|
||||
// listen detect
|
||||
this.websocket.send(JSON.stringify({
|
||||
session_id: sessionId,
|
||||
type: 'listen',
|
||||
state: 'detect',
|
||||
text: '嘿，你好呀'
|
||||
}));
|
||||
log('发送listen detect消息，唤醒词: 嘿，你好呀', 'info');
|
||||
|
||||
// listen start：开始监听
|
||||
this.websocket.send(JSON.stringify({
|
||||
session_id: sessionId,
|
||||
type: 'listen',
|
||||
state: 'start',
|
||||
mode: 'auto'
|
||||
}));
|
||||
log('发送listen start消息', 'info');
|
||||
}
|
||||
|
||||
// 处理文本消息
|
||||
handleTextMessage(message) {
|
||||
if (message.type === 'hello') {
|
||||
log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success');
|
||||
window.cameraAvailable = true;
|
||||
log('连接成功，摄像头已可用', 'success');
|
||||
uiController.updateDialButton(true);
|
||||
|
||||
this._sendWakeupMessages(message.session_id);
|
||||
|
||||
uiController.startAIChatSession();
|
||||
} else if (message.type === 'tts') {
|
||||
this.handleTTSMessage(message);
|
||||
} else if (message.type === 'audio') {
|
||||
log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
|
||||
} else if (message.type === 'stt') {
|
||||
log(`识别结果: ${message.text}`, 'info');
|
||||
// 检查是否需要绑定设备
|
||||
if (message.text && (message.text.includes('绑定') || message.text.includes('bind'))) {
|
||||
log('收到设备绑定提示，更新摄像头状态', 'warning');
|
||||
window.cameraAvailable = false;
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
window.stopCamera();
|
||||
}
|
||||
// 更新摄像头按钮状态
|
||||
const cameraBtn = document.getElementById('cameraBtn');
|
||||
if (cameraBtn) {
|
||||
cameraBtn.classList.remove('camera-active');
|
||||
cameraBtn.querySelector('.btn-text').textContent = '摄像头';
|
||||
cameraBtn.disabled = true;
|
||||
cameraBtn.title = '请先绑定验证码';
|
||||
}
|
||||
}
|
||||
// 使用新的聊天消息回调显示STT消息
|
||||
if (this.onChatMessage && message.text) {
|
||||
this.onChatMessage(message.text, true);
|
||||
}
|
||||
} else if (message.type === 'llm') {
|
||||
log(`大模型回复: ${message.text}`, 'info');
|
||||
// 使用新的聊天消息回调显示LLM回复
|
||||
if (this.onChatMessage && message.text) {
|
||||
this.onChatMessage(message.text, false);
|
||||
}
|
||||
|
||||
// 如果包含表情，更新sessionStatus表情并触发Live2D动作
|
||||
if (message.text && /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u.test(message.text)) {
|
||||
// 提取表情符号
|
||||
const emojiMatch = message.text.match(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u);
|
||||
if (emojiMatch && this.onSessionEmotionChange) {
|
||||
this.onSessionEmotionChange(emojiMatch[0]);
|
||||
}
|
||||
|
||||
// 触发Live2D情绪动作
|
||||
if (message.emotion) {
|
||||
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
|
||||
this.triggerLive2DEmotionAction(message.emotion);
|
||||
}
|
||||
}
|
||||
|
||||
// 只有当文本不仅仅是表情时，才添加到对话中
|
||||
// 移除文本中的表情后检查是否还有内容
|
||||
const textWithoutEmoji = message.text ? message.text.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim() : '';
|
||||
if (textWithoutEmoji && this.onChatMessage) {
|
||||
this.onChatMessage(message.text, false);
|
||||
}
|
||||
} else if (message.type === 'mcp') {
|
||||
this.handleMCPMessage(message);
|
||||
} else {
|
||||
log(`未知消息类型: ${message.type}`, 'info');
|
||||
if (this.onChatMessage) {
|
||||
this.onChatMessage(`未知消息类型: ${message.type}\n${JSON.stringify(message, null, 2)}`, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 处理TTS消息
|
||||
handleTTSMessage(message) {
|
||||
if (message.state === 'start') {
|
||||
log('服务器开始发送语音', 'info');
|
||||
this.currentSessionId = message.session_id;
|
||||
this.isRemoteSpeaking = true;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(true);
|
||||
}
|
||||
|
||||
// 启动Live2D说话动画
|
||||
this.startLive2DTalking();
|
||||
} else if (message.state === 'sentence_start') {
|
||||
log(`服务器发送语音段: ${message.text}`, 'info');
|
||||
this.ttsSentenceCount = (this.ttsSentenceCount || 0) + 1;
|
||||
|
||||
if (message.text && this.onChatMessage) {
|
||||
this.onChatMessage(message.text, false);
|
||||
}
|
||||
|
||||
// 确保动画在句子开始时运行
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager && !live2dManager.isTalking) {
|
||||
this.startLive2DTalking();
|
||||
}
|
||||
} else if (message.state === 'sentence_end') {
|
||||
log(`语音段结束: ${message.text}`, 'info');
|
||||
|
||||
// 句子结束时不清除动画，等待下一个句子或最终停止
|
||||
} else if (message.state === 'stop') {
|
||||
log('服务器语音传输结束，清空所有音频缓冲', 'info');
|
||||
|
||||
// 清空所有音频缓冲并停止播放
|
||||
const audioPlayer = getAudioPlayer();
|
||||
audioPlayer.clearAllAudio();
|
||||
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onRecordButtonStateChange) {
|
||||
this.onRecordButtonStateChange(false);
|
||||
}
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
|
||||
// 延迟停止Live2D说话动画，确保所有句子都播放完毕
|
||||
setTimeout(() => {
|
||||
this.stopLive2DTalking();
|
||||
this.ttsSentenceCount = 0; // 重置计数器
|
||||
}, 1000); // 1秒延迟，确保所有句子都完成
|
||||
}
|
||||
}
|
||||
|
||||
// 启动Live2D说话动画
|
||||
startLive2DTalking() {
|
||||
try {
|
||||
// 获取Live2D管理器实例
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager && live2dManager.live2dModel) {
|
||||
// 使用音频播放器的分析器节点
|
||||
live2dManager.startTalking();
|
||||
log('Live2D说话动画已启动', 'info');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`启动Live2D说话动画失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 停止Live2D说话动画
|
||||
stopLive2DTalking() {
|
||||
try {
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager) {
|
||||
live2dManager.stopTalking();
|
||||
log('Live2D说话动画已停止', 'info');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`停止Live2D说话动画失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化Live2D音频分析器
|
||||
initializeLive2DAudioAnalyzer() {
|
||||
try {
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager) {
|
||||
// 初始化音频分析器（使用音频播放器的上下文）
|
||||
if (live2dManager.initializeAudioAnalyzer()) {
|
||||
log('Live2D音频分析器初始化完成，已连接到音频播放器', 'success');
|
||||
} else {
|
||||
log('Live2D音频分析器初始化失败，将使用模拟动画', 'warning');
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log(`初始化Live2D音频分析器失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 处理MCP消息
|
||||
handleMCPMessage(message) {
|
||||
const payload = message.payload || {};
|
||||
log(`服务器下发: ${JSON.stringify(message)}`, 'info');
|
||||
|
||||
if (payload.method === 'tools/list') {
|
||||
const tools = getMcpTools();
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"tools": tools
|
||||
}
|
||||
}
|
||||
});
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
log(`回复MCP工具列表: ${tools.length} 个工具`, 'info');
|
||||
|
||||
} else if (payload.method === 'tools/call') {
|
||||
const toolName = payload.params?.name;
|
||||
const toolArgs = payload.params?.arguments;
|
||||
|
||||
log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info');
|
||||
|
||||
executeMcpTool(toolName, toolArgs).then(result => {
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": JSON.stringify(result)
|
||||
}
|
||||
],
|
||||
"isError": false
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
}).catch(error => {
|
||||
log(`工具执行失败: ${error.message}`, 'error');
|
||||
const errorReply = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"error": {
|
||||
"code": -32603,
|
||||
"message": error.message
|
||||
}
|
||||
}
|
||||
});
|
||||
this.websocket.send(errorReply);
|
||||
});
|
||||
} else if (payload.method === 'initialize') {
|
||||
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
|
||||
// 保存视觉分析接口地址
|
||||
const visionUrl = document.getElementById('visionUrl');
|
||||
const visionConfig = payload?.params?.capabilities?.vision;
|
||||
if (visionConfig && typeof visionConfig === 'object' && visionConfig.url && visionConfig.token) {
|
||||
const visionConfigStr = JSON.stringify(visionConfig);
|
||||
localStorage.setItem('xz_tester_vision', visionConfigStr);
|
||||
if (visionUrl) visionUrl.value = visionConfig.url;
|
||||
} else {
|
||||
localStorage.removeItem('xz_tester_vision');
|
||||
if (visionUrl) visionUrl.value = '';
|
||||
}
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {
|
||||
"tools": {}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "xiaozhi-web-test",
|
||||
"version": "2.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
log(`回复初始化响应`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
} else {
|
||||
log(`未知的MCP方法: ${payload.method}`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 处理二进制消息
|
||||
async handleBinaryMessage(data) {
|
||||
try {
|
||||
let arrayBuffer;
|
||||
if (data instanceof ArrayBuffer) {
|
||||
arrayBuffer = data;
|
||||
} else if (data instanceof Blob) {
|
||||
arrayBuffer = await data.arrayBuffer();
|
||||
log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`, 'debug');
|
||||
} else {
|
||||
log(`收到未知类型的二进制数据: ${typeof data}`, 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
const opusData = new Uint8Array(arrayBuffer);
|
||||
const audioPlayer = getAudioPlayer();
|
||||
audioPlayer.enqueueAudioData(opusData);
|
||||
} catch (error) {
|
||||
log(`处理二进制消息出错: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 连接WebSocket服务器
|
||||
async connect() {
|
||||
const config = getConfig();
|
||||
log('正在检查OTA状态...', 'info');
|
||||
saveConnectionUrls();
|
||||
|
||||
try {
|
||||
const otaUrl = document.getElementById('otaUrl').value.trim();
|
||||
const ws = await webSocketConnect(otaUrl, config);
|
||||
if (ws === undefined) {
|
||||
return false;
|
||||
}
|
||||
this.websocket = ws;
|
||||
|
||||
// 设置接收二进制数据的类型为ArrayBuffer
|
||||
this.websocket.binaryType = 'arraybuffer';
|
||||
|
||||
// 设置 MCP 模块的 WebSocket 实例
|
||||
setMcpWebSocket(this.websocket);
|
||||
|
||||
// 设置录音器的WebSocket
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.setWebSocket(this.websocket);
|
||||
|
||||
this.setupEventHandlers();
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`连接错误: ${error.message}`, 'error');
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 设置事件处理器
|
||||
setupEventHandlers() {
|
||||
this.websocket.onopen = async () => {
|
||||
const url = document.getElementById('serverUrl').value;
|
||||
log(`已连接到服务器: ${url}`, 'success');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(true);
|
||||
}
|
||||
|
||||
// 连接成功后，默认状态为聆听中
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
|
||||
// 在WebSocket连接成功时初始化Live2D音频分析器
|
||||
this.initializeLive2DAudioAnalyzer();
|
||||
|
||||
await this.sendHelloMessage();
|
||||
};
|
||||
|
||||
this.websocket.onclose = () => {
|
||||
log('已断开连接', 'info');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
window.stopCamera();
|
||||
}
|
||||
|
||||
// 隐藏摄像头显示区域
|
||||
const cameraContainer = document.getElementById('cameraContainer');
|
||||
if (cameraContainer) {
|
||||
cameraContainer.classList.remove('active');
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.onerror = (error) => {
|
||||
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
|
||||
uiController.addChatMessage(`⚠️ WebSocket错误: ${error.message || '未知错误'}`, false);
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.onmessage = (event) => {
|
||||
try {
|
||||
if (typeof event.data === 'string') {
|
||||
const message = JSON.parse(event.data);
|
||||
this.handleTextMessage(message);
|
||||
} else {
|
||||
this.handleBinaryMessage(event.data);
|
||||
}
|
||||
} catch (error) {
|
||||
log(`WebSocket消息处理错误: ${error.message}`, 'error');
|
||||
// 不再使用旧的addMessage函数，因为conversationDiv元素不存在
|
||||
// 错误消息将通过其他方式显示
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// 断开连接
|
||||
disconnect() {
|
||||
if (!this.websocket) return;
|
||||
|
||||
this.websocket.close();
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
window.stopCamera();
|
||||
}
|
||||
|
||||
// 隐藏摄像头显示区域
|
||||
const cameraContainer = document.getElementById('cameraContainer');
|
||||
if (cameraContainer) {
|
||||
cameraContainer.classList.remove('active');
|
||||
}
|
||||
}
|
||||
|
||||
// 发送文本消息
|
||||
sendTextMessage(text) {
|
||||
if (text === '' || !this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// 如果对方正在说话，先发送打断消息
|
||||
if (this.isRemoteSpeaking && this.currentSessionId) {
|
||||
const abortMessage = {
|
||||
session_id: this.currentSessionId,
|
||||
type: 'abort',
|
||||
reason: 'wake_word_detected'
|
||||
};
|
||||
this.websocket.send(JSON.stringify(abortMessage));
|
||||
log('发送打断消息', 'info');
|
||||
}
|
||||
|
||||
const listenMessage = {
|
||||
type: 'listen',
|
||||
state: 'detect',
|
||||
text: text
|
||||
};
|
||||
|
||||
this.websocket.send(JSON.stringify(listenMessage));
|
||||
log(`发送文本消息: ${text}`, 'info');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`发送消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 触发Live2D情绪动作
|
||||
* @param {string} emotion - 情绪名称
|
||||
*/
|
||||
triggerLive2DEmotionAction(emotion) {
|
||||
try {
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager && typeof live2dManager.triggerEmotionAction === 'function') {
|
||||
live2dManager.triggerEmotionAction(emotion);
|
||||
log(`触发Live2D情绪动作: ${emotion}`, 'info');
|
||||
} else {
|
||||
log(`无法触发Live2D情绪动作: Live2D管理器未找到或方法不可用`, 'warning');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`触发Live2D情绪动作失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 获取WebSocket实例
|
||||
getWebSocket() {
|
||||
return this.websocket;
|
||||
}
|
||||
|
||||
// 检查是否已连接
|
||||
isConnected() {
|
||||
return this.websocket && this.websocket.readyState === WebSocket.OPEN;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let wsHandlerInstance = null;
|
||||
|
||||
export function getWebSocketHandler() {
|
||||
if (!wsHandlerInstance) {
|
||||
wsHandlerInstance = new WebSocketHandler();
|
||||
}
|
||||
return wsHandlerInstance;
|
||||
}
|
||||
+1
File diff suppressed because one or more lines are too long
@@ -0,0 +1,824 @@
|
||||
/**
|
||||
* Live2D 管理器
|
||||
* 负责 Live2D 模型的初始化、嘴部动画控制等功能
|
||||
*/
|
||||
class Live2DManager {
|
||||
constructor() {
|
||||
this.live2dApp = null;
|
||||
this.live2dModel = null;
|
||||
this.isTalking = false;
|
||||
this.mouthAnimationId = null;
|
||||
this.mouthParam = 'ParamMouthOpenY';
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
this.dataArray = null;
|
||||
this.lastEmotionActionTime = null;
|
||||
this.currentModelName = null;
|
||||
|
||||
// 模型特定配置
|
||||
this.modelConfig = {
|
||||
'hiyori_pro_zh': {
|
||||
mouthParam: 'ParamMouthOpenY',
|
||||
mouthAmplitude: 1.0,
|
||||
mouthThresholds: { low: 0.3, high: 0.7 },
|
||||
motionMap: {
|
||||
'FlickUp': 'FlickUp',
|
||||
'FlickDown': 'FlickDown',
|
||||
'Tap': 'Tap',
|
||||
'Tap@Body': 'Tap@Body',
|
||||
'Flick': 'Flick',
|
||||
'Flick@Body': 'Flick@Body'
|
||||
}
|
||||
},
|
||||
'natori_pro_zh': {
|
||||
mouthParam: 'ParamMouthOpenY',
|
||||
mouthAmplitude: 1.0,
|
||||
mouthThresholds: { low: 0.1, high: 0.4 },
|
||||
mouthFormParam: 'ParamMouthForm',
|
||||
mouthFormAmplitude: 1.0,
|
||||
mouthForm2Param: 'ParamMouthForm2',
|
||||
mouthForm2Amplitude: 0.8,
|
||||
motionMap: {
|
||||
'FlickUp': 'FlickUp',
|
||||
'FlickDown': 'Flick@Body',
|
||||
'Tap': 'Tap',
|
||||
'Tap@Body': 'Tap@Head',
|
||||
'Flick': 'Tap',
|
||||
'Flick@Body': 'Flick@Body'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 情绪到动作的映射
|
||||
this.emotionToActionMap = {
|
||||
'happy': 'FlickUp', // 开心-向上轻扫动作
|
||||
'laughing': 'FlickUp', // 大笑-向上轻扫动作
|
||||
'funny': 'FlickUp', // 搞笑-向上轻扫动作
|
||||
'sad': 'FlickDown', // 伤心-向下轻扫动作
|
||||
'crying': 'FlickDown', // 哭泣-向下轻扫动作
|
||||
'angry': 'Tap@Body', // 生气-身体点击动作
|
||||
'surprised': 'Tap', // 惊讶-点击动作
|
||||
'neutral': 'Flick', // 平常-轻扫动作
|
||||
'default': 'Flick@Body' // 默认-身体轻扫动作
|
||||
};
|
||||
|
||||
// 单/双击判定配置与状态
|
||||
this._lastClickTime = 0;
|
||||
this._lastClickPos = { x: 0, y: 0 };
|
||||
this._singleClickTimer = null;
|
||||
this._doubleClickMs = 280; // 双击时间阈值(ms)
|
||||
this._doubleClickDist = 16; // 双击允许的最大位移(px)
|
||||
// 滑动判定
|
||||
this._pointerDown = false;
|
||||
this._downPos = { x: 0, y: 0 };
|
||||
this._downTime = 0;
|
||||
this._downArea = 'Body';
|
||||
this._movedBeyondClick = false;
|
||||
this._swipeMinDist = 24; // 触发滑动的最小距离
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化 Live2D
|
||||
*/
|
||||
async initializeLive2D() {
|
||||
try {
|
||||
const canvas = document.getElementById('live2d-stage');
|
||||
|
||||
// 供内部使用
|
||||
window.PIXI = PIXI;
|
||||
|
||||
this.live2dApp = new PIXI.Application({
|
||||
view: canvas,
|
||||
height: window.innerHeight,
|
||||
width: window.innerWidth,
|
||||
resolution: window.devicePixelRatio,
|
||||
autoDensity: true,
|
||||
antialias: true,
|
||||
backgroundAlpha: 0,
|
||||
});
|
||||
|
||||
// 加载 Live2D 模型 - 动态检测当前目录，适配不同环境
|
||||
// 获取当前HTML文件所在的目录路径
|
||||
const currentPath = window.location.pathname;
|
||||
const lastSlashIndex = currentPath.lastIndexOf('/');
|
||||
const basePath = currentPath.substring(0, lastSlashIndex + 1);
|
||||
|
||||
// 从 localStorage 读取上次选择的模型，如果没有则使用默认
|
||||
const savedModelName = localStorage.getItem('live2dModel') || 'hiyori_pro_zh';
|
||||
const modelFileMap = {
|
||||
'hiyori_pro_zh': 'hiyori_pro_t11.model3.json',
|
||||
'natori_pro_zh': 'natori_pro_t06.model3.json'
|
||||
};
|
||||
const modelFileName = modelFileMap[savedModelName] || 'hiyori_pro_t11.model3.json';
|
||||
const modelPath = basePath + 'resources/' + savedModelName + '/runtime/' + modelFileName;
|
||||
|
||||
this.live2dModel = await PIXI.live2d.Live2DModel.from(modelPath);
|
||||
this.live2dApp.stage.addChild(this.live2dModel);
|
||||
|
||||
// 保存当前模型名称
|
||||
this.currentModelName = savedModelName;
|
||||
|
||||
// 更新下拉框显示
|
||||
const modelSelect = document.getElementById('live2dModelSelect');
|
||||
if (modelSelect) {
|
||||
modelSelect.value = savedModelName;
|
||||
}
|
||||
|
||||
// 设置模型特定的嘴部参数名
|
||||
if (this.modelConfig[savedModelName]) {
|
||||
this.mouthParam = this.modelConfig[savedModelName].mouthParam || 'ParamMouthOpenY';
|
||||
}
|
||||
|
||||
// 设置模型属性
|
||||
this.live2dModel.scale.set(0.33);
|
||||
this.live2dModel.x = (window.innerWidth - this.live2dModel.width) * 0.5;
|
||||
this.live2dModel.y = -50;
|
||||
|
||||
// 启用交互并监听点击命中（头部/身体等）
|
||||
|
||||
this.live2dModel.interactive = true;
|
||||
|
||||
|
||||
this.live2dModel.on('doublehit', (args) => {
|
||||
const area = Array.isArray(args) ? args[0] : args;
|
||||
|
||||
// 触发双击动作
|
||||
if (area === 'Body') {
|
||||
this.motion('Flick@Body');
|
||||
} else if (area === 'Head' || area === 'Face') {
|
||||
this.motion('Flick');
|
||||
}
|
||||
|
||||
const app = window.chatApp;
|
||||
const payload = JSON.stringify({ type: 'live2d', event: 'doublehit', area });
|
||||
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
|
||||
app.dataChannel.send(payload);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
this.live2dModel.on('singlehit', (args) => {
|
||||
const area = Array.isArray(args) ? args[0] : args;
|
||||
|
||||
// 触发单击动作
|
||||
if (area === 'Body') {
|
||||
this.motion('Tap@Body');
|
||||
} else if (area === 'Head' || area === 'Face') {
|
||||
this.motion('Tap');
|
||||
}
|
||||
|
||||
const app = window.chatApp;
|
||||
const payload = JSON.stringify({ type: 'live2d', event: 'singlehit', area });
|
||||
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
|
||||
app.dataChannel.send(payload);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
this.live2dModel.on('swipe', (args) => {
|
||||
const area = Array.isArray(args) ? args[0] : args;
|
||||
const dir = Array.isArray(args) ? args[1] : undefined;
|
||||
|
||||
// 触发滑动动作
|
||||
if (area === 'Body') {
|
||||
if (dir === 'up') {
|
||||
this.motion('FlickUp');
|
||||
} else if (dir === 'down') {
|
||||
this.motion('FlickDown');
|
||||
}
|
||||
} else if (area === 'Head' || area === 'Face') {
|
||||
if (dir === 'up') {
|
||||
this.motion('FlickUp');
|
||||
} else if (dir === 'down') {
|
||||
this.motion('FlickDown');
|
||||
}
|
||||
}
|
||||
|
||||
const app = window.chatApp;
|
||||
const payload = JSON.stringify({ type: 'live2d', event: 'swipe', area, dir });
|
||||
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
|
||||
app.dataChannel.send(payload);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
// 兜底：自定义"头部/身体"命中区域 + 单/双击/滑动区分
|
||||
this.live2dModel.on('pointerdown', (event) => {
|
||||
try {
|
||||
const global = event.data.global;
|
||||
const bounds = this.live2dModel.getBounds();
|
||||
// 仅在点击落在模型可见范围内时判定
|
||||
if (!bounds || !bounds.contains(global.x, global.y)) return;
|
||||
|
||||
const relX = (global.x - bounds.x) / (bounds.width || 1);
|
||||
const relY = (global.y - bounds.y) / (bounds.height || 1);
|
||||
let area = '';
|
||||
// 经验阈值：模型可见矩形的上部 20% 视为"头部"区域
|
||||
if (relX >= 0.4 && relX <= 0.6) {
|
||||
if (relY <= 0.15) {
|
||||
area = 'Head';
|
||||
} else if (relY <= 0.23) {
|
||||
area = 'Face';
|
||||
} else {
|
||||
area = 'Body';
|
||||
}
|
||||
}
|
||||
if (area === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
// 记录按下状态用于滑动判定
|
||||
this._pointerDown = true;
|
||||
this._downPos = { x: global.x, y: global.y };
|
||||
this._downTime = performance.now();
|
||||
this._downArea = area;
|
||||
this._movedBeyondClick = false;
|
||||
|
||||
const now = performance.now();
|
||||
const dt = now - (this._lastClickTime || 0);
|
||||
const dx = global.x - (this._lastClickPos?.x || 0);
|
||||
const dy = global.y - (this._lastClickPos?.y || 0);
|
||||
const dist = Math.hypot(dx, dy);
|
||||
|
||||
// 命中确认：仅当点击在模型上时做单/双击判断
|
||||
if (this._lastClickTime && dt <= this._doubleClickMs && dist <= this._doubleClickDist) {
|
||||
// 判定为双击：取消待触发的单击事件
|
||||
if (this._singleClickTimer) {
|
||||
clearTimeout(this._singleClickTimer);
|
||||
this._singleClickTimer = null;
|
||||
}
|
||||
if (typeof this.live2dModel.emit === 'function') {
|
||||
this.live2dModel.emit('doublehit', [area]);
|
||||
}
|
||||
this._lastClickTime = 0;
|
||||
this._pointerDown = false; // 双击完成，重置状态
|
||||
return;
|
||||
}
|
||||
|
||||
// 可能是单击：记录并延迟确认
|
||||
this._lastClickTime = now;
|
||||
this._lastClickPos = { x: global.x, y: global.y };
|
||||
if (this._singleClickTimer) {
|
||||
clearTimeout(this._singleClickTimer);
|
||||
this._singleClickTimer = null;
|
||||
}
|
||||
this._singleClickTimer = setTimeout(() => {
|
||||
// 若在等待期间发生了移动超过阈值，则不再当作单击
|
||||
if (!this._movedBeyondClick && typeof this.live2dModel.emit === 'function') {
|
||||
this.live2dModel.emit('singlehit', [area]);
|
||||
}
|
||||
this._singleClickTimer = null;
|
||||
this._lastClickTime = 0;
|
||||
}, this._doubleClickMs);
|
||||
} catch (e) {
|
||||
// 忽略自定义命中判断中的异常，避免影响主流程
|
||||
}
|
||||
});
|
||||
|
||||
// 指针移动：用于判定是否从"点击"升级为"滑动"
|
||||
this.live2dModel.on('pointermove', (event) => {
|
||||
try {
|
||||
if (!this._pointerDown) return;
|
||||
const global = event.data.global;
|
||||
const dx = global.x - this._downPos.x;
|
||||
const dy = global.y - this._downPos.y;
|
||||
const dist = Math.hypot(dx, dy);
|
||||
|
||||
// 使用 _doubleClickDist 作为点击/滑动的判定阈值
|
||||
if (dist > this._doubleClickDist) {
|
||||
this._movedBeyondClick = true;
|
||||
// 若已超出点击阈值，取消可能的单击触发
|
||||
if (this._singleClickTimer) {
|
||||
clearTimeout(this._singleClickTimer);
|
||||
this._singleClickTimer = null;
|
||||
}
|
||||
this._lastClickTime = 0;
|
||||
}
|
||||
} catch (e) {
|
||||
// 忽略移动判定中的异常
|
||||
}
|
||||
});
|
||||
|
||||
// 指针抬起：确认是否为滑动
|
||||
const handlePointerUp = (event) => {
|
||||
try {
|
||||
if (!this._pointerDown) return;
|
||||
const global = (event && event.data && event.data.global) ? event.data.global : { x: this._downPos.x, y: this._downPos.y };
|
||||
const dx = global.x - this._downPos.x;
|
||||
const dy = global.y - this._downPos.y;
|
||||
const dist = Math.hypot(dx, dy);
|
||||
|
||||
// 滑动：超过滑动最小距离则触发 swipe 事件（携带方向与区域）
|
||||
if (this._movedBeyondClick && dist >= this._swipeMinDist) {
|
||||
if (typeof this.live2dModel.emit === 'function') {
|
||||
const dir = Math.abs(dx) >= Math.abs(dy)
|
||||
? (dx > 0 ? 'right' : 'left')
|
||||
: (dy > 0 ? 'down' : 'up');
|
||||
this.live2dModel.emit('swipe', [this._downArea, dir]);
|
||||
}
|
||||
// 终止：不再让单击/双击触发
|
||||
if (this._singleClickTimer) {
|
||||
clearTimeout(this._singleClickTimer);
|
||||
this._singleClickTimer = null;
|
||||
}
|
||||
this._lastClickTime = 0;
|
||||
}
|
||||
} catch (e) {
|
||||
// 忽略抬起判定中的异常
|
||||
}
|
||||
finally {
|
||||
this._pointerDown = false;
|
||||
this._movedBeyondClick = false;
|
||||
}
|
||||
};
|
||||
|
||||
this.live2dModel.on('pointerup', handlePointerUp);
|
||||
this.live2dModel.on('pointerupoutside', handlePointerUp);
|
||||
|
||||
// 添加窗口大小变化监听器，保持模型在Canvas中间和底部
|
||||
window.addEventListener('resize', () => {
|
||||
if (this.live2dModel) {
|
||||
// 使用窗口实际尺寸重新计算模型位置
|
||||
this.live2dModel.x = (window.innerWidth - this.live2dModel.width) * 0.5;
|
||||
this.live2dModel.y = -50;
|
||||
}
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
console.error('加载 Live2D 模型失败:', err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化音频分析器 - 使用音频播放器的分析器节点
|
||||
*/
|
||||
initializeAudioAnalyzer() {
|
||||
try {
|
||||
// 获取音频播放器实例
|
||||
const audioPlayer = window.chatApp?.audioPlayer;
|
||||
if (!audioPlayer) {
|
||||
console.warn('音频播放器未初始化，无法获取分析器节点');
|
||||
return false;
|
||||
}
|
||||
|
||||
// 获取音频播放器的音频上下文
|
||||
this.audioContext = audioPlayer.getAudioContext();
|
||||
if (!this.audioContext) {
|
||||
console.warn('无法获取音频播放器的音频上下文');
|
||||
return false;
|
||||
}
|
||||
|
||||
// 创建分析器节点
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 256;
|
||||
this.dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('初始化音频分析器失败:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接到音频播放器的输出节点
|
||||
*/
|
||||
connectToAudioPlayer() {
|
||||
try {
|
||||
// 获取音频播放器的流上下文
|
||||
const audioPlayer = window.chatApp?.audioPlayer;
|
||||
if (!audioPlayer || !audioPlayer.streamingContext) {
|
||||
console.warn('音频播放器或流上下文未初始化');
|
||||
return false;
|
||||
}
|
||||
|
||||
// 获取音频播放器的流上下文
|
||||
const streamingContext = audioPlayer.streamingContext;
|
||||
|
||||
// 获取分析器节点
|
||||
const analyser = streamingContext.getAnalyser();
|
||||
if (!analyser) {
|
||||
console.warn('音频播放器尚未创建分析器节点，无法连接');
|
||||
return false;
|
||||
}
|
||||
|
||||
// 使用音频播放器的分析器节点
|
||||
this.analyser = analyser;
|
||||
this.dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('连接到音频播放器失败:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 嘴部动画循环
|
||||
*/
|
||||
animateMouth() {
|
||||
if (!this.isTalking) return;
|
||||
if (!this.live2dModel) return;
|
||||
const internal = this.live2dModel && this.live2dModel.internalModel;
|
||||
if (internal && internal.coreModel) {
|
||||
const coreModel = internal.coreModel;
|
||||
|
||||
let mouthOpenY = 0;
|
||||
let mouthForm = 0;
|
||||
let mouthForm2 = 0;
|
||||
let average = 0;
|
||||
|
||||
if (this.analyser && this.dataArray) {
|
||||
this.analyser.getByteFrequencyData(this.dataArray);
|
||||
average = this.dataArray.reduce((a, b) => a + b) / this.dataArray.length;
|
||||
|
||||
const normalizedVolume = average / 255;
|
||||
|
||||
// 获取模型特定的阈值
|
||||
let lowThreshold = 0.3;
|
||||
let highThreshold = 0.7;
|
||||
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
|
||||
lowThreshold = this.modelConfig[this.currentModelName].mouthThresholds?.low || 0.3;
|
||||
highThreshold = this.modelConfig[this.currentModelName].mouthThresholds?.high || 0.7;
|
||||
}
|
||||
|
||||
// 使用模型特定的阈值进行映射
|
||||
let minOpenY = 0.1;
|
||||
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
|
||||
minOpenY = this.modelConfig[this.currentModelName].mouthMinOpenY || 0.1;
|
||||
}
|
||||
|
||||
if (normalizedVolume < lowThreshold) {
|
||||
mouthOpenY = minOpenY + Math.pow(normalizedVolume / lowThreshold, 1.5) * (0.4 - minOpenY);
|
||||
} else if (normalizedVolume < highThreshold) {
|
||||
mouthOpenY = 0.4 + (normalizedVolume - lowThreshold) / (highThreshold - lowThreshold) * 0.4;
|
||||
} else {
|
||||
mouthOpenY = 0.8 + Math.pow((normalizedVolume - highThreshold) / (1 - highThreshold), 1.2) * 0.2;
|
||||
}
|
||||
|
||||
// 应用模型特定的嘴部开合幅度
|
||||
let amplitudeMultiplier = 1.0;
|
||||
let maxOpenY = 2.5;
|
||||
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
|
||||
amplitudeMultiplier = this.modelConfig[this.currentModelName].mouthAmplitude;
|
||||
maxOpenY = this.modelConfig[this.currentModelName].maxOpenY || 2.5;
|
||||
}
|
||||
mouthOpenY = mouthOpenY * amplitudeMultiplier;
|
||||
mouthOpenY = Math.min(Math.max(mouthOpenY, 0), maxOpenY);
|
||||
|
||||
// 计算嘴型参数（仅对支持嘴型变化的模型）
|
||||
if (this.currentModelName && this.modelConfig[this.currentModelName]?.mouthFormParam) {
|
||||
const config = this.modelConfig[this.currentModelName];
|
||||
const formAmplitude = config.mouthFormAmplitude || 0.5;
|
||||
const form2Amplitude = config.mouthForm2Amplitude || 0;
|
||||
|
||||
// 嘴型随音量变化：
|
||||
// 低音量：嘴型偏"一"字（负值）
|
||||
// 高音量：嘴型偏"o"字（正值）
|
||||
// 音量=0时：嘴型=0（自然状态）
|
||||
mouthForm = (normalizedVolume - 0.5) * 2 * formAmplitude;
|
||||
mouthForm = Math.max(-formAmplitude, Math.min(formAmplitude, mouthForm));
|
||||
|
||||
// 第二嘴型参数（natori特有）
|
||||
if (config.mouthForm2Param) {
|
||||
mouthForm2 = (normalizedVolume - 0.3) * 2 * form2Amplitude;
|
||||
mouthForm2 = Math.max(-form2Amplitude, Math.min(form2Amplitude, mouthForm2));
|
||||
}
|
||||
}
|
||||
|
||||
// 调试日志：输出嘴部参数
|
||||
console.log(`[Live2D] 模型: ${this.currentModelName || 'unknown'}, 音量: ${average?.toFixed(0)}, OpenY: ${mouthOpenY.toFixed(3)}, Form: ${mouthForm.toFixed(3)}, Form2: ${mouthForm2.toFixed(3)}`);
|
||||
}
|
||||
|
||||
// 设置嘴部开合参数
|
||||
coreModel.setParameterValueById(this.mouthParam, mouthOpenY);
|
||||
|
||||
// 设置嘴型参数（仅对支持嘴型变化的模型）
|
||||
if (this.currentModelName && this.modelConfig[this.currentModelName]?.mouthFormParam) {
|
||||
const config = this.modelConfig[this.currentModelName];
|
||||
const formParam = config.mouthFormParam;
|
||||
coreModel.setParameterValueById(formParam, mouthForm);
|
||||
|
||||
// 设置第二嘴型参数（natori特有）
|
||||
if (config.mouthForm2Param) {
|
||||
coreModel.setParameterValueById(config.mouthForm2Param, mouthForm2);
|
||||
}
|
||||
}
|
||||
|
||||
coreModel.update();
|
||||
}
|
||||
this.mouthAnimationId = requestAnimationFrame(() => this.animateMouth());
|
||||
}
|
||||
|
||||
/**
|
||||
* 开始说话动画
|
||||
*/
|
||||
startTalking() {
|
||||
if (this.isTalking || !this.live2dModel) return;
|
||||
|
||||
// 确保音频分析器已初始化
|
||||
if (!this.analyser) {
|
||||
if (!this.initializeAudioAnalyzer()) {
|
||||
console.warn('音频分析器初始化失败，将使用模拟动画');
|
||||
// 即使分析器初始化失败，也启动动画（使用模拟数据）
|
||||
this.isTalking = true;
|
||||
this.animateMouth();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 连接到音频播放器输出
|
||||
if (!this.connectToAudioPlayer()) {
|
||||
console.warn('无法连接到音频播放器输出，将使用模拟动画');
|
||||
}
|
||||
|
||||
this.isTalking = true;
|
||||
this.animateMouth();
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止说话动画
|
||||
*/
|
||||
stopTalking() {
|
||||
this.isTalking = false;
|
||||
if (this.mouthAnimationId) {
|
||||
cancelAnimationFrame(this.mouthAnimationId);
|
||||
this.mouthAnimationId = null;
|
||||
}
|
||||
|
||||
// 重置嘴部参数
|
||||
if (this.live2dModel) {
|
||||
const internal = this.live2dModel.internalModel;
|
||||
if (internal && internal.coreModel) {
|
||||
const coreModel = internal.coreModel;
|
||||
coreModel.setParameterValueById(this.mouthParam, 0);
|
||||
coreModel.update();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 基于情绪触发动作
|
||||
* @param {string} emotion - 情绪名称
|
||||
*/
|
||||
triggerEmotionAction(emotion) {
|
||||
if (!this.live2dModel) return;
|
||||
|
||||
// 添加冷却时间控制，避免过于频繁触发
|
||||
const now = Date.now();
|
||||
if (this.lastEmotionActionTime && now - this.lastEmotionActionTime < 5000) { // 5秒冷却时间
|
||||
return;
|
||||
}
|
||||
|
||||
// 根据情绪获取对应的动作
|
||||
const action = this.emotionToActionMap[emotion] || this.emotionToActionMap['default'];
|
||||
|
||||
// 触发动作并记录时间
|
||||
this.motion(action);
|
||||
this.lastEmotionActionTime = now;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 触发模型动作（Motion）
|
||||
* @param {string} name - 动作分组名称，如 'TapBody'、'FlickUp'、'Idle' 等
|
||||
*/
|
||||
motion(name) {
|
||||
try {
|
||||
if (!this.live2dModel) return;
|
||||
|
||||
// 根据当前模型获取对应的动作名称
|
||||
let actualMotionName = name;
|
||||
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
|
||||
const motionMap = this.modelConfig[this.currentModelName].motionMap;
|
||||
actualMotionName = motionMap[name] || name;
|
||||
}
|
||||
|
||||
this.live2dModel.motion(actualMotionName);
|
||||
} catch (error) {
|
||||
console.error('触发动作失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置模型交互事件
|
||||
*/
|
||||
setupModelInteractions() {
|
||||
if (!this.live2dModel) return;
|
||||
|
||||
this.live2dModel.interactive = true;
|
||||
|
||||
this.live2dModel.on('doublehit', (args) => {
|
||||
const area = Array.isArray(args) ? args[0] : args;
|
||||
|
||||
if (area === 'Body') {
|
||||
this.motion('Flick@Body');
|
||||
} else if (area === 'Head' || area === 'Face') {
|
||||
this.motion('Flick');
|
||||
}
|
||||
|
||||
const app = window.chatApp;
|
||||
const payload = JSON.stringify({ type: 'live2d', event: 'doublehit', area });
|
||||
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
|
||||
app.dataChannel.send(payload);
|
||||
}
|
||||
});
|
||||
|
||||
this.live2dModel.on('singlehit', (args) => {
|
||||
const area = Array.isArray(args) ? args[0] : args;
|
||||
|
||||
if (area === 'Body') {
|
||||
this.motion('Tap@Body');
|
||||
} else if (area === 'Head' || area === 'Face') {
|
||||
this.motion('Tap');
|
||||
}
|
||||
|
||||
const app = window.chatApp;
|
||||
const payload = JSON.stringify({ type: 'live2d', event: 'singlehit', area });
|
||||
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
|
||||
app.dataChannel.send(payload);
|
||||
}
|
||||
});
|
||||
|
||||
this.live2dModel.on('swipe', (args) => {
|
||||
const area = Array.isArray(args) ? args[0] : args;
|
||||
const dir = Array.isArray(args) ? args[1] : undefined;
|
||||
|
||||
if (area === 'Body') {
|
||||
if (dir === 'up') {
|
||||
this.motion('FlickUp');
|
||||
} else if (dir === 'down') {
|
||||
this.motion('FlickDown');
|
||||
}
|
||||
}
|
||||
|
||||
const app = window.chatApp;
|
||||
const payload = JSON.stringify({ type: 'live2d', event: 'swipe', area, dir });
|
||||
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
|
||||
app.dataChannel.send(payload);
|
||||
}
|
||||
});
|
||||
|
||||
this.live2dModel.on('pointerdown', (event) => {
|
||||
try {
|
||||
const global = event.data.global;
|
||||
const bounds = this.live2dModel.getBounds();
|
||||
if (!bounds || !bounds.contains(global.x, global.y)) return;
|
||||
|
||||
const relX = (global.x - bounds.x) / (bounds.width || 1);
|
||||
const relY = (global.y - bounds.y) / (bounds.height || 1);
|
||||
let area = '';
|
||||
|
||||
if (relX >= 0.4 && relX <= 0.6) {
|
||||
if (relY <= 0.15) {
|
||||
area = 'Head';
|
||||
} else if (relY >= 0.7) {
|
||||
area = 'Body';
|
||||
}
|
||||
}
|
||||
|
||||
if (!area) return;
|
||||
|
||||
const now = Date.now();
|
||||
const dt = now - (this._lastClickTime || 0);
|
||||
const dx = global.x - (this._lastClickPos?.x || 0);
|
||||
const dy = global.y - (this._lastClickPos?.y || 0);
|
||||
const dist = Math.hypot(dx, dy);
|
||||
|
||||
if (this._lastClickTime && dt <= this._doubleClickMs && dist <= this._doubleClickDist) {
|
||||
if (this._singleClickTimer) {
|
||||
clearTimeout(this._singleClickTimer);
|
||||
this._singleClickTimer = null;
|
||||
}
|
||||
|
||||
this.live2dModel.emit('doublehit', area);
|
||||
this._lastClickTime = null;
|
||||
this._lastClickPos = null;
|
||||
} else {
|
||||
this._lastClickTime = now;
|
||||
this._lastClickPos = { x: global.x, y: global.y };
|
||||
|
||||
this._singleClickTimer = setTimeout(() => {
|
||||
this._singleClickTimer = null;
|
||||
this.live2dModel.emit('singlehit', area);
|
||||
}, this._doubleClickMs);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('pointerdown 处理出错:', e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理资源
|
||||
*/
|
||||
destroy() {
|
||||
this.stopTalking();
|
||||
|
||||
// 清理音频分析器
|
||||
if (this.audioContext) {
|
||||
this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
}
|
||||
this.analyser = null;
|
||||
this.dataArray = null;
|
||||
|
||||
// 清理 Live2D 应用
|
||||
if (this.live2dApp) {
|
||||
this.live2dApp.destroy(true);
|
||||
this.live2dApp = null;
|
||||
}
|
||||
this.live2dModel = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 切换 Live2D 模型
|
||||
* @param {string} modelName - 模型目录名称，如 'hiyori_pro_zh'、'natori_pro_zh'
|
||||
* @returns {Promise<boolean>} - 切换是否成功
|
||||
*/
|
||||
async switchModel(modelName) {
|
||||
try {
|
||||
// 获取模型文件名映射
|
||||
const modelFileMap = {
|
||||
'hiyori_pro_zh': 'hiyori_pro_t11.model3.json',
|
||||
'natori_pro_zh': 'natori_pro_t06.model3.json',
|
||||
'chitose': 'chitose.model3.json',
|
||||
'haru_greeter_pro_jp': 'haru_greeter_t05.model3.json'
|
||||
};
|
||||
|
||||
const modelFileName = modelFileMap[modelName];
|
||||
if (!modelFileName) {
|
||||
console.error('未知的模型名称:', modelName);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 获取基础路径
|
||||
const currentPath = window.location.pathname;
|
||||
const lastSlashIndex = currentPath.lastIndexOf('/');
|
||||
const basePath = currentPath.substring(0, lastSlashIndex + 1);
|
||||
const modelPath = basePath + 'resources/' + modelName + '/runtime/' + modelFileName;
|
||||
|
||||
// 如果已存在模型，先移除
|
||||
if (this.live2dModel) {
|
||||
this.live2dApp.stage.removeChild(this.live2dModel);
|
||||
this.live2dModel.destroy();
|
||||
this.live2dModel = null;
|
||||
}
|
||||
|
||||
// 显示加载状态
|
||||
const app = window.chatApp;
|
||||
if (app) {
|
||||
app.setModelLoadingStatus(true);
|
||||
}
|
||||
|
||||
// 加载新模型
|
||||
this.live2dModel = await PIXI.live2d.Live2DModel.from(modelPath);
|
||||
this.live2dApp.stage.addChild(this.live2dModel);
|
||||
|
||||
// 设置模型属性
|
||||
this.live2dModel.scale.set(0.33);
|
||||
this.live2dModel.x = (window.innerWidth - this.live2dModel.width) * 0.5;
|
||||
this.live2dModel.y = -50;
|
||||
|
||||
// 重新绑定交互事件
|
||||
this.setupModelInteractions();
|
||||
|
||||
// 隐藏加载状态
|
||||
if (app) {
|
||||
app.setModelLoadingStatus(false);
|
||||
}
|
||||
|
||||
// 保存当前模型名称
|
||||
this.currentModelName = modelName;
|
||||
|
||||
// 设置模型特定的嘴部参数名
|
||||
if (this.modelConfig[modelName]) {
|
||||
this.mouthParam = this.modelConfig[modelName].mouthParam || 'ParamMouthOpenY';
|
||||
}
|
||||
|
||||
// 保存到 localStorage
|
||||
localStorage.setItem('live2dModel', modelName);
|
||||
|
||||
// 更新下拉框显示
|
||||
const modelSelect = document.getElementById('live2dModelSelect');
|
||||
if (modelSelect) {
|
||||
modelSelect.value = modelName;
|
||||
}
|
||||
|
||||
console.log('模型切换成功:', modelName);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('切换模型失败:', error);
|
||||
const app = window.chatApp;
|
||||
if (app) {
|
||||
app.setModelLoadingStatus(false);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
// 导出全局实例
|
||||
window.Live2DManager = Live2DManager;
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,27 @@
|
||||
// 背景图加载检测
|
||||
(function() {
|
||||
const backgroundContainer = document.getElementById('backgroundContainer');
|
||||
|
||||
// 提取背景图片URL
|
||||
let bgImageUrl = window.getComputedStyle(backgroundContainer).backgroundImage;
|
||||
const urlMatch = bgImageUrl && bgImageUrl.match(/url\(["']?(.*?)["']?\)/);
|
||||
|
||||
if (!urlMatch || !urlMatch[1]) {
|
||||
console.warn('未提取到有效的背景图片URL');
|
||||
return;
|
||||
}
|
||||
|
||||
bgImageUrl = urlMatch[1];
|
||||
|
||||
const bgImage = new Image();
|
||||
bgImage.onerror = function() {
|
||||
console.error('背景图片加载失败:', bgImageUrl);
|
||||
};
|
||||
|
||||
// 加载成功显示模型加载
|
||||
bgImage.onload = function() {
|
||||
modelLoading.style.display = 'flex';
|
||||
};
|
||||
|
||||
bgImage.src = bgImageUrl;
|
||||
})();
|
||||
@@ -0,0 +1,929 @@
|
||||
// UI controller module
|
||||
import { loadConfig, saveConfig } from '../config/manager.js?v=0205';
|
||||
import { getAudioPlayer } from '../core/audio/player.js?v=0205';
|
||||
import { getAudioRecorder } from '../core/audio/recorder.js?v=0205';
|
||||
import { requestWakewordBridge, stopWakewordBridgeListener, startWakewordBridgeListener, getWakewordBridgeUrl, onNextBridgeConnected } from '../core/network/wakeword-bridge.js?v=0205';
|
||||
import { getWebSocketHandler } from '../core/network/websocket.js?v=0205';
|
||||
import { log } from '../utils/logger.js?v=0205';
|
||||
|
||||
// UI controller class
|
||||
class UIController {
|
||||
constructor() {
|
||||
this.isEditing = false;
|
||||
this.visualizerCanvas = null;
|
||||
this.visualizerContext = null;
|
||||
this.audioStatsTimer = null;
|
||||
this.currentBackgroundIndex = localStorage.getItem('backgroundIndex') ? parseInt(localStorage.getItem('backgroundIndex')) : 0;
|
||||
this.backgroundImages = ['1.png', '2.png', '3.png'];
|
||||
this.dialBtnDisabled = false;
|
||||
this.isConnecting = false;
|
||||
this.lastWakewordDialTime = 0;
|
||||
|
||||
// Bind methods
|
||||
this.init = this.init.bind(this);
|
||||
this.initEventListeners = this.initEventListeners.bind(this);
|
||||
this.updateDialButton = this.updateDialButton.bind(this);
|
||||
this.addChatMessage = this.addChatMessage.bind(this);
|
||||
this.switchBackground = this.switchBackground.bind(this);
|
||||
this.switchLive2DModel = this.switchLive2DModel.bind(this);
|
||||
this.showModal = this.showModal.bind(this);
|
||||
this.hideModal = this.hideModal.bind(this);
|
||||
this.switchTab = this.switchTab.bind(this);
|
||||
this.applyWakewordConfig = this.applyWakewordConfig.bind(this);
|
||||
this.handleApplyWakeword = this.handleApplyWakeword.bind(this);
|
||||
this.triggerWakewordDial = this.triggerWakewordDial.bind(this);
|
||||
}
|
||||
|
||||
// Initialize
|
||||
init() {
|
||||
console.log('UIController init started');
|
||||
|
||||
this.visualizerCanvas = document.getElementById('audioVisualizer');
|
||||
if (this.visualizerCanvas) {
|
||||
this.visualizerContext = this.visualizerCanvas.getContext('2d');
|
||||
this.initVisualizer();
|
||||
}
|
||||
|
||||
// Check if connect button exists during initialization
|
||||
const connectBtn = document.getElementById('connectBtn');
|
||||
console.log('connectBtn during init:', connectBtn);
|
||||
|
||||
this.initEventListeners();
|
||||
this.startAudioStatsMonitor();
|
||||
loadConfig();
|
||||
|
||||
// Register recording callback
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.onRecordingStart = (seconds) => {
|
||||
this.updateRecordButtonState(true, seconds);
|
||||
};
|
||||
|
||||
// Initialize status display
|
||||
this.updateConnectionUI(false);
|
||||
// Apply saved background
|
||||
const backgroundContainer = document.querySelector('.background-container');
|
||||
if (backgroundContainer) {
|
||||
backgroundContainer.style.backgroundImage = `url('./images/${this.backgroundImages[this.currentBackgroundIndex]}')`;
|
||||
}
|
||||
|
||||
this.updateDialButton(false);
|
||||
|
||||
console.log('UIController init completed');
|
||||
}
|
||||
|
||||
// Initialize visualizer
|
||||
initVisualizer() {
|
||||
if (this.visualizerCanvas) {
|
||||
this.visualizerCanvas.width = this.visualizerCanvas.clientWidth;
|
||||
this.visualizerCanvas.height = this.visualizerCanvas.clientHeight;
|
||||
this.visualizerContext.fillStyle = '#fafafa';
|
||||
this.visualizerContext.fillRect(0, 0, this.visualizerCanvas.width, this.visualizerCanvas.height);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize event listeners
|
||||
initEventListeners() {
|
||||
// Settings button
|
||||
const settingsBtn = document.getElementById('settingsBtn');
|
||||
if (settingsBtn) {
|
||||
settingsBtn.addEventListener('click', () => {
|
||||
this.showModal('settingsModal');
|
||||
});
|
||||
}
|
||||
|
||||
// Background switch button
|
||||
const backgroundBtn = document.getElementById('backgroundBtn');
|
||||
if (backgroundBtn) {
|
||||
backgroundBtn.addEventListener('click', this.switchBackground);
|
||||
}
|
||||
|
||||
// Model select change event
|
||||
const modelSelect = document.getElementById('live2dModelSelect');
|
||||
if (modelSelect) {
|
||||
modelSelect.addEventListener('change', () => {
|
||||
this.switchLive2DModel();
|
||||
});
|
||||
}
|
||||
|
||||
// Camera switch button
|
||||
const cameraSwitch = document.getElementById('cameraSwitch');
|
||||
const cameraSwitchMask = document.getElementById('cameraSwitchMask');
|
||||
if (cameraSwitchMask) {
|
||||
cameraSwitchMask.addEventListener('click', () => {
|
||||
const isCameraActive = cameraSwitch.classList.contains('active');
|
||||
if (isCameraActive) {
|
||||
window.switchCamera();
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Dial button
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
dialBtn.addEventListener('click', () => {
|
||||
dialBtn.disabled = true;
|
||||
this.dialBtnDisabled = true;
|
||||
setTimeout(() => {
|
||||
dialBtn.disabled = false;
|
||||
this.dialBtnDisabled = false;
|
||||
}, 3000);
|
||||
|
||||
const wsHandler = getWebSocketHandler();
|
||||
const isConnected = wsHandler.isConnected();
|
||||
|
||||
if (isConnected) {
|
||||
wsHandler.disconnect();
|
||||
this.updateDialButton(false);
|
||||
if (cameraSwitch) cameraSwitch.classList.remove('active');
|
||||
this.addChatMessage('Disconnected, see you next time~😊', false);
|
||||
} else {
|
||||
// Check if OTA URL is filled
|
||||
const otaUrlInput = document.getElementById('otaUrl');
|
||||
if (!otaUrlInput || !otaUrlInput.value.trim()) {
|
||||
// If OTA URL is not filled, show settings modal and switch to device tab
|
||||
this.showModal('settingsModal');
|
||||
this.switchTab('device');
|
||||
this.addChatMessage('Please fill in OTA server URL', false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Start connection process
|
||||
this.handleConnect();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Camera button
|
||||
const cameraBtn = document.getElementById('cameraBtn');
|
||||
let cameraTimer = null;
|
||||
if (cameraBtn) {
|
||||
cameraBtn.addEventListener('click', () => {
|
||||
if (cameraTimer) {
|
||||
clearTimeout(cameraTimer);
|
||||
cameraTimer = null;
|
||||
}
|
||||
cameraTimer = setTimeout(() => {
|
||||
const cameraContainer = document.getElementById('cameraContainer');
|
||||
if (!cameraContainer) {
|
||||
log('摄像头容器不存在', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
const isActive = cameraContainer.classList.contains('active');
|
||||
if (isActive) {
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
if (cameraSwitch) cameraSwitch.classList.remove('active');
|
||||
window.stopCamera();
|
||||
}
|
||||
cameraContainer.classList.remove('active');
|
||||
cameraBtn.classList.remove('camera-active');
|
||||
cameraBtn.querySelector('.btn-text').textContent = '摄像头';
|
||||
log('摄像头已关闭', 'info');
|
||||
} else {
|
||||
// 打开摄像头
|
||||
if (typeof window.startCamera === 'function') {
|
||||
window.startCamera().then(success => {
|
||||
if (success) {
|
||||
cameraBtn.classList.add('camera-active');
|
||||
cameraBtn.querySelector('.btn-text').textContent = '关闭';
|
||||
} else {
|
||||
this.addChatMessage('⚠️ 摄像头启动失败，请检查浏览器权限', false);
|
||||
}
|
||||
}).catch(error => {
|
||||
log(`启动摄像头异常: ${error.message}`, 'error');
|
||||
});
|
||||
} else {
|
||||
log('startCamera函数未定义', 'warning');
|
||||
}
|
||||
}
|
||||
}, 300);
|
||||
});
|
||||
}
|
||||
|
||||
// Record button
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
let recordTimer = null;
|
||||
recordBtn.addEventListener('click', () => {
|
||||
if (recordTimer) {
|
||||
clearTimeout(recordTimer);
|
||||
recordTimer = null;
|
||||
}
|
||||
recordTimer = setTimeout(() => {
|
||||
const audioRecorder = getAudioRecorder();
|
||||
if (audioRecorder.isRecording) {
|
||||
audioRecorder.stop();
|
||||
// Restore record button to normal state
|
||||
recordBtn.classList.remove('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
} else {
|
||||
// Update button state to recording
|
||||
recordBtn.classList.add('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音中';
|
||||
|
||||
// Start recording, update button state after delay
|
||||
setTimeout(() => {
|
||||
audioRecorder.start();
|
||||
}, 100);
|
||||
}
|
||||
}, 300);
|
||||
});
|
||||
}
|
||||
|
||||
// Chat input event listener
|
||||
const chatIpt = document.getElementById('chatIpt');
|
||||
if (chatIpt) {
|
||||
const wsHandler = getWebSocketHandler();
|
||||
chatIpt.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter') {
|
||||
if (e.target.value) {
|
||||
wsHandler.sendTextMessage(e.target.value);
|
||||
e.target.value = '';
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Close button
|
||||
const closeButtons = document.querySelectorAll('.close-btn');
|
||||
closeButtons.forEach(btn => {
|
||||
btn.addEventListener('click', (e) => {
|
||||
e.stopPropagation();
|
||||
const modal = e.target.closest('.modal');
|
||||
if (modal) {
|
||||
if (modal.id === 'settingsModal') {
|
||||
saveConfig();
|
||||
}
|
||||
this.hideModal(modal.id);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Settings tab switch
|
||||
const tabBtns = document.querySelectorAll('.tab-btn');
|
||||
tabBtns.forEach(btn => {
|
||||
btn.addEventListener('click', (e) => {
|
||||
this.switchTab(e.target.dataset.tab);
|
||||
});
|
||||
});
|
||||
|
||||
const applyWakewordBtn = document.getElementById('applyWakewordBtn');
|
||||
if (applyWakewordBtn) {
|
||||
applyWakewordBtn.addEventListener('click', this.handleApplyWakeword);
|
||||
}
|
||||
|
||||
// 点击模态框背景关闭（仅对特定模态框禁用此功能）
|
||||
const modals = document.querySelectorAll('.modal');
|
||||
modals.forEach(modal => {
|
||||
modal.addEventListener('click', (e) => {
|
||||
if (e.target === modal) {
|
||||
// settingsModal、mcpToolModal、mcpPropertyModal 只能通过点击X关闭
|
||||
const nonClosableModals = ['settingsModal', 'mcpToolModal', 'mcpPropertyModal'];
|
||||
if (nonClosableModals.includes(modal.id)) {
|
||||
return; // 禁止点击背景关闭
|
||||
}
|
||||
this.hideModal(modal.id);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Add MCP tool button
|
||||
const addMCPToolBtn = document.getElementById('addMCPToolBtn');
|
||||
if (addMCPToolBtn) {
|
||||
addMCPToolBtn.addEventListener('click', (e) => {
|
||||
e.stopPropagation();
|
||||
this.addMCPTool();
|
||||
});
|
||||
}
|
||||
|
||||
// Connect button and send button are not removed, can be added to dial button later
|
||||
}
|
||||
|
||||
// Update connection status UI
|
||||
updateConnectionUI(isConnected) {
|
||||
const connectionStatus = document.getElementById('connectionStatus');
|
||||
const statusDot = document.querySelector('.status-dot');
|
||||
|
||||
if (connectionStatus) {
|
||||
if (isConnected) {
|
||||
connectionStatus.textContent = '已连接';
|
||||
if (statusDot) {
|
||||
statusDot.className = 'status-dot status-connected';
|
||||
}
|
||||
} else {
|
||||
connectionStatus.textContent = '离线';
|
||||
if (statusDot) {
|
||||
statusDot.className = 'status-dot status-disconnected';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update dial button state
|
||||
updateDialButton(isConnected) {
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
const cameraBtn = document.getElementById('cameraBtn');
|
||||
|
||||
if (dialBtn) {
|
||||
if (isConnected) {
|
||||
dialBtn.classList.add('dial-active');
|
||||
dialBtn.querySelector('.btn-text').textContent = '挂断';
|
||||
// Update dial button icon to hang up icon
|
||||
dialBtn.querySelector('svg').innerHTML = `
|
||||
<path d="M12,9C10.4,9 9,10.4 9,12C9,13.6 10.4,15 12,15C13.6,15 15,13.6 15,12C15,10.4 13.6,9 12,9M12,17C9.2,17 7,14.8 7,12C7,9.2 9.2,7 12,7C14.8,7 17,9.2 17,12C17,14.8 14.8,17 12,17M12,4.5C7,4.5 2.7,7.6 1,12C2.7,16.4 7,19.5 12,19.5C17,19.5 21.3,16.4 23,12C21.3,7.6 17,4.5 12,4.5Z"/>
|
||||
`;
|
||||
} else {
|
||||
dialBtn.classList.remove('dial-active');
|
||||
dialBtn.querySelector('.btn-text').textContent = '拨号';
|
||||
// Restore dial button icon
|
||||
dialBtn.querySelector('svg').innerHTML = `
|
||||
<path d="M6.62,10.79C8.06,13.62 10.38,15.94 13.21,17.38L15.41,15.18C15.69,14.9 16.08,14.82 16.43,14.93C17.55,15.3 18.75,15.5 20,15.5A1,1 0 0,1 21,16.5V20A1,1 0 0,1 20,21A17,17 0 0,1 3,4A1,1 0 0,1 4,3H7.5A1,1 0 0,1 8.5,4C8.5,5.25 8.7,6.45 9.07,7.57C9.18,7.92 9.1,8.31 8.82,8.59L6.62,10.79Z"/>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
// Update camera button state - reset to default when disconnected
|
||||
if (cameraBtn && !isConnected) {
|
||||
const cameraContainer = document.getElementById('cameraContainer');
|
||||
if (cameraContainer && cameraContainer.classList.contains('active')) {
|
||||
cameraContainer.classList.remove('active');
|
||||
}
|
||||
cameraBtn.classList.remove('camera-active');
|
||||
cameraBtn.querySelector('.btn-text').textContent = '摄像头';
|
||||
cameraBtn.disabled = true;
|
||||
cameraBtn.title = '请先连接服务器';
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
window.stopCamera();
|
||||
}
|
||||
}
|
||||
|
||||
// Update camera button state - enable when connected and camera is available
|
||||
if (cameraBtn && isConnected) {
|
||||
if (window.cameraAvailable) {
|
||||
cameraBtn.disabled = false;
|
||||
cameraBtn.title = '打开/关闭摄像头';
|
||||
} else {
|
||||
cameraBtn.disabled = true;
|
||||
cameraBtn.title = '请先绑定验证码';
|
||||
}
|
||||
}
|
||||
|
||||
// Update record button state
|
||||
if (recordBtn) {
|
||||
const microphoneAvailable = window.microphoneAvailable !== false;
|
||||
if (isConnected && microphoneAvailable) {
|
||||
recordBtn.disabled = false;
|
||||
recordBtn.title = '开始录音';
|
||||
// Restore record button to normal state
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.classList.remove('recording');
|
||||
} else {
|
||||
recordBtn.disabled = true;
|
||||
if (!microphoneAvailable) {
|
||||
recordBtn.title = window.isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
|
||||
} else {
|
||||
recordBtn.title = '请先连接服务器';
|
||||
}
|
||||
// Restore record button to normal state
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.classList.remove('recording');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update record button state
|
||||
updateRecordButtonState(isRecording, seconds = 0) {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
if (isRecording) {
|
||||
recordBtn.querySelector('.btn-text').textContent = `录音中`;
|
||||
recordBtn.classList.add('recording');
|
||||
} else {
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.classList.remove('recording');
|
||||
}
|
||||
// Only enable button when microphone is available
|
||||
recordBtn.disabled = window.microphoneAvailable === false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update microphone availability state
|
||||
* @param {boolean} isAvailable - Whether microphone is available
|
||||
* @param {boolean} isHttpNonLocalhost - Whether it is HTTP non-localhost access
|
||||
*/
|
||||
updateMicrophoneAvailability(isAvailable, isHttpNonLocalhost) {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (!recordBtn) return;
|
||||
if (!isAvailable) {
|
||||
// Disable record button
|
||||
recordBtn.disabled = true;
|
||||
// Update button text and title
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.title = isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
|
||||
|
||||
} else {
|
||||
// If connected, enable record button
|
||||
const wsHandler = getWebSocketHandler();
|
||||
if (wsHandler && wsHandler.isConnected()) {
|
||||
recordBtn.disabled = false;
|
||||
recordBtn.title = '开始录音';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add chat message
|
||||
addChatMessage(content, isUser = false) {
|
||||
const chatStream = document.getElementById('chatStream');
|
||||
if (!chatStream) return;
|
||||
|
||||
const messageDiv = document.createElement('div');
|
||||
messageDiv.className = `chat-message ${isUser ? 'user' : 'ai'}`;
|
||||
messageDiv.innerHTML = `<div class="message-bubble">${content}</div>`;
|
||||
chatStream.appendChild(messageDiv);
|
||||
|
||||
// Scroll to bottom
|
||||
chatStream.scrollTop = chatStream.scrollHeight;
|
||||
}
|
||||
|
||||
// Switch background
|
||||
switchBackground() {
|
||||
this.currentBackgroundIndex = (this.currentBackgroundIndex + 1) % this.backgroundImages.length;
|
||||
const backgroundContainer = document.querySelector('.background-container');
|
||||
if (backgroundContainer) {
|
||||
backgroundContainer.style.backgroundImage = `url('./images/${this.backgroundImages[this.currentBackgroundIndex]}')`;
|
||||
}
|
||||
localStorage.setItem('backgroundIndex', this.currentBackgroundIndex);
|
||||
}
|
||||
|
||||
// Switch Live2D model
|
||||
switchLive2DModel() {
|
||||
const modelSelect = document.getElementById('live2dModelSelect');
|
||||
if (!modelSelect) {
|
||||
console.error('模型选择下拉框不存在');
|
||||
return;
|
||||
}
|
||||
|
||||
const selectedModel = modelSelect.value;
|
||||
const app = window.chatApp;
|
||||
|
||||
if (app && app.live2dManager) {
|
||||
app.live2dManager.switchModel(selectedModel)
|
||||
.then(success => {
|
||||
if (success) {
|
||||
this.addChatMessage(`已切换到模型: ${selectedModel}`, false);
|
||||
} else {
|
||||
this.addChatMessage('模型切换失败', false);
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('模型切换错误:', error);
|
||||
this.addChatMessage('模型切换出错', false);
|
||||
});
|
||||
} else {
|
||||
this.addChatMessage('Live2D管理器未初始化', false);
|
||||
}
|
||||
}
|
||||
|
||||
// Show modal
|
||||
showModal(modalId) {
|
||||
const modal = document.getElementById(modalId);
|
||||
if (modal) {
|
||||
modal.style.display = 'flex';
|
||||
}
|
||||
}
|
||||
|
||||
// Hide modal
|
||||
hideModal(modalId) {
|
||||
const modal = document.getElementById(modalId);
|
||||
if (modal) {
|
||||
modal.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
// Switch tab
|
||||
switchTab(tabName) {
|
||||
// Remove active class from all tabs
|
||||
const tabBtns = document.querySelectorAll('.tab-btn');
|
||||
const tabContents = document.querySelectorAll('.tab-content');
|
||||
|
||||
tabBtns.forEach(btn => btn.classList.remove('active'));
|
||||
tabContents.forEach(content => content.classList.remove('active'));
|
||||
|
||||
// Activate selected tab
|
||||
const activeTabBtn = document.querySelector(`[data-tab="${tabName}"]`);
|
||||
const activeTabContent = document.getElementById(`${tabName}Tab`);
|
||||
|
||||
if (activeTabBtn && activeTabContent) {
|
||||
activeTabBtn.classList.add('active');
|
||||
activeTabContent.classList.add('active');
|
||||
}
|
||||
}
|
||||
|
||||
applyWakewordConfig(config = {}) {
|
||||
const wakewordEnabledInput = document.getElementById('wakewordEnabled');
|
||||
const wakewordListInput = document.getElementById('wakewordList');
|
||||
|
||||
if (!wakewordEnabledInput || !wakewordListInput) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wakeWords = Array.isArray(config.wakeWords)
|
||||
? config.wakeWords.filter(item => typeof item === 'string' && item.trim())
|
||||
: [];
|
||||
|
||||
wakewordEnabledInput.value = config.enabled === false ? 'false' : 'true';
|
||||
wakewordListInput.value = wakeWords.join('\n');
|
||||
saveConfig();
|
||||
}
|
||||
|
||||
async handleApplyWakeword() {
|
||||
const wakewordEnabledInput = document.getElementById('wakewordEnabled');
|
||||
const wakewordListInput = document.getElementById('wakewordList');
|
||||
const wakewordWsUrlInput = document.getElementById('wakewordWsUrl');
|
||||
if (!wakewordEnabledInput || !wakewordListInput) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wakeWords = wakewordListInput.value
|
||||
.split(/\r?\n/u)
|
||||
.map(item => item.trim())
|
||||
.filter(Boolean)
|
||||
.filter((item, index, items) => items.indexOf(item) === index);
|
||||
|
||||
const payload = {
|
||||
enabled: wakewordEnabledInput.value !== 'false',
|
||||
wakeWords,
|
||||
};
|
||||
|
||||
if (payload.enabled && payload.wakeWords.length === 0) {
|
||||
this.addChatMessage('启用唤醒词时，至少需要填写一个唤醒词。', false);
|
||||
return;
|
||||
}
|
||||
|
||||
const applyWakewordBtn = document.getElementById('applyWakewordBtn');
|
||||
if (applyWakewordBtn) {
|
||||
applyWakewordBtn.disabled = true;
|
||||
applyWakewordBtn.textContent = '应用中...';
|
||||
}
|
||||
|
||||
try {
|
||||
// 保存地址到 localStorage
|
||||
if (wakewordWsUrlInput && wakewordWsUrlInput.value.trim()) {
|
||||
localStorage.setItem('xz_tester_wakewordWsUrl', wakewordWsUrlInput.value.trim());
|
||||
}
|
||||
|
||||
// 比较新地址和当前连接地址
|
||||
const newWsUrl = localStorage.getItem('xz_tester_wakewordWsUrl');
|
||||
const currentWsUrl = getWakewordBridgeUrl();
|
||||
const urlChanged = newWsUrl !== currentWsUrl;
|
||||
|
||||
if (urlChanged) {
|
||||
// 地址变了，先确认
|
||||
const shouldRestart = window.confirm('地址已变更，是否继续？（将断开旧连接并重新连接）');
|
||||
if (!shouldRestart) {
|
||||
// 恢复 localStorage 里的旧地址
|
||||
localStorage.setItem('xz_tester_wakewordWsUrl', currentWsUrl);
|
||||
this.addChatMessage('已取消地址变更。', false);
|
||||
return;
|
||||
}
|
||||
|
||||
// 断开旧连接
|
||||
stopWakewordBridgeListener();
|
||||
|
||||
// 启动新连接（自动用新地址）
|
||||
startWakewordBridgeListener();
|
||||
|
||||
// 等 bridge_connected
|
||||
await new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
reject(new Error('连接新服务器超时'));
|
||||
}, 5000);
|
||||
|
||||
onNextBridgeConnected(() => {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// 发配置和重启
|
||||
await requestWakewordBridge('set_wakeword_config', payload);
|
||||
this.applyWakewordConfig(payload);
|
||||
await requestWakewordBridge('restart_wakeword_service');
|
||||
this.addChatMessage('唤醒词配置已保存，唤醒词服务正在重启。', false);
|
||||
} else {
|
||||
// 地址没变：直接在当前连接操作
|
||||
const response = await requestWakewordBridge('set_wakeword_config', payload);
|
||||
this.applyWakewordConfig(response.payload || payload);
|
||||
|
||||
const shouldRestart = window.confirm('唤醒词已保存。是否现在重启唤醒词服务以立即生效？');
|
||||
if (!shouldRestart) {
|
||||
this.addChatMessage('唤醒词配置已保存，可稍后手动重启服务后生效。', false);
|
||||
return;
|
||||
}
|
||||
|
||||
await requestWakewordBridge('restart_wakeword_service');
|
||||
this.addChatMessage('唤醒词配置已保存，唤醒词服务正在重启。', false);
|
||||
}
|
||||
} catch (error) {
|
||||
this.addChatMessage(`应用唤醒词失败: ${error.message}`, false);
|
||||
} finally {
|
||||
if (applyWakewordBtn) {
|
||||
applyWakewordBtn.disabled = false;
|
||||
applyWakewordBtn.textContent = '应用唤醒词';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start AI chat session after connection
|
||||
startAIChatSession() {
|
||||
this.addChatMessage('连接成功，开始聊天吧~😊', false);
|
||||
// Check microphone availability and show error messages if needed
|
||||
if (!window.microphoneAvailable) {
|
||||
if (window.isHttpNonLocalhost) {
|
||||
this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
|
||||
} else {
|
||||
this.addChatMessage('⚠️ 麦克风不可用，请检查权限设置，只能用文字交互', false);
|
||||
}
|
||||
}
|
||||
// Start recording only if microphone is available
|
||||
if (window.microphoneAvailable) {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
recordBtn.click();
|
||||
}
|
||||
}
|
||||
// Start camera only if camera is available (bound with verification code)
|
||||
if (window.cameraAvailable && typeof window.startCamera === 'function') {
|
||||
window.startCamera().then(success => {
|
||||
if (success) {
|
||||
const cameraBtn = document.getElementById('cameraBtn');
|
||||
if (cameraBtn) {
|
||||
cameraBtn.classList.add('camera-active');
|
||||
cameraBtn.querySelector('.btn-text').textContent = '关闭';
|
||||
}
|
||||
} else {
|
||||
this.addChatMessage('⚠️ 摄像头启动失败，可能被浏览器拒绝', false);
|
||||
}
|
||||
}).catch(error => {
|
||||
log(`启动摄像头异常: ${error.message}`, 'error');
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Handle connect button click
|
||||
async handleConnect() {
|
||||
const wsHandler = getWebSocketHandler();
|
||||
if (this.isConnecting || (wsHandler && wsHandler.isConnected())) {
|
||||
log('连接已存在或正在进行，忽略本次拨号请求', 'info');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isConnecting = true;
|
||||
console.log('handleConnect called');
|
||||
|
||||
try {
|
||||
// Switch to device settings tab
|
||||
this.switchTab('device');
|
||||
|
||||
// Wait for DOM update
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
|
||||
const otaUrlInput = document.getElementById('otaUrl');
|
||||
|
||||
console.log('otaUrl element:', otaUrlInput);
|
||||
|
||||
if (!otaUrlInput || !otaUrlInput.value) {
|
||||
this.addChatMessage('请输入OTA服务器地址', false);
|
||||
return;
|
||||
}
|
||||
|
||||
const otaUrl = otaUrlInput.value;
|
||||
console.log('otaUrl value:', otaUrl);
|
||||
|
||||
// Update dial button state to connecting
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
dialBtn.classList.add('dial-active');
|
||||
dialBtn.querySelector('.btn-text').textContent = '连接中...';
|
||||
dialBtn.disabled = true;
|
||||
}
|
||||
|
||||
// Show connecting message
|
||||
this.addChatMessage('正在连接服务器...', false);
|
||||
|
||||
const chatIpt = document.getElementById('chatIpt');
|
||||
if (chatIpt) {
|
||||
chatIpt.style.display = 'flex';
|
||||
}
|
||||
|
||||
// Get WebSocket handler instance
|
||||
// Register connection state callback BEFORE connecting
|
||||
wsHandler.onConnectionStateChange = (isConnected) => {
|
||||
this.updateConnectionUI(isConnected);
|
||||
this.updateDialButton(isConnected);
|
||||
};
|
||||
|
||||
// Register chat message callback BEFORE connecting
|
||||
wsHandler.onChatMessage = (text, isUser) => {
|
||||
this.addChatMessage(text, isUser);
|
||||
};
|
||||
|
||||
// Register record button state callback BEFORE connecting
|
||||
wsHandler.onRecordButtonStateChange = (isRecording) => {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
if (isRecording) {
|
||||
recordBtn.classList.add('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音中';
|
||||
} else {
|
||||
recordBtn.classList.remove('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const isConnected = await wsHandler.connect();
|
||||
|
||||
if (isConnected) {
|
||||
// Check microphone availability (check again after connection)
|
||||
const { checkMicrophoneAvailability } = await import('../core/audio/recorder.js?v=0205');
|
||||
const micAvailable = await checkMicrophoneAvailability();
|
||||
|
||||
if (!micAvailable) {
|
||||
const isHttp = window.isHttpNonLocalhost;
|
||||
if (isHttp) {
|
||||
this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
|
||||
}
|
||||
// Update global state
|
||||
window.microphoneAvailable = false;
|
||||
}
|
||||
|
||||
// Update dial button state
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
if (!this.dialBtnDisabled) {
|
||||
dialBtn.disabled = false;
|
||||
}
|
||||
dialBtn.querySelector('.btn-text').textContent = '挂断';
|
||||
dialBtn.classList.add('dial-active');
|
||||
}
|
||||
|
||||
this.hideModal('settingsModal');
|
||||
} else {
|
||||
throw new Error('OTA连接失败');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Connection error details:', {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
name: error.name
|
||||
});
|
||||
|
||||
// Show error message
|
||||
const errorMessage = error.message.includes('Cannot set properties of null')
|
||||
? '连接失败：请检查设备连接'
|
||||
: `连接失败: ${error.message}`;
|
||||
|
||||
this.addChatMessage(errorMessage, false);
|
||||
|
||||
// Restore dial button state
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
if (!this.dialBtnDisabled) {
|
||||
dialBtn.disabled = false;
|
||||
}
|
||||
dialBtn.querySelector('.btn-text').textContent = '拨号';
|
||||
dialBtn.classList.remove('dial-active');
|
||||
console.log('Dial button state restored successfully');
|
||||
}
|
||||
} finally {
|
||||
this.isConnecting = false;
|
||||
}
|
||||
}
|
||||
|
||||
async triggerWakewordDial(wakeWord = '唤醒词') {
|
||||
const wsHandler = getWebSocketHandler();
|
||||
const now = Date.now();
|
||||
|
||||
if (wsHandler && wsHandler.isConnected()) {
|
||||
log('页面已连接，忽略自动拨号', 'info');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (this.isConnecting || this.dialBtnDisabled) {
|
||||
log('页面正在连接中，忽略重复唤醒', 'info');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (now - this.lastWakewordDialTime < 3000) {
|
||||
log('唤醒触发过于频繁，忽略本次自动拨号', 'warning');
|
||||
return false;
|
||||
}
|
||||
|
||||
this.lastWakewordDialTime = now;
|
||||
this.addChatMessage(`检测到唤醒词“${wakeWord}”，准备连接服务器...`, false);
|
||||
await this.handleConnect();
|
||||
return true;
|
||||
}
|
||||
|
||||
// Add MCP tool
|
||||
addMCPTool() {
|
||||
const mcpToolsList = document.getElementById('mcpToolsList');
|
||||
if (!mcpToolsList) return;
|
||||
|
||||
const toolId = `mcp-tool-${Date.now()}`;
|
||||
const toolDiv = document.createElement('div');
|
||||
toolDiv.className = 'properties-container';
|
||||
toolDiv.innerHTML = `
|
||||
<div class="property-item">
|
||||
<input type="text" placeholder="工具名称" value="新工具">
|
||||
<input type="text" placeholder="工具描述" value="工具描述">
|
||||
<button class="remove-property" onclick="uiController.removeMCPTool('${toolId}')">删除</button>
|
||||
</div>
|
||||
`;
|
||||
|
||||
mcpToolsList.appendChild(toolDiv);
|
||||
}
|
||||
|
||||
// Remove MCP tool
|
||||
removeMCPTool(toolId) {
|
||||
const toolElement = document.getElementById(toolId);
|
||||
if (toolElement) {
|
||||
toolElement.remove();
|
||||
}
|
||||
}
|
||||
|
||||
// Update audio statistics display
|
||||
updateAudioStats() {
|
||||
const audioPlayer = getAudioPlayer();
|
||||
if (!audioPlayer) return;
|
||||
|
||||
const stats = audioPlayer.getAudioStats();
|
||||
// Here can add audio statistics UI update logic
|
||||
}
|
||||
|
||||
// Start audio statistics monitor
|
||||
startAudioStatsMonitor() {
|
||||
// Update audio statistics every 100ms
|
||||
this.audioStatsTimer = setInterval(() => {
|
||||
this.updateAudioStats();
|
||||
}, 100);
|
||||
}
|
||||
|
||||
// Stop audio statistics monitor
|
||||
stopAudioStatsMonitor() {
|
||||
if (this.audioStatsTimer) {
|
||||
clearInterval(this.audioStatsTimer);
|
||||
this.audioStatsTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Draw audio visualizer waveform
|
||||
drawVisualizer(dataArray) {
|
||||
if (!this.visualizerContext || !this.visualizerCanvas) return;
|
||||
|
||||
this.visualizerContext.fillStyle = '#fafafa';
|
||||
this.visualizerContext.fillRect(0, 0, this.visualizerCanvas.width, this.visualizerCanvas.height);
|
||||
|
||||
const barWidth = (this.visualizerCanvas.width / dataArray.length) * 2.5;
|
||||
let barHeight;
|
||||
let x = 0;
|
||||
|
||||
for (let i = 0; i < dataArray.length; i++) {
|
||||
barHeight = dataArray[i] / 2;
|
||||
|
||||
// Create gradient color: from purple to blue to green
|
||||
const gradient = this.visualizerContext.createLinearGradient(0, 0, 0, this.visualizerCanvas.height);
|
||||
gradient.addColorStop(0, '#8e44ad');
|
||||
gradient.addColorStop(0.5, '#3498db');
|
||||
gradient.addColorStop(1, '#1abc9c');
|
||||
|
||||
this.visualizerContext.fillStyle = gradient;
|
||||
this.visualizerContext.fillRect(x, this.visualizerCanvas.height - barHeight, barWidth, barHeight);
|
||||
x += barWidth + 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Update session status UI
|
||||
updateSessionStatus(isSpeaking) {
|
||||
// Here can add session status UI update logic
|
||||
// For example: update Live2D model's mouth movement status
|
||||
}
|
||||
|
||||
// Update session emotion
|
||||
updateSessionEmotion(emoji) {
|
||||
// Here can add emotion update logic
|
||||
// For example: display emoji in status indicator
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
export const uiController = new UIController();
|
||||
|
||||
// Export class for module usage
|
||||
export { UIController };
|
||||
@@ -0,0 +1,103 @@
|
||||
export default class BlockingQueue {
|
||||
#items = [];
|
||||
#waiters = []; // {resolve, reject, min, timer, onTimeout}
|
||||
|
||||
/* 空队列一次性闸门 */
|
||||
#emptyPromise = null;
|
||||
#emptyResolve = null;
|
||||
|
||||
/* 生产者：把数据塞进去 */
|
||||
enqueue(item, ...restItems) {
|
||||
if (restItems.length === 0) {
|
||||
this.#items.push(item);
|
||||
}
|
||||
// 如果有额外参数，批量处理所有项
|
||||
else {
|
||||
const items = [item, ...restItems].filter(i => i);
|
||||
if (items.length === 0) return;
|
||||
this.#items.push(...items);
|
||||
}
|
||||
// 若有空队列闸门，一次性放行所有等待者
|
||||
if (this.#emptyResolve) {
|
||||
this.#emptyResolve();
|
||||
this.#emptyResolve = null;
|
||||
this.#emptyPromise = null;
|
||||
}
|
||||
|
||||
// 唤醒所有正在等的 waiter
|
||||
this.#wakeWaiters();
|
||||
}
|
||||
|
||||
/* 消费者：min 条或 timeout ms 先到谁 */
|
||||
async dequeue(min = 1, timeout = Infinity, onTimeout = null) {
|
||||
// 1. 若空，等第一次数据到达（所有调用共享同一个 promise）
|
||||
if (this.#items.length === 0) {
|
||||
await this.#waitForFirstItem();
|
||||
}
|
||||
|
||||
// 立即满足
|
||||
if (this.#items.length >= min) {
|
||||
return this.#flush();
|
||||
}
|
||||
|
||||
// 需要等待
|
||||
return new Promise((resolve, reject) => {
|
||||
let timer = null;
|
||||
const waiter = { resolve, reject, min, onTimeout, timer };
|
||||
|
||||
// 超时逻辑
|
||||
if (Number.isFinite(timeout)) {
|
||||
waiter.timer = setTimeout(() => {
|
||||
this.#removeWaiter(waiter);
|
||||
if (onTimeout) onTimeout(this.#items.length);
|
||||
resolve(this.#flush());
|
||||
}, timeout);
|
||||
}
|
||||
|
||||
this.#waiters.push(waiter);
|
||||
});
|
||||
}
|
||||
|
||||
/* 空队列闸门生成器 */
|
||||
#waitForFirstItem() {
|
||||
if (!this.#emptyPromise) {
|
||||
this.#emptyPromise = new Promise(r => (this.#emptyResolve = r));
|
||||
}
|
||||
return this.#emptyPromise;
|
||||
}
|
||||
|
||||
/* 内部：每次数据变动后，检查哪些 waiter 已满足 */
|
||||
#wakeWaiters() {
|
||||
for (let i = this.#waiters.length - 1; i >= 0; i--) {
|
||||
const w = this.#waiters[i];
|
||||
if (this.#items.length >= w.min) {
|
||||
this.#removeWaiter(w);
|
||||
w.resolve(this.#flush());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#removeWaiter(waiter) {
|
||||
const idx = this.#waiters.indexOf(waiter);
|
||||
if (idx !== -1) {
|
||||
this.#waiters.splice(idx, 1);
|
||||
if (waiter.timer) clearTimeout(waiter.timer);
|
||||
}
|
||||
}
|
||||
|
||||
#flush() {
|
||||
const snapshot = [...this.#items];
|
||||
this.#items.length = 0;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/* 当前缓存长度（不含等待者） */
|
||||
get length() {
|
||||
return this.#items.length;
|
||||
}
|
||||
|
||||
/* 清空队列（保持对象引用，不影响等待者） */
|
||||
clear() {
|
||||
this.#items.length = 0;
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,43 @@
|
||||
// 日志记录函数
|
||||
export function log(message, type = 'info') {
|
||||
// 将消息按换行符分割成多行
|
||||
const lines = message.split('\n');
|
||||
const now = new Date();
|
||||
// const timestamp = `[${now.toLocaleTimeString()}] `;
|
||||
const timestamp = `[${now.toLocaleTimeString()}.${now.getMilliseconds().toString().padStart(3, '0')}] `;
|
||||
|
||||
// 检查是否存在日志容器
|
||||
const logContainer = document.getElementById('logContainer');
|
||||
if (!logContainer) {
|
||||
// 如果日志容器不存在，只输出到控制台
|
||||
console.log(`[${type.toUpperCase()}] ${message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 为每一行创建日志条目
|
||||
lines.forEach((line, index) => {
|
||||
const logEntry = document.createElement('div');
|
||||
logEntry.className = `log-entry log-${type}`;
|
||||
// 如果是第一条日志，显示时间戳
|
||||
const prefix = index === 0 ? timestamp : ' '.repeat(timestamp.length);
|
||||
logEntry.textContent = `${prefix}${line}`;
|
||||
// logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
|
||||
// logEntry.style 保留起始的空格
|
||||
logEntry.style.whiteSpace = 'pre';
|
||||
if (type === 'error') {
|
||||
logEntry.style.color = 'red';
|
||||
} else if (type === 'debug') {
|
||||
logEntry.style.color = 'gray';
|
||||
return;
|
||||
} else if (type === 'warning') {
|
||||
logEntry.style.color = 'orange';
|
||||
} else if (type === 'success') {
|
||||
logEntry.style.color = 'green';
|
||||
} else {
|
||||
logEntry.style.color = 'black';
|
||||
}
|
||||
logContainer.appendChild(logEntry);
|
||||
});
|
||||
|
||||
logContainer.scrollTop = logContainer.scrollHeight;
|
||||
}
|
||||
Reference in New Issue
Block a user