mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 01:03:51 +08:00
update:test迁移重命名为digital-human
This commit is contained in:
3030332422
@@ -0,0 +1,182 @@
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
export function checkOpusLoaded() {
|
||||
try {
|
||||
// 检查Module是否存在（本地库导出的全局变量）
|
||||
if (typeof Module === 'undefined') {
|
||||
throw new Error('Opus库未加载，Module对象不存在');
|
||||
}
|
||||
|
||||
// 尝试先使用Module.instance（libopus.js最后一行导出方式）
|
||||
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
|
||||
// 使用Module.instance对象替换全局Module对象
|
||||
window.ModuleInstance = Module.instance;
|
||||
log('Opus库加载成功（使用Module.instance）', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
// 如果没有Module.instance，检查全局Module函数
|
||||
if (typeof Module._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module;
|
||||
log('Opus库加载成功（使用全局Module）', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error('Opus解码函数未找到，可能Module结构不正确');
|
||||
} catch (err) {
|
||||
log(`Opus库加载失败，请检查libopus.js文件是否存在且正确: ${err.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 创建一个Opus编码器
|
||||
let opusEncoder = null;
|
||||
export function initOpusEncoder() {
|
||||
try {
|
||||
if (opusEncoder) {
|
||||
return opusEncoder; // 已经初始化过
|
||||
}
|
||||
|
||||
if (!window.ModuleInstance) {
|
||||
log('无法创建Opus编码器：ModuleInstance不可用', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 初始化一个Opus编码器
|
||||
const mod = window.ModuleInstance;
|
||||
const sampleRate = 16000; // 16kHz采样率
|
||||
const channels = 1; // 单声道
|
||||
const application = 2048; // OPUS_APPLICATION_VOIP = 2048
|
||||
|
||||
// 创建编码器
|
||||
opusEncoder = {
|
||||
channels: channels,
|
||||
sampleRate: sampleRate,
|
||||
frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
|
||||
maxPacketSize: 4000, // 最大包大小
|
||||
module: mod,
|
||||
|
||||
// 初始化编码器
|
||||
init: function () {
|
||||
try {
|
||||
// 获取编码器大小
|
||||
const encoderSize = mod._opus_encoder_get_size(this.channels);
|
||||
log(`Opus编码器大小: ${encoderSize}字节`, 'info');
|
||||
|
||||
// 分配内存
|
||||
this.encoderPtr = mod._malloc(encoderSize);
|
||||
if (!this.encoderPtr) {
|
||||
throw new Error("无法分配编码器内存");
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
const err = mod._opus_encoder_init(
|
||||
this.encoderPtr,
|
||||
this.sampleRate,
|
||||
this.channels,
|
||||
application
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
throw new Error(`Opus编码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
// 设置位率 (16kbps)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4002, 16000); // OPUS_SET_BITRATE
|
||||
|
||||
// 设置复杂度 (0-10, 越高质量越好但CPU使用越多)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4010, 5); // OPUS_SET_COMPLEXITY
|
||||
|
||||
// 设置使用DTX (不传输静音帧)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4016, 1); // OPUS_SET_DTX
|
||||
|
||||
log("Opus编码器初始化成功", 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (this.encoderPtr) {
|
||||
mod._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
log(`Opus编码器初始化失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
// 编码PCM数据为Opus
|
||||
encode: function (pcmData) {
|
||||
if (!this.encoderPtr) {
|
||||
if (!this.init()) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
// 为PCM数据分配内存
|
||||
const pcmPtr = mod._malloc(pcmData.length * 2); // 2字节/int16
|
||||
|
||||
// 将PCM数据复制到HEAP
|
||||
for (let i = 0; i < pcmData.length; i++) {
|
||||
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
|
||||
}
|
||||
|
||||
// 为输出分配内存
|
||||
const outPtr = mod._malloc(this.maxPacketSize);
|
||||
|
||||
// 进行编码
|
||||
const encodedLen = mod._opus_encode(
|
||||
this.encoderPtr,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
outPtr,
|
||||
this.maxPacketSize
|
||||
);
|
||||
|
||||
if (encodedLen < 0) {
|
||||
throw new Error(`Opus编码失败: ${encodedLen}`);
|
||||
}
|
||||
|
||||
// 复制编码后的数据
|
||||
const opusData = new Uint8Array(encodedLen);
|
||||
for (let i = 0; i < encodedLen; i++) {
|
||||
opusData[i] = mod.HEAPU8[outPtr + i];
|
||||
}
|
||||
|
||||
// 释放内存
|
||||
mod._free(pcmPtr);
|
||||
mod._free(outPtr);
|
||||
|
||||
return opusData;
|
||||
} catch (error) {
|
||||
log(`Opus编码出错: ${error.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
},
|
||||
|
||||
// 销毁编码器
|
||||
destroy: function () {
|
||||
if (this.encoderPtr) {
|
||||
this.module._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
opusEncoder.init();
|
||||
return opusEncoder;
|
||||
} catch (error) {
|
||||
log(`创建Opus编码器失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
// 音频播放模块
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { createStreamingContext } from './stream-context.js?v=0205';
|
||||
|
||||
// 音频播放器类
|
||||
export class AudioPlayer {
|
||||
constructor() {
|
||||
// 音频参数
|
||||
this.SAMPLE_RATE = 16000;
|
||||
this.CHANNELS = 1;
|
||||
this.FRAME_SIZE = 960;
|
||||
this.MIN_AUDIO_DURATION = 0.12;
|
||||
|
||||
// 状态
|
||||
this.audioContext = null;
|
||||
this.opusDecoder = null;
|
||||
this.streamingContext = null;
|
||||
this.queue = new BlockingQueue();
|
||||
this.isPlaying = false;
|
||||
}
|
||||
|
||||
// 获取或创建AudioContext
|
||||
getAudioContext() {
|
||||
if (!this.audioContext) {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: this.SAMPLE_RATE,
|
||||
latencyHint: 'interactive'
|
||||
});
|
||||
log('创建音频上下文，采样率: ' + this.SAMPLE_RATE + 'Hz', 'debug');
|
||||
}
|
||||
return this.audioContext;
|
||||
}
|
||||
|
||||
// 初始化Opus解码器
|
||||
async initOpusDecoder() {
|
||||
if (this.opusDecoder) return this.opusDecoder;
|
||||
|
||||
try {
|
||||
if (typeof window.ModuleInstance === 'undefined') {
|
||||
if (typeof Module !== 'undefined') {
|
||||
window.ModuleInstance = Module;
|
||||
log('使用全局Module作为ModuleInstance', 'info');
|
||||
} else {
|
||||
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
|
||||
}
|
||||
}
|
||||
|
||||
const mod = window.ModuleInstance;
|
||||
|
||||
this.opusDecoder = {
|
||||
channels: this.CHANNELS,
|
||||
rate: this.SAMPLE_RATE,
|
||||
frameSize: this.FRAME_SIZE,
|
||||
module: mod,
|
||||
decoderPtr: null,
|
||||
|
||||
init: function () {
|
||||
if (this.decoderPtr) return true;
|
||||
|
||||
const decoderSize = mod._opus_decoder_get_size(this.channels);
|
||||
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
|
||||
|
||||
this.decoderPtr = mod._malloc(decoderSize);
|
||||
if (!this.decoderPtr) {
|
||||
throw new Error("无法分配解码器内存");
|
||||
}
|
||||
|
||||
const err = mod._opus_decoder_init(
|
||||
this.decoderPtr,
|
||||
this.rate,
|
||||
this.channels
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
this.destroy();
|
||||
throw new Error(`Opus解码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
log("Opus解码器初始化成功", 'success');
|
||||
return true;
|
||||
},
|
||||
|
||||
decode: function (opusData) {
|
||||
if (!this.decoderPtr) {
|
||||
if (!this.init()) {
|
||||
throw new Error("解码器未初始化且无法初始化");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
const opusPtr = mod._malloc(opusData.length);
|
||||
mod.HEAPU8.set(opusData, opusPtr);
|
||||
|
||||
const pcmPtr = mod._malloc(this.frameSize * 2);
|
||||
|
||||
const decodedSamples = mod._opus_decode(
|
||||
this.decoderPtr,
|
||||
opusPtr,
|
||||
opusData.length,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
0
|
||||
);
|
||||
|
||||
if (decodedSamples < 0) {
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
throw new Error(`Opus解码失败: ${decodedSamples}`);
|
||||
}
|
||||
|
||||
const decodedData = new Int16Array(decodedSamples);
|
||||
for (let i = 0; i < decodedSamples; i++) {
|
||||
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
|
||||
}
|
||||
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
|
||||
return decodedData;
|
||||
} catch (error) {
|
||||
log(`Opus解码错误: ${error.message}`, 'error');
|
||||
return new Int16Array(0);
|
||||
}
|
||||
},
|
||||
|
||||
destroy: function () {
|
||||
if (this.decoderPtr) {
|
||||
this.module._free(this.decoderPtr);
|
||||
this.decoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (!this.opusDecoder.init()) {
|
||||
throw new Error("Opus解码器初始化失败");
|
||||
}
|
||||
|
||||
return this.opusDecoder;
|
||||
|
||||
} catch (error) {
|
||||
log(`Opus解码器初始化失败: ${error.message}`, 'error');
|
||||
this.opusDecoder = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 启动音频缓冲
|
||||
async startAudioBuffering() {
|
||||
log("开始音频缓冲...", 'info');
|
||||
|
||||
this.initOpusDecoder().catch(error => {
|
||||
log(`预初始化Opus解码器失败: ${error.message}`, 'warning');
|
||||
});
|
||||
|
||||
const timeout = 400;
|
||||
while (true) {
|
||||
const packets = await this.queue.dequeue(
|
||||
6,
|
||||
timeout,
|
||||
(count) => {
|
||||
log(`缓冲超时，当前缓冲包数: ${count}，开始播放`, 'info');
|
||||
}
|
||||
);
|
||||
if (packets.length) {
|
||||
log(`已缓冲 ${packets.length} 个音频包，开始播放`, 'info');
|
||||
this.streamingContext.pushAudioBuffer(packets);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const data = await this.queue.dequeue(99, 30);
|
||||
if (data.length) {
|
||||
this.streamingContext.pushAudioBuffer(data);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 播放已缓冲的音频
|
||||
async playBufferedAudio() {
|
||||
try {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (!this.opusDecoder) {
|
||||
log('初始化Opus解码器...', 'info');
|
||||
try {
|
||||
this.opusDecoder = await this.initOpusDecoder();
|
||||
if (!this.opusDecoder) {
|
||||
throw new Error('解码器初始化失败');
|
||||
}
|
||||
log('Opus解码器初始化成功', 'success');
|
||||
} catch (error) {
|
||||
log('Opus解码器初始化失败: ' + error.message, 'error');
|
||||
this.isPlaying = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.streamingContext) {
|
||||
this.streamingContext = createStreamingContext(
|
||||
this.opusDecoder,
|
||||
this.audioContext,
|
||||
this.SAMPLE_RATE,
|
||||
this.CHANNELS,
|
||||
this.MIN_AUDIO_DURATION
|
||||
);
|
||||
}
|
||||
|
||||
this.streamingContext.decodeOpusFrames();
|
||||
this.streamingContext.startPlaying();
|
||||
|
||||
} catch (error) {
|
||||
log(`播放已缓冲的音频出错: ${error.message}`, 'error');
|
||||
this.isPlaying = false;
|
||||
this.streamingContext = null;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加音频数据到队列
|
||||
enqueueAudioData(opusData) {
|
||||
if (opusData.length > 0) {
|
||||
this.queue.enqueue(opusData);
|
||||
} else {
|
||||
log('收到空音频数据帧，可能是结束标志', 'warning');
|
||||
if (this.isPlaying && this.streamingContext) {
|
||||
this.streamingContext.endOfStream = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 预加载解码器
|
||||
async preload() {
|
||||
log('预加载Opus解码器...', 'info');
|
||||
try {
|
||||
await this.initOpusDecoder();
|
||||
log('Opus解码器预加载成功', 'success');
|
||||
} catch (error) {
|
||||
log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 启动播放系统
|
||||
async start() {
|
||||
await this.preload();
|
||||
this.playBufferedAudio();
|
||||
this.startAudioBuffering();
|
||||
}
|
||||
|
||||
// 获取音频包统计信息
|
||||
getAudioStats() {
|
||||
if (!this.streamingContext) {
|
||||
return {
|
||||
pendingDecode: 0,
|
||||
pendingPlay: 0,
|
||||
totalPending: 0
|
||||
};
|
||||
}
|
||||
|
||||
const pendingDecode = this.streamingContext.getPendingDecodeCount();
|
||||
const pendingPlay = this.streamingContext.getPendingPlayCount();
|
||||
|
||||
return {
|
||||
pendingDecode, // 待解码包数
|
||||
pendingPlay, // 待播放包数
|
||||
totalPending: pendingDecode + pendingPlay // 总待处理包数
|
||||
};
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲并停止播放
|
||||
clearAllAudio() {
|
||||
log('AudioPlayer: 清空所有音频', 'info');
|
||||
|
||||
// 清空接收队列（使用clear方法保持对象引用）
|
||||
this.queue.clear();
|
||||
|
||||
// 清空流上下文的所有缓冲
|
||||
if (this.streamingContext) {
|
||||
this.streamingContext.clearAllBuffers();
|
||||
}
|
||||
|
||||
log('AudioPlayer: 音频已清空', 'success');
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let audioPlayerInstance = null;
|
||||
|
||||
export function getAudioPlayer() {
|
||||
if (!audioPlayerInstance) {
|
||||
audioPlayerInstance = new AudioPlayer();
|
||||
}
|
||||
return audioPlayerInstance;
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
// Audio recording module
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { initOpusEncoder } from './opus-codec.js?v=0205';
|
||||
import { getAudioPlayer } from './player.js?v=0205';
|
||||
|
||||
// Audio recorder class
|
||||
export class AudioRecorder {
|
||||
constructor() {
|
||||
this.isRecording = false;
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
this.audioProcessor = null;
|
||||
this.audioProcessorType = null;
|
||||
this.audioSource = null;
|
||||
this.opusEncoder = null;
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.visualizationRequest = null;
|
||||
this.recordingTimer = null;
|
||||
this.websocket = null;
|
||||
// Callback functions
|
||||
this.onRecordingStart = null;
|
||||
this.onRecordingStop = null;
|
||||
this.onVisualizerUpdate = null;
|
||||
}
|
||||
|
||||
// Set WebSocket instance
|
||||
setWebSocket(ws) {
|
||||
this.websocket = ws;
|
||||
}
|
||||
|
||||
// Get AudioContext instance
|
||||
getAudioContext() {
|
||||
return getAudioPlayer().getAudioContext();
|
||||
}
|
||||
|
||||
// Initialize encoder
|
||||
initEncoder() {
|
||||
if (!this.opusEncoder) {
|
||||
this.opusEncoder = initOpusEncoder();
|
||||
}
|
||||
return this.opusEncoder;
|
||||
}
|
||||
|
||||
// PCM processor code
|
||||
getAudioProcessorCode() {
|
||||
return `
|
||||
class AudioRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffers = [];
|
||||
this.frameSize = 960;
|
||||
this.buffer = new Int16Array(this.frameSize);
|
||||
this.bufferIndex = 0;
|
||||
this.isRecording = false;
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.command === 'start') {
|
||||
this.isRecording = true;
|
||||
this.port.postMessage({ type: 'status', status: 'started' });
|
||||
} else if (event.data.command === 'stop') {
|
||||
this.isRecording = false;
|
||||
if (this.bufferIndex > 0) {
|
||||
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
||||
this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
this.port.postMessage({ type: 'status', status: 'stopped' });
|
||||
}
|
||||
};
|
||||
}
|
||||
process(inputs, outputs, parameters) {
|
||||
if (!this.isRecording) return true;
|
||||
const input = inputs[0][0];
|
||||
if (!input) return true;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (this.bufferIndex >= this.frameSize) {
|
||||
this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
|
||||
`;
|
||||
}
|
||||
|
||||
// Create audio processor
|
||||
async createAudioProcessor() {
|
||||
this.audioContext = this.getAudioContext();
|
||||
try {
|
||||
if (this.audioContext.audioWorklet) {
|
||||
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
await this.audioContext.audioWorklet.addModule(url);
|
||||
URL.revokeObjectURL(url);
|
||||
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
|
||||
audioProcessor.port.onmessage = (event) => {
|
||||
if (event.data.type === 'buffer') {
|
||||
this.processPCMBuffer(event.data.buffer);
|
||||
}
|
||||
};
|
||||
log('使用AudioWorklet处理音频', 'success');
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
audioProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
return { node: audioProcessor, type: 'worklet' };
|
||||
} else {
|
||||
log('AudioWorklet不可用，使用ScriptProcessorNode作为后备方案', 'warning');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
} catch (error) {
|
||||
log(`创建音频处理器失败: ${error.message}，尝试后备方案`, 'error');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
// Create ScriptProcessor as fallback
|
||||
createScriptProcessor() {
|
||||
try {
|
||||
const frameSize = 4096;
|
||||
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
|
||||
scriptProcessor.onaudioprocess = (event) => {
|
||||
if (!this.isRecording) return;
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
const buffer = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
this.processPCMBuffer(buffer);
|
||||
};
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
scriptProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
log('使用ScriptProcessorNode作为后备方案成功', 'warning');
|
||||
return { node: scriptProcessor, type: 'processor' };
|
||||
} catch (fallbackError) {
|
||||
log(`后备方案也失败: ${fallbackError.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Process PCM buffer data
|
||||
processPCMBuffer(buffer) {
|
||||
if (!this.isRecording) return;
|
||||
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
|
||||
newBuffer.set(this.pcmDataBuffer);
|
||||
newBuffer.set(buffer, this.pcmDataBuffer.length);
|
||||
this.pcmDataBuffer = newBuffer;
|
||||
const samplesPerFrame = 960;
|
||||
while (this.pcmDataBuffer.length >= samplesPerFrame) {
|
||||
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
|
||||
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
|
||||
this.encodeAndSendOpus(frameData);
|
||||
}
|
||||
}
|
||||
|
||||
// Encode and send Opus data
|
||||
encodeAndSendOpus(pcmData = null) {
|
||||
if (!this.opusEncoder) {
|
||||
log('Opus编码器未初始化', 'error');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (pcmData) {
|
||||
const opusData = this.opusEncoder.encode(pcmData);
|
||||
if (opusData && opusData.length > 0) {
|
||||
this.audioBuffers.push(opusData.buffer);
|
||||
this.totalAudioSize += opusData.length;
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.websocket.send(opusData.buffer);
|
||||
} catch (error) {
|
||||
log(`WebSocket发送错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('Opus编码失败，未返回有效数据', 'error');
|
||||
}
|
||||
} else {
|
||||
if (this.pcmDataBuffer.length > 0) {
|
||||
const samplesPerFrame = 960;
|
||||
if (this.pcmDataBuffer.length < samplesPerFrame) {
|
||||
const paddedBuffer = new Int16Array(samplesPerFrame);
|
||||
paddedBuffer.set(this.pcmDataBuffer);
|
||||
this.encodeAndSendOpus(paddedBuffer);
|
||||
} else {
|
||||
this.encodeAndSendOpus(this.pcmDataBuffer.slice(0, samplesPerFrame));
|
||||
}
|
||||
this.pcmDataBuffer = new Int16Array(0);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log(`Opus编码错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// Start recording
|
||||
async start() {
|
||||
if (this.isRecording) return false;
|
||||
try {
|
||||
if (!this.initEncoder()) {
|
||||
log('无法开始录音: Opus编码器初始化失败', 'error');
|
||||
return false;
|
||||
}
|
||||
log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
this.audioContext = this.getAudioContext();
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
const processorResult = await this.createAudioProcessor();
|
||||
if (!processorResult) {
|
||||
log('无法创建音频处理器', 'error');
|
||||
return false;
|
||||
}
|
||||
this.audioProcessor = processorResult.node;
|
||||
this.audioProcessorType = processorResult.type;
|
||||
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 2048;
|
||||
this.audioSource.connect(this.analyser);
|
||||
this.audioSource.connect(this.audioProcessor);
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.isRecording = true;
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'start' });
|
||||
}
|
||||
// Send listening start message
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
log(`已发送录音开始消息`, 'info');
|
||||
} else {
|
||||
log('WebSocket未连接，无法发送开始消息', 'error');
|
||||
return false;
|
||||
}
|
||||
// Start visualization
|
||||
if (this.onVisualizerUpdate) {
|
||||
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
this.startVisualization(dataArray);
|
||||
}
|
||||
// Immediately notify recording start, update button state
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(0);
|
||||
}
|
||||
// Start recording timer
|
||||
let recordingSeconds = 0;
|
||||
this.recordingTimer = setInterval(() => {
|
||||
recordingSeconds += 0.1;
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(recordingSeconds);
|
||||
}
|
||||
}, 100);
|
||||
log('已开始PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音启动错误: ${error.message}`, 'error');
|
||||
this.isRecording = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Start visualization
|
||||
startVisualization(dataArray) {
|
||||
const draw = () => {
|
||||
this.visualizationRequest = requestAnimationFrame(() => draw());
|
||||
if (!this.isRecording) return;
|
||||
this.analyser.getByteFrequencyData(dataArray);
|
||||
if (this.onVisualizerUpdate) {
|
||||
this.onVisualizerUpdate(dataArray);
|
||||
}
|
||||
};
|
||||
draw();
|
||||
}
|
||||
|
||||
// Stop recording
|
||||
stop() {
|
||||
if (!this.isRecording) return false;
|
||||
try {
|
||||
this.isRecording = false;
|
||||
if (this.audioProcessor) {
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'stop' });
|
||||
}
|
||||
this.audioProcessor.disconnect();
|
||||
this.audioProcessor = null;
|
||||
}
|
||||
if (this.audioSource) {
|
||||
this.audioSource.disconnect();
|
||||
this.audioSource = null;
|
||||
}
|
||||
if (this.visualizationRequest) {
|
||||
cancelAnimationFrame(this.visualizationRequest);
|
||||
this.visualizationRequest = null;
|
||||
}
|
||||
if (this.recordingTimer) {
|
||||
clearInterval(this.recordingTimer);
|
||||
this.recordingTimer = null;
|
||||
}
|
||||
// Encode and send remaining data
|
||||
this.encodeAndSendOpus();
|
||||
// Send end signal
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const emptyOpusFrame = new Uint8Array(0);
|
||||
this.websocket.send(emptyOpusFrame);
|
||||
log('已发送录音停止信号', 'info');
|
||||
}
|
||||
if (this.onRecordingStop) {
|
||||
this.onRecordingStop();
|
||||
}
|
||||
log('已停止PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音停止错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Get analyser
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
let audioRecorderInstance = null;
|
||||
|
||||
export function getAudioRecorder() {
|
||||
if (!audioRecorderInstance) {
|
||||
audioRecorderInstance = new AudioRecorder();
|
||||
}
|
||||
return audioRecorderInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if microphone is available
|
||||
* @returns {Promise<boolean>} Returns true if available, false if not available
|
||||
*/
|
||||
export async function checkMicrophoneAvailability() {
|
||||
// Check if browser supports getUserMedia API
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log('浏览器不支持getUserMedia API', 'warning');
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
// Try to access microphone
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
// Immediately stop all tracks to release microphone
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
log('麦克风可用性检查成功', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`麦克风不可用: ${error.message}`, 'warning');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if it is HTTP non-localhost access
|
||||
* @returns {boolean} Returns true if it is HTTP non-localhost access
|
||||
*/
|
||||
export function isHttpNonLocalhost() {
|
||||
const protocol = window.location.protocol;
|
||||
const hostname = window.location.hostname;
|
||||
// Check if it is HTTP protocol
|
||||
if (protocol !== 'http:') {
|
||||
return false;
|
||||
}
|
||||
// localhost and 127.0.0.1 can use microphone
|
||||
if (hostname === 'localhost' || hostname === '127.0.0.1') {
|
||||
return false;
|
||||
}
|
||||
// Private IP addresses can also use microphone (browser allows)
|
||||
if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
|
||||
return false;
|
||||
}
|
||||
// Other HTTP access is considered non-localhost
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
// 音频流播放上下文类
|
||||
export class StreamingContext {
|
||||
constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
this.opusDecoder = opusDecoder;
|
||||
this.audioContext = audioContext;
|
||||
|
||||
// 音频参数
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
this.minAudioDuration = minAudioDuration;
|
||||
|
||||
// 初始化队列和状态
|
||||
this.queue = []; // 已解码的PCM队列。正在播放
|
||||
this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放
|
||||
this.pendingAudioBufferQueue = []; // 待处理的缓存队列
|
||||
this.audioBufferQueue = new BlockingQueue(); // 缓存队列
|
||||
this.playing = false; // 是否正在播放
|
||||
this.endOfStream = false; // 是否收到结束信号
|
||||
this.source = null; // 当前音频源
|
||||
this.totalSamples = 0; // 累积的总样本数
|
||||
this.lastPlayTime = 0; // 上次播放的时间戳
|
||||
this.scheduledEndTime = 0; // 已调度音频的结束时间
|
||||
|
||||
// 初始化分析器节点（供Live2D使用）
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 256;
|
||||
}
|
||||
|
||||
// 缓存音频数组
|
||||
pushAudioBuffer(item) {
|
||||
this.audioBufferQueue.enqueue(...item);
|
||||
}
|
||||
|
||||
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
|
||||
async getPendingAudioBufferQueue() {
|
||||
// 等待数据到达并获取
|
||||
const data = await this.audioBufferQueue.dequeue();
|
||||
// 赋值给待处理队列
|
||||
this.pendingAudioBufferQueue = data;
|
||||
}
|
||||
|
||||
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
|
||||
async getQueue(minSamples) {
|
||||
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
|
||||
|
||||
// 等待数据并获取
|
||||
const tempArray = await this.activeQueue.dequeue(num);
|
||||
this.queue.push(...tempArray);
|
||||
}
|
||||
|
||||
// 将Int16音频数据转换为Float32音频数据
|
||||
convertInt16ToFloat32(int16Data) {
|
||||
const float32Data = new Float32Array(int16Data.length);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
// 将[-32768,32767]范围转换为[-1,1]，统一使用32768.0避免不对称失真
|
||||
float32Data[i] = int16Data[i] / 32768.0;
|
||||
}
|
||||
return float32Data;
|
||||
}
|
||||
|
||||
// 获取待解码包数
|
||||
getPendingDecodeCount() {
|
||||
return this.audioBufferQueue.length + this.pendingAudioBufferQueue.length;
|
||||
}
|
||||
|
||||
// 获取待播放样本数（转换为包数，每包960样本）
|
||||
getPendingPlayCount() {
|
||||
// 计算已在队列中的样本
|
||||
const queuedSamples = this.activeQueue.length + this.queue.length;
|
||||
|
||||
// 计算已调度但未播放的样本（在Web Audio缓冲区中）
|
||||
let scheduledSamples = 0;
|
||||
if (this.playing && this.scheduledEndTime) {
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const remainingTime = Math.max(0, this.scheduledEndTime - currentTime);
|
||||
scheduledSamples = Math.floor(remainingTime * this.sampleRate);
|
||||
}
|
||||
|
||||
const totalSamples = queuedSamples + scheduledSamples;
|
||||
return Math.ceil(totalSamples / 960);
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲
|
||||
clearAllBuffers() {
|
||||
log('清空所有音频缓冲', 'info');
|
||||
|
||||
// 清空所有队列（使用clear方法保持对象引用）
|
||||
this.audioBufferQueue.clear();
|
||||
this.pendingAudioBufferQueue = [];
|
||||
this.activeQueue.clear();
|
||||
this.queue = [];
|
||||
|
||||
// 停止当前播放的音频源
|
||||
if (this.source) {
|
||||
try {
|
||||
this.source.stop();
|
||||
this.source.disconnect();
|
||||
} catch (e) {
|
||||
// 忽略已经停止的错误
|
||||
}
|
||||
this.source = null;
|
||||
}
|
||||
|
||||
// 重置状态
|
||||
this.playing = false;
|
||||
this.scheduledEndTime = this.audioContext.currentTime;
|
||||
this.totalSamples = 0;
|
||||
|
||||
log('音频缓冲已清空', 'success');
|
||||
}
|
||||
|
||||
// 获取分析器节点（供Live2D使用）
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
|
||||
// 将Opus数据解码为PCM
|
||||
async decodeOpusFrames() {
|
||||
if (!this.opusDecoder) {
|
||||
log('Opus解码器未初始化，无法解码', 'error');
|
||||
return;
|
||||
} else {
|
||||
log('Opus解码器启动', 'info');
|
||||
}
|
||||
|
||||
while (true) {
|
||||
let decodedSamples = [];
|
||||
for (const frame of this.pendingAudioBufferQueue) {
|
||||
try {
|
||||
// 使用Opus解码器解码
|
||||
const frameData = this.opusDecoder.decode(frame);
|
||||
if (frameData && frameData.length > 0) {
|
||||
// 转换为Float32
|
||||
const floatData = this.convertInt16ToFloat32(frameData);
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < floatData.length; i++) {
|
||||
decodedSamples.push(floatData[i]);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log("Opus解码失败: " + error.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedSamples.length > 0) {
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < decodedSamples.length; i++) {
|
||||
this.activeQueue.enqueue(decodedSamples[i]);
|
||||
}
|
||||
this.totalSamples += decodedSamples.length;
|
||||
} else {
|
||||
log('没有成功解码的样本', 'warning');
|
||||
}
|
||||
await this.getPendingAudioBufferQueue();
|
||||
}
|
||||
}
|
||||
|
||||
// 开始播放音频
|
||||
async startPlaying() {
|
||||
this.scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
|
||||
|
||||
while (true) {
|
||||
// 初始缓冲：等待足够的样本再开始播放
|
||||
const minSamples = this.sampleRate * this.minAudioDuration * 2;
|
||||
if (!this.playing && this.queue.length < minSamples) {
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
this.playing = true;
|
||||
|
||||
// 持续播放队列中的音频，每次播放一个小块
|
||||
while (this.playing && this.queue.length > 0) {
|
||||
// 每次播放120ms的音频（2个Opus包）
|
||||
const playDuration = 0.12;
|
||||
const targetSamples = Math.floor(this.sampleRate * playDuration);
|
||||
const actualSamples = Math.min(this.queue.length, targetSamples);
|
||||
|
||||
if (actualSamples === 0) break;
|
||||
|
||||
const currentSamples = this.queue.splice(0, actualSamples);
|
||||
const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate);
|
||||
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
|
||||
|
||||
// 创建音频源
|
||||
this.source = this.audioContext.createBufferSource();
|
||||
this.source.buffer = audioBuffer;
|
||||
|
||||
// 精确调度播放时间
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const startTime = Math.max(this.scheduledEndTime, currentTime);
|
||||
|
||||
// 连接到分析器和输出
|
||||
this.source.connect(this.analyser);
|
||||
this.source.connect(this.audioContext.destination);
|
||||
|
||||
log(`调度播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'debug');
|
||||
this.source.start(startTime);
|
||||
|
||||
// 更新下一个音频块的调度时间
|
||||
const duration = audioBuffer.duration;
|
||||
this.scheduledEndTime = startTime + duration;
|
||||
this.lastPlayTime = startTime;
|
||||
|
||||
// 如果队列中数据不足，等待新数据
|
||||
if (this.queue.length < targetSamples) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 等待新数据
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 创建streamingContext实例的工厂函数
|
||||
export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration);
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
// ==========================================
|
||||
// MCP 工具管理逻辑
|
||||
// ==========================================
|
||||
|
||||
// 全局变量
|
||||
let mcpTools = [];
|
||||
let mcpEditingIndex = null;
|
||||
let mcpProperties = [];
|
||||
let websocket = null; // 将从外部设置
|
||||
|
||||
/**
|
||||
* 设置 WebSocket 实例
|
||||
* @param {WebSocket} ws - WebSocket 连接实例
|
||||
*/
|
||||
export function setWebSocket(ws) {
|
||||
websocket = ws;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化 MCP 工具
|
||||
*/
|
||||
export async function initMcpTools() {
|
||||
// 加载默认工具数据
|
||||
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
|
||||
const savedTools = localStorage.getItem('mcpTools');
|
||||
if (savedTools) {
|
||||
try {
|
||||
const parsedTools = JSON.parse(savedTools);
|
||||
// 合并默认工具和用户保存的工具，保留用户自定义的工具
|
||||
const defaultToolNames = new Set(defaultMcpTools.map(t => t.name));
|
||||
// 添加默认工具中不存在的新工具
|
||||
parsedTools.forEach(tool => {
|
||||
if (!defaultToolNames.has(tool.name)) {
|
||||
defaultMcpTools.push(tool);
|
||||
}
|
||||
});
|
||||
mcpTools = defaultMcpTools;
|
||||
} catch (e) {
|
||||
log('加载MCP工具失败，使用默认工具', 'warning');
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
} else {
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
renderMcpTools();
|
||||
setupMcpEventListeners();
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染工具列表
|
||||
*/
|
||||
function renderMcpTools() {
|
||||
const container = document.getElementById('mcpToolsContainer');
|
||||
const countSpan = document.getElementById('mcpToolsCount');
|
||||
if (!container) {
|
||||
return; // Container not found, skip rendering
|
||||
}
|
||||
if (countSpan) {
|
||||
countSpan.textContent = `${mcpTools.length} 个工具`;
|
||||
}
|
||||
if (mcpTools.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
|
||||
return;
|
||||
}
|
||||
container.innerHTML = mcpTools.map((tool, index) => {
|
||||
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
|
||||
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
|
||||
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
|
||||
return `
|
||||
<div class="mcp-tool-card">
|
||||
<div class="mcp-tool-header">
|
||||
<div class="mcp-tool-name">${tool.name}</div>
|
||||
<div class="mcp-tool-actions">
|
||||
<button class="mcp-edit-btn" onclick="window.mcpModule.editMcpTool(${index})">
|
||||
✏️ 编辑
|
||||
</button>
|
||||
<button class="mcp-delete-btn" onclick="window.mcpModule.deleteMcpTool(${index})">
|
||||
🗑️ 删除
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mcp-tool-description">${tool.description}</div>
|
||||
<div class="mcp-tool-info">
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">参数数量:</span>
|
||||
<span class="mcp-tool-info-value">${paramCount} 个 ${requiredCount > 0 ? `(${requiredCount} 个必填)` : ''}</span>
|
||||
</div>
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">模拟返回:</span>
|
||||
<span class="mcp-tool-info-value">${hasMockResponse ? '✅ 已配置: ' + JSON.stringify(tool.mockResponse) : '⚪ 使用默认'}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染参数列表
|
||||
*/
|
||||
function renderMcpProperties() {
|
||||
const container = document.getElementById('mcpPropertiesContainer');
|
||||
const emptyState = document.getElementById('mcpEmptyState');
|
||||
if (!container) {
|
||||
return; // Container not found, skip rendering
|
||||
}
|
||||
if (mcpProperties.length === 0) {
|
||||
if (emptyState) {
|
||||
emptyState.style.display = 'block';
|
||||
}
|
||||
container.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
if (emptyState) {
|
||||
emptyState.style.display = 'none';
|
||||
}
|
||||
container.innerHTML = mcpProperties.map((prop, index) => `
|
||||
<div class="mcp-property-card" onclick="window.mcpModule.editMcpProperty(${index})">
|
||||
<div class="mcp-property-row-label">
|
||||
<span class="mcp-property-label">参数名称</span>
|
||||
<span class="mcp-property-value">${prop.name}${prop.required ? ' <span class="mcp-property-required-badge">[必填]</span>' : ''}</span>
|
||||
</div>
|
||||
<div class="mcp-property-row-label">
|
||||
<span class="mcp-property-label">数据类型</span>
|
||||
<span class="mcp-property-value">${getTypeLabel(prop.type)}</span>
|
||||
</div>
|
||||
<div class="mcp-property-row-label">
|
||||
<span class="mcp-property-label">描述</span>
|
||||
<span class="mcp-property-value">${prop.description || '-'}</span>
|
||||
</div>
|
||||
<div class="mcp-property-row-action">
|
||||
<button class="mcp-property-delete-btn" onclick="event.stopPropagation(); window.mcpModule.deleteMcpProperty(${index})">删除</button>
|
||||
</div>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取数据类型标签
|
||||
*/
|
||||
function getTypeLabel(type) {
|
||||
const typeMap = {
|
||||
'string': '字符串',
|
||||
'integer': '整数',
|
||||
'number': '数字',
|
||||
'boolean': '布尔值',
|
||||
'array': '数组',
|
||||
'object': '对象'
|
||||
};
|
||||
return typeMap[type] || type;
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加参数 - 打开参数编辑模态框
|
||||
*/
|
||||
function addMcpProperty() {
|
||||
openPropertyModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 编辑参数 - 打开参数编辑模态框
|
||||
*/
|
||||
function editMcpProperty(index) {
|
||||
openPropertyModal(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开参数编辑模态框
|
||||
*/
|
||||
function openPropertyModal(index = null) {
|
||||
const form = document.getElementById('mcpPropertyForm');
|
||||
const title = document.getElementById('mcpPropertyModalTitle');
|
||||
document.getElementById('mcpPropertyIndex').value = index !== null ? index : -1;
|
||||
|
||||
if (index !== null) {
|
||||
const prop = mcpProperties[index];
|
||||
title.textContent = '编辑参数';
|
||||
document.getElementById('mcpPropertyName').value = prop.name;
|
||||
document.getElementById('mcpPropertyType').value = prop.type || 'string';
|
||||
document.getElementById('mcpPropertyMinimum').value = prop.minimum !== undefined ? prop.minimum : '';
|
||||
document.getElementById('mcpPropertyMaximum').value = prop.maximum !== undefined ? prop.maximum : '';
|
||||
document.getElementById('mcpPropertyDescription').value = prop.description || '';
|
||||
document.getElementById('mcpPropertyRequired').checked = prop.required || false;
|
||||
} else {
|
||||
title.textContent = '添加参数';
|
||||
form.reset();
|
||||
document.getElementById('mcpPropertyName').value = `param_${mcpProperties.length + 1}`;
|
||||
document.getElementById('mcpPropertyType').value = 'string';
|
||||
document.getElementById('mcpPropertyMinimum').value = '';
|
||||
document.getElementById('mcpPropertyMaximum').value = '';
|
||||
document.getElementById('mcpPropertyDescription').value = '';
|
||||
document.getElementById('mcpPropertyRequired').checked = false;
|
||||
}
|
||||
|
||||
updatePropertyRangeVisibility();
|
||||
document.getElementById('mcpPropertyModal').style.display = 'flex';
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭参数编辑模态框
|
||||
*/
|
||||
function closePropertyModal() {
|
||||
document.getElementById('mcpPropertyModal').style.display = 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新数值范围输入框的可见性
|
||||
*/
|
||||
function updatePropertyRangeVisibility() {
|
||||
const type = document.getElementById('mcpPropertyType').value;
|
||||
const rangeGroup = document.getElementById('mcpPropertyRangeGroup');
|
||||
if (type === 'integer' || type === 'number') {
|
||||
rangeGroup.style.display = 'block';
|
||||
} else {
|
||||
rangeGroup.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理参数表单提交
|
||||
*/
|
||||
function handlePropertySubmit(e) {
|
||||
e.preventDefault();
|
||||
const index = parseInt(document.getElementById('mcpPropertyIndex').value);
|
||||
const name = document.getElementById('mcpPropertyName').value.trim();
|
||||
const type = document.getElementById('mcpPropertyType').value;
|
||||
const minimum = document.getElementById('mcpPropertyMinimum').value;
|
||||
const maximum = document.getElementById('mcpPropertyMaximum').value;
|
||||
const description = document.getElementById('mcpPropertyDescription').value.trim();
|
||||
const required = document.getElementById('mcpPropertyRequired').checked;
|
||||
|
||||
// 检查名称重复
|
||||
const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === name);
|
||||
if (isDuplicate) {
|
||||
alert('参数名称已存在，请使用不同的名称');
|
||||
return;
|
||||
}
|
||||
|
||||
const propData = {
|
||||
name,
|
||||
type,
|
||||
description,
|
||||
required
|
||||
};
|
||||
|
||||
// 数值类型添加范围限制
|
||||
if (type === 'integer' || type === 'number') {
|
||||
if (minimum !== '') {
|
||||
propData.minimum = parseFloat(minimum);
|
||||
}
|
||||
if (maximum !== '') {
|
||||
propData.maximum = parseFloat(maximum);
|
||||
}
|
||||
}
|
||||
|
||||
if (index >= 0) {
|
||||
mcpProperties[index] = propData;
|
||||
} else {
|
||||
mcpProperties.push(propData);
|
||||
}
|
||||
|
||||
renderMcpProperties();
|
||||
closePropertyModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除参数
|
||||
*/
|
||||
function deleteMcpProperty(index) {
|
||||
mcpProperties.splice(index, 1);
|
||||
renderMcpProperties();
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置事件监听
|
||||
*/
|
||||
function setupMcpEventListeners() {
|
||||
const panel = document.getElementById('mcpToolsPanel');
|
||||
const addBtn = document.getElementById('addMcpToolBtn');
|
||||
const modal = document.getElementById('mcpToolModal');
|
||||
const closeBtn = document.getElementById('closeMcpModalBtn');
|
||||
const cancelBtn = document.getElementById('cancelMcpBtn');
|
||||
const form = document.getElementById('mcpToolForm');
|
||||
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
|
||||
|
||||
// 参数编辑模态框相关元素
|
||||
const propertyModal = document.getElementById('mcpPropertyModal');
|
||||
const closePropertyBtn = document.getElementById('closeMcpPropertyModalBtn');
|
||||
const cancelPropertyBtn = document.getElementById('cancelMcpPropertyBtn');
|
||||
const propertyForm = document.getElementById('mcpPropertyForm');
|
||||
const propertyTypeSelect = document.getElementById('mcpPropertyType');
|
||||
|
||||
// Return early if required elements don't exist (e.g., in test environment)
|
||||
if (!panel || !addBtn || !modal || !closeBtn || !cancelBtn || !form || !addPropertyBtn) {
|
||||
return;
|
||||
}
|
||||
addBtn.addEventListener('click', () => openMcpModal());
|
||||
closeBtn.addEventListener('click', closeMcpModal);
|
||||
cancelBtn.addEventListener('click', closeMcpModal);
|
||||
addPropertyBtn.addEventListener('click', addMcpProperty);
|
||||
form.addEventListener('submit', handleMcpSubmit);
|
||||
|
||||
// 参数编辑模态框事件
|
||||
if (propertyModal && closePropertyBtn && cancelPropertyBtn && propertyForm && propertyTypeSelect) {
|
||||
closePropertyBtn.addEventListener('click', closePropertyModal);
|
||||
cancelPropertyBtn.addEventListener('click', closePropertyModal);
|
||||
propertyForm.addEventListener('submit', handlePropertySubmit);
|
||||
propertyTypeSelect.addEventListener('change', updatePropertyRangeVisibility);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开模态框
|
||||
*/
|
||||
function openMcpModal(index = null) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
mcpEditingIndex = index;
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
if (index !== null) {
|
||||
document.getElementById('mcpModalTitle').textContent = '编辑工具';
|
||||
const tool = mcpTools[index];
|
||||
document.getElementById('mcpToolName').value = tool.name;
|
||||
document.getElementById('mcpToolDescription').value = tool.description;
|
||||
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
|
||||
mcpProperties = [];
|
||||
const schema = tool.inputSchema;
|
||||
if (schema.properties) {
|
||||
Object.keys(schema.properties).forEach(key => {
|
||||
const prop = schema.properties[key];
|
||||
mcpProperties.push({
|
||||
name: key,
|
||||
type: prop.type || 'string',
|
||||
minimum: prop.minimum,
|
||||
maximum: prop.maximum,
|
||||
description: prop.description || '',
|
||||
required: schema.required && schema.required.includes(key)
|
||||
});
|
||||
});
|
||||
}
|
||||
} else {
|
||||
document.getElementById('mcpModalTitle').textContent = '添加工具';
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
}
|
||||
renderMcpProperties();
|
||||
document.getElementById('mcpToolModal').style.display = 'flex';
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭模态框
|
||||
*/
|
||||
function closeMcpModal() {
|
||||
document.getElementById('mcpToolModal').style.display = 'none';
|
||||
mcpEditingIndex = null;
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
document.getElementById('mcpErrorContainer').innerHTML = '';
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理表单提交
|
||||
*/
|
||||
function handleMcpSubmit(e) {
|
||||
e.preventDefault();
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
const name = document.getElementById('mcpToolName').value.trim();
|
||||
const description = document.getElementById('mcpToolDescription').value.trim();
|
||||
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
|
||||
// 检查名称重复
|
||||
const isDuplicate = mcpTools.some((tool, index) => tool.name === name && index !== mcpEditingIndex);
|
||||
if (isDuplicate) {
|
||||
showMcpError('工具名称已存在，请使用不同的名称');
|
||||
return;
|
||||
}
|
||||
// 解析模拟返回结果
|
||||
let mockResponse = null;
|
||||
if (mockResponseText) {
|
||||
try {
|
||||
mockResponse = JSON.parse(mockResponseText);
|
||||
} catch (e) {
|
||||
showMcpError('模拟返回结果不是有效的 JSON 格式: ' + e.message);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// 构建 inputSchema
|
||||
const inputSchema = { type: "object", properties: {}, required: [] };
|
||||
mcpProperties.forEach(prop => {
|
||||
const propSchema = { type: prop.type };
|
||||
if (prop.description) {
|
||||
propSchema.description = prop.description;
|
||||
}
|
||||
if ((prop.type === 'integer' || prop.type === 'number')) {
|
||||
if (prop.minimum !== undefined && prop.minimum !== '') {
|
||||
propSchema.minimum = prop.minimum;
|
||||
}
|
||||
if (prop.maximum !== undefined && prop.maximum !== '') {
|
||||
propSchema.maximum = prop.maximum;
|
||||
}
|
||||
}
|
||||
inputSchema.properties[prop.name] = propSchema;
|
||||
if (prop.required) {
|
||||
inputSchema.required.push(prop.name);
|
||||
}
|
||||
});
|
||||
if (inputSchema.required.length === 0) {
|
||||
delete inputSchema.required;
|
||||
}
|
||||
const tool = { name, description, inputSchema, mockResponse };
|
||||
if (mcpEditingIndex !== null) {
|
||||
mcpTools[mcpEditingIndex] = tool;
|
||||
log(`已更新工具: ${name}`, 'success');
|
||||
} else {
|
||||
mcpTools.push(tool);
|
||||
log(`已添加工具: ${name}`, 'success');
|
||||
}
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
closeMcpModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 显示错误
|
||||
*/
|
||||
function showMcpError(message) {
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = `<div class="mcp-error">${message}</div>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编辑工具
|
||||
*/
|
||||
function editMcpTool(index) {
|
||||
openMcpModal(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除工具
|
||||
*/
|
||||
function deleteMcpTool(index) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
if (confirm(`确定要删除工具 "${mcpTools[index].name}" 吗？`)) {
|
||||
const toolName = mcpTools[index].name;
|
||||
mcpTools.splice(index, 1);
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
log(`已删除工具: ${toolName}`, 'info');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存工具
|
||||
*/
|
||||
function saveMcpTools() {
|
||||
localStorage.setItem('mcpTools', JSON.stringify(mcpTools));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取工具列表
|
||||
*/
|
||||
export function getMcpTools() {
|
||||
return mcpTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行工具调用
|
||||
*/
|
||||
export async function executeMcpTool(toolName, toolArgs) {
|
||||
const tool = mcpTools.find(t => t.name === toolName);
|
||||
if (!tool) {
|
||||
log(`未找到工具: ${toolName}`, 'error');
|
||||
return { success: false, error: `未知工具: ${toolName}` };
|
||||
}
|
||||
|
||||
// 处理拍照工具
|
||||
if (toolName === 'self_camera_take_photo') {
|
||||
if (typeof window.takePhoto === 'function') {
|
||||
const question = toolArgs && toolArgs.question ? toolArgs.question : '描述一下看到的物品';
|
||||
log(`正在执行拍照: ${question}`, 'info');
|
||||
const result = await window.takePhoto(question);
|
||||
return result;
|
||||
} else {
|
||||
log('拍照功能不可用', 'warning');
|
||||
return { success: false, error: '摄像头未启动或不支持拍照功能' };
|
||||
}
|
||||
}
|
||||
|
||||
// 如果有模拟返回结果，使用它
|
||||
if (tool.mockResponse) {
|
||||
// 替换模板变量
|
||||
let responseStr = JSON.stringify(tool.mockResponse);
|
||||
// 替换 ${paramName} 格式的变量
|
||||
if (toolArgs) {
|
||||
Object.keys(toolArgs).forEach(key => {
|
||||
const regex = new RegExp(`\\$\\{${key}\\}`, 'g');
|
||||
responseStr = responseStr.replace(regex, toolArgs[key]);
|
||||
});
|
||||
}
|
||||
try {
|
||||
const response = JSON.parse(responseStr);
|
||||
log(`工具 ${toolName} 执行成功，返回模拟结果: ${responseStr}`, 'success');
|
||||
return response;
|
||||
} catch (e) {
|
||||
log(`解析模拟返回结果失败: ${e.message}`, 'error');
|
||||
return tool.mockResponse;
|
||||
}
|
||||
}
|
||||
// 没有模拟返回结果，返回默认成功消息
|
||||
log(`工具 ${toolName} 执行成功，返回默认结果`, 'success');
|
||||
return { success: true, message: `工具 ${toolName} 执行成功`, tool: toolName, arguments: toolArgs };
|
||||
}
|
||||
|
||||
// 暴露全局方法供 HTML 内联事件调用
|
||||
window.mcpModule = { addMcpProperty, editMcpProperty, deleteMcpProperty, editMcpTool, deleteMcpTool };
|
||||
@@ -0,0 +1,109 @@
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
// WebSocket 连接
|
||||
export async function webSocketConnect(otaUrl, config) {
|
||||
|
||||
if (!validateConfig(config)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 发送OTA请求并获取返回的websocket信息
|
||||
const otaResult = await sendOTA(otaUrl, config);
|
||||
if (!otaResult) {
|
||||
log('无法从OTA服务器获取信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 从OTA响应中提取websocket信息
|
||||
const { websocket } = otaResult;
|
||||
if (!websocket || !websocket.url) {
|
||||
log('OTA响应中缺少websocket信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 使用OTA返回的websocket URL
|
||||
let connUrl = new URL(websocket.url);
|
||||
|
||||
// 添加token参数（从OTA响应中获取）
|
||||
if (websocket.token) {
|
||||
if (websocket.token.startsWith("Bearer ")) {
|
||||
connUrl.searchParams.append('authorization', websocket.token);
|
||||
} else {
|
||||
connUrl.searchParams.append('authorization', 'Bearer ' + websocket.token);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加认证参数（保持原有逻辑）
|
||||
connUrl.searchParams.append('device-id', config.deviceId);
|
||||
connUrl.searchParams.append('client-id', config.clientId);
|
||||
|
||||
const wsurl = connUrl.toString()
|
||||
|
||||
log(`正在连接: ${wsurl}`, 'info');
|
||||
|
||||
if (wsurl) {
|
||||
document.getElementById('serverUrl').value = wsurl;
|
||||
}
|
||||
|
||||
return new WebSocket(connUrl.toString());
|
||||
}
|
||||
|
||||
// 验证配置
|
||||
function validateConfig(config) {
|
||||
if (!config.deviceMac) {
|
||||
log('设备MAC地址不能为空', 'error');
|
||||
return false;
|
||||
}
|
||||
if (!config.clientId) {
|
||||
log('客户端ID不能为空', 'error');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// OTA发送请求，验证状态，并返回响应数据
|
||||
async function sendOTA(otaUrl, config) {
|
||||
try {
|
||||
const res = await fetch(otaUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Device-Id': config.deviceId,
|
||||
'Client-Id': config.clientId
|
||||
},
|
||||
body: JSON.stringify({
|
||||
version: 0,
|
||||
uuid: '',
|
||||
application: {
|
||||
name: 'xiaozhi-web-test',
|
||||
version: '1.0.0',
|
||||
compile_time: '2025-04-16 10:00:00',
|
||||
idf_version: '4.4.3',
|
||||
elf_sha256: '1234567890abcdef1234567890abcdef1234567890abcdef'
|
||||
},
|
||||
ota: { label: 'xiaozhi-web-test' },
|
||||
board: {
|
||||
type: config.deviceName,
|
||||
ssid: 'xiaozhi-web-test',
|
||||
rssi: 0,
|
||||
channel: 0,
|
||||
ip: '192.168.1.1',
|
||||
mac: config.deviceMac
|
||||
},
|
||||
flash_size: 0,
|
||||
minimum_free_heap_size: 0,
|
||||
mac_address: config.deviceMac,
|
||||
chip_model_name: '',
|
||||
chip_info: { model: 0, cores: 0, revision: 0, features: 0 },
|
||||
partition_table: [{ label: '', type: 0, subtype: 0, address: 0, size: 0 }]
|
||||
})
|
||||
});
|
||||
|
||||
if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
|
||||
|
||||
const result = await res.json();
|
||||
return result; // 返回完整的响应数据
|
||||
} catch (err) {
|
||||
return null; // 失败返回null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { uiController } from '../../ui/controller.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
let wakewordSocket = null;
|
||||
let reconnectTimer = null;
|
||||
let reconnectAttempts = 0;
|
||||
let shouldReconnect = true;
|
||||
let wakewordRequestSeq = 0;
|
||||
let onNextBridgeConnectedCallback = null;
|
||||
|
||||
const pendingWakewordRequests = new Map();
|
||||
|
||||
export function startWakewordBridgeListener() {
|
||||
if (wakewordSocket) {
|
||||
return wakewordSocket;
|
||||
}
|
||||
|
||||
shouldReconnect = true;
|
||||
log('正在连接本地唤醒事件桥...', 'info');
|
||||
tryConnect();
|
||||
return wakewordSocket;
|
||||
}
|
||||
|
||||
function tryConnect() {
|
||||
const bridgeUrl = buildWakewordBridgeUrl();
|
||||
|
||||
try {
|
||||
wakewordSocket = new WebSocket(bridgeUrl);
|
||||
wakewordSocket.onopen = () => {
|
||||
reconnectAttempts = 0;
|
||||
log(`本地唤醒事件桥已连接: ${bridgeUrl}`, 'success');
|
||||
// 连接成功后自动保存地址，刷新后仍能记住
|
||||
localStorage.setItem('xz_tester_wakewordWsUrl', bridgeUrl);
|
||||
const urlInput = document.getElementById('wakewordWsUrl');
|
||||
if (urlInput) urlInput.value = bridgeUrl;
|
||||
};
|
||||
|
||||
wakewordSocket.onerror = () => {
|
||||
log(`本地唤醒事件桥连接失败: ${bridgeUrl}`, 'error');
|
||||
};
|
||||
|
||||
wakewordSocket.onmessage = async (event) => {
|
||||
try {
|
||||
const message = parseWakewordBridgeMessage(event.data);
|
||||
if (message.requestId && pendingWakewordRequests.has(message.requestId)) {
|
||||
settleWakewordRequest(message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.success === false) {
|
||||
log(`本地唤醒事件桥返回错误: ${message.error || '未知错误'}`, 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'bridge_connected') {
|
||||
log('本地唤醒监听已就绪', 'info');
|
||||
if (onNextBridgeConnectedCallback) {
|
||||
const cb = onNextBridgeConnectedCallback;
|
||||
onNextBridgeConnectedCallback = null;
|
||||
cb(message);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'service_ready') {
|
||||
log('本地唤醒服务已启动', 'info');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'wakeword_config') {
|
||||
uiController.applyWakewordConfig(message.payload || {});
|
||||
log('已同步本地唤醒词配置', 'info');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'service_stopping') {
|
||||
log('本地唤醒服务正在停止', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type === 'wake_word_detected') {
|
||||
const wakeWord = message.payload?.wake_word || '唤醒词';
|
||||
log(`检测到本地唤醒事件: ${wakeWord}`, 'info');
|
||||
await uiController.triggerWakewordDial(wakeWord);
|
||||
}
|
||||
} catch (error) {
|
||||
log(`解析本地唤醒事件失败: ${error.message}`, 'error');
|
||||
}
|
||||
};
|
||||
|
||||
wakewordSocket.onclose = () => {
|
||||
if (wakewordSocket) {
|
||||
wakewordSocket = null;
|
||||
}
|
||||
|
||||
rejectAllWakewordRequests('本地唤醒事件桥已断开');
|
||||
|
||||
if (!shouldReconnect) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (reconnectTimer) {
|
||||
return;
|
||||
}
|
||||
|
||||
reconnectAttempts += 1;
|
||||
const delay = Math.min(1000 * reconnectAttempts, 5000);
|
||||
log(`本地唤醒事件桥将在 ${delay}ms 后重连: ${bridgeUrl}`, 'warning');
|
||||
reconnectTimer = window.setTimeout(() => {
|
||||
reconnectTimer = null;
|
||||
tryConnect();
|
||||
}, delay);
|
||||
};
|
||||
|
||||
return wakewordSocket;
|
||||
} catch (error) {
|
||||
log(`启动本地唤醒监听失败: ${error.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function stopWakewordBridgeListener() {
|
||||
shouldReconnect = false;
|
||||
|
||||
if (reconnectTimer) {
|
||||
window.clearTimeout(reconnectTimer);
|
||||
reconnectTimer = null;
|
||||
}
|
||||
|
||||
if (!wakewordSocket) {
|
||||
return;
|
||||
}
|
||||
|
||||
wakewordSocket.onclose = null;
|
||||
wakewordSocket.close();
|
||||
wakewordSocket = null;
|
||||
}
|
||||
|
||||
export function sendWakewordBridgeMessage(type, payload = {}, requestId = null) {
|
||||
if (!wakewordSocket || wakewordSocket.readyState !== WebSocket.OPEN) {
|
||||
log('本地唤醒事件桥未连接，无法发送消息', 'warning');
|
||||
return false;
|
||||
}
|
||||
|
||||
wakewordSocket.send(JSON.stringify({
|
||||
type,
|
||||
requestId,
|
||||
payload,
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
|
||||
export function requestWakewordBridge(type, payload = {}, timeout = 5000) {
|
||||
const requestId = `wakeword-${Date.now()}-${++wakewordRequestSeq}`;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = window.setTimeout(() => {
|
||||
pendingWakewordRequests.delete(requestId);
|
||||
reject(new Error('本地唤醒服务响应超时'));
|
||||
}, timeout);
|
||||
|
||||
pendingWakewordRequests.set(requestId, { resolve, reject, timer });
|
||||
|
||||
if (!sendWakewordBridgeMessage(type, payload, requestId)) {
|
||||
window.clearTimeout(timer);
|
||||
pendingWakewordRequests.delete(requestId);
|
||||
reject(new Error('本地唤醒事件桥未连接'));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function getWakewordBridgeUrl() {
|
||||
if (wakewordSocket && wakewordSocket.url) {
|
||||
return wakewordSocket.url;
|
||||
}
|
||||
return buildWakewordBridgeUrl();
|
||||
}
|
||||
|
||||
export function onNextBridgeConnected(callback) {
|
||||
onNextBridgeConnectedCallback = callback;
|
||||
}
|
||||
|
||||
function buildWakewordBridgeUrl() {
|
||||
const configured = localStorage.getItem('xz_tester_wakewordWsUrl');
|
||||
if (configured && configured.trim()) {
|
||||
return configured.trim();
|
||||
}
|
||||
return 'ws://127.0.0.1:8006/wakeword-ws';
|
||||
}
|
||||
|
||||
function parseWakewordBridgeMessage(rawData) {
|
||||
const message = JSON.parse(rawData);
|
||||
return {
|
||||
type: message.type || '',
|
||||
requestId: message.requestId || null,
|
||||
success: message.success !== false,
|
||||
payload: message.payload || {},
|
||||
error: message.error || null,
|
||||
};
|
||||
}
|
||||
|
||||
function settleWakewordRequest(message) {
|
||||
const pendingRequest = pendingWakewordRequests.get(message.requestId);
|
||||
if (!pendingRequest) {
|
||||
return;
|
||||
}
|
||||
|
||||
window.clearTimeout(pendingRequest.timer);
|
||||
pendingWakewordRequests.delete(message.requestId);
|
||||
|
||||
if (message.success === false) {
|
||||
pendingRequest.reject(new Error(message.error || '本地唤醒服务返回失败'));
|
||||
return;
|
||||
}
|
||||
|
||||
pendingRequest.resolve(message);
|
||||
}
|
||||
|
||||
function rejectAllWakewordRequests(errorMessage) {
|
||||
pendingWakewordRequests.forEach((pendingRequest) => {
|
||||
window.clearTimeout(pendingRequest.timer);
|
||||
pendingRequest.reject(new Error(errorMessage));
|
||||
});
|
||||
pendingWakewordRequests.clear();
|
||||
}
|
||||
@@ -0,0 +1,591 @@
|
||||
// WebSocket消息处理模块
|
||||
import { getConfig, saveConnectionUrls } from '../../config/manager.js?v=0205';
|
||||
import { uiController } from '../../ui/controller.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { getAudioPlayer } from '../audio/player.js?v=0205';
|
||||
import { getAudioRecorder } from '../audio/recorder.js?v=0205';
|
||||
import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js?v=0205';
|
||||
import { webSocketConnect } from './ota-connector.js?v=0205';
|
||||
|
||||
// WebSocket处理器类
|
||||
export class WebSocketHandler {
|
||||
constructor() {
|
||||
this.websocket = null;
|
||||
this.onConnectionStateChange = null;
|
||||
this.onRecordButtonStateChange = null;
|
||||
this.onSessionStateChange = null;
|
||||
this.onSessionEmotionChange = null;
|
||||
this.onChatMessage = null; // 新增：聊天消息回调
|
||||
this.currentSessionId = null;
|
||||
this.isRemoteSpeaking = false;
|
||||
}
|
||||
|
||||
// 发送hello握手消息
|
||||
async sendHelloMessage() {
|
||||
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) return false;
|
||||
|
||||
try {
|
||||
const config = getConfig();
|
||||
|
||||
const helloMessage = {
|
||||
type: 'hello',
|
||||
device_id: config.deviceId,
|
||||
device_name: config.deviceName,
|
||||
device_mac: config.deviceMac,
|
||||
token: config.token,
|
||||
features: {
|
||||
mcp: true,
|
||||
emoji: config.emojiEnabled
|
||||
}
|
||||
};
|
||||
|
||||
log('发送hello握手消息', 'info');
|
||||
this.websocket.send(JSON.stringify(helloMessage));
|
||||
|
||||
return new Promise(resolve => {
|
||||
const timeout = setTimeout(() => {
|
||||
log('等待hello响应超时', 'error');
|
||||
log('提示: 请尝试点击"测试认证"按钮进行连接排查', 'info');
|
||||
resolve(false);
|
||||
}, 5000);
|
||||
|
||||
const onMessageHandler = (event) => {
|
||||
try {
|
||||
const response = JSON.parse(event.data);
|
||||
if (response.type === 'hello' && response.session_id) {
|
||||
log(`服务器握手成功，会话ID: ${response.session_id}`, 'success');
|
||||
clearTimeout(timeout);
|
||||
this.websocket.removeEventListener('message', onMessageHandler);
|
||||
resolve(true);
|
||||
}
|
||||
} catch (e) {
|
||||
// 忽略非JSON消息
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.addEventListener('message', onMessageHandler);
|
||||
});
|
||||
} catch (error) {
|
||||
log(`发送hello消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
_sendWakeupMessages(sessionId) {
|
||||
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) return;
|
||||
|
||||
// listen detect
|
||||
this.websocket.send(JSON.stringify({
|
||||
session_id: sessionId,
|
||||
type: 'listen',
|
||||
state: 'detect',
|
||||
text: '嘿，你好呀'
|
||||
}));
|
||||
log('发送listen detect消息，唤醒词: 嘿，你好呀', 'info');
|
||||
|
||||
// listen start：开始监听
|
||||
this.websocket.send(JSON.stringify({
|
||||
session_id: sessionId,
|
||||
type: 'listen',
|
||||
state: 'start',
|
||||
mode: 'auto'
|
||||
}));
|
||||
log('发送listen start消息', 'info');
|
||||
}
|
||||
|
||||
// 处理文本消息
|
||||
handleTextMessage(message) {
|
||||
if (message.type === 'hello') {
|
||||
log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success');
|
||||
window.cameraAvailable = true;
|
||||
log('连接成功，摄像头已可用', 'success');
|
||||
uiController.updateDialButton(true);
|
||||
|
||||
this._sendWakeupMessages(message.session_id);
|
||||
|
||||
uiController.startAIChatSession();
|
||||
} else if (message.type === 'tts') {
|
||||
this.handleTTSMessage(message);
|
||||
} else if (message.type === 'audio') {
|
||||
log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
|
||||
} else if (message.type === 'stt') {
|
||||
log(`识别结果: ${message.text}`, 'info');
|
||||
// 检查是否需要绑定设备
|
||||
if (message.text && (message.text.includes('绑定') || message.text.includes('bind'))) {
|
||||
log('收到设备绑定提示，更新摄像头状态', 'warning');
|
||||
window.cameraAvailable = false;
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
window.stopCamera();
|
||||
}
|
||||
// 更新摄像头按钮状态
|
||||
const cameraBtn = document.getElementById('cameraBtn');
|
||||
if (cameraBtn) {
|
||||
cameraBtn.classList.remove('camera-active');
|
||||
cameraBtn.querySelector('.btn-text').textContent = '摄像头';
|
||||
cameraBtn.disabled = true;
|
||||
cameraBtn.title = '请先绑定验证码';
|
||||
}
|
||||
}
|
||||
// 使用新的聊天消息回调显示STT消息
|
||||
if (this.onChatMessage && message.text) {
|
||||
this.onChatMessage(message.text, true);
|
||||
}
|
||||
} else if (message.type === 'llm') {
|
||||
log(`大模型回复: ${message.text}`, 'info');
|
||||
// 使用新的聊天消息回调显示LLM回复
|
||||
if (this.onChatMessage && message.text) {
|
||||
this.onChatMessage(message.text, false);
|
||||
}
|
||||
|
||||
// 如果包含表情，更新sessionStatus表情并触发Live2D动作
|
||||
if (message.text && /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u.test(message.text)) {
|
||||
// 提取表情符号
|
||||
const emojiMatch = message.text.match(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u);
|
||||
if (emojiMatch && this.onSessionEmotionChange) {
|
||||
this.onSessionEmotionChange(emojiMatch[0]);
|
||||
}
|
||||
|
||||
// 触发Live2D情绪动作
|
||||
if (message.emotion) {
|
||||
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
|
||||
this.triggerLive2DEmotionAction(message.emotion);
|
||||
}
|
||||
}
|
||||
|
||||
// 只有当文本不仅仅是表情时，才添加到对话中
|
||||
// 移除文本中的表情后检查是否还有内容
|
||||
const textWithoutEmoji = message.text ? message.text.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim() : '';
|
||||
if (textWithoutEmoji && this.onChatMessage) {
|
||||
this.onChatMessage(message.text, false);
|
||||
}
|
||||
} else if (message.type === 'mcp') {
|
||||
this.handleMCPMessage(message);
|
||||
} else {
|
||||
log(`未知消息类型: ${message.type}`, 'info');
|
||||
if (this.onChatMessage) {
|
||||
this.onChatMessage(`未知消息类型: ${message.type}\n${JSON.stringify(message, null, 2)}`, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 处理TTS消息
|
||||
handleTTSMessage(message) {
|
||||
if (message.state === 'start') {
|
||||
log('服务器开始发送语音', 'info');
|
||||
this.currentSessionId = message.session_id;
|
||||
this.isRemoteSpeaking = true;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(true);
|
||||
}
|
||||
|
||||
// 启动Live2D说话动画
|
||||
this.startLive2DTalking();
|
||||
} else if (message.state === 'sentence_start') {
|
||||
log(`服务器发送语音段: ${message.text}`, 'info');
|
||||
this.ttsSentenceCount = (this.ttsSentenceCount || 0) + 1;
|
||||
|
||||
if (message.text && this.onChatMessage) {
|
||||
this.onChatMessage(message.text, false);
|
||||
}
|
||||
|
||||
// 确保动画在句子开始时运行
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager && !live2dManager.isTalking) {
|
||||
this.startLive2DTalking();
|
||||
}
|
||||
} else if (message.state === 'sentence_end') {
|
||||
log(`语音段结束: ${message.text}`, 'info');
|
||||
|
||||
// 句子结束时不清除动画，等待下一个句子或最终停止
|
||||
} else if (message.state === 'stop') {
|
||||
log('服务器语音传输结束，清空所有音频缓冲', 'info');
|
||||
|
||||
// 清空所有音频缓冲并停止播放
|
||||
const audioPlayer = getAudioPlayer();
|
||||
audioPlayer.clearAllAudio();
|
||||
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onRecordButtonStateChange) {
|
||||
this.onRecordButtonStateChange(false);
|
||||
}
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
|
||||
// 延迟停止Live2D说话动画，确保所有句子都播放完毕
|
||||
setTimeout(() => {
|
||||
this.stopLive2DTalking();
|
||||
this.ttsSentenceCount = 0; // 重置计数器
|
||||
}, 1000); // 1秒延迟，确保所有句子都完成
|
||||
}
|
||||
}
|
||||
|
||||
// 启动Live2D说话动画
|
||||
startLive2DTalking() {
|
||||
try {
|
||||
// 获取Live2D管理器实例
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager && live2dManager.live2dModel) {
|
||||
// 使用音频播放器的分析器节点
|
||||
live2dManager.startTalking();
|
||||
log('Live2D说话动画已启动', 'info');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`启动Live2D说话动画失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 停止Live2D说话动画
|
||||
stopLive2DTalking() {
|
||||
try {
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager) {
|
||||
live2dManager.stopTalking();
|
||||
log('Live2D说话动画已停止', 'info');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`停止Live2D说话动画失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化Live2D音频分析器
|
||||
initializeLive2DAudioAnalyzer() {
|
||||
try {
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager) {
|
||||
// 初始化音频分析器（使用音频播放器的上下文）
|
||||
if (live2dManager.initializeAudioAnalyzer()) {
|
||||
log('Live2D音频分析器初始化完成，已连接到音频播放器', 'success');
|
||||
} else {
|
||||
log('Live2D音频分析器初始化失败，将使用模拟动画', 'warning');
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log(`初始化Live2D音频分析器失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 处理MCP消息
|
||||
handleMCPMessage(message) {
|
||||
const payload = message.payload || {};
|
||||
log(`服务器下发: ${JSON.stringify(message)}`, 'info');
|
||||
|
||||
if (payload.method === 'tools/list') {
|
||||
const tools = getMcpTools();
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"tools": tools
|
||||
}
|
||||
}
|
||||
});
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
log(`回复MCP工具列表: ${tools.length} 个工具`, 'info');
|
||||
|
||||
} else if (payload.method === 'tools/call') {
|
||||
const toolName = payload.params?.name;
|
||||
const toolArgs = payload.params?.arguments;
|
||||
|
||||
log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info');
|
||||
|
||||
executeMcpTool(toolName, toolArgs).then(result => {
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": JSON.stringify(result)
|
||||
}
|
||||
],
|
||||
"isError": false
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
}).catch(error => {
|
||||
log(`工具执行失败: ${error.message}`, 'error');
|
||||
const errorReply = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"error": {
|
||||
"code": -32603,
|
||||
"message": error.message
|
||||
}
|
||||
}
|
||||
});
|
||||
this.websocket.send(errorReply);
|
||||
});
|
||||
} else if (payload.method === 'initialize') {
|
||||
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
|
||||
// 保存视觉分析接口地址
|
||||
const visionUrl = document.getElementById('visionUrl');
|
||||
const visionConfig = payload?.params?.capabilities?.vision;
|
||||
if (visionConfig && typeof visionConfig === 'object' && visionConfig.url && visionConfig.token) {
|
||||
const visionConfigStr = JSON.stringify(visionConfig);
|
||||
localStorage.setItem('xz_tester_vision', visionConfigStr);
|
||||
if (visionUrl) visionUrl.value = visionConfig.url;
|
||||
} else {
|
||||
localStorage.removeItem('xz_tester_vision');
|
||||
if (visionUrl) visionUrl.value = '';
|
||||
}
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {
|
||||
"tools": {}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "xiaozhi-web-test",
|
||||
"version": "2.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
log(`回复初始化响应`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
} else {
|
||||
log(`未知的MCP方法: ${payload.method}`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 处理二进制消息
|
||||
async handleBinaryMessage(data) {
|
||||
try {
|
||||
let arrayBuffer;
|
||||
if (data instanceof ArrayBuffer) {
|
||||
arrayBuffer = data;
|
||||
} else if (data instanceof Blob) {
|
||||
arrayBuffer = await data.arrayBuffer();
|
||||
log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`, 'debug');
|
||||
} else {
|
||||
log(`收到未知类型的二进制数据: ${typeof data}`, 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
const opusData = new Uint8Array(arrayBuffer);
|
||||
const audioPlayer = getAudioPlayer();
|
||||
audioPlayer.enqueueAudioData(opusData);
|
||||
} catch (error) {
|
||||
log(`处理二进制消息出错: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 连接WebSocket服务器
|
||||
async connect() {
|
||||
const config = getConfig();
|
||||
log('正在检查OTA状态...', 'info');
|
||||
saveConnectionUrls();
|
||||
|
||||
try {
|
||||
const otaUrl = document.getElementById('otaUrl').value.trim();
|
||||
const ws = await webSocketConnect(otaUrl, config);
|
||||
if (ws === undefined) {
|
||||
return false;
|
||||
}
|
||||
this.websocket = ws;
|
||||
|
||||
// 设置接收二进制数据的类型为ArrayBuffer
|
||||
this.websocket.binaryType = 'arraybuffer';
|
||||
|
||||
// 设置 MCP 模块的 WebSocket 实例
|
||||
setMcpWebSocket(this.websocket);
|
||||
|
||||
// 设置录音器的WebSocket
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.setWebSocket(this.websocket);
|
||||
|
||||
this.setupEventHandlers();
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`连接错误: ${error.message}`, 'error');
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 设置事件处理器
|
||||
setupEventHandlers() {
|
||||
this.websocket.onopen = async () => {
|
||||
const url = document.getElementById('serverUrl').value;
|
||||
log(`已连接到服务器: ${url}`, 'success');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(true);
|
||||
}
|
||||
|
||||
// 连接成功后，默认状态为聆听中
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
|
||||
// 在WebSocket连接成功时初始化Live2D音频分析器
|
||||
this.initializeLive2DAudioAnalyzer();
|
||||
|
||||
await this.sendHelloMessage();
|
||||
};
|
||||
|
||||
this.websocket.onclose = () => {
|
||||
log('已断开连接', 'info');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
window.stopCamera();
|
||||
}
|
||||
|
||||
// 隐藏摄像头显示区域
|
||||
const cameraContainer = document.getElementById('cameraContainer');
|
||||
if (cameraContainer) {
|
||||
cameraContainer.classList.remove('active');
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.onerror = (error) => {
|
||||
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
|
||||
uiController.addChatMessage(`⚠️ WebSocket错误: ${error.message || '未知错误'}`, false);
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.onmessage = (event) => {
|
||||
try {
|
||||
if (typeof event.data === 'string') {
|
||||
const message = JSON.parse(event.data);
|
||||
this.handleTextMessage(message);
|
||||
} else {
|
||||
this.handleBinaryMessage(event.data);
|
||||
}
|
||||
} catch (error) {
|
||||
log(`WebSocket消息处理错误: ${error.message}`, 'error');
|
||||
// 不再使用旧的addMessage函数，因为conversationDiv元素不存在
|
||||
// 错误消息将通过其他方式显示
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// 断开连接
|
||||
disconnect() {
|
||||
if (!this.websocket) return;
|
||||
|
||||
this.websocket.close();
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
|
||||
// 关闭摄像头
|
||||
if (typeof window.stopCamera === 'function') {
|
||||
window.stopCamera();
|
||||
}
|
||||
|
||||
// 隐藏摄像头显示区域
|
||||
const cameraContainer = document.getElementById('cameraContainer');
|
||||
if (cameraContainer) {
|
||||
cameraContainer.classList.remove('active');
|
||||
}
|
||||
}
|
||||
|
||||
// 发送文本消息
|
||||
sendTextMessage(text) {
|
||||
if (text === '' || !this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// 如果对方正在说话，先发送打断消息
|
||||
if (this.isRemoteSpeaking && this.currentSessionId) {
|
||||
const abortMessage = {
|
||||
session_id: this.currentSessionId,
|
||||
type: 'abort',
|
||||
reason: 'wake_word_detected'
|
||||
};
|
||||
this.websocket.send(JSON.stringify(abortMessage));
|
||||
log('发送打断消息', 'info');
|
||||
}
|
||||
|
||||
const listenMessage = {
|
||||
type: 'listen',
|
||||
state: 'detect',
|
||||
text: text
|
||||
};
|
||||
|
||||
this.websocket.send(JSON.stringify(listenMessage));
|
||||
log(`发送文本消息: ${text}`, 'info');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`发送消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 触发Live2D情绪动作
|
||||
* @param {string} emotion - 情绪名称
|
||||
*/
|
||||
triggerLive2DEmotionAction(emotion) {
|
||||
try {
|
||||
const live2dManager = window.chatApp?.live2dManager;
|
||||
if (live2dManager && typeof live2dManager.triggerEmotionAction === 'function') {
|
||||
live2dManager.triggerEmotionAction(emotion);
|
||||
log(`触发Live2D情绪动作: ${emotion}`, 'info');
|
||||
} else {
|
||||
log(`无法触发Live2D情绪动作: Live2D管理器未找到或方法不可用`, 'warning');
|
||||
}
|
||||
} catch (error) {
|
||||
log(`触发Live2D情绪动作失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 获取WebSocket实例
|
||||
getWebSocket() {
|
||||
return this.websocket;
|
||||
}
|
||||
|
||||
// 检查是否已连接
|
||||
isConnected() {
|
||||
return this.websocket && this.websocket.readyState === WebSocket.OPEN;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let wsHandlerInstance = null;
|
||||
|
||||
export function getWebSocketHandler() {
|
||||
if (!wsHandlerInstance) {
|
||||
wsHandlerInstance = new WebSocketHandler();
|
||||
}
|
||||
return wsHandlerInstance;
|
||||
}
|
||||
Reference in New Issue
Block a user