mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-27 09:33:55 +08:00
update:音频测试页面增加打断功能
This commit is contained in:
hrz
@@ -0,0 +1,186 @@
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { updateScriptStatus } from '../../ui/dom-helper.js'
|
||||
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
export function checkOpusLoaded() {
|
||||
try {
|
||||
// 检查Module是否存在（本地库导出的全局变量）
|
||||
if (typeof Module === 'undefined') {
|
||||
throw new Error('Opus库未加载，Module对象不存在');
|
||||
}
|
||||
|
||||
// 尝试先使用Module.instance（libopus.js最后一行导出方式）
|
||||
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
|
||||
// 使用Module.instance对象替换全局Module对象
|
||||
window.ModuleInstance = Module.instance;
|
||||
log('Opus库加载成功（使用Module.instance）', 'success');
|
||||
updateScriptStatus('Opus库加载成功', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
// 如果没有Module.instance，检查全局Module函数
|
||||
if (typeof Module._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module;
|
||||
log('Opus库加载成功（使用全局Module）', 'success');
|
||||
updateScriptStatus('Opus库加载成功', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error('Opus解码函数未找到，可能Module结构不正确');
|
||||
} catch (err) {
|
||||
log(`Opus库加载失败，请检查libopus.js文件是否存在且正确: ${err.message}`, 'error');
|
||||
updateScriptStatus('Opus库加载失败，请检查libopus.js文件是否存在且正确', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 创建一个Opus编码器
|
||||
let opusEncoder = null;
|
||||
export function initOpusEncoder() {
|
||||
try {
|
||||
if (opusEncoder) {
|
||||
return opusEncoder; // 已经初始化过
|
||||
}
|
||||
|
||||
if (!window.ModuleInstance) {
|
||||
log('无法创建Opus编码器：ModuleInstance不可用', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 初始化一个Opus编码器
|
||||
const mod = window.ModuleInstance;
|
||||
const sampleRate = 16000; // 16kHz采样率
|
||||
const channels = 1; // 单声道
|
||||
const application = 2048; // OPUS_APPLICATION_VOIP = 2048
|
||||
|
||||
// 创建编码器
|
||||
opusEncoder = {
|
||||
channels: channels,
|
||||
sampleRate: sampleRate,
|
||||
frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
|
||||
maxPacketSize: 4000, // 最大包大小
|
||||
module: mod,
|
||||
|
||||
// 初始化编码器
|
||||
init: function () {
|
||||
try {
|
||||
// 获取编码器大小
|
||||
const encoderSize = mod._opus_encoder_get_size(this.channels);
|
||||
log(`Opus编码器大小: ${encoderSize}字节`, 'info');
|
||||
|
||||
// 分配内存
|
||||
this.encoderPtr = mod._malloc(encoderSize);
|
||||
if (!this.encoderPtr) {
|
||||
throw new Error("无法分配编码器内存");
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
const err = mod._opus_encoder_init(
|
||||
this.encoderPtr,
|
||||
this.sampleRate,
|
||||
this.channels,
|
||||
application
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
throw new Error(`Opus编码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
// 设置位率 (16kbps)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4002, 16000); // OPUS_SET_BITRATE
|
||||
|
||||
// 设置复杂度 (0-10, 越高质量越好但CPU使用越多)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4010, 5); // OPUS_SET_COMPLEXITY
|
||||
|
||||
// 设置使用DTX (不传输静音帧)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4016, 1); // OPUS_SET_DTX
|
||||
|
||||
log("Opus编码器初始化成功", 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (this.encoderPtr) {
|
||||
mod._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
log(`Opus编码器初始化失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
// 编码PCM数据为Opus
|
||||
encode: function (pcmData) {
|
||||
if (!this.encoderPtr) {
|
||||
if (!this.init()) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
// 为PCM数据分配内存
|
||||
const pcmPtr = mod._malloc(pcmData.length * 2); // 2字节/int16
|
||||
|
||||
// 将PCM数据复制到HEAP
|
||||
for (let i = 0; i < pcmData.length; i++) {
|
||||
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
|
||||
}
|
||||
|
||||
// 为输出分配内存
|
||||
const outPtr = mod._malloc(this.maxPacketSize);
|
||||
|
||||
// 进行编码
|
||||
const encodedLen = mod._opus_encode(
|
||||
this.encoderPtr,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
outPtr,
|
||||
this.maxPacketSize
|
||||
);
|
||||
|
||||
if (encodedLen < 0) {
|
||||
throw new Error(`Opus编码失败: ${encodedLen}`);
|
||||
}
|
||||
|
||||
// 复制编码后的数据
|
||||
const opusData = new Uint8Array(encodedLen);
|
||||
for (let i = 0; i < encodedLen; i++) {
|
||||
opusData[i] = mod.HEAPU8[outPtr + i];
|
||||
}
|
||||
|
||||
// 释放内存
|
||||
mod._free(pcmPtr);
|
||||
mod._free(outPtr);
|
||||
|
||||
return opusData;
|
||||
} catch (error) {
|
||||
log(`Opus编码出错: ${error.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
},
|
||||
|
||||
// 销毁编码器
|
||||
destroy: function () {
|
||||
if (this.encoderPtr) {
|
||||
this.module._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
opusEncoder.init();
|
||||
return opusEncoder;
|
||||
} catch (error) {
|
||||
log(`创建Opus编码器失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
// 音频播放模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import BlockingQueue from '../../utils/blocking-queue.js';
|
||||
import { createStreamingContext } from './stream-context.js';
|
||||
|
||||
// 音频播放器类
|
||||
export class AudioPlayer {
|
||||
constructor() {
|
||||
// 音频参数
|
||||
this.SAMPLE_RATE = 16000;
|
||||
this.CHANNELS = 1;
|
||||
this.FRAME_SIZE = 960;
|
||||
this.MIN_AUDIO_DURATION = 0.12;
|
||||
|
||||
// 状态
|
||||
this.audioContext = null;
|
||||
this.opusDecoder = null;
|
||||
this.streamingContext = null;
|
||||
this.queue = new BlockingQueue();
|
||||
this.isPlaying = false;
|
||||
}
|
||||
|
||||
// 获取或创建AudioContext
|
||||
getAudioContext() {
|
||||
if (!this.audioContext) {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: this.SAMPLE_RATE,
|
||||
latencyHint: 'interactive'
|
||||
});
|
||||
log('创建音频上下文，采样率: ' + this.SAMPLE_RATE + 'Hz', 'debug');
|
||||
}
|
||||
return this.audioContext;
|
||||
}
|
||||
|
||||
// 初始化Opus解码器
|
||||
async initOpusDecoder() {
|
||||
if (this.opusDecoder) return this.opusDecoder;
|
||||
|
||||
try {
|
||||
if (typeof window.ModuleInstance === 'undefined') {
|
||||
if (typeof Module !== 'undefined') {
|
||||
window.ModuleInstance = Module;
|
||||
log('使用全局Module作为ModuleInstance', 'info');
|
||||
} else {
|
||||
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
|
||||
}
|
||||
}
|
||||
|
||||
const mod = window.ModuleInstance;
|
||||
|
||||
this.opusDecoder = {
|
||||
channels: this.CHANNELS,
|
||||
rate: this.SAMPLE_RATE,
|
||||
frameSize: this.FRAME_SIZE,
|
||||
module: mod,
|
||||
decoderPtr: null,
|
||||
|
||||
init: function () {
|
||||
if (this.decoderPtr) return true;
|
||||
|
||||
const decoderSize = mod._opus_decoder_get_size(this.channels);
|
||||
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
|
||||
|
||||
this.decoderPtr = mod._malloc(decoderSize);
|
||||
if (!this.decoderPtr) {
|
||||
throw new Error("无法分配解码器内存");
|
||||
}
|
||||
|
||||
const err = mod._opus_decoder_init(
|
||||
this.decoderPtr,
|
||||
this.rate,
|
||||
this.channels
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
this.destroy();
|
||||
throw new Error(`Opus解码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
log("Opus解码器初始化成功", 'success');
|
||||
return true;
|
||||
},
|
||||
|
||||
decode: function (opusData) {
|
||||
if (!this.decoderPtr) {
|
||||
if (!this.init()) {
|
||||
throw new Error("解码器未初始化且无法初始化");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
const opusPtr = mod._malloc(opusData.length);
|
||||
mod.HEAPU8.set(opusData, opusPtr);
|
||||
|
||||
const pcmPtr = mod._malloc(this.frameSize * 2);
|
||||
|
||||
const decodedSamples = mod._opus_decode(
|
||||
this.decoderPtr,
|
||||
opusPtr,
|
||||
opusData.length,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
0
|
||||
);
|
||||
|
||||
if (decodedSamples < 0) {
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
throw new Error(`Opus解码失败: ${decodedSamples}`);
|
||||
}
|
||||
|
||||
const decodedData = new Int16Array(decodedSamples);
|
||||
for (let i = 0; i < decodedSamples; i++) {
|
||||
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
|
||||
}
|
||||
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
|
||||
return decodedData;
|
||||
} catch (error) {
|
||||
log(`Opus解码错误: ${error.message}`, 'error');
|
||||
return new Int16Array(0);
|
||||
}
|
||||
},
|
||||
|
||||
destroy: function () {
|
||||
if (this.decoderPtr) {
|
||||
this.module._free(this.decoderPtr);
|
||||
this.decoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (!this.opusDecoder.init()) {
|
||||
throw new Error("Opus解码器初始化失败");
|
||||
}
|
||||
|
||||
return this.opusDecoder;
|
||||
|
||||
} catch (error) {
|
||||
log(`Opus解码器初始化失败: ${error.message}`, 'error');
|
||||
this.opusDecoder = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 启动音频缓冲
|
||||
async startAudioBuffering() {
|
||||
log("开始音频缓冲...", 'info');
|
||||
|
||||
this.initOpusDecoder().catch(error => {
|
||||
log(`预初始化Opus解码器失败: ${error.message}`, 'warning');
|
||||
});
|
||||
|
||||
const timeout = 400;
|
||||
while (true) {
|
||||
const packets = await this.queue.dequeue(
|
||||
6,
|
||||
timeout,
|
||||
(count) => {
|
||||
log(`缓冲超时，当前缓冲包数: ${count}，开始播放`, 'info');
|
||||
}
|
||||
);
|
||||
if (packets.length) {
|
||||
log(`已缓冲 ${packets.length} 个音频包，开始播放`, 'info');
|
||||
this.streamingContext.pushAudioBuffer(packets);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const data = await this.queue.dequeue(99, 30);
|
||||
if (data.length) {
|
||||
this.streamingContext.pushAudioBuffer(data);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 播放已缓冲的音频
|
||||
async playBufferedAudio() {
|
||||
try {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (!this.opusDecoder) {
|
||||
log('初始化Opus解码器...', 'info');
|
||||
try {
|
||||
this.opusDecoder = await this.initOpusDecoder();
|
||||
if (!this.opusDecoder) {
|
||||
throw new Error('解码器初始化失败');
|
||||
}
|
||||
log('Opus解码器初始化成功', 'success');
|
||||
} catch (error) {
|
||||
log('Opus解码器初始化失败: ' + error.message, 'error');
|
||||
this.isPlaying = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.streamingContext) {
|
||||
this.streamingContext = createStreamingContext(
|
||||
this.opusDecoder,
|
||||
this.audioContext,
|
||||
this.SAMPLE_RATE,
|
||||
this.CHANNELS,
|
||||
this.MIN_AUDIO_DURATION
|
||||
);
|
||||
}
|
||||
|
||||
this.streamingContext.decodeOpusFrames();
|
||||
this.streamingContext.startPlaying();
|
||||
|
||||
} catch (error) {
|
||||
log(`播放已缓冲的音频出错: ${error.message}`, 'error');
|
||||
this.isPlaying = false;
|
||||
this.streamingContext = null;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加音频数据到队列
|
||||
enqueueAudioData(opusData) {
|
||||
if (opusData.length > 0) {
|
||||
this.queue.enqueue(opusData);
|
||||
} else {
|
||||
log('收到空音频数据帧，可能是结束标志', 'warning');
|
||||
if (this.isPlaying && this.streamingContext) {
|
||||
this.streamingContext.endOfStream = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 预加载解码器
|
||||
async preload() {
|
||||
log('预加载Opus解码器...', 'info');
|
||||
try {
|
||||
await this.initOpusDecoder();
|
||||
log('Opus解码器预加载成功', 'success');
|
||||
} catch (error) {
|
||||
log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 启动播放系统
|
||||
async start() {
|
||||
await this.preload();
|
||||
this.playBufferedAudio();
|
||||
this.startAudioBuffering();
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let audioPlayerInstance = null;
|
||||
|
||||
export function getAudioPlayer() {
|
||||
if (!audioPlayerInstance) {
|
||||
audioPlayerInstance = new AudioPlayer();
|
||||
}
|
||||
return audioPlayerInstance;
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
// 音频录制模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { initOpusEncoder } from './opus-codec.js';
|
||||
import { getAudioPlayer } from './player.js';
|
||||
|
||||
// 音频录制器类
|
||||
export class AudioRecorder {
|
||||
constructor() {
|
||||
this.isRecording = false;
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
this.audioProcessor = null;
|
||||
this.audioProcessorType = null;
|
||||
this.audioSource = null;
|
||||
this.opusEncoder = null;
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.visualizationRequest = null;
|
||||
this.recordingTimer = null;
|
||||
this.websocket = null;
|
||||
|
||||
// 回调函数
|
||||
this.onRecordingStart = null;
|
||||
this.onRecordingStop = null;
|
||||
this.onVisualizerUpdate = null;
|
||||
}
|
||||
|
||||
// 设置WebSocket实例
|
||||
setWebSocket(ws) {
|
||||
this.websocket = ws;
|
||||
}
|
||||
|
||||
// 获取AudioContext实例
|
||||
getAudioContext() {
|
||||
const audioPlayer = getAudioPlayer();
|
||||
return audioPlayer.getAudioContext();
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
initEncoder() {
|
||||
if (!this.opusEncoder) {
|
||||
this.opusEncoder = initOpusEncoder();
|
||||
}
|
||||
return this.opusEncoder;
|
||||
}
|
||||
|
||||
// PCM处理器代码
|
||||
getAudioProcessorCode() {
|
||||
return `
|
||||
class AudioRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffers = [];
|
||||
this.frameSize = 960;
|
||||
this.buffer = new Int16Array(this.frameSize);
|
||||
this.bufferIndex = 0;
|
||||
this.isRecording = false;
|
||||
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.command === 'start') {
|
||||
this.isRecording = true;
|
||||
this.port.postMessage({ type: 'status', status: 'started' });
|
||||
} else if (event.data.command === 'stop') {
|
||||
this.isRecording = false;
|
||||
|
||||
if (this.bufferIndex > 0) {
|
||||
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
||||
this.port.postMessage({
|
||||
type: 'buffer',
|
||||
buffer: finalBuffer
|
||||
});
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
|
||||
this.port.postMessage({ type: 'status', status: 'stopped' });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
process(inputs, outputs, parameters) {
|
||||
if (!this.isRecording) return true;
|
||||
|
||||
const input = inputs[0][0];
|
||||
if (!input) return true;
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (this.bufferIndex >= this.frameSize) {
|
||||
this.port.postMessage({
|
||||
type: 'buffer',
|
||||
buffer: this.buffer.slice(0)
|
||||
});
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
|
||||
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
|
||||
`;
|
||||
}
|
||||
|
||||
// 创建音频处理器
|
||||
async createAudioProcessor() {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
try {
|
||||
if (this.audioContext.audioWorklet) {
|
||||
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
await this.audioContext.audioWorklet.addModule(url);
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
|
||||
|
||||
audioProcessor.port.onmessage = (event) => {
|
||||
if (event.data.type === 'buffer') {
|
||||
this.processPCMBuffer(event.data.buffer);
|
||||
}
|
||||
};
|
||||
|
||||
log('使用AudioWorklet处理音频', 'success');
|
||||
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
audioProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
return { node: audioProcessor, type: 'worklet' };
|
||||
} else {
|
||||
log('AudioWorklet不可用，使用ScriptProcessorNode作为回退方案', 'warning');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
} catch (error) {
|
||||
log(`创建音频处理器失败: ${error.message}，尝试回退方案`, 'error');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
// 创建ScriptProcessor作为回退
|
||||
createScriptProcessor() {
|
||||
try {
|
||||
const frameSize = 4096;
|
||||
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
|
||||
|
||||
scriptProcessor.onaudioprocess = (event) => {
|
||||
if (!this.isRecording) return;
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
const buffer = new Int16Array(input.length);
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
|
||||
this.processPCMBuffer(buffer);
|
||||
};
|
||||
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
scriptProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
|
||||
log('使用ScriptProcessorNode作为回退方案成功', 'warning');
|
||||
return { node: scriptProcessor, type: 'processor' };
|
||||
} catch (fallbackError) {
|
||||
log(`回退方案也失败: ${fallbackError.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 处理PCM缓冲数据
|
||||
processPCMBuffer(buffer) {
|
||||
if (!this.isRecording) return;
|
||||
|
||||
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
|
||||
newBuffer.set(this.pcmDataBuffer);
|
||||
newBuffer.set(buffer, this.pcmDataBuffer.length);
|
||||
this.pcmDataBuffer = newBuffer;
|
||||
|
||||
const samplesPerFrame = 960;
|
||||
|
||||
while (this.pcmDataBuffer.length >= samplesPerFrame) {
|
||||
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
|
||||
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
|
||||
|
||||
this.encodeAndSendOpus(frameData);
|
||||
}
|
||||
}
|
||||
|
||||
// 编码并发送Opus数据
|
||||
encodeAndSendOpus(pcmData = null) {
|
||||
if (!this.opusEncoder) {
|
||||
log('Opus编码器未初始化', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (pcmData) {
|
||||
const opusData = this.opusEncoder.encode(pcmData);
|
||||
|
||||
if (opusData && opusData.length > 0) {
|
||||
this.audioBuffers.push(opusData.buffer);
|
||||
this.totalAudioSize += opusData.length;
|
||||
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.websocket.send(opusData.buffer);
|
||||
log(`发送Opus帧，大小：${opusData.length}字节`, 'debug');
|
||||
} catch (error) {
|
||||
log(`WebSocket发送错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('Opus编码失败，无有效数据返回', 'error');
|
||||
}
|
||||
} else {
|
||||
if (this.pcmDataBuffer.length > 0) {
|
||||
const samplesPerFrame = 960;
|
||||
if (this.pcmDataBuffer.length < samplesPerFrame) {
|
||||
const paddedBuffer = new Int16Array(samplesPerFrame);
|
||||
paddedBuffer.set(this.pcmDataBuffer);
|
||||
this.encodeAndSendOpus(paddedBuffer);
|
||||
} else {
|
||||
this.encodeAndSendOpus(this.pcmDataBuffer.slice(0, samplesPerFrame));
|
||||
}
|
||||
this.pcmDataBuffer = new Int16Array(0);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log(`Opus编码错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 开始录音
|
||||
async start() {
|
||||
if (this.isRecording) return false;
|
||||
|
||||
try {
|
||||
if (!this.initEncoder()) {
|
||||
log('无法启动录音: Opus编码器初始化失败', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
log('请至少录制1-2秒钟的音频，确保采集到足够数据', 'info');
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
sampleRate: 16000,
|
||||
channelCount: 1
|
||||
}
|
||||
});
|
||||
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
|
||||
const processorResult = await this.createAudioProcessor();
|
||||
if (!processorResult) {
|
||||
log('无法创建音频处理器', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
this.audioProcessor = processorResult.node;
|
||||
this.audioProcessorType = processorResult.type;
|
||||
|
||||
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 2048;
|
||||
|
||||
this.audioSource.connect(this.analyser);
|
||||
this.audioSource.connect(this.audioProcessor);
|
||||
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.isRecording = true;
|
||||
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'start' });
|
||||
}
|
||||
|
||||
// 发送监听开始消息
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const listenMessage = {
|
||||
type: 'listen',
|
||||
mode: 'manual',
|
||||
state: 'start'
|
||||
};
|
||||
|
||||
log(`发送录音开始消息: ${JSON.stringify(listenMessage)}`, 'info');
|
||||
this.websocket.send(JSON.stringify(listenMessage));
|
||||
} else {
|
||||
log('WebSocket未连接，无法发送开始消息', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
// 开始可视化
|
||||
if (this.onVisualizerUpdate) {
|
||||
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
this.startVisualization(dataArray);
|
||||
}
|
||||
|
||||
// 启动录音计时器
|
||||
let recordingSeconds = 0;
|
||||
this.recordingTimer = setInterval(() => {
|
||||
recordingSeconds += 0.1;
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(recordingSeconds);
|
||||
}
|
||||
}, 100);
|
||||
|
||||
log('开始PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音启动错误: ${error.message}`, 'error');
|
||||
this.isRecording = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 开始可视化
|
||||
startVisualization(dataArray) {
|
||||
const draw = () => {
|
||||
this.visualizationRequest = requestAnimationFrame(() => draw());
|
||||
|
||||
if (!this.isRecording) return;
|
||||
|
||||
this.analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
if (this.onVisualizerUpdate) {
|
||||
this.onVisualizerUpdate(dataArray);
|
||||
}
|
||||
};
|
||||
draw();
|
||||
}
|
||||
|
||||
// 停止录音
|
||||
stop() {
|
||||
if (!this.isRecording) return false;
|
||||
|
||||
try {
|
||||
this.isRecording = false;
|
||||
|
||||
if (this.audioProcessor) {
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'stop' });
|
||||
}
|
||||
|
||||
this.audioProcessor.disconnect();
|
||||
this.audioProcessor = null;
|
||||
}
|
||||
|
||||
if (this.audioSource) {
|
||||
this.audioSource.disconnect();
|
||||
this.audioSource = null;
|
||||
}
|
||||
|
||||
if (this.visualizationRequest) {
|
||||
cancelAnimationFrame(this.visualizationRequest);
|
||||
this.visualizationRequest = null;
|
||||
}
|
||||
|
||||
if (this.recordingTimer) {
|
||||
clearInterval(this.recordingTimer);
|
||||
this.recordingTimer = null;
|
||||
}
|
||||
|
||||
// 编码并发送剩余的数据
|
||||
this.encodeAndSendOpus();
|
||||
|
||||
// 发送结束信号
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const emptyOpusFrame = new Uint8Array(0);
|
||||
this.websocket.send(emptyOpusFrame);
|
||||
|
||||
const stopMessage = {
|
||||
type: 'listen',
|
||||
mode: 'manual',
|
||||
state: 'stop'
|
||||
};
|
||||
|
||||
this.websocket.send(JSON.stringify(stopMessage));
|
||||
log('已发送录音停止信号', 'info');
|
||||
}
|
||||
|
||||
if (this.onRecordingStop) {
|
||||
this.onRecordingStop();
|
||||
}
|
||||
|
||||
log('停止PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音停止错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 获取分析器
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let audioRecorderInstance = null;
|
||||
|
||||
export function getAudioRecorder() {
|
||||
if (!audioRecorderInstance) {
|
||||
audioRecorderInstance = new AudioRecorder();
|
||||
}
|
||||
return audioRecorderInstance;
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import BlockingQueue from '../../utils/blocking-queue.js';
|
||||
import { log } from '../../utils/logger.js';
|
||||
|
||||
// 音频流播放上下文类
|
||||
export class StreamingContext {
|
||||
constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
this.opusDecoder = opusDecoder;
|
||||
this.audioContext = audioContext;
|
||||
|
||||
// 音频参数
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
this.minAudioDuration = minAudioDuration;
|
||||
|
||||
// 初始化队列和状态
|
||||
this.queue = []; // 已解码的PCM队列。正在播放
|
||||
this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放
|
||||
this.pendingAudioBufferQueue = []; // 待处理的缓存队列
|
||||
this.audioBufferQueue = new BlockingQueue(); // 缓存队列
|
||||
this.playing = false; // 是否正在播放
|
||||
this.endOfStream = false; // 是否收到结束信号
|
||||
this.source = null; // 当前音频源
|
||||
this.totalSamples = 0; // 累积的总样本数
|
||||
this.lastPlayTime = 0; // 上次播放的时间戳
|
||||
}
|
||||
|
||||
// 缓存音频数组
|
||||
pushAudioBuffer(item) {
|
||||
this.audioBufferQueue.enqueue(...item);
|
||||
}
|
||||
|
||||
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
|
||||
async getPendingAudioBufferQueue() {
|
||||
// 原子交换 + 清空
|
||||
[this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()];
|
||||
}
|
||||
|
||||
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
|
||||
async getQueue(minSamples) {
|
||||
let TepArray = [];
|
||||
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
|
||||
// 原子交换 + 清空
|
||||
[TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()];
|
||||
this.queue.push(...TepArray);
|
||||
}
|
||||
|
||||
// 将Int16音频数据转换为Float32音频数据
|
||||
convertInt16ToFloat32(int16Data) {
|
||||
const float32Data = new Float32Array(int16Data.length);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
// 将[-32768,32767]范围转换为[-1,1]，统一使用32768.0避免不对称失真
|
||||
float32Data[i] = int16Data[i] / 32768.0;
|
||||
}
|
||||
return float32Data;
|
||||
}
|
||||
|
||||
// 将Opus数据解码为PCM
|
||||
async decodeOpusFrames() {
|
||||
if (!this.opusDecoder) {
|
||||
log('Opus解码器未初始化，无法解码', 'error');
|
||||
return;
|
||||
} else {
|
||||
log('Opus解码器启动', 'info');
|
||||
}
|
||||
|
||||
while (true) {
|
||||
let decodedSamples = [];
|
||||
for (const frame of this.pendingAudioBufferQueue) {
|
||||
try {
|
||||
// 使用Opus解码器解码
|
||||
const frameData = this.opusDecoder.decode(frame);
|
||||
if (frameData && frameData.length > 0) {
|
||||
// 转换为Float32
|
||||
const floatData = this.convertInt16ToFloat32(frameData);
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < floatData.length; i++) {
|
||||
decodedSamples.push(floatData[i]);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log("Opus解码失败: " + error.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedSamples.length > 0) {
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < decodedSamples.length; i++) {
|
||||
this.activeQueue.enqueue(decodedSamples[i]);
|
||||
}
|
||||
this.totalSamples += decodedSamples.length;
|
||||
} else {
|
||||
log('没有成功解码的样本', 'warning');
|
||||
}
|
||||
await this.getPendingAudioBufferQueue();
|
||||
}
|
||||
}
|
||||
|
||||
// 开始播放音频
|
||||
async startPlaying() {
|
||||
let scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
|
||||
|
||||
while (true) {
|
||||
// 初始缓冲：等待足够的样本再开始播放
|
||||
const minSamples = this.sampleRate * this.minAudioDuration * 2;
|
||||
if (!this.playing && this.queue.length < minSamples) {
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
this.playing = true;
|
||||
|
||||
// 持续播放队列中的音频，每次播放一个小块
|
||||
while (this.playing && this.queue.length > 0) {
|
||||
// 每次播放120ms的音频（2个Opus包）
|
||||
const playDuration = 0.12;
|
||||
const targetSamples = Math.floor(this.sampleRate * playDuration);
|
||||
const actualSamples = Math.min(this.queue.length, targetSamples);
|
||||
|
||||
if (actualSamples === 0) break;
|
||||
|
||||
const currentSamples = this.queue.splice(0, actualSamples);
|
||||
const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate);
|
||||
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
|
||||
|
||||
// 创建音频源
|
||||
this.source = this.audioContext.createBufferSource();
|
||||
this.source.buffer = audioBuffer;
|
||||
|
||||
// 精确调度播放时间
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const startTime = Math.max(scheduledEndTime, currentTime);
|
||||
|
||||
// 直接连接到输出
|
||||
this.source.connect(this.audioContext.destination);
|
||||
|
||||
log(`调度播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'debug');
|
||||
this.source.start(startTime);
|
||||
|
||||
// 更新下一个音频块的调度时间
|
||||
const duration = audioBuffer.duration;
|
||||
scheduledEndTime = startTime + duration;
|
||||
this.lastPlayTime = startTime;
|
||||
|
||||
// 如果队列中数据不足，等待新数据
|
||||
if (this.queue.length < targetSamples) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 等待新数据
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 创建streamingContext实例的工厂函数
|
||||
export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration);
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
import { log } from '../../utils/logger.js';
|
||||
|
||||
// ==========================================
|
||||
// MCP 工具管理逻辑
|
||||
// ==========================================
|
||||
|
||||
// 全局变量
|
||||
let mcpTools = [];
|
||||
let mcpEditingIndex = null;
|
||||
let mcpProperties = [];
|
||||
let websocket = null; // 将从外部设置
|
||||
|
||||
/**
|
||||
* 设置 WebSocket 实例
|
||||
* @param {WebSocket} ws - WebSocket 连接实例
|
||||
*/
|
||||
export function setWebSocket(ws) {
|
||||
websocket = ws;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化 MCP 工具
|
||||
*/
|
||||
export async function initMcpTools() {
|
||||
// 加载默认工具数据
|
||||
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
|
||||
|
||||
const savedTools = localStorage.getItem('mcpTools');
|
||||
if (savedTools) {
|
||||
try {
|
||||
mcpTools = JSON.parse(savedTools);
|
||||
} catch (e) {
|
||||
log('加载MCP工具失败，使用默认工具', 'warning');
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
} else {
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
|
||||
renderMcpTools();
|
||||
setupMcpEventListeners();
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染工具列表
|
||||
*/
|
||||
function renderMcpTools() {
|
||||
const container = document.getElementById('mcpToolsContainer');
|
||||
const countSpan = document.getElementById('mcpToolsCount');
|
||||
|
||||
countSpan.textContent = `${mcpTools.length} 个工具`;
|
||||
|
||||
if (mcpTools.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = mcpTools.map((tool, index) => {
|
||||
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
|
||||
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
|
||||
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
|
||||
|
||||
return `
|
||||
<div class="mcp-tool-card">
|
||||
<div class="mcp-tool-header">
|
||||
<div class="mcp-tool-name">${tool.name}</div>
|
||||
<div class="mcp-tool-actions">
|
||||
<button onclick="window.mcpModule.editMcpTool(${index})"
|
||||
style="padding: 4px 10px; border: none; border-radius: 4px; background-color: #2196f3; color: white; cursor: pointer; font-size: 12px;">
|
||||
✏️ 编辑
|
||||
</button>
|
||||
<button onclick="window.mcpModule.deleteMcpTool(${index})"
|
||||
style="padding: 4px 10px; border: none; border-radius: 4px; background-color: #f44336; color: white; cursor: pointer; font-size: 12px;">
|
||||
🗑️ 删除
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mcp-tool-description">${tool.description}</div>
|
||||
<div class="mcp-tool-info">
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">参数数量:</span>
|
||||
<span class="mcp-tool-info-value">${paramCount} 个 ${requiredCount > 0 ? `(${requiredCount} 个必填)` : ''}</span>
|
||||
</div>
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">模拟返回:</span>
|
||||
<span class="mcp-tool-info-value">${hasMockResponse ? '✅ 已配置: ' + JSON.stringify(tool.mockResponse) : '⚪ 使用默认'}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染参数列表
|
||||
*/
|
||||
function renderMcpProperties() {
|
||||
const container = document.getElementById('mcpPropertiesContainer');
|
||||
|
||||
if (mcpProperties.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 20px; color: #999; font-size: 14px;">暂无参数，点击下方按钮添加参数</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = mcpProperties.map((prop, index) => `
|
||||
<div class="mcp-property-item">
|
||||
<div class="mcp-property-header">
|
||||
<span class="mcp-property-name">${prop.name}</span>
|
||||
<button type="button" onclick="window.mcpModule.deleteMcpProperty(${index})"
|
||||
style="padding: 3px 8px; border: none; border-radius: 3px; background-color: #f44336; color: white; cursor: pointer; font-size: 11px;">
|
||||
删除
|
||||
</button>
|
||||
</div>
|
||||
<div class="mcp-property-row">
|
||||
<div>
|
||||
<label class="mcp-small-label">参数名称 *</label>
|
||||
<input type="text" class="mcp-small-input" value="${prop.name}"
|
||||
onchange="window.mcpModule.updateMcpProperty(${index}, 'name', this.value)" required>
|
||||
</div>
|
||||
<div>
|
||||
<label class="mcp-small-label">数据类型 *</label>
|
||||
<select class="mcp-small-input" onchange="window.mcpModule.updateMcpProperty(${index}, 'type', this.value)">
|
||||
<option value="string" ${prop.type === 'string' ? 'selected' : ''}>字符串</option>
|
||||
<option value="integer" ${prop.type === 'integer' ? 'selected' : ''}>整数</option>
|
||||
<option value="number" ${prop.type === 'number' ? 'selected' : ''}>数字</option>
|
||||
<option value="boolean" ${prop.type === 'boolean' ? 'selected' : ''}>布尔值</option>
|
||||
<option value="array" ${prop.type === 'array' ? 'selected' : ''}>数组</option>
|
||||
<option value="object" ${prop.type === 'object' ? 'selected' : ''}>对象</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
${(prop.type === 'integer' || prop.type === 'number') ? `
|
||||
<div class="mcp-property-row">
|
||||
<div>
|
||||
<label class="mcp-small-label">最小值</label>
|
||||
<input type="number" class="mcp-small-input" value="${prop.minimum !== undefined ? prop.minimum : ''}"
|
||||
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'minimum', this.value ? parseFloat(this.value) : undefined)">
|
||||
</div>
|
||||
<div>
|
||||
<label class="mcp-small-label">最大值</label>
|
||||
<input type="number" class="mcp-small-input" value="${prop.maximum !== undefined ? prop.maximum : ''}"
|
||||
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'maximum', this.value ? parseFloat(this.value) : undefined)">
|
||||
</div>
|
||||
</div>
|
||||
` : ''}
|
||||
<div class="mcp-property-row-full">
|
||||
<label class="mcp-small-label">参数描述</label>
|
||||
<input type="text" class="mcp-small-input" value="${prop.description || ''}"
|
||||
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'description', this.value)">
|
||||
</div>
|
||||
<label class="mcp-checkbox-label">
|
||||
<input type="checkbox" ${prop.required ? 'checked' : ''}
|
||||
onchange="window.mcpModule.updateMcpProperty(${index}, 'required', this.checked)">
|
||||
必填参数
|
||||
</label>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加参数
|
||||
*/
|
||||
function addMcpProperty() {
|
||||
mcpProperties.push({
|
||||
name: `param_${mcpProperties.length + 1}`,
|
||||
type: 'string',
|
||||
required: false,
|
||||
description: ''
|
||||
});
|
||||
renderMcpProperties();
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新参数
|
||||
*/
|
||||
function updateMcpProperty(index, field, value) {
|
||||
if (field === 'name') {
|
||||
const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === value);
|
||||
if (isDuplicate) {
|
||||
alert('参数名称已存在，请使用不同的名称');
|
||||
renderMcpProperties();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
mcpProperties[index][field] = value;
|
||||
|
||||
if (field === 'type' && value !== 'integer' && value !== 'number') {
|
||||
delete mcpProperties[index].minimum;
|
||||
delete mcpProperties[index].maximum;
|
||||
renderMcpProperties();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除参数
|
||||
*/
|
||||
function deleteMcpProperty(index) {
|
||||
mcpProperties.splice(index, 1);
|
||||
renderMcpProperties();
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置事件监听
|
||||
*/
|
||||
function setupMcpEventListeners() {
|
||||
const toggleBtn = document.getElementById('toggleMcpTools');
|
||||
const panel = document.getElementById('mcpToolsPanel');
|
||||
const addBtn = document.getElementById('addMcpToolBtn');
|
||||
const modal = document.getElementById('mcpToolModal');
|
||||
const closeBtn = document.getElementById('closeMcpModalBtn');
|
||||
const cancelBtn = document.getElementById('cancelMcpBtn');
|
||||
const form = document.getElementById('mcpToolForm');
|
||||
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
|
||||
|
||||
toggleBtn.addEventListener('click', () => {
|
||||
const isExpanded = panel.classList.contains('expanded');
|
||||
panel.classList.toggle('expanded');
|
||||
toggleBtn.textContent = isExpanded ? '展开' : '收起';
|
||||
});
|
||||
|
||||
addBtn.addEventListener('click', () => openMcpModal());
|
||||
closeBtn.addEventListener('click', closeMcpModal);
|
||||
cancelBtn.addEventListener('click', closeMcpModal);
|
||||
addPropertyBtn.addEventListener('click', addMcpProperty);
|
||||
|
||||
modal.addEventListener('click', (e) => {
|
||||
if (e.target === modal) closeMcpModal();
|
||||
});
|
||||
|
||||
form.addEventListener('submit', handleMcpSubmit);
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开模态框
|
||||
*/
|
||||
function openMcpModal(index = null) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
|
||||
mcpEditingIndex = index;
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
|
||||
if (index !== null) {
|
||||
document.getElementById('mcpModalTitle').textContent = '编辑工具';
|
||||
const tool = mcpTools[index];
|
||||
document.getElementById('mcpToolName').value = tool.name;
|
||||
document.getElementById('mcpToolDescription').value = tool.description;
|
||||
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
|
||||
|
||||
mcpProperties = [];
|
||||
const schema = tool.inputSchema;
|
||||
if (schema.properties) {
|
||||
Object.keys(schema.properties).forEach(key => {
|
||||
const prop = schema.properties[key];
|
||||
mcpProperties.push({
|
||||
name: key,
|
||||
type: prop.type || 'string',
|
||||
minimum: prop.minimum,
|
||||
maximum: prop.maximum,
|
||||
description: prop.description || '',
|
||||
required: schema.required && schema.required.includes(key)
|
||||
});
|
||||
});
|
||||
}
|
||||
} else {
|
||||
document.getElementById('mcpModalTitle').textContent = '添加工具';
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
}
|
||||
|
||||
renderMcpProperties();
|
||||
document.getElementById('mcpToolModal').style.display = 'block';
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭模态框
|
||||
*/
|
||||
function closeMcpModal() {
|
||||
document.getElementById('mcpToolModal').style.display = 'none';
|
||||
mcpEditingIndex = null;
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
document.getElementById('mcpErrorContainer').innerHTML = '';
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理表单提交
|
||||
*/
|
||||
function handleMcpSubmit(e) {
|
||||
e.preventDefault();
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
|
||||
const name = document.getElementById('mcpToolName').value.trim();
|
||||
const description = document.getElementById('mcpToolDescription').value.trim();
|
||||
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
|
||||
|
||||
// 检查名称重复
|
||||
const isDuplicate = mcpTools.some((tool, index) =>
|
||||
tool.name === name && index !== mcpEditingIndex
|
||||
);
|
||||
|
||||
if (isDuplicate) {
|
||||
showMcpError('工具名称已存在，请使用不同的名称');
|
||||
return;
|
||||
}
|
||||
|
||||
// 解析模拟返回结果
|
||||
let mockResponse = null;
|
||||
if (mockResponseText) {
|
||||
try {
|
||||
mockResponse = JSON.parse(mockResponseText);
|
||||
} catch (e) {
|
||||
showMcpError('模拟返回结果不是有效的 JSON 格式: ' + e.message);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 构建 inputSchema
|
||||
const inputSchema = {
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: []
|
||||
};
|
||||
|
||||
mcpProperties.forEach(prop => {
|
||||
const propSchema = { type: prop.type };
|
||||
|
||||
if (prop.description) {
|
||||
propSchema.description = prop.description;
|
||||
}
|
||||
|
||||
if ((prop.type === 'integer' || prop.type === 'number')) {
|
||||
if (prop.minimum !== undefined && prop.minimum !== '') {
|
||||
propSchema.minimum = prop.minimum;
|
||||
}
|
||||
if (prop.maximum !== undefined && prop.maximum !== '') {
|
||||
propSchema.maximum = prop.maximum;
|
||||
}
|
||||
}
|
||||
|
||||
inputSchema.properties[prop.name] = propSchema;
|
||||
|
||||
if (prop.required) {
|
||||
inputSchema.required.push(prop.name);
|
||||
}
|
||||
});
|
||||
|
||||
if (inputSchema.required.length === 0) {
|
||||
delete inputSchema.required;
|
||||
}
|
||||
|
||||
const tool = { name, description, inputSchema, mockResponse };
|
||||
|
||||
if (mcpEditingIndex !== null) {
|
||||
mcpTools[mcpEditingIndex] = tool;
|
||||
log(`已更新工具: ${name}`, 'success');
|
||||
} else {
|
||||
mcpTools.push(tool);
|
||||
log(`已添加工具: ${name}`, 'success');
|
||||
}
|
||||
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
closeMcpModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 显示错误
|
||||
*/
|
||||
function showMcpError(message) {
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = `<div class="mcp-error">${message}</div>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编辑工具
|
||||
*/
|
||||
function editMcpTool(index) {
|
||||
openMcpModal(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除工具
|
||||
*/
|
||||
function deleteMcpTool(index) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
if (confirm(`确定要删除工具 "${mcpTools[index].name}" 吗？`)) {
|
||||
const toolName = mcpTools[index].name;
|
||||
mcpTools.splice(index, 1);
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
log(`已删除工具: ${toolName}`, 'info');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存工具
|
||||
*/
|
||||
function saveMcpTools() {
|
||||
localStorage.setItem('mcpTools', JSON.stringify(mcpTools));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取工具列表
|
||||
*/
|
||||
export function getMcpTools() {
|
||||
return mcpTools.map(tool => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: tool.inputSchema
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行工具调用
|
||||
*/
|
||||
export function executeMcpTool(toolName, toolArgs) {
|
||||
const tool = mcpTools.find(t => t.name === toolName);
|
||||
|
||||
if (!tool) {
|
||||
log(`未找到工具: ${toolName}`, 'error');
|
||||
return {
|
||||
success: false,
|
||||
error: `未知工具: ${toolName}`
|
||||
};
|
||||
}
|
||||
|
||||
// 如果有模拟返回结果，使用它
|
||||
if (tool.mockResponse) {
|
||||
// 替换模板变量
|
||||
let responseStr = JSON.stringify(tool.mockResponse);
|
||||
|
||||
// 替换 ${paramName} 格式的变量
|
||||
if (toolArgs) {
|
||||
Object.keys(toolArgs).forEach(key => {
|
||||
const regex = new RegExp(`\\$\\{${key}\\}`, 'g');
|
||||
responseStr = responseStr.replace(regex, toolArgs[key]);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const response = JSON.parse(responseStr);
|
||||
log(`工具 ${toolName} 执行成功，返回模拟结果: ${responseStr}`, 'success');
|
||||
return response;
|
||||
} catch (e) {
|
||||
log(`解析模拟返回结果失败: ${e.message}`, 'error');
|
||||
return tool.mockResponse;
|
||||
}
|
||||
}
|
||||
|
||||
// 没有模拟返回结果，返回默认成功消息
|
||||
log(`工具 ${toolName} 执行成功，返回默认结果`, 'success');
|
||||
return {
|
||||
success: true,
|
||||
message: `工具 ${toolName} 执行成功`,
|
||||
tool: toolName,
|
||||
arguments: toolArgs
|
||||
};
|
||||
}
|
||||
|
||||
// 暴露全局方法供 HTML 内联事件调用
|
||||
window.mcpModule = {
|
||||
updateMcpProperty,
|
||||
deleteMcpProperty,
|
||||
editMcpTool,
|
||||
deleteMcpTool
|
||||
};
|
||||
@@ -0,0 +1,124 @@
|
||||
import { otaStatusStyle } from '../../ui/dom-helper.js';
|
||||
import { log } from '../../utils/logger.js';
|
||||
|
||||
// WebSocket 连接
|
||||
export async function webSocketConnect(otaUrl, config) {
|
||||
|
||||
if (!validateConfig(config)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 发送OTA请求并获取返回的websocket信息
|
||||
const otaResult = await sendOTA(otaUrl, config);
|
||||
if (!otaResult) {
|
||||
log('无法从OTA服务器获取信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 从OTA响应中提取websocket信息
|
||||
const { websocket } = otaResult;
|
||||
if (!websocket || !websocket.url) {
|
||||
log('OTA响应中缺少websocket信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 使用OTA返回的websocket URL
|
||||
let connUrl = new URL(websocket.url);
|
||||
|
||||
// 添加token参数（从OTA响应中获取）
|
||||
if (websocket.token) {
|
||||
if (websocket.token.startsWith("Bearer ")) {
|
||||
connUrl.searchParams.append('authorization', websocket.token);
|
||||
} else {
|
||||
connUrl.searchParams.append('authorization', 'Bearer ' + websocket.token);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加认证参数（保持原有逻辑）
|
||||
connUrl.searchParams.append('device-id', config.deviceId);
|
||||
connUrl.searchParams.append('client-id', config.clientId);
|
||||
|
||||
const wsurl = connUrl.toString()
|
||||
|
||||
log(`正在连接: ${wsurl}`, 'info');
|
||||
|
||||
if (wsurl) {
|
||||
document.getElementById('serverUrl').value = wsurl;
|
||||
}
|
||||
|
||||
return new WebSocket(connUrl.toString());
|
||||
}
|
||||
|
||||
// 验证配置
|
||||
function validateConfig(config) {
|
||||
if (!config.deviceMac) {
|
||||
log('设备MAC地址不能为空', 'error');
|
||||
return false;
|
||||
}
|
||||
if (!config.clientId) {
|
||||
log('客户端ID不能为空', 'error');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 判断wsUrl路径是否存在错误
|
||||
function validateWsUrl(wsUrl) {
|
||||
if (wsUrl === '') return false;
|
||||
// 检查URL格式
|
||||
if (!wsUrl.startsWith('ws://') && !wsUrl.startsWith('wss://')) {
|
||||
log('URL格式错误，必须以ws://或wss://开头', 'error');
|
||||
return false;
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
// OTA发送请求，验证状态，并返回响应数据
|
||||
async function sendOTA(otaUrl, config) {
|
||||
try {
|
||||
const res = await fetch(otaUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Device-Id': config.deviceId,
|
||||
'Client-Id': config.clientId
|
||||
},
|
||||
body: JSON.stringify({
|
||||
version: 0,
|
||||
uuid: '',
|
||||
application: {
|
||||
name: 'xiaozhi-web-test',
|
||||
version: '1.0.0',
|
||||
compile_time: '2025-04-16 10:00:00',
|
||||
idf_version: '4.4.3',
|
||||
elf_sha256: '1234567890abcdef1234567890abcdef1234567890abcdef'
|
||||
},
|
||||
ota: { label: 'xiaozhi-web-test' },
|
||||
board: {
|
||||
type: 'xiaozhi-web-test',
|
||||
ssid: 'xiaozhi-web-test',
|
||||
rssi: 0,
|
||||
channel: 0,
|
||||
ip: '192.168.1.1',
|
||||
mac: config.deviceMac
|
||||
},
|
||||
flash_size: 0,
|
||||
minimum_free_heap_size: 0,
|
||||
mac_address: config.deviceMac,
|
||||
chip_model_name: '',
|
||||
chip_info: { model: 0, cores: 0, revision: 0, features: 0 },
|
||||
partition_table: [{ label: '', type: 0, subtype: 0, address: 0, size: 0 }]
|
||||
})
|
||||
});
|
||||
|
||||
if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
|
||||
|
||||
const result = await res.json();
|
||||
otaStatusStyle(true)
|
||||
return result; // 返回完整的响应数据
|
||||
} catch (err) {
|
||||
otaStatusStyle(false)
|
||||
return null; // 失败返回null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
// WebSocket消息处理模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { addMessage } from '../../ui/dom-helper.js';
|
||||
import { webSocketConnect } from './ota-connector.js';
|
||||
import { getConfig, saveConnectionUrls } from '../../config/manager.js';
|
||||
import { getAudioPlayer } from '../audio/player.js';
|
||||
import { getAudioRecorder } from '../audio/recorder.js';
|
||||
import { getMcpTools, executeMcpTool, setWebSocket as setMcpWebSocket } from '../mcp/tools.js';
|
||||
|
||||
// WebSocket处理器类
|
||||
export class WebSocketHandler {
|
||||
constructor() {
|
||||
this.websocket = null;
|
||||
this.onConnectionStateChange = null;
|
||||
this.onRecordButtonStateChange = null;
|
||||
this.onSessionStateChange = null;
|
||||
this.onSessionEmotionChange = null;
|
||||
this.currentSessionId = null;
|
||||
this.isRemoteSpeaking = false;
|
||||
}
|
||||
|
||||
// 发送hello握手消息
|
||||
async sendHelloMessage() {
|
||||
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) return false;
|
||||
|
||||
try {
|
||||
const config = getConfig();
|
||||
|
||||
const helloMessage = {
|
||||
type: 'hello',
|
||||
device_id: config.deviceId,
|
||||
device_name: config.deviceName,
|
||||
device_mac: config.deviceMac,
|
||||
token: config.token,
|
||||
features: {
|
||||
mcp: true
|
||||
}
|
||||
};
|
||||
|
||||
log('发送hello握手消息', 'info');
|
||||
this.websocket.send(JSON.stringify(helloMessage));
|
||||
|
||||
return new Promise(resolve => {
|
||||
const timeout = setTimeout(() => {
|
||||
log('等待hello响应超时', 'error');
|
||||
log('提示: 请尝试点击"测试认证"按钮进行连接排查', 'info');
|
||||
resolve(false);
|
||||
}, 5000);
|
||||
|
||||
const onMessageHandler = (event) => {
|
||||
try {
|
||||
const response = JSON.parse(event.data);
|
||||
if (response.type === 'hello' && response.session_id) {
|
||||
log(`服务器握手成功，会话ID: ${response.session_id}`, 'success');
|
||||
clearTimeout(timeout);
|
||||
this.websocket.removeEventListener('message', onMessageHandler);
|
||||
resolve(true);
|
||||
}
|
||||
} catch (e) {
|
||||
// 忽略非JSON消息
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.addEventListener('message', onMessageHandler);
|
||||
});
|
||||
} catch (error) {
|
||||
log(`发送hello消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 处理文本消息
|
||||
handleTextMessage(message) {
|
||||
if (message.type === 'hello') {
|
||||
log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success');
|
||||
} else if (message.type === 'tts') {
|
||||
this.handleTTSMessage(message);
|
||||
} else if (message.type === 'audio') {
|
||||
log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
|
||||
} else if (message.type === 'stt') {
|
||||
log(`识别结果: ${message.text}`, 'info');
|
||||
addMessage(`${message.text}`, true);
|
||||
} else if (message.type === 'llm') {
|
||||
log(`大模型回复: ${message.text}`, 'info');
|
||||
|
||||
// 如果包含表情，更新sessionStatus表情
|
||||
if (message.text && /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u.test(message.text)) {
|
||||
// 提取表情符号
|
||||
const emojiMatch = message.text.match(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u);
|
||||
if (emojiMatch && this.onSessionEmotionChange) {
|
||||
this.onSessionEmotionChange(emojiMatch[0]);
|
||||
}
|
||||
}
|
||||
|
||||
// 只有当文本不仅仅是表情时，才添加到对话中
|
||||
// 移除文本中的表情后检查是否还有内容
|
||||
const textWithoutEmoji = message.text ? message.text.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim() : '';
|
||||
if (textWithoutEmoji) {
|
||||
addMessage(message.text);
|
||||
}
|
||||
} else if (message.type === 'mcp') {
|
||||
this.handleMCPMessage(message);
|
||||
} else {
|
||||
log(`未知消息类型: ${message.type}`, 'info');
|
||||
addMessage(JSON.stringify(message, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
// 处理TTS消息
|
||||
handleTTSMessage(message) {
|
||||
if (message.state === 'start') {
|
||||
log('服务器开始发送语音', 'info');
|
||||
this.currentSessionId = message.session_id;
|
||||
this.isRemoteSpeaking = true;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(true);
|
||||
}
|
||||
} else if (message.state === 'sentence_start') {
|
||||
log(`服务器发送语音段: ${message.text}`, 'info');
|
||||
if (message.text) {
|
||||
addMessage(message.text);
|
||||
}
|
||||
} else if (message.state === 'sentence_end') {
|
||||
log(`语音段结束: ${message.text}`, 'info');
|
||||
} else if (message.state === 'stop') {
|
||||
log('服务器语音传输结束', 'info');
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onRecordButtonStateChange) {
|
||||
this.onRecordButtonStateChange(false);
|
||||
}
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 处理MCP消息
|
||||
handleMCPMessage(message) {
|
||||
const payload = message.payload || {};
|
||||
log(`服务器下发: ${JSON.stringify(message)}`, 'info');
|
||||
|
||||
if (payload.method === 'tools/list') {
|
||||
const tools = getMcpTools();
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"tools": tools
|
||||
}
|
||||
}
|
||||
});
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
log(`回复MCP工具列表: ${tools.length} 个工具`, 'info');
|
||||
|
||||
} else if (payload.method === 'tools/call') {
|
||||
const toolName = payload.params?.name;
|
||||
const toolArgs = payload.params?.arguments;
|
||||
|
||||
log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info');
|
||||
|
||||
const result = executeMcpTool(toolName, toolArgs);
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": JSON.stringify(result)
|
||||
}
|
||||
],
|
||||
"isError": false
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
} else if (payload.method === 'initialize') {
|
||||
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
|
||||
} else {
|
||||
log(`未知的MCP方法: ${payload.method}`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 处理二进制消息
|
||||
async handleBinaryMessage(data) {
|
||||
try {
|
||||
let arrayBuffer;
|
||||
if (data instanceof ArrayBuffer) {
|
||||
arrayBuffer = data;
|
||||
log(`收到ArrayBuffer音频数据，大小: ${data.byteLength}字节`, 'debug');
|
||||
} else if (data instanceof Blob) {
|
||||
arrayBuffer = await data.arrayBuffer();
|
||||
log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`, 'debug');
|
||||
} else {
|
||||
log(`收到未知类型的二进制数据: ${typeof data}`, 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
const opusData = new Uint8Array(arrayBuffer);
|
||||
const audioPlayer = getAudioPlayer();
|
||||
audioPlayer.enqueueAudioData(opusData);
|
||||
} catch (error) {
|
||||
log(`处理二进制消息出错: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 连接WebSocket服务器
|
||||
async connect() {
|
||||
const config = getConfig();
|
||||
log('正在检查OTA状态...', 'info');
|
||||
saveConnectionUrls();
|
||||
|
||||
try {
|
||||
const otaUrl = document.getElementById('otaUrl').value.trim();
|
||||
const ws = await webSocketConnect(otaUrl, config);
|
||||
if (ws === undefined) {
|
||||
return false;
|
||||
}
|
||||
this.websocket = ws;
|
||||
|
||||
// 设置接收二进制数据的类型为ArrayBuffer
|
||||
this.websocket.binaryType = 'arraybuffer';
|
||||
|
||||
// 设置 MCP 模块的 WebSocket 实例
|
||||
setMcpWebSocket(this.websocket);
|
||||
|
||||
// 设置录音器的WebSocket
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.setWebSocket(this.websocket);
|
||||
|
||||
this.setupEventHandlers();
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`连接错误: ${error.message}`, 'error');
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 设置事件处理器
|
||||
setupEventHandlers() {
|
||||
this.websocket.onopen = async () => {
|
||||
const url = document.getElementById('serverUrl').value;
|
||||
log(`已连接到服务器: ${url}`, 'success');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(true);
|
||||
}
|
||||
|
||||
// 连接成功后，默认状态为聆听中
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
|
||||
await this.sendHelloMessage();
|
||||
};
|
||||
|
||||
this.websocket.onclose = () => {
|
||||
log('已断开连接', 'info');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
};
|
||||
|
||||
this.websocket.onerror = (error) => {
|
||||
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.onmessage = (event) => {
|
||||
try {
|
||||
if (typeof event.data === 'string') {
|
||||
const message = JSON.parse(event.data);
|
||||
this.handleTextMessage(message);
|
||||
} else {
|
||||
this.handleBinaryMessage(event.data);
|
||||
}
|
||||
} catch (error) {
|
||||
log(`WebSocket消息处理错误: ${error.message}`, 'error');
|
||||
if (typeof event.data === 'string') {
|
||||
addMessage(event.data);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// 断开连接
|
||||
disconnect() {
|
||||
if (!this.websocket) return;
|
||||
|
||||
this.websocket.close();
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
}
|
||||
|
||||
// 发送文本消息
|
||||
sendTextMessage(text) {
|
||||
if (text === '' || !this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// 如果对方正在说话，先发送打断消息
|
||||
if (this.isRemoteSpeaking && this.currentSessionId) {
|
||||
const abortMessage = {
|
||||
session_id: this.currentSessionId,
|
||||
type: 'abort',
|
||||
reason: 'wake_word_detected'
|
||||
};
|
||||
this.websocket.send(JSON.stringify(abortMessage));
|
||||
log('发送打断消息', 'info');
|
||||
}
|
||||
|
||||
const listenMessage = {
|
||||
type: 'listen',
|
||||
mode: 'manual',
|
||||
state: 'detect',
|
||||
text: text
|
||||
};
|
||||
|
||||
this.websocket.send(JSON.stringify(listenMessage));
|
||||
log(`发送文本消息: ${text}`, 'info');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`发送消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 获取WebSocket实例
|
||||
getWebSocket() {
|
||||
return this.websocket;
|
||||
}
|
||||
|
||||
// 检查是否已连接
|
||||
isConnected() {
|
||||
return this.websocket && this.websocket.readyState === WebSocket.OPEN;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let wsHandlerInstance = null;
|
||||
|
||||
export function getWebSocketHandler() {
|
||||
if (!wsHandlerInstance) {
|
||||
wsHandlerInstance = new WebSocketHandler();
|
||||
}
|
||||
return wsHandlerInstance;
|
||||
}
|
||||
Reference in New Issue
Block a user