mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 02:43:55 +08:00
update:test迁移重命名为digital-human
This commit is contained in:
3030332422
@@ -0,0 +1,182 @@
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
export function checkOpusLoaded() {
|
||||
try {
|
||||
// 检查Module是否存在（本地库导出的全局变量）
|
||||
if (typeof Module === 'undefined') {
|
||||
throw new Error('Opus库未加载，Module对象不存在');
|
||||
}
|
||||
|
||||
// 尝试先使用Module.instance（libopus.js最后一行导出方式）
|
||||
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
|
||||
// 使用Module.instance对象替换全局Module对象
|
||||
window.ModuleInstance = Module.instance;
|
||||
log('Opus库加载成功（使用Module.instance）', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
// 如果没有Module.instance，检查全局Module函数
|
||||
if (typeof Module._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module;
|
||||
log('Opus库加载成功（使用全局Module）', 'success');
|
||||
|
||||
// 3秒后隐藏状态
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error('Opus解码函数未找到，可能Module结构不正确');
|
||||
} catch (err) {
|
||||
log(`Opus库加载失败，请检查libopus.js文件是否存在且正确: ${err.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 创建一个Opus编码器
|
||||
let opusEncoder = null;
|
||||
export function initOpusEncoder() {
|
||||
try {
|
||||
if (opusEncoder) {
|
||||
return opusEncoder; // 已经初始化过
|
||||
}
|
||||
|
||||
if (!window.ModuleInstance) {
|
||||
log('无法创建Opus编码器：ModuleInstance不可用', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 初始化一个Opus编码器
|
||||
const mod = window.ModuleInstance;
|
||||
const sampleRate = 16000; // 16kHz采样率
|
||||
const channels = 1; // 单声道
|
||||
const application = 2048; // OPUS_APPLICATION_VOIP = 2048
|
||||
|
||||
// 创建编码器
|
||||
opusEncoder = {
|
||||
channels: channels,
|
||||
sampleRate: sampleRate,
|
||||
frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
|
||||
maxPacketSize: 4000, // 最大包大小
|
||||
module: mod,
|
||||
|
||||
// 初始化编码器
|
||||
init: function () {
|
||||
try {
|
||||
// 获取编码器大小
|
||||
const encoderSize = mod._opus_encoder_get_size(this.channels);
|
||||
log(`Opus编码器大小: ${encoderSize}字节`, 'info');
|
||||
|
||||
// 分配内存
|
||||
this.encoderPtr = mod._malloc(encoderSize);
|
||||
if (!this.encoderPtr) {
|
||||
throw new Error("无法分配编码器内存");
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
const err = mod._opus_encoder_init(
|
||||
this.encoderPtr,
|
||||
this.sampleRate,
|
||||
this.channels,
|
||||
application
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
throw new Error(`Opus编码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
// 设置位率 (16kbps)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4002, 16000); // OPUS_SET_BITRATE
|
||||
|
||||
// 设置复杂度 (0-10, 越高质量越好但CPU使用越多)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4010, 5); // OPUS_SET_COMPLEXITY
|
||||
|
||||
// 设置使用DTX (不传输静音帧)
|
||||
mod._opus_encoder_ctl(this.encoderPtr, 4016, 1); // OPUS_SET_DTX
|
||||
|
||||
log("Opus编码器初始化成功", 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (this.encoderPtr) {
|
||||
mod._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
log(`Opus编码器初始化失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
// 编码PCM数据为Opus
|
||||
encode: function (pcmData) {
|
||||
if (!this.encoderPtr) {
|
||||
if (!this.init()) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
// 为PCM数据分配内存
|
||||
const pcmPtr = mod._malloc(pcmData.length * 2); // 2字节/int16
|
||||
|
||||
// 将PCM数据复制到HEAP
|
||||
for (let i = 0; i < pcmData.length; i++) {
|
||||
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
|
||||
}
|
||||
|
||||
// 为输出分配内存
|
||||
const outPtr = mod._malloc(this.maxPacketSize);
|
||||
|
||||
// 进行编码
|
||||
const encodedLen = mod._opus_encode(
|
||||
this.encoderPtr,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
outPtr,
|
||||
this.maxPacketSize
|
||||
);
|
||||
|
||||
if (encodedLen < 0) {
|
||||
throw new Error(`Opus编码失败: ${encodedLen}`);
|
||||
}
|
||||
|
||||
// 复制编码后的数据
|
||||
const opusData = new Uint8Array(encodedLen);
|
||||
for (let i = 0; i < encodedLen; i++) {
|
||||
opusData[i] = mod.HEAPU8[outPtr + i];
|
||||
}
|
||||
|
||||
// 释放内存
|
||||
mod._free(pcmPtr);
|
||||
mod._free(outPtr);
|
||||
|
||||
return opusData;
|
||||
} catch (error) {
|
||||
log(`Opus编码出错: ${error.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
},
|
||||
|
||||
// 销毁编码器
|
||||
destroy: function () {
|
||||
if (this.encoderPtr) {
|
||||
this.module._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
opusEncoder.init();
|
||||
return opusEncoder;
|
||||
} catch (error) {
|
||||
log(`创建Opus编码器失败: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
// 音频播放模块
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { createStreamingContext } from './stream-context.js?v=0205';
|
||||
|
||||
// 音频播放器类
|
||||
export class AudioPlayer {
|
||||
constructor() {
|
||||
// 音频参数
|
||||
this.SAMPLE_RATE = 16000;
|
||||
this.CHANNELS = 1;
|
||||
this.FRAME_SIZE = 960;
|
||||
this.MIN_AUDIO_DURATION = 0.12;
|
||||
|
||||
// 状态
|
||||
this.audioContext = null;
|
||||
this.opusDecoder = null;
|
||||
this.streamingContext = null;
|
||||
this.queue = new BlockingQueue();
|
||||
this.isPlaying = false;
|
||||
}
|
||||
|
||||
// 获取或创建AudioContext
|
||||
getAudioContext() {
|
||||
if (!this.audioContext) {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: this.SAMPLE_RATE,
|
||||
latencyHint: 'interactive'
|
||||
});
|
||||
log('创建音频上下文，采样率: ' + this.SAMPLE_RATE + 'Hz', 'debug');
|
||||
}
|
||||
return this.audioContext;
|
||||
}
|
||||
|
||||
// 初始化Opus解码器
|
||||
async initOpusDecoder() {
|
||||
if (this.opusDecoder) return this.opusDecoder;
|
||||
|
||||
try {
|
||||
if (typeof window.ModuleInstance === 'undefined') {
|
||||
if (typeof Module !== 'undefined') {
|
||||
window.ModuleInstance = Module;
|
||||
log('使用全局Module作为ModuleInstance', 'info');
|
||||
} else {
|
||||
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
|
||||
}
|
||||
}
|
||||
|
||||
const mod = window.ModuleInstance;
|
||||
|
||||
this.opusDecoder = {
|
||||
channels: this.CHANNELS,
|
||||
rate: this.SAMPLE_RATE,
|
||||
frameSize: this.FRAME_SIZE,
|
||||
module: mod,
|
||||
decoderPtr: null,
|
||||
|
||||
init: function () {
|
||||
if (this.decoderPtr) return true;
|
||||
|
||||
const decoderSize = mod._opus_decoder_get_size(this.channels);
|
||||
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
|
||||
|
||||
this.decoderPtr = mod._malloc(decoderSize);
|
||||
if (!this.decoderPtr) {
|
||||
throw new Error("无法分配解码器内存");
|
||||
}
|
||||
|
||||
const err = mod._opus_decoder_init(
|
||||
this.decoderPtr,
|
||||
this.rate,
|
||||
this.channels
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
this.destroy();
|
||||
throw new Error(`Opus解码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
log("Opus解码器初始化成功", 'success');
|
||||
return true;
|
||||
},
|
||||
|
||||
decode: function (opusData) {
|
||||
if (!this.decoderPtr) {
|
||||
if (!this.init()) {
|
||||
throw new Error("解码器未初始化且无法初始化");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
const opusPtr = mod._malloc(opusData.length);
|
||||
mod.HEAPU8.set(opusData, opusPtr);
|
||||
|
||||
const pcmPtr = mod._malloc(this.frameSize * 2);
|
||||
|
||||
const decodedSamples = mod._opus_decode(
|
||||
this.decoderPtr,
|
||||
opusPtr,
|
||||
opusData.length,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
0
|
||||
);
|
||||
|
||||
if (decodedSamples < 0) {
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
throw new Error(`Opus解码失败: ${decodedSamples}`);
|
||||
}
|
||||
|
||||
const decodedData = new Int16Array(decodedSamples);
|
||||
for (let i = 0; i < decodedSamples; i++) {
|
||||
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
|
||||
}
|
||||
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
|
||||
return decodedData;
|
||||
} catch (error) {
|
||||
log(`Opus解码错误: ${error.message}`, 'error');
|
||||
return new Int16Array(0);
|
||||
}
|
||||
},
|
||||
|
||||
destroy: function () {
|
||||
if (this.decoderPtr) {
|
||||
this.module._free(this.decoderPtr);
|
||||
this.decoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (!this.opusDecoder.init()) {
|
||||
throw new Error("Opus解码器初始化失败");
|
||||
}
|
||||
|
||||
return this.opusDecoder;
|
||||
|
||||
} catch (error) {
|
||||
log(`Opus解码器初始化失败: ${error.message}`, 'error');
|
||||
this.opusDecoder = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 启动音频缓冲
|
||||
async startAudioBuffering() {
|
||||
log("开始音频缓冲...", 'info');
|
||||
|
||||
this.initOpusDecoder().catch(error => {
|
||||
log(`预初始化Opus解码器失败: ${error.message}`, 'warning');
|
||||
});
|
||||
|
||||
const timeout = 400;
|
||||
while (true) {
|
||||
const packets = await this.queue.dequeue(
|
||||
6,
|
||||
timeout,
|
||||
(count) => {
|
||||
log(`缓冲超时，当前缓冲包数: ${count}，开始播放`, 'info');
|
||||
}
|
||||
);
|
||||
if (packets.length) {
|
||||
log(`已缓冲 ${packets.length} 个音频包，开始播放`, 'info');
|
||||
this.streamingContext.pushAudioBuffer(packets);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const data = await this.queue.dequeue(99, 30);
|
||||
if (data.length) {
|
||||
this.streamingContext.pushAudioBuffer(data);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 播放已缓冲的音频
|
||||
async playBufferedAudio() {
|
||||
try {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (!this.opusDecoder) {
|
||||
log('初始化Opus解码器...', 'info');
|
||||
try {
|
||||
this.opusDecoder = await this.initOpusDecoder();
|
||||
if (!this.opusDecoder) {
|
||||
throw new Error('解码器初始化失败');
|
||||
}
|
||||
log('Opus解码器初始化成功', 'success');
|
||||
} catch (error) {
|
||||
log('Opus解码器初始化失败: ' + error.message, 'error');
|
||||
this.isPlaying = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.streamingContext) {
|
||||
this.streamingContext = createStreamingContext(
|
||||
this.opusDecoder,
|
||||
this.audioContext,
|
||||
this.SAMPLE_RATE,
|
||||
this.CHANNELS,
|
||||
this.MIN_AUDIO_DURATION
|
||||
);
|
||||
}
|
||||
|
||||
this.streamingContext.decodeOpusFrames();
|
||||
this.streamingContext.startPlaying();
|
||||
|
||||
} catch (error) {
|
||||
log(`播放已缓冲的音频出错: ${error.message}`, 'error');
|
||||
this.isPlaying = false;
|
||||
this.streamingContext = null;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加音频数据到队列
|
||||
enqueueAudioData(opusData) {
|
||||
if (opusData.length > 0) {
|
||||
this.queue.enqueue(opusData);
|
||||
} else {
|
||||
log('收到空音频数据帧，可能是结束标志', 'warning');
|
||||
if (this.isPlaying && this.streamingContext) {
|
||||
this.streamingContext.endOfStream = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 预加载解码器
|
||||
async preload() {
|
||||
log('预加载Opus解码器...', 'info');
|
||||
try {
|
||||
await this.initOpusDecoder();
|
||||
log('Opus解码器预加载成功', 'success');
|
||||
} catch (error) {
|
||||
log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 启动播放系统
|
||||
async start() {
|
||||
await this.preload();
|
||||
this.playBufferedAudio();
|
||||
this.startAudioBuffering();
|
||||
}
|
||||
|
||||
// 获取音频包统计信息
|
||||
getAudioStats() {
|
||||
if (!this.streamingContext) {
|
||||
return {
|
||||
pendingDecode: 0,
|
||||
pendingPlay: 0,
|
||||
totalPending: 0
|
||||
};
|
||||
}
|
||||
|
||||
const pendingDecode = this.streamingContext.getPendingDecodeCount();
|
||||
const pendingPlay = this.streamingContext.getPendingPlayCount();
|
||||
|
||||
return {
|
||||
pendingDecode, // 待解码包数
|
||||
pendingPlay, // 待播放包数
|
||||
totalPending: pendingDecode + pendingPlay // 总待处理包数
|
||||
};
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲并停止播放
|
||||
clearAllAudio() {
|
||||
log('AudioPlayer: 清空所有音频', 'info');
|
||||
|
||||
// 清空接收队列（使用clear方法保持对象引用）
|
||||
this.queue.clear();
|
||||
|
||||
// 清空流上下文的所有缓冲
|
||||
if (this.streamingContext) {
|
||||
this.streamingContext.clearAllBuffers();
|
||||
}
|
||||
|
||||
log('AudioPlayer: 音频已清空', 'success');
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let audioPlayerInstance = null;
|
||||
|
||||
export function getAudioPlayer() {
|
||||
if (!audioPlayerInstance) {
|
||||
audioPlayerInstance = new AudioPlayer();
|
||||
}
|
||||
return audioPlayerInstance;
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
// Audio recording module
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
import { initOpusEncoder } from './opus-codec.js?v=0205';
|
||||
import { getAudioPlayer } from './player.js?v=0205';
|
||||
|
||||
// Audio recorder class
|
||||
export class AudioRecorder {
|
||||
constructor() {
|
||||
this.isRecording = false;
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
this.audioProcessor = null;
|
||||
this.audioProcessorType = null;
|
||||
this.audioSource = null;
|
||||
this.opusEncoder = null;
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.visualizationRequest = null;
|
||||
this.recordingTimer = null;
|
||||
this.websocket = null;
|
||||
// Callback functions
|
||||
this.onRecordingStart = null;
|
||||
this.onRecordingStop = null;
|
||||
this.onVisualizerUpdate = null;
|
||||
}
|
||||
|
||||
// Set WebSocket instance
|
||||
setWebSocket(ws) {
|
||||
this.websocket = ws;
|
||||
}
|
||||
|
||||
// Get AudioContext instance
|
||||
getAudioContext() {
|
||||
return getAudioPlayer().getAudioContext();
|
||||
}
|
||||
|
||||
// Initialize encoder
|
||||
initEncoder() {
|
||||
if (!this.opusEncoder) {
|
||||
this.opusEncoder = initOpusEncoder();
|
||||
}
|
||||
return this.opusEncoder;
|
||||
}
|
||||
|
||||
// PCM processor code
|
||||
getAudioProcessorCode() {
|
||||
return `
|
||||
class AudioRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffers = [];
|
||||
this.frameSize = 960;
|
||||
this.buffer = new Int16Array(this.frameSize);
|
||||
this.bufferIndex = 0;
|
||||
this.isRecording = false;
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.command === 'start') {
|
||||
this.isRecording = true;
|
||||
this.port.postMessage({ type: 'status', status: 'started' });
|
||||
} else if (event.data.command === 'stop') {
|
||||
this.isRecording = false;
|
||||
if (this.bufferIndex > 0) {
|
||||
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
||||
this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
this.port.postMessage({ type: 'status', status: 'stopped' });
|
||||
}
|
||||
};
|
||||
}
|
||||
process(inputs, outputs, parameters) {
|
||||
if (!this.isRecording) return true;
|
||||
const input = inputs[0][0];
|
||||
if (!input) return true;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (this.bufferIndex >= this.frameSize) {
|
||||
this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
|
||||
`;
|
||||
}
|
||||
|
||||
// Create audio processor
|
||||
async createAudioProcessor() {
|
||||
this.audioContext = this.getAudioContext();
|
||||
try {
|
||||
if (this.audioContext.audioWorklet) {
|
||||
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
await this.audioContext.audioWorklet.addModule(url);
|
||||
URL.revokeObjectURL(url);
|
||||
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
|
||||
audioProcessor.port.onmessage = (event) => {
|
||||
if (event.data.type === 'buffer') {
|
||||
this.processPCMBuffer(event.data.buffer);
|
||||
}
|
||||
};
|
||||
log('使用AudioWorklet处理音频', 'success');
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
audioProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
return { node: audioProcessor, type: 'worklet' };
|
||||
} else {
|
||||
log('AudioWorklet不可用，使用ScriptProcessorNode作为后备方案', 'warning');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
} catch (error) {
|
||||
log(`创建音频处理器失败: ${error.message}，尝试后备方案`, 'error');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
// Create ScriptProcessor as fallback
|
||||
createScriptProcessor() {
|
||||
try {
|
||||
const frameSize = 4096;
|
||||
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
|
||||
scriptProcessor.onaudioprocess = (event) => {
|
||||
if (!this.isRecording) return;
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
const buffer = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
this.processPCMBuffer(buffer);
|
||||
};
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
scriptProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
log('使用ScriptProcessorNode作为后备方案成功', 'warning');
|
||||
return { node: scriptProcessor, type: 'processor' };
|
||||
} catch (fallbackError) {
|
||||
log(`后备方案也失败: ${fallbackError.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Process PCM buffer data
|
||||
processPCMBuffer(buffer) {
|
||||
if (!this.isRecording) return;
|
||||
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
|
||||
newBuffer.set(this.pcmDataBuffer);
|
||||
newBuffer.set(buffer, this.pcmDataBuffer.length);
|
||||
this.pcmDataBuffer = newBuffer;
|
||||
const samplesPerFrame = 960;
|
||||
while (this.pcmDataBuffer.length >= samplesPerFrame) {
|
||||
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
|
||||
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
|
||||
this.encodeAndSendOpus(frameData);
|
||||
}
|
||||
}
|
||||
|
||||
// Encode and send Opus data
|
||||
encodeAndSendOpus(pcmData = null) {
|
||||
if (!this.opusEncoder) {
|
||||
log('Opus编码器未初始化', 'error');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (pcmData) {
|
||||
const opusData = this.opusEncoder.encode(pcmData);
|
||||
if (opusData && opusData.length > 0) {
|
||||
this.audioBuffers.push(opusData.buffer);
|
||||
this.totalAudioSize += opusData.length;
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.websocket.send(opusData.buffer);
|
||||
} catch (error) {
|
||||
log(`WebSocket发送错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('Opus编码失败，未返回有效数据', 'error');
|
||||
}
|
||||
} else {
|
||||
if (this.pcmDataBuffer.length > 0) {
|
||||
const samplesPerFrame = 960;
|
||||
if (this.pcmDataBuffer.length < samplesPerFrame) {
|
||||
const paddedBuffer = new Int16Array(samplesPerFrame);
|
||||
paddedBuffer.set(this.pcmDataBuffer);
|
||||
this.encodeAndSendOpus(paddedBuffer);
|
||||
} else {
|
||||
this.encodeAndSendOpus(this.pcmDataBuffer.slice(0, samplesPerFrame));
|
||||
}
|
||||
this.pcmDataBuffer = new Int16Array(0);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log(`Opus编码错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// Start recording
|
||||
async start() {
|
||||
if (this.isRecording) return false;
|
||||
try {
|
||||
if (!this.initEncoder()) {
|
||||
log('无法开始录音: Opus编码器初始化失败', 'error');
|
||||
return false;
|
||||
}
|
||||
log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
this.audioContext = this.getAudioContext();
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
const processorResult = await this.createAudioProcessor();
|
||||
if (!processorResult) {
|
||||
log('无法创建音频处理器', 'error');
|
||||
return false;
|
||||
}
|
||||
this.audioProcessor = processorResult.node;
|
||||
this.audioProcessorType = processorResult.type;
|
||||
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 2048;
|
||||
this.audioSource.connect(this.analyser);
|
||||
this.audioSource.connect(this.audioProcessor);
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.isRecording = true;
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'start' });
|
||||
}
|
||||
// Send listening start message
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
log(`已发送录音开始消息`, 'info');
|
||||
} else {
|
||||
log('WebSocket未连接，无法发送开始消息', 'error');
|
||||
return false;
|
||||
}
|
||||
// Start visualization
|
||||
if (this.onVisualizerUpdate) {
|
||||
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
this.startVisualization(dataArray);
|
||||
}
|
||||
// Immediately notify recording start, update button state
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(0);
|
||||
}
|
||||
// Start recording timer
|
||||
let recordingSeconds = 0;
|
||||
this.recordingTimer = setInterval(() => {
|
||||
recordingSeconds += 0.1;
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(recordingSeconds);
|
||||
}
|
||||
}, 100);
|
||||
log('已开始PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音启动错误: ${error.message}`, 'error');
|
||||
this.isRecording = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Start visualization
|
||||
startVisualization(dataArray) {
|
||||
const draw = () => {
|
||||
this.visualizationRequest = requestAnimationFrame(() => draw());
|
||||
if (!this.isRecording) return;
|
||||
this.analyser.getByteFrequencyData(dataArray);
|
||||
if (this.onVisualizerUpdate) {
|
||||
this.onVisualizerUpdate(dataArray);
|
||||
}
|
||||
};
|
||||
draw();
|
||||
}
|
||||
|
||||
// Stop recording
|
||||
stop() {
|
||||
if (!this.isRecording) return false;
|
||||
try {
|
||||
this.isRecording = false;
|
||||
if (this.audioProcessor) {
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'stop' });
|
||||
}
|
||||
this.audioProcessor.disconnect();
|
||||
this.audioProcessor = null;
|
||||
}
|
||||
if (this.audioSource) {
|
||||
this.audioSource.disconnect();
|
||||
this.audioSource = null;
|
||||
}
|
||||
if (this.visualizationRequest) {
|
||||
cancelAnimationFrame(this.visualizationRequest);
|
||||
this.visualizationRequest = null;
|
||||
}
|
||||
if (this.recordingTimer) {
|
||||
clearInterval(this.recordingTimer);
|
||||
this.recordingTimer = null;
|
||||
}
|
||||
// Encode and send remaining data
|
||||
this.encodeAndSendOpus();
|
||||
// Send end signal
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const emptyOpusFrame = new Uint8Array(0);
|
||||
this.websocket.send(emptyOpusFrame);
|
||||
log('已发送录音停止信号', 'info');
|
||||
}
|
||||
if (this.onRecordingStop) {
|
||||
this.onRecordingStop();
|
||||
}
|
||||
log('已停止PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音停止错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Get analyser
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
let audioRecorderInstance = null;
|
||||
|
||||
export function getAudioRecorder() {
|
||||
if (!audioRecorderInstance) {
|
||||
audioRecorderInstance = new AudioRecorder();
|
||||
}
|
||||
return audioRecorderInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if microphone is available
|
||||
* @returns {Promise<boolean>} Returns true if available, false if not available
|
||||
*/
|
||||
export async function checkMicrophoneAvailability() {
|
||||
// Check if browser supports getUserMedia API
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log('浏览器不支持getUserMedia API', 'warning');
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
// Try to access microphone
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
// Immediately stop all tracks to release microphone
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
log('麦克风可用性检查成功', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`麦克风不可用: ${error.message}`, 'warning');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if it is HTTP non-localhost access
|
||||
* @returns {boolean} Returns true if it is HTTP non-localhost access
|
||||
*/
|
||||
export function isHttpNonLocalhost() {
|
||||
const protocol = window.location.protocol;
|
||||
const hostname = window.location.hostname;
|
||||
// Check if it is HTTP protocol
|
||||
if (protocol !== 'http:') {
|
||||
return false;
|
||||
}
|
||||
// localhost and 127.0.0.1 can use microphone
|
||||
if (hostname === 'localhost' || hostname === '127.0.0.1') {
|
||||
return false;
|
||||
}
|
||||
// Private IP addresses can also use microphone (browser allows)
|
||||
if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
|
||||
return false;
|
||||
}
|
||||
// Other HTTP access is considered non-localhost
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0205';
|
||||
import { log } from '../../utils/logger.js?v=0205';
|
||||
|
||||
// 音频流播放上下文类
|
||||
export class StreamingContext {
|
||||
constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
this.opusDecoder = opusDecoder;
|
||||
this.audioContext = audioContext;
|
||||
|
||||
// 音频参数
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
this.minAudioDuration = minAudioDuration;
|
||||
|
||||
// 初始化队列和状态
|
||||
this.queue = []; // 已解码的PCM队列。正在播放
|
||||
this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放
|
||||
this.pendingAudioBufferQueue = []; // 待处理的缓存队列
|
||||
this.audioBufferQueue = new BlockingQueue(); // 缓存队列
|
||||
this.playing = false; // 是否正在播放
|
||||
this.endOfStream = false; // 是否收到结束信号
|
||||
this.source = null; // 当前音频源
|
||||
this.totalSamples = 0; // 累积的总样本数
|
||||
this.lastPlayTime = 0; // 上次播放的时间戳
|
||||
this.scheduledEndTime = 0; // 已调度音频的结束时间
|
||||
|
||||
// 初始化分析器节点（供Live2D使用）
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 256;
|
||||
}
|
||||
|
||||
// 缓存音频数组
|
||||
pushAudioBuffer(item) {
|
||||
this.audioBufferQueue.enqueue(...item);
|
||||
}
|
||||
|
||||
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
|
||||
async getPendingAudioBufferQueue() {
|
||||
// 等待数据到达并获取
|
||||
const data = await this.audioBufferQueue.dequeue();
|
||||
// 赋值给待处理队列
|
||||
this.pendingAudioBufferQueue = data;
|
||||
}
|
||||
|
||||
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
|
||||
async getQueue(minSamples) {
|
||||
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
|
||||
|
||||
// 等待数据并获取
|
||||
const tempArray = await this.activeQueue.dequeue(num);
|
||||
this.queue.push(...tempArray);
|
||||
}
|
||||
|
||||
// 将Int16音频数据转换为Float32音频数据
|
||||
convertInt16ToFloat32(int16Data) {
|
||||
const float32Data = new Float32Array(int16Data.length);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
// 将[-32768,32767]范围转换为[-1,1]，统一使用32768.0避免不对称失真
|
||||
float32Data[i] = int16Data[i] / 32768.0;
|
||||
}
|
||||
return float32Data;
|
||||
}
|
||||
|
||||
// 获取待解码包数
|
||||
getPendingDecodeCount() {
|
||||
return this.audioBufferQueue.length + this.pendingAudioBufferQueue.length;
|
||||
}
|
||||
|
||||
// 获取待播放样本数（转换为包数，每包960样本）
|
||||
getPendingPlayCount() {
|
||||
// 计算已在队列中的样本
|
||||
const queuedSamples = this.activeQueue.length + this.queue.length;
|
||||
|
||||
// 计算已调度但未播放的样本（在Web Audio缓冲区中）
|
||||
let scheduledSamples = 0;
|
||||
if (this.playing && this.scheduledEndTime) {
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const remainingTime = Math.max(0, this.scheduledEndTime - currentTime);
|
||||
scheduledSamples = Math.floor(remainingTime * this.sampleRate);
|
||||
}
|
||||
|
||||
const totalSamples = queuedSamples + scheduledSamples;
|
||||
return Math.ceil(totalSamples / 960);
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲
|
||||
clearAllBuffers() {
|
||||
log('清空所有音频缓冲', 'info');
|
||||
|
||||
// 清空所有队列（使用clear方法保持对象引用）
|
||||
this.audioBufferQueue.clear();
|
||||
this.pendingAudioBufferQueue = [];
|
||||
this.activeQueue.clear();
|
||||
this.queue = [];
|
||||
|
||||
// 停止当前播放的音频源
|
||||
if (this.source) {
|
||||
try {
|
||||
this.source.stop();
|
||||
this.source.disconnect();
|
||||
} catch (e) {
|
||||
// 忽略已经停止的错误
|
||||
}
|
||||
this.source = null;
|
||||
}
|
||||
|
||||
// 重置状态
|
||||
this.playing = false;
|
||||
this.scheduledEndTime = this.audioContext.currentTime;
|
||||
this.totalSamples = 0;
|
||||
|
||||
log('音频缓冲已清空', 'success');
|
||||
}
|
||||
|
||||
// 获取分析器节点（供Live2D使用）
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
|
||||
// 将Opus数据解码为PCM
|
||||
async decodeOpusFrames() {
|
||||
if (!this.opusDecoder) {
|
||||
log('Opus解码器未初始化，无法解码', 'error');
|
||||
return;
|
||||
} else {
|
||||
log('Opus解码器启动', 'info');
|
||||
}
|
||||
|
||||
while (true) {
|
||||
let decodedSamples = [];
|
||||
for (const frame of this.pendingAudioBufferQueue) {
|
||||
try {
|
||||
// 使用Opus解码器解码
|
||||
const frameData = this.opusDecoder.decode(frame);
|
||||
if (frameData && frameData.length > 0) {
|
||||
// 转换为Float32
|
||||
const floatData = this.convertInt16ToFloat32(frameData);
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < floatData.length; i++) {
|
||||
decodedSamples.push(floatData[i]);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log("Opus解码失败: " + error.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedSamples.length > 0) {
|
||||
// 使用循环替代展开运算符
|
||||
for (let i = 0; i < decodedSamples.length; i++) {
|
||||
this.activeQueue.enqueue(decodedSamples[i]);
|
||||
}
|
||||
this.totalSamples += decodedSamples.length;
|
||||
} else {
|
||||
log('没有成功解码的样本', 'warning');
|
||||
}
|
||||
await this.getPendingAudioBufferQueue();
|
||||
}
|
||||
}
|
||||
|
||||
// 开始播放音频
|
||||
async startPlaying() {
|
||||
this.scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
|
||||
|
||||
while (true) {
|
||||
// 初始缓冲：等待足够的样本再开始播放
|
||||
const minSamples = this.sampleRate * this.minAudioDuration * 2;
|
||||
if (!this.playing && this.queue.length < minSamples) {
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
this.playing = true;
|
||||
|
||||
// 持续播放队列中的音频，每次播放一个小块
|
||||
while (this.playing && this.queue.length > 0) {
|
||||
// 每次播放120ms的音频（2个Opus包）
|
||||
const playDuration = 0.12;
|
||||
const targetSamples = Math.floor(this.sampleRate * playDuration);
|
||||
const actualSamples = Math.min(this.queue.length, targetSamples);
|
||||
|
||||
if (actualSamples === 0) break;
|
||||
|
||||
const currentSamples = this.queue.splice(0, actualSamples);
|
||||
const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate);
|
||||
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
|
||||
|
||||
// 创建音频源
|
||||
this.source = this.audioContext.createBufferSource();
|
||||
this.source.buffer = audioBuffer;
|
||||
|
||||
// 精确调度播放时间
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const startTime = Math.max(this.scheduledEndTime, currentTime);
|
||||
|
||||
// 连接到分析器和输出
|
||||
this.source.connect(this.analyser);
|
||||
this.source.connect(this.audioContext.destination);
|
||||
|
||||
log(`调度播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'debug');
|
||||
this.source.start(startTime);
|
||||
|
||||
// 更新下一个音频块的调度时间
|
||||
const duration = audioBuffer.duration;
|
||||
this.scheduledEndTime = startTime + duration;
|
||||
this.lastPlayTime = startTime;
|
||||
|
||||
// 如果队列中数据不足，等待新数据
|
||||
if (this.queue.length < targetSamples) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 等待新数据
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 创建streamingContext实例的工厂函数
|
||||
export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
|
||||
return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration);
|
||||
}
|
||||
Reference in New Issue
Block a user