From 18acec1a8144c535b6435aaceca65420da1ba27c Mon Sep 17 00:00:00 2001 From: Minamiyama Date: Sun, 24 Aug 2025 11:31:30 +0800 Subject: [PATCH 01/13] =?UTF-8?q?refactor(audio):=20=E5=B0=86=E6=B5=81?= =?UTF-8?q?=E5=BC=8F=E9=9F=B3=E9=A2=91=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E6=8F=90=E5=8F=96=E4=B8=BA=E7=8B=AC=E7=AB=8B=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将原本内联在test_page.html中的流式音频处理逻辑重构为独立的StreamingContext类，提高代码可维护性和复用性 --- .../test/js/StreamingContext.js | 149 ++++++++++++++++++ main/xiaozhi-server/test/test_page.html | 131 +-------------- 2 files changed, 152 insertions(+), 128 deletions(-) create mode 100644 main/xiaozhi-server/test/js/StreamingContext.js diff --git a/main/xiaozhi-server/test/js/StreamingContext.js b/main/xiaozhi-server/test/js/StreamingContext.js new file mode 100644 index 00000000..576d9761 --- /dev/null +++ b/main/xiaozhi-server/test/js/StreamingContext.js @@ -0,0 +1,149 @@ +import BlockingQueue from './utils/BlockingQueue.js'; +import { log } from './utils/logger.js'; + +// 音频流播放上下文类 +export class StreamingContext { + constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) { + this.opusDecoder = opusDecoder; + this.audioContext = audioContext; + + // 音频参数 + this.sampleRate = sampleRate; + this.channels = channels; + this.minAudioDuration = minAudioDuration; + + // 初始化队列和状态 + this.queue = []; // 已解码的PCM队列。正在播放 + this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放 + this.pendingAudioBufferQueue = []; // 待处理的缓存队列 + this.audioBufferQueue = new BlockingQueue(); // 缓存队列 + this.playing = false; // 是否正在播放 + this.endOfStream = false; // 是否收到结束信号 + this.source = null; // 当前音频源 + this.totalSamples = 0; // 累积的总样本数 + this.lastPlayTime = 0; // 上次播放的时间戳 + } + + // 缓存音频数组 + pushAudioBuffer(item) { + this.audioBufferQueue.enqueue(...item); + } + + // 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题 + async getPendingAudioBufferQueue() { + // 原子交换 + 清空 + [this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()]; + } + + // 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题 + async getQueue(minSamples) { + let TepArray = []; + const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1; + // 原子交换 + 清空 + [TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()]; + this.queue.push(...TepArray); + } + + // 将Int16音频数据转换为Float32音频数据 + convertInt16ToFloat32(int16Data) { + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + // 将[-32768,32767]范围转换为[-1,1] + float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF); + } + return float32Data; + } + + // 将Opus数据解码为PCM + async decodeOpusFrames() { + if (!this.opusDecoder) { + log('Opus解码器未初始化，无法解码', 'error'); + return; + } else { + log('Opus解码器启动', 'info'); + } + + while (true) { + let decodedSamples = []; + for (const frame of this.pendingAudioBufferQueue) { + try { + // 使用Opus解码器解码 + const frameData = this.opusDecoder.decode(frame); + if (frameData && frameData.length > 0) { + // 转换为Float32 + const floatData = this.convertInt16ToFloat32(frameData); + // 使用循环替代展开运算符 + for (let i = 0; i < floatData.length; i++) { + decodedSamples.push(floatData[i]); + } + } + } catch (error) { + log("Opus解码失败: " + error.message, 'error'); + } + } + + if (decodedSamples.length > 0) { + // 使用循环替代展开运算符 + for (let i = 0; i < decodedSamples.length; i++) { + this.activeQueue.enqueue(decodedSamples[i]); + } + this.totalSamples += decodedSamples.length; + } else { + log('没有成功解码的样本', 'warning'); + } + await this.getPendingAudioBufferQueue(); + } + } + + // 开始播放音频 + async startPlaying() { + while (true) { + // 如果累积了至少0.3秒的音频，开始播放 + const minSamples = this.sampleRate * this.minAudioDuration * 3; + if (!this.playing && this.queue.length < minSamples) { + await this.getQueue(minSamples); + } + this.playing = true; + while (this.playing && this.queue.length) { + // 创建新的音频缓冲区 + const minPlaySamples = Math.min(this.queue.length, this.sampleRate); + const currentSamples = this.queue.splice(0, minPlaySamples); + + const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate); + audioBuffer.copyToChannel(new Float32Array(currentSamples), 0); + + // 创建音频源 + this.source = this.audioContext.createBufferSource(); + this.source.buffer = audioBuffer; + + // 创建增益节点用于平滑过渡 + const gainNode = this.audioContext.createGain(); + + // 应用淡入淡出效果避免爆音 + const fadeDuration = 0.02; // 20毫秒 + gainNode.gain.setValueAtTime(0, this.audioContext.currentTime); + gainNode.gain.linearRampToValueAtTime(1, this.audioContext.currentTime + fadeDuration); + + const duration = audioBuffer.duration; + if (duration > fadeDuration * 2) { + gainNode.gain.setValueAtTime(1, this.audioContext.currentTime + duration - fadeDuration); + gainNode.gain.linearRampToValueAtTime(0, this.audioContext.currentTime + duration); + } + + // 连接节点并开始播放 + this.source.connect(gainNode); + gainNode.connect(this.audioContext.destination); + + this.lastPlayTime = this.audioContext.currentTime; + log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'info'); + this.source.start(); + } + await this.getQueue(minSamples); + } + } +} + +// 创建streamingContext实例的工厂函数 +export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) { + return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration); +} \ No newline at end of file diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index 983d3b9a..547f2b31 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -181,6 +181,7 @@ import { checkOpusLoaded, initOpusEncoder } from './js/opus.js'; import { addMessage } from './js/document.js' import BlockingQueue from './js/utils/BlockingQueue.js' + import { createStreamingContext } from './js/StreamingContext.js' // 需要加载的脚本列表 - 移除Opus依赖 const scriptFiles = []; @@ -336,125 +337,7 @@ // 创建流式播放上下文 if (!streamingContext) { - streamingContext = { - queue: [], // 已解码的PCM队列。正在播放 - activeQueue: new BlockingQueue(), // 已解码的PCM队列。准备播放 - pendingAudioBufferQueue: [], // 待处理的缓存队列 - audioBufferQueue: new BlockingQueue(), // 缓存队列 - playing: false, // 是否正在播放 - endOfStream: false, // 是否收到结束信号 - source: null, // 当前音频源 - totalSamples: 0, // 累积的总样本数 - lastPlayTime: 0, // 上次播放的时间戳 - - - // 缓存音频数组 - pushAudioBuffer: function (item) { - this.audioBufferQueue.enqueue(...item) - }, - - // 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题 - getPendingAudioBufferQueue: async function () { - // 原子交换 + 清空 - [this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()]; - - }, - // 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题 - getQueue: async function (minSamples) { - let TepArray = [] - const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1; - // 原子交换 + 清空 - [TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()]; - this.queue.push(...TepArray) - }, - // 将Opus数据解码为PCM - decodeOpusFrames: async function () { - if (!opusDecoder) { - log('Opus解码器未初始化，无法解码', 'error'); - return; - } else { - log('Opus解码器启动', 'info'); - } - - while (true) { - let decodedSamples = []; - for (const frame of this.pendingAudioBufferQueue) { - try { - // 使用Opus解码器解码 - const frameData = opusDecoder.decode(frame); - if (frameData && frameData.length > 0) { - // 转换为Float32 - const floatData = convertInt16ToFloat32(frameData); - // 使用循环替代展开运算符 - for (let i = 0; i < floatData.length; i++) { - decodedSamples.push(floatData[i]); - } - } - } catch (error) { - log("Opus解码失败: " + error.message, 'error'); - } - } - - if (decodedSamples.length > 0) { - // 使用循环替代展开运算符 - for (let i = 0; i < decodedSamples.length; i++) { - this.activeQueue.enqueue(decodedSamples[i]); - } - this.totalSamples += decodedSamples.length; - } else { - log('没有成功解码的样本', 'warning'); - } - await this.getPendingAudioBufferQueue(); - } - }, - - // 开始播放音频 - startPlaying: async function () { - while (true) { - // 如果累积了至少0.3秒的音频，开始播放 - const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION * 3; - if (!this.playing && this.queue.length < minSamples) { - await this.getQueue(minSamples) - } - this.playing = true; - while (this.playing && this.queue.length) { - // 创建新的音频缓冲区 - const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE); - const currentSamples = this.queue.splice(0, minPlaySamples); - - const audioBuffer = audioContext.createBuffer(CHANNELS, currentSamples.length, SAMPLE_RATE); - audioBuffer.copyToChannel(new Float32Array(currentSamples), 0); - - // 创建音频源 - this.source = audioContext.createBufferSource(); - this.source.buffer = audioBuffer; - - // 创建增益节点用于平滑过渡 - const gainNode = audioContext.createGain(); - - // 应用淡入淡出效果避免爆音 - const fadeDuration = 0.02; // 20毫秒 - gainNode.gain.setValueAtTime(0, audioContext.currentTime); - gainNode.gain.linearRampToValueAtTime(1, audioContext.currentTime + fadeDuration); - - const duration = audioBuffer.duration; - if (duration > fadeDuration * 2) { - gainNode.gain.setValueAtTime(1, audioContext.currentTime + duration - fadeDuration); - gainNode.gain.linearRampToValueAtTime(0, audioContext.currentTime + duration); - } - - // 连接节点并开始播放 - this.source.connect(gainNode); - gainNode.connect(audioContext.destination); - - this.lastPlayTime = audioContext.currentTime; - log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info'); - this.source.start(); - } - await this.getQueue(minSamples) - } - } - }; + streamingContext = createStreamingContext(opusDecoder, audioContext, SAMPLE_RATE, CHANNELS, MIN_AUDIO_DURATION); } streamingContext.decodeOpusFrames(); @@ -467,15 +350,7 @@ } } - // 将Int16音频数据转换为Float32音频数据 - function convertInt16ToFloat32(int16Data) { - const float32Data = new Float32Array(int16Data.length); - for (let i = 0; i < int16Data.length; i++) { - // 将[-32768,32767]范围转换为[-1,1] - float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF); - } - return float32Data; - } + // 初始化Opus解码器 - 确保完全初始化完成后才返回 async function initOpusDecoder() { From 5993470d3f304be545a8a1f5e7ee728dd23e4a05 Mon Sep 17 00:00:00 2001 From: Minamiyama Date: Sun, 24 Aug 2025 23:40:54 +0800 Subject: [PATCH 02/13] =?UTF-8?q?feat(=E6=A8=A1=E5=9E=8B=E7=AE=A1=E7=90=86?= =?UTF-8?q?):=20=E6=B7=BB=E5=8A=A0=E6=A8=A1=E5=9E=8B=E5=89=AF=E6=9C=AC?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在模型编辑对话框中增加副本创建功能，当选择创建副本时自动在原模型名称和代码后添加'_副本'后缀，并通过新增API接口保存为新模型 --- .../src/components/ModelEditDialog.vue | 7 +++- main/manager-web/src/views/ModelConfig.vue | 37 +++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/main/manager-web/src/components/ModelEditDialog.vue b/main/manager-web/src/components/ModelEditDialog.vue index 0bc2f7cf..ec29c30e 100644 --- a/main/manager-web/src/components/ModelEditDialog.vue +++ b/main/manager-web/src/components/ModelEditDialog.vue @@ -3,7 +3,7 @@ class="center-dialog" >
- 修改模型 + {{ modelData.duplicateMode ? '创建副本' : '修改模型' }}