From d2e3a63418700180b85bde7aafec62c3f17e0ba5 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 14 Dec 2025 14:24:39 +0800 Subject: [PATCH] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E7=BC=93=E5=86=B2=E9=9F=B3=E9=A2=91=E6=92=AD?= =?UTF-8?q?=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/test/css/test_page.css | 3 +- .../test/js/core/audio/player.js | 35 +++++++++ .../test/js/core/audio/stream-context.js | 72 ++++++++++++++--- .../test/js/core/network/websocket.js | 7 +- main/xiaozhi-server/test/js/ui/controller.js | 78 ++++++++++++++++++- .../test/js/utils/blocking-queue.js | 5 ++ main/xiaozhi-server/test/test_page.html | 8 +- 7 files changed, 191 insertions(+), 17 deletions(-) diff --git a/main/xiaozhi-server/test/css/test_page.css b/main/xiaozhi-server/test/css/test_page.css index c0564191..b197c60d 100644 --- a/main/xiaozhi-server/test/css/test_page.css +++ b/main/xiaozhi-server/test/css/test_page.css @@ -267,6 +267,7 @@ span.connection-status.llm-emoji { .llm-emoji .status { font-size: 14px !important; padding: 8px 20px !important; + line-height: 1.2 !important; } .emoji-large { @@ -393,7 +394,7 @@ span.connection-status.llm-emoji { /* ==================== 会话记录和日志 ==================== */ .flex-container { display: flex; - margin-top: 10px; + margin-top: 20px; background-color: #f9fafb; } diff --git a/main/xiaozhi-server/test/js/core/audio/player.js b/main/xiaozhi-server/test/js/core/audio/player.js index 459fbb54..bc89d58a 100644 --- a/main/xiaozhi-server/test/js/core/audio/player.js +++ b/main/xiaozhi-server/test/js/core/audio/player.js @@ -249,6 +249,41 @@ export class AudioPlayer { this.playBufferedAudio(); this.startAudioBuffering(); } + + // 获取音频包统计信息 + getAudioStats() { + if (!this.streamingContext) { + return { + pendingDecode: 0, + pendingPlay: 0, + totalPending: 0 + }; + } + + const pendingDecode = this.streamingContext.getPendingDecodeCount(); + const pendingPlay = this.streamingContext.getPendingPlayCount(); + + return { + pendingDecode, // 待解码包数 + pendingPlay, // 待播放包数 + totalPending: pendingDecode + pendingPlay // 总待处理包数 + }; + } + + // 清空所有音频缓冲并停止播放 + clearAllAudio() { + log('AudioPlayer: 清空所有音频', 'info'); + + // 清空接收队列（使用clear方法保持对象引用） + this.queue.clear(); + + // 清空流上下文的所有缓冲 + if (this.streamingContext) { + this.streamingContext.clearAllBuffers(); + } + + log('AudioPlayer: 音频已清空', 'success'); + } } // 创建单例 diff --git a/main/xiaozhi-server/test/js/core/audio/stream-context.js b/main/xiaozhi-server/test/js/core/audio/stream-context.js index 9c722505..05b8c1db 100644 --- a/main/xiaozhi-server/test/js/core/audio/stream-context.js +++ b/main/xiaozhi-server/test/js/core/audio/stream-context.js @@ -22,6 +22,7 @@ export class StreamingContext { this.source = null; // 当前音频源 this.totalSamples = 0; // 累积的总样本数 this.lastPlayTime = 0; // 上次播放的时间戳 + this.scheduledEndTime = 0; // 已调度音频的结束时间 } // 缓存音频数组 @@ -31,17 +32,19 @@ export class StreamingContext { // 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题 async getPendingAudioBufferQueue() { - // 原子交换 + 清空 - [this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()]; + // 等待数据到达并获取 + const data = await this.audioBufferQueue.dequeue(); + // 赋值给待处理队列 + this.pendingAudioBufferQueue = data; } // 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题 async getQueue(minSamples) { - let TepArray = []; const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1; - // 原子交换 + 清空 - [TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()]; - this.queue.push(...TepArray); + + // 等待数据并获取 + const tempArray = await this.activeQueue.dequeue(num); + this.queue.push(...tempArray); } // 将Int16音频数据转换为Float32音频数据 @@ -54,6 +57,57 @@ export class StreamingContext { return float32Data; } + // 获取待解码包数 + getPendingDecodeCount() { + return this.audioBufferQueue.length + this.pendingAudioBufferQueue.length; + } + + // 获取待播放样本数（转换为包数，每包960样本） + getPendingPlayCount() { + // 计算已在队列中的样本 + const queuedSamples = this.activeQueue.length + this.queue.length; + + // 计算已调度但未播放的样本（在Web Audio缓冲区中） + let scheduledSamples = 0; + if (this.playing && this.scheduledEndTime) { + const currentTime = this.audioContext.currentTime; + const remainingTime = Math.max(0, this.scheduledEndTime - currentTime); + scheduledSamples = Math.floor(remainingTime * this.sampleRate); + } + + const totalSamples = queuedSamples + scheduledSamples; + return Math.ceil(totalSamples / 960); + } + + // 清空所有音频缓冲 + clearAllBuffers() { + log('清空所有音频缓冲', 'info'); + + // 清空所有队列（使用clear方法保持对象引用） + this.audioBufferQueue.clear(); + this.pendingAudioBufferQueue = []; + this.activeQueue.clear(); + this.queue = []; + + // 停止当前播放的音频源 + if (this.source) { + try { + this.source.stop(); + this.source.disconnect(); + } catch (e) { + // 忽略已经停止的错误 + } + this.source = null; + } + + // 重置状态 + this.playing = false; + this.scheduledEndTime = this.audioContext.currentTime; + this.totalSamples = 0; + + log('音频缓冲已清空', 'success'); + } + // 将Opus数据解码为PCM async decodeOpusFrames() { if (!this.opusDecoder) { @@ -97,7 +151,7 @@ export class StreamingContext { // 开始播放音频 async startPlaying() { - let scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间 + this.scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间 while (true) { // 初始缓冲：等待足够的样本再开始播放 @@ -126,7 +180,7 @@ export class StreamingContext { // 精确调度播放时间 const currentTime = this.audioContext.currentTime; - const startTime = Math.max(scheduledEndTime, currentTime); + const startTime = Math.max(this.scheduledEndTime, currentTime); // 直接连接到输出 this.source.connect(this.audioContext.destination); @@ -136,7 +190,7 @@ export class StreamingContext { // 更新下一个音频块的调度时间 const duration = audioBuffer.duration; - scheduledEndTime = startTime + duration; + this.scheduledEndTime = startTime + duration; this.lastPlayTime = startTime; // 如果队列中数据不足，等待新数据 diff --git a/main/xiaozhi-server/test/js/core/network/websocket.js b/main/xiaozhi-server/test/js/core/network/websocket.js index b62c710b..a605e6f1 100644 --- a/main/xiaozhi-server/test/js/core/network/websocket.js +++ b/main/xiaozhi-server/test/js/core/network/websocket.js @@ -123,7 +123,12 @@ export class WebSocketHandler { } else if (message.state === 'sentence_end') { log(`语音段结束: ${message.text}`, 'info'); } else if (message.state === 'stop') { - log('服务器语音传输结束', 'info'); + log('服务器语音传输结束，清空所有音频缓冲', 'info'); + + // 清空所有音频缓冲并停止播放 + const audioPlayer = getAudioPlayer(); + audioPlayer.clearAllAudio(); + this.isRemoteSpeaking = false; if (this.onRecordButtonStateChange) { this.onRecordButtonStateChange(false); diff --git a/main/xiaozhi-server/test/js/ui/controller.js b/main/xiaozhi-server/test/js/ui/controller.js index 8fc9dd31..3e4f6563 100644 --- a/main/xiaozhi-server/test/js/ui/controller.js +++ b/main/xiaozhi-server/test/js/ui/controller.js @@ -2,6 +2,7 @@ import { loadConfig, saveConfig } from '../config/manager.js'; import { getAudioRecorder } from '../core/audio/recorder.js'; import { getWebSocketHandler } from '../core/network/websocket.js'; +import { getAudioPlayer } from '../core/audio/player.js'; // UI控制器类 export class UIController { @@ -9,6 +10,7 @@ export class UIController { this.isEditing = false; this.visualizerCanvas = null; this.visualizerContext = null; + this.audioStatsTimer = null; } // 初始化 @@ -18,6 +20,7 @@ export class UIController { this.initVisualizer(); this.initEventListeners(); + this.startAudioStatsMonitor(); loadConfig(); } @@ -86,17 +89,20 @@ export class UIController { const sessionStatus = document.getElementById('sessionStatus'); if (!sessionStatus) return; + // 保留背景元素 + const bgHtml = ''; + if (isSpeaking === null) { // 离线状态 - sessionStatus.innerHTML = '😶 小智离线中'; + sessionStatus.innerHTML = bgHtml + '😶 小智离线中'; sessionStatus.className = 'status offline'; } else if (isSpeaking) { // 说话中 - sessionStatus.innerHTML = '😶 小智说话中'; + sessionStatus.innerHTML = bgHtml + '😶 小智说话中'; sessionStatus.className = 'status speaking'; } else { // 聆听中 - sessionStatus.innerHTML = '😶 小智聆听中'; + sessionStatus.innerHTML = bgHtml + '😶 小智聆听中'; sessionStatus.className = 'status listening'; } } @@ -110,8 +116,72 @@ export class UIController { let currentText = sessionStatus.textContent; // 移除现有的表情符号 currentText = currentText.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim(); + + // 保留背景元素 + const bgHtml = ''; + // 使用 innerHTML 添加带样式的表情 - sessionStatus.innerHTML = `${emoji} ${currentText}`; + sessionStatus.innerHTML = bgHtml + `${emoji} ${currentText}`; + } + + // 更新音频统计信息 + updateAudioStats() { + const audioPlayer = getAudioPlayer(); + const stats = audioPlayer.getAudioStats(); + + const sessionStatus = document.getElementById('sessionStatus'); + const sessionStatusBg = document.getElementById('sessionStatusBg'); + + // 只在说话状态下显示背景进度 + if (sessionStatus && sessionStatus.classList.contains('speaking') && sessionStatusBg) { + if (stats.pendingPlay > 0) { + // 计算进度：5包=50%，10包及以上=100% + let percentage; + if (stats.pendingPlay >= 10) { + percentage = 100; + } else { + percentage = (stats.pendingPlay / 10) * 100; + } + + sessionStatusBg.style.width = `${percentage}%`; + + // 根据缓冲量改变背景颜色 + if (stats.pendingPlay < 5) { + // 缓冲不足：橙红色半透明 + sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(255, 152, 0, 0.25), rgba(255, 87, 34, 0.25))'; + } else if (stats.pendingPlay < 10) { + // 一般：黄绿色半透明 + sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(205, 220, 57, 0.25), rgba(76, 175, 80, 0.25))'; + } else { + // 充足：绿蓝色半透明 + sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(76, 175, 80, 0.25), rgba(33, 150, 243, 0.25))'; + } + } else { + // 没有缓冲，隐藏背景 + sessionStatusBg.style.width = '0%'; + } + } else { + // 非说话状态，隐藏背景 + if (sessionStatusBg) { + sessionStatusBg.style.width = '0%'; + } + } + } + + // 启动音频统计监控 + startAudioStatsMonitor() { + // 每100ms更新一次音频统计 + this.audioStatsTimer = setInterval(() => { + this.updateAudioStats(); + }, 100); + } + + // 停止音频统计监控 + stopAudioStatsMonitor() { + if (this.audioStatsTimer) { + clearInterval(this.audioStatsTimer); + this.audioStatsTimer = null; + } } // 绘制音频可视化效果 diff --git a/main/xiaozhi-server/test/js/utils/blocking-queue.js b/main/xiaozhi-server/test/js/utils/blocking-queue.js index 31a43872..738a3e75 100644 --- a/main/xiaozhi-server/test/js/utils/blocking-queue.js +++ b/main/xiaozhi-server/test/js/utils/blocking-queue.js @@ -95,4 +95,9 @@ export default class BlockingQueue { get length() { return this.#items.length; } + + /* 清空队列（保持对象引用，不影响等待者） */ + clear() { + this.#items.length = 0; + } } \ No newline at end of file diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index 97af77ac..f3f583c9 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -91,6 +91,7 @@ +
@@ -113,9 +114,12 @@
-
+
- 😶 小智离线中 + + + 😶 小智离线中 +