Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2701 from xinnan-tech/test_page_pre_buffer

Browse Source 
update:优化测试页面缓冲音频播放
This commit is contained in:
欣南科技
2025-12-14 14:25:25 +08:00
committed by GitHub
parent 15b3f0a8f5 d2e3a63418
commit 0decedd6a1
7 changed files with 191 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/test/css/test_page.css
+2 -1
View File
@@ -267,6 +267,7 @@ span.connection-status.llm-emoji {
.llm-emoji .status {
font-size: 14px !important;
padding: 8px 20px !important;
line-height: 1.2 !important;
}
.emoji-large {
@@ -393,7 +394,7 @@ span.connection-status.llm-emoji {
/* ==================== 会话记录和日志 ==================== */
.flex-container {
display: flex;
margin-top: 10px;
margin-top: 20px;
background-color: #f9fafb;
}
main/xiaozhi-server/test/js/core/audio/player.js
+35
View File
@@ -249,6 +249,41 @@ export class AudioPlayer {
this.playBufferedAudio();
this.startAudioBuffering();
}
// 获取音频包统计信息
getAudioStats() {
if (!this.streamingContext) {
return {
pendingDecode: 0,
pendingPlay: 0,
totalPending: 0
};
}
const pendingDecode = this.streamingContext.getPendingDecodeCount();
const pendingPlay = this.streamingContext.getPendingPlayCount();
return {
pendingDecode, // 待解码包数
pendingPlay, // 待播放包数
totalPending: pendingDecode + pendingPlay // 总待处理包数
};
}
// 清空所有音频缓冲并停止播放
clearAllAudio() {
log('AudioPlayer: 清空所有音频', 'info');
// 清空接收队列（使用clear方法保持对象引用）
this.queue.clear();
// 清空流上下文的所有缓冲
if (this.streamingContext) {
this.streamingContext.clearAllBuffers();
}
log('AudioPlayer: 音频已清空', 'success');
}
}
// 创建单例
main/xiaozhi-server/test/js/core/audio/stream-context.js
+63 -9
View File
@@ -22,6 +22,7 @@ export class StreamingContext {
this.source = null; // 当前音频源
this.totalSamples = 0; // 累积的总样本数
this.lastPlayTime = 0; // 上次播放的时间戳
this.scheduledEndTime = 0; // 已调度音频的结束时间
}
// 缓存音频数组
@@ -31,17 +32,19 @@ export class StreamingContext {
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
async getPendingAudioBufferQueue() {
// 原子交换 + 清空
[this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()];
// 等待数据到达并获取
const data = await this.audioBufferQueue.dequeue();
// 赋值给待处理队列
this.pendingAudioBufferQueue = data;
}
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
async getQueue(minSamples) {
let TepArray = [];
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
// 原子交换 + 清空
[TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()];
this.queue.push(...TepArray);
// 等待数据并获取
const tempArray = await this.activeQueue.dequeue(num);
this.queue.push(...tempArray);
}
// 将Int16音频数据转换为Float32音频数据
@@ -54,6 +57,57 @@ export class StreamingContext {
return float32Data;
}
// 获取待解码包数
getPendingDecodeCount() {
return this.audioBufferQueue.length + this.pendingAudioBufferQueue.length;
}
// 获取待播放样本数（转换为包数，每包960样本）
getPendingPlayCount() {
// 计算已在队列中的样本
const queuedSamples = this.activeQueue.length + this.queue.length;
// 计算已调度但未播放的样本（在Web Audio缓冲区中）
let scheduledSamples = 0;
if (this.playing && this.scheduledEndTime) {
const currentTime = this.audioContext.currentTime;
const remainingTime = Math.max(0, this.scheduledEndTime - currentTime);
scheduledSamples = Math.floor(remainingTime * this.sampleRate);
}
const totalSamples = queuedSamples + scheduledSamples;
return Math.ceil(totalSamples / 960);
}
// 清空所有音频缓冲
clearAllBuffers() {
log('清空所有音频缓冲', 'info');
// 清空所有队列（使用clear方法保持对象引用）
this.audioBufferQueue.clear();
this.pendingAudioBufferQueue = [];
this.activeQueue.clear();
this.queue = [];
// 停止当前播放的音频源
if (this.source) {
try {
this.source.stop();
this.source.disconnect();
} catch (e) {
// 忽略已经停止的错误
}
this.source = null;
}
// 重置状态
this.playing = false;
this.scheduledEndTime = this.audioContext.currentTime;
this.totalSamples = 0;
log('音频缓冲已清空', 'success');
}
// 将Opus数据解码为PCM
async decodeOpusFrames() {
if (!this.opusDecoder) {
@@ -97,7 +151,7 @@ export class StreamingContext {
// 开始播放音频
async startPlaying() {
let scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
this.scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
while (true) {
// 初始缓冲：等待足够的样本再开始播放
@@ -126,7 +180,7 @@ export class StreamingContext {
// 精确调度播放时间
const currentTime = this.audioContext.currentTime;
const startTime = Math.max(scheduledEndTime, currentTime);
const startTime = Math.max(this.scheduledEndTime, currentTime);
// 直接连接到输出
this.source.connect(this.audioContext.destination);
@@ -136,7 +190,7 @@ export class StreamingContext {
// 更新下一个音频块的调度时间
const duration = audioBuffer.duration;
scheduledEndTime = startTime + duration;
this.scheduledEndTime = startTime + duration;
this.lastPlayTime = startTime;
// 如果队列中数据不足，等待新数据
main/xiaozhi-server/test/js/core/network/websocket.js
+6 -1
View File
@@ -123,7 +123,12 @@ export class WebSocketHandler {
} else if (message.state === 'sentence_end') {
log(`语音段结束: ${message.text}`, 'info');
} else if (message.state === 'stop') {
log('服务器语音传输结束', 'info');
log('服务器语音传输结束，清空所有音频缓冲', 'info');
// 清空所有音频缓冲并停止播放
const audioPlayer = getAudioPlayer();
audioPlayer.clearAllAudio();
this.isRemoteSpeaking = false;
if (this.onRecordButtonStateChange) {
this.onRecordButtonStateChange(false);
main/xiaozhi-server/test/js/ui/controller.js
+74 -4
View File
@@ -2,6 +2,7 @@
import { loadConfig, saveConfig } from '../config/manager.js';
import { getAudioRecorder } from '../core/audio/recorder.js';
import { getWebSocketHandler } from '../core/network/websocket.js';
import { getAudioPlayer } from '../core/audio/player.js';
// UI控制器类
export class UIController {
@@ -9,6 +10,7 @@ export class UIController {
this.isEditing = false;
this.visualizerCanvas = null;
this.visualizerContext = null;
this.audioStatsTimer = null;
}
// 初始化
@@ -18,6 +20,7 @@ export class UIController {
this.initVisualizer();
this.initEventListeners();
this.startAudioStatsMonitor();
loadConfig();
}
@@ -86,17 +89,20 @@ export class UIController {
const sessionStatus = document.getElementById('sessionStatus');
if (!sessionStatus) return;
// 保留背景元素
const bgHtml = '<span id="sessionStatusBg" style="position: absolute; left: 0; top: 0; bottom: 0; width: 0%; background: linear-gradient(90deg, rgba(76, 175, 80, 0.2), rgba(33, 150, 243, 0.2)); transition: width 0.15s ease-out, background 0.3s ease; z-index: 0; border-radius: 20px;"></span>';
if (isSpeaking === null) {
// 离线状态
sessionStatus.innerHTML = '<span class="emoji-large">😶</span> 小智离线中';
sessionStatus.innerHTML = bgHtml + '<span style="position: relative; z-index: 1;"><span class="emoji-large">😶</span> 小智离线中</span>';
sessionStatus.className = 'status offline';
} else if (isSpeaking) {
// 说话中
sessionStatus.innerHTML = '<span class="emoji-large">😶</span> 小智说话中';
sessionStatus.innerHTML = bgHtml + '<span style="position: relative; z-index: 1;"><span class="emoji-large">😶</span> 小智说话中</span>';
sessionStatus.className = 'status speaking';
} else {
// 聆听中
sessionStatus.innerHTML = '<span class="emoji-large">😶</span> 小智聆听中';
sessionStatus.innerHTML = bgHtml + '<span style="position: relative; z-index: 1;"><span class="emoji-large">😶</span> 小智聆听中</span>';
sessionStatus.className = 'status listening';
}
}
@@ -110,8 +116,72 @@ export class UIController {
let currentText = sessionStatus.textContent;
// 移除现有的表情符号
currentText = currentText.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim();
// 保留背景元素
const bgHtml = '<span id="sessionStatusBg" style="position: absolute; left: 0; top: 0; bottom: 0; width: 0%; background: linear-gradient(90deg, rgba(76, 175, 80, 0.2), rgba(33, 150, 243, 0.2)); transition: width 0.15s ease-out, background 0.3s ease; z-index: 0; border-radius: 20px;"></span>';
// 使用 innerHTML 添加带样式的表情
sessionStatus.innerHTML = `<span class="emoji-large">${emoji}</span> ${currentText}`;
sessionStatus.innerHTML = bgHtml + `<span style="position: relative; z-index: 1;"><span class="emoji-large">${emoji}</span> ${currentText}</span>`;
}
// 更新音频统计信息
updateAudioStats() {
const audioPlayer = getAudioPlayer();
const stats = audioPlayer.getAudioStats();
const sessionStatus = document.getElementById('sessionStatus');
const sessionStatusBg = document.getElementById('sessionStatusBg');
// 只在说话状态下显示背景进度
if (sessionStatus && sessionStatus.classList.contains('speaking') && sessionStatusBg) {
if (stats.pendingPlay > 0) {
// 计算进度：5包=50%，10包及以上=100%
let percentage;
if (stats.pendingPlay >= 10) {
percentage = 100;
} else {
percentage = (stats.pendingPlay / 10) * 100;
}
sessionStatusBg.style.width = `${percentage}%`;
// 根据缓冲量改变背景颜色
if (stats.pendingPlay < 5) {
// 缓冲不足：橙红色半透明
sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(255, 152, 0, 0.25), rgba(255, 87, 34, 0.25))';
} else if (stats.pendingPlay < 10) {
// 一般：黄绿色半透明
sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(205, 220, 57, 0.25), rgba(76, 175, 80, 0.25))';
} else {
// 充足：绿蓝色半透明
sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(76, 175, 80, 0.25), rgba(33, 150, 243, 0.25))';
}
} else {
// 没有缓冲，隐藏背景
sessionStatusBg.style.width = '0%';
}
} else {
// 非说话状态，隐藏背景
if (sessionStatusBg) {
sessionStatusBg.style.width = '0%';
}
}
}
// 启动音频统计监控
startAudioStatsMonitor() {
// 每100ms更新一次音频统计
this.audioStatsTimer = setInterval(() => {
this.updateAudioStats();
}, 100);
}
// 停止音频统计监控
stopAudioStatsMonitor() {
if (this.audioStatsTimer) {
clearInterval(this.audioStatsTimer);
this.audioStatsTimer = null;
}
}
// 绘制音频可视化效果
main/xiaozhi-server/test/js/utils/blocking-queue.js
+5
View File
@@ -95,4 +95,9 @@ export default class BlockingQueue {
get length() {
return this.#items.length;
}
/* 清空队列（保持对象引用，不影响等待者） */
clear() {
this.#items.length = 0;
}
}
main/xiaozhi-server/test/test_page.html
+6 -2
View File
@@ -91,6 +91,7 @@
<input type="text" id="serverUrl" value="" readonly disabled placeholder="点击连接按钮后，自动从OTA接口获取" />
</div>
</div>
</div>
<div class="section">
@@ -113,9 +114,12 @@
<canvas id="audioVisualizer" class="audio-visualizer"></canvas>
</div>
<div style="margin: -10px 0px 5px 0px;">
<div style="margin: -10px 0px 10px 0px;">
<span class="connection-status llm-emoji">
<span id="sessionStatus" class="status offline"><span class="emoji-large">😶</span> 小智离线中</span>
<span id="sessionStatus" class="status offline" style="position: relative; display: inline-block;">
<span id="sessionStatusBg" style="position: absolute; left: 0; top: 0; bottom: 0; width: 0%; background: linear-gradient(90deg, rgba(76, 175, 80, 0.2), rgba(33, 150, 243, 0.2)); transition: width 0.15s ease-out, background 0.3s ease; z-index: 0; border-radius: 20px;"></span>
<span style="position: relative; z-index: 1;"><span class="emoji-large">😶</span> 小智离线中</span>
</span>
</span>
</div>
<div class="flex-container">