Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-30 17:33:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化:优化播放逻辑去除多个方法共享属性导致的逻辑错误

Browse Source
This commit is contained in:
JianYu Zheng
2025-08-15 11:26:14 +08:00
parent 7c0df908e9
commit 4423cb85c2
2 changed files with 1525 additions and 1454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/test/js/utils/BlockingQueue.js
+9 -2
View File
@@ -7,9 +7,16 @@ export default class BlockingQueue {
#emptyResolve = null; #emptyResolve = null;
/* 生产者：把数据塞进去 */ /* 生产者：把数据塞进去 */
enqueue(item) { enqueue(item, ...restItems) {
if (restItems.length === 0) {
this.#items.push(item); this.#items.push(item);
}
// 如果有额外参数，批量处理所有项
else {
const items = [item, ...restItems].filter(i => i);
if (items.length === 0) return;
this.#items.push(...items);
}
// 若有空队列闸门，一次性放行所有等待者 // 若有空队列闸门，一次性放行所有等待者
if (this.#emptyResolve) { if (this.#emptyResolve) {
this.#emptyResolve(); this.#emptyResolve();
main/xiaozhi-server/test/test_page.html
+181 -117
View File
@@ -8,11 +8,12 @@
<link rel="stylesheet" href="test_page.css"> <link rel="stylesheet" href="test_page.css">
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<h1>小智服务器测试页面</h1> <h1>小智服务器测试页面</h1>
<div id="scriptStatus" style="position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%);"> <div id="scriptStatus" style="position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%);">
正在加载Opus库...</div> 正在加载Opus库...
</div>
<!-- 添加配置面板 --> <!-- 添加配置面板 -->
<div class="section"> <div class="section">
@@ -56,9 +57,9 @@
</h2> </h2>
<div class="connection-controls"> <div class="connection-controls">
<input type="text" id="otaUrl" value="http://127.0.0.1:8002/xiaozhi/ota/" <input type="text" id="otaUrl" value="http://127.0.0.1:8002/xiaozhi/ota/"
placeholder="OTA服务器地址，如：http://127.0.0.1:8002/xiaozhi/ota/" /> placeholder="OTA服务器地址，如：http://127.0.0.1:8002/xiaozhi/ota/"/>
<input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/" <input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/"
placeholder="WebSocket服务器地址，如：ws://127.0.0.1:8000/xiaozhi/v1/" /> placeholder="WebSocket服务器地址，如：ws://127.0.0.1:8000/xiaozhi/v1/"/>
<button id="connectButton">连接</button> <button id="connectButton">连接</button>
<button id="authTestButton">测试认证</button> <button id="authTestButton">测试认证</button>
</div> </div>
@@ -94,16 +95,16 @@
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<!-- Opus解码库 --> <!-- Opus解码库 -->
<script src="libopus.js"></script> <script src="libopus.js"></script>
<script type="module"> <script type="module">
import { log } from './js/utils/logger.js'; import {log} from './js/utils/logger.js';
import { webSocketConnect } from './js/xiaoZhiConnect.js'; import {webSocketConnect} from './js/xiaoZhiConnect.js';
import { checkOpusLoaded , initOpusEncoder } from './js/opus.js'; import {checkOpusLoaded, initOpusEncoder} from './js/opus.js';
import { addMessage } from './js/document.js' import {addMessage} from './js/document.js'
import BlockingQueue from './js/utils/BlockingQueue.js' import BlockingQueue from './js/utils/BlockingQueue.js'
// 需要加载的脚本列表 - 移除Opus依赖 // 需要加载的脚本列表 - 移除Opus依赖
const scriptFiles = []; const scriptFiles = [];
@@ -189,7 +190,8 @@
} }
const queue = new BlockingQueue(); const queue = new BlockingQueue();
// 开始音频缓冲过程
// 启动缓存进程
async function startAudioBuffering() { async function startAudioBuffering() {
log("开始音频缓冲...", 'info'); log("开始音频缓冲...", 'info');
@@ -200,6 +202,7 @@
}); });
const timeout = 300; const timeout = 300;
while (true) { while (true) {
// 每次数据空的时候等三条数据
const packets = await queue.dequeue( const packets = await queue.dequeue(
3, // 至少 3 条 3, // 至少 3 条
timeout, // 最多等 300 ms timeout, // 最多等 300 ms
@@ -209,28 +212,23 @@
); );
if (packets.length) { if (packets.length) {
log(`已缓冲 ${packets.length} 个音频包，开始播放`, 'info'); log(`已缓冲 ${packets.length} 个音频包，开始播放`, 'info');
audioBufferQueue.push(...packets) streamingContext.pushAudioBuffer(packets)
playBufferedAudio();
} }
console.log(queue.length) // 50毫秒里，有多少给多少
while (true){ while (true) {
const data = await queue.dequeue(99,50) const data = await queue.dequeue(99, 50)
if(data.length){ if (data.length) {
audioBufferQueue.push(...data) streamingContext.pushAudioBuffer(data)
playBufferedAudio(); } else {
break
} }
} }
} }
} }
// 播放已缓冲的音频 // 播放已缓冲的音频
function playBufferedAudio() { async function playBufferedAudio() {
if (isAudioPlaying || audioBufferQueue.length === 0) return;
isAudioPlaying = true;
// 确保Opus解码器已初始化 // 确保Opus解码器已初始化
const initDecoderAndPlay = async () => {
try { try {
// 确保音频上下文存在 // 确保音频上下文存在
if (!audioContext) { if (!audioContext) {
@@ -259,24 +257,48 @@
// 创建流式播放上下文 // 创建流式播放上下文
if (!streamingContext) { if (!streamingContext) {
streamingContext = { streamingContext = {
queue: [], // 已解码的PCM队列 queue: [], // 已解码的PCM队列。正在播放
activeQueue: new BlockingQueue(), // 已解码的PCM队列。准备播放
pendingAudioBufferQueue: [], // 待处理的缓存队列
audioBufferQueue: new BlockingQueue(), // 缓存队列
playing: false, // 是否正在播放 playing: false, // 是否正在播放
endOfStream: false, // 是否收到结束信号 endOfStream: false, // 是否收到结束信号
source: null, // 当前音频源 source: null, // 当前音频源
totalSamples: 0, // 累积的总样本数 totalSamples: 0, // 累积的总样本数
lastPlayTime: 0, // 上次播放的时间戳 lastPlayTime: 0, // 上次播放的时间戳
i: 1,
// 缓存音频数组
pushAudioBuffer: function (item) {
this.audioBufferQueue.enqueue(...item)
},
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
getPendingAudioBufferQueue: async function () {
// 原子交换 + 清空
[this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()];
},
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
getQueue: async function (minSamples) {
let TepArray = []
const num = minSamples - this.queue.length > 0 ? minSamples -this.queue.length : 1;
// 原子交换 + 清空
[TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()];
this.queue.push(...TepArray)
},
// 将Opus数据解码为PCM // 将Opus数据解码为PCM
decodeOpusFrames: async function (opusFrames) { decodeOpusFrames: async function () {
if (!opusDecoder) { if (!opusDecoder) {
log('Opus解码器未初始化，无法解码', 'error'); log('Opus解码器未初始化，无法解码', 'error');
return; return;
} else {
log('Opus解码器启动', 'info');
} }
while (true) {
let decodedSamples = []; let decodedSamples = [];
for (const frame of this.pendingAudioBufferQueue) {
for (const frame of opusFrames) {
try { try {
// 使用Opus解码器解码 // 使用Opus解码器解码
const frameData = opusDecoder.decode(frame); const frameData = opusDecoder.decode(frame);
@@ -296,26 +318,26 @@
if (decodedSamples.length > 0) { if (decodedSamples.length > 0) {
// 使用循环替代展开运算符 // 使用循环替代展开运算符
for (let i = 0; i < decodedSamples.length; i++) { for (let i = 0; i < decodedSamples.length; i++) {
this.queue.push(decodedSamples[i]); this.activeQueue.enqueue(decodedSamples[i]);
} }
this.totalSamples += decodedSamples.length; this.totalSamples += decodedSamples.length;
// 如果累积了至少0.2秒的音频，开始播放
const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION;
if (!this.playing && this.queue.length >= minSamples) {
this.startPlaying();
}
} else { } else {
log('没有成功解码的样本', 'warning'); log('没有成功解码的样本', 'warning');
} }
await this.getPendingAudioBufferQueue();
}
}, },
// 开始播放音频 // 开始播放音频
startPlaying: function () { startPlaying: async function () {
if (this.playing || this.queue.length === 0) return; while (true) {
// 如果累积了至少0.2秒的音频，开始播放
const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION;
if (!this.playing && this.queue.length < minSamples) {
await this.getQueue(minSamples)
}
this.playing = true; this.playing = true;
while (this.playing && this.queue.length) {
// 创建新的音频缓冲区 // 创建新的音频缓冲区
const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE); const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE);
const currentSamples = this.queue.splice(0, minPlaySamples); const currentSamples = this.queue.splice(0, minPlaySamples);
@@ -347,68 +369,61 @@
this.lastPlayTime = audioContext.currentTime; this.lastPlayTime = audioContext.currentTime;
log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info'); log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info');
// 播放结束后的处理
this.source.onended = () => {
this.source = null;
this.playing = false;
// 使用setTimeout避免递归调用
setTimeout(() => {
// 如果队列中还有数据，继续播放
if (this.queue.length > 0) {
this.startPlaying();
} else if (audioBufferQueue.length > 0) {
// 缓冲区有新数据，进行解码
const frames = [...audioBufferQueue];
audioBufferQueue = [];
this.decodeOpusFrames(frames);
} else if (this.endOfStream) {
// 流已结束且没有更多数据
log("音频播放完成", 'info');
isAudioPlaying = false;
this.endOfStream = false;
streamingContext = null;
} else {
// 等待更多数据
setTimeout(() => {
// 如果仍然没有新数据，但有更多的包到达
if (this.queue.length === 0 && audioBufferQueue.length > 0) {
const frames = [...audioBufferQueue];
audioBufferQueue = [];
this.decodeOpusFrames(frames);
} else if (this.queue.length === 0 && audioBufferQueue.length === 0) {
// 真的没有更多数据了
log("音频播放完成 (超时)", 'info');
isAudioPlaying = false;
streamingContext = null;
}
}, 500); // 500ms超时
}
}, 10); // 10ms延迟，避免立即递归
};
this.source.start(); this.source.start();
} }
await this.getQueue(minSamples)
}
// // 播放结束后的处理
// this.source.onended = () => {
// this.source = null;
// this.playing = false;
//
// // 使用setTimeout避免递归调用
// setTimeout(() => {
// // 如果队列中还有数据，继续播放
// if (this.queue.length > 0) {
// this.startPlaying();
// } else if (audioBufferQueue.length > 0) {
// // 缓冲区有新数据，进行解码
// const frames = [...audioBufferQueue];
// audioBufferQueue = [];
// this.decodeOpusFrames(frames);
// } else if (this.endOfStream) {
// // 流已结束且没有更多数据
// log("音频播放完成", 'info');
// isAudioPlaying = false;
// this.endOfStream = false;
// streamingContext = null;
// } else {
// // 等待更多数据
// setTimeout(() => {
// // 如果仍然没有新数据，但有更多的包到达
// if (this.queue.length === 0 && audioBufferQueue.length > 0) {
// const frames = [...audioBufferQueue];
// audioBufferQueue = [];
// this.decodeOpusFrames(frames);
// } else if (this.queue.length === 0 && audioBufferQueue.length === 0) {
// // 真的没有更多数据了
// log("音频播放完成 (超时)", 'info');
// isAudioPlaying = false;
// streamingContext = null;
// }
// }, 500); // 500ms超时
// }
// }, 10); // 10ms延迟，避免立即递归
// };
}
}; };
} }
// 开始处理缓冲的数据 streamingContext.decodeOpusFrames();
const frames = [...audioBufferQueue]; streamingContext.startPlaying();
audioBufferQueue = []; // 清空缓冲队列
// 解码并播放
await streamingContext.decodeOpusFrames(frames);
} catch (error) { } catch (error) {
log(`播放已缓冲的音频出错: ${error.message}`, 'error'); log(`播放已缓冲的音频出错: ${error.message}`, 'error');
isAudioPlaying = false; isAudioPlaying = false;
streamingContext = null; streamingContext = null;
} }
};
// 执行初始化和播放
initDecoderAndPlay();
} }
// 将Int16音频数据转换为Float32音频数据 // 将Int16音频数据转换为Float32音频数据
@@ -642,7 +657,7 @@
} }
// 创建完整的录音blob // 创建完整的录音blob
const blob = new Blob(audioChunks, { type: audioChunks[0].type }); const blob = new Blob(audioChunks, {type: audioChunks[0].type});
log(`已创建音频Blob，MIME类型: ${audioChunks[0].type}，大小: ${(blob.size / 1024).toFixed(2)} KB`, 'info'); log(`已创建音频Blob，MIME类型: ${audioChunks[0].type}，大小: ${(blob.size / 1024).toFixed(2)} KB`, 'info');
// 保存原始块，以防清空后需要调试 // 保存原始块，以防清空后需要调试
@@ -786,8 +801,8 @@
localStorage.setItem('wsUrl', url); localStorage.setItem('wsUrl', url);
try { try {
const ws = await webSocketConnect(otaUrl,url,config) const ws = await webSocketConnect(otaUrl, url, config)
if (ws === undefined){ if (ws === undefined) {
return return
} }
websocket = ws websocket = ws
@@ -881,18 +896,72 @@
if (message.text && message.text !== '😊') { if (message.text && message.text !== '😊') {
addMessage(message.text); addMessage(message.text);
} }
}else if (message.type === 'mcp') { } else if (message.type === 'mcp') {
const payload = message.payload || {}; const payload = message.payload || {};
log(`服务器下发: ${JSON.stringify(message)}`, 'info'); log(`服务器下发: ${JSON.stringify(message)}`, 'info');
if (payload) { if (payload) {
// 模拟小智客户端行为 // 模拟小智客户端行为
if(payload.method === 'tools/list'){ if (payload.method === 'tools/list') {
const replay_message = JSON.stringify({"session_id":"","type":"mcp","payload":{"jsonrpc":"2.0","id":2,"result":{"tools":[{"name":"self.get_device_status","description":"Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)","inputSchema":{"type":"object","properties":{}}},{"name":"self.audio_speaker.set_volume","description":"Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.","inputSchema":{"type":"object","properties":{"volume":{"type":"integer","minimum":0,"maximum":100}},"required":["volume"]}},{"name":"self.screen.set_brightness","description":"Set the brightness of the screen.","inputSchema":{"type":"object","properties":{"brightness":{"type":"integer","minimum":0,"maximum":100}},"required":["brightness"]}},{"name":"self.screen.set_theme","description":"Set the theme of the screen. The theme can be 'light' or 'dark'.","inputSchema":{"type":"object","properties":{"theme":{"type":"string"}},"required":["theme"]}}]}}}) const replay_message = JSON.stringify({
"session_id": "", "type": "mcp", "payload": {
"jsonrpc": "2.0", "id": 2, "result": {
"tools": [{
"name": "self.get_device_status",
"description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
"inputSchema": {"type": "object", "properties": {}}
}, {
"name": "self.audio_speaker.set_volume",
"description": "Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
"inputSchema": {
"type": "object",
"properties": {
"volume": {
"type": "integer",
"minimum": 0,
"maximum": 100
}
},
"required": ["volume"]
}
}, {
"name": "self.screen.set_brightness",
"description": "Set the brightness of the screen.",
"inputSchema": {
"type": "object",
"properties": {
"brightness": {
"type": "integer",
"minimum": 0,
"maximum": 100
}
},
"required": ["brightness"]
}
}, {
"name": "self.screen.set_theme",
"description": "Set the theme of the screen. The theme can be 'light' or 'dark'.",
"inputSchema": {
"type": "object",
"properties": {"theme": {"type": "string"}},
"required": ["theme"]
}
}]
}
}
})
websocket.send(replay_message); websocket.send(replay_message);
log(`回复MCP消息: ${replay_message}`, 'info'); log(`回复MCP消息: ${replay_message}`, 'info');
} else if(payload.method === 'tools/call'){ } else if (payload.method === 'tools/call') {
// 模拟回复 // 模拟回复
const replay_message = JSON.stringify({"session_id":"9f261599","type":"mcp","payload":{"jsonrpc":"2.0","id": payload.id,"result":{"content":[{"type":"text","text":"true"}],"isError":false}}}) const replay_message = JSON.stringify({
"session_id": "9f261599",
"type": "mcp",
"payload": {
"jsonrpc": "2.0",
"id": payload.id,
"result": {"content": [{"type": "text", "text": "true"}], "isError": false}
}
})
websocket.send(replay_message); websocket.send(replay_message);
log(`回复MCP消息: ${replay_message}`, 'info'); log(`回复MCP消息: ${replay_message}`, 'info');
} }
@@ -991,8 +1060,6 @@
const message = messageInput.value.trim(); const message = messageInput.value.trim();
if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return; if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return;
audioBufferQueue = [];
isAudioPlaying = false;
try { try {
// 直接发送listen消息，不需要重复发送hello // 直接发送listen消息，不需要重复发送hello
@@ -1244,7 +1311,9 @@
}).catch(error => { }).catch(error => {
log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning'); log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning');
}); });
playBufferedAudio()
startAudioBuffering() startAudioBuffering()
} }
// PCM录音处理器代码 - 会被注入到AudioWorklet中 // PCM录音处理器代码 - 会被注入到AudioWorklet中
@@ -1322,7 +1391,7 @@
// 检查是否支持AudioWorklet // 检查是否支持AudioWorklet
if (audioContext.audioWorklet) { if (audioContext.audioWorklet) {
// 注册音频处理器 // 注册音频处理器
const blob = new Blob([audioProcessorCode], { type: 'application/javascript' }); const blob = new Blob([audioProcessorCode], {type: 'application/javascript'});
const url = URL.createObjectURL(blob); const url = URL.createObjectURL(blob);
await audioContext.audioWorklet.addModule(url); await audioContext.audioWorklet.addModule(url);
URL.revokeObjectURL(url); URL.revokeObjectURL(url);
@@ -1339,7 +1408,7 @@
}; };
log('使用AudioWorklet处理音频', 'success'); log('使用AudioWorklet处理音频', 'success');
return { node: audioProcessor, type: 'worklet' }; return {node: audioProcessor, type: 'worklet'};
} else { } else {
// 使用旧版ScriptProcessorNode作为回退方案 // 使用旧版ScriptProcessorNode作为回退方案
log('AudioWorklet不可用，使用ScriptProcessorNode作为回退方案', 'warning'); log('AudioWorklet不可用，使用ScriptProcessorNode作为回退方案', 'warning');
@@ -1370,7 +1439,7 @@
scriptProcessor.connect(silent); scriptProcessor.connect(silent);
silent.connect(audioContext.destination); silent.connect(audioContext.destination);
return { node: scriptProcessor, type: 'processor' }; return {node: scriptProcessor, type: 'processor'};
} }
} catch (error) { } catch (error) {
log(`创建音频处理器失败: ${error.message}，尝试回退方案`, 'error'); log(`创建音频处理器失败: ${error.message}，尝试回退方案`, 'error');
@@ -1399,7 +1468,7 @@
silent.connect(audioContext.destination); silent.connect(audioContext.destination);
log('使用ScriptProcessorNode作为回退方案成功', 'warning'); log('使用ScriptProcessorNode作为回退方案成功', 'warning');
return { node: scriptProcessor, type: 'processor' }; return {node: scriptProcessor, type: 'processor'};
} catch (fallbackError) { } catch (fallbackError) {
log(`回退方案也失败: ${fallbackError.message}`, 'error'); log(`回退方案也失败: ${fallbackError.message}`, 'error');
return null; return null;
@@ -1414,6 +1483,7 @@
// 处理PCM缓冲数据 // 处理PCM缓冲数据
let pcmDataBuffer = new Int16Array(); let pcmDataBuffer = new Int16Array();
function processPCMBuffer(buffer) { function processPCMBuffer(buffer) {
if (!isRecording) return; if (!isRecording) return;
@@ -1543,7 +1613,7 @@
// 启动音频处理器的录音 - 只有AudioWorklet才需要发送消息 // 启动音频处理器的录音 - 只有AudioWorklet才需要发送消息
if (audioProcessorType === 'worklet' && audioProcessor.port) { if (audioProcessorType === 'worklet' && audioProcessor.port) {
audioProcessor.port.postMessage({ command: 'start' }); audioProcessor.port.postMessage({command: 'start'});
} }
// 发送监听开始消息 // 发送监听开始消息
@@ -1600,7 +1670,7 @@
if (audioProcessor) { if (audioProcessor) {
// 只有AudioWorklet才需要发送停止消息 // 只有AudioWorklet才需要发送停止消息
if (audioProcessorType === 'worklet' && audioProcessor.port) { if (audioProcessorType === 'worklet' && audioProcessor.port) {
audioProcessor.port.postMessage({ command: 'stop' }); audioProcessor.port.postMessage({command: 'stop'});
} }
audioProcessor.disconnect(); audioProcessor.disconnect();
@@ -1680,12 +1750,6 @@
queue.enqueue(opusData); queue.enqueue(opusData);
} else { } else {
log('收到空音频数据帧，可能是结束标志', 'warning'); log('收到空音频数据帧，可能是结束标志', 'warning');
// 如果缓冲队列中有数据且没有在播放，立即开始播放
if (audioBufferQueue.length > 0 && !isAudioPlaying) {
playBufferedAudio();
}
// 如果正在播放，发送结束信号 // 如果正在播放，发送结束信号
if (isAudioPlaying && streamingContext) { if (isAudioPlaying && streamingContext) {
streamingContext.endOfStream = true; streamingContext.endOfStream = true;
@@ -1709,7 +1773,7 @@
} }
initApp(); initApp();
</script> </script>
</body> </body>
</html> </html>