Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #665 from xinnan-tech/test-server-pr

Browse Source 
优化test_page.html播放卡顿问题
This commit is contained in:
欣南科技
2025-04-04 19:11:06 +08:00
committed by GitHub
parent b042a34726 70ea24ac83
commit efd22b36ef
8 changed files with 3272 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+2 -2
View File
@@ -73,7 +73,7 @@ class ConnectionHandler:
self.intent = _intent
# vad相关变量
self.client_audio_buffer = bytes()
self.client_audio_buffer = bytearray()
self.client_have_voice = False
self.client_have_voice_last_time = 0.0
self.client_no_voice_last_time = 0.0
@@ -804,7 +804,7 @@ class ConnectionHandler:
# q.queue.put(None)
def reset_vad_states(self):
self.client_audio_buffer = bytes()
self.client_audio_buffer = bytearray()
self.client_have_voice = False
self.client_have_voice_last_time = 0
self.client_voice_stop = False
main/xiaozhi-server/core/utils/vad.py
+1 -1
View File
@@ -31,7 +31,7 @@ class SileroVAD(VAD):
def is_vad(self, conn, opus_packet):
try:
pcm_frame = self.decoder.decode(opus_packet, 960)
conn.client_audio_buffer += pcm_frame # 将新数据加入缓冲区
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区
# 处理缓冲区中的完整帧（每次处理512采样点）
client_have_voice = False
main/xiaozhi-server/requirements.txt
+1
View File
@@ -24,3 +24,4 @@ modelscope==1.23.2
sherpa_onnx==1.11.0
mcp==1.4.1
cnlunar==0.2.0
PySocks==1.7.1
main/xiaozhi-server/test/abbreviated_version/app.js
+1250
View File
File diff suppressed because it is too large Load Diff
main/xiaozhi-server/test/abbreviated_version/test.html
+503
View File
@@ -0,0 +1,503 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>小智语音服务测试</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 20px;
}
.container {
background-color: #f5f5f5;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
margin-bottom: 20px;
}
button {
padding: 8px 15px;
margin-right: 10px;
border: none;
border-radius: 5px;
background-color: #4285f4;
color: white;
cursor: pointer;
transition: background-color 0.2s;
}
button:hover {
background-color: #3367d6;
}
button:disabled {
background-color: #cccccc;
cursor: not-allowed;
}
#status {
font-weight: bold;
}
#scriptStatus {
position: fixed;
top: 10px;
left: 50%;
transform: translateX(-50%);
padding: 15px;
border-radius: 5px;
display: block;
z-index: 100;
transition: all 0.3s ease;
}
#scriptStatus.info {
background-color: #e8f0fe;
color: #4285f4;
border-left: 4px solid #4285f4;
}
#scriptStatus.success {
background-color: #e6f4ea;
color: #0f9d58;
border-left: 4px solid #0f9d58;
}
#scriptStatus.error {
background-color: #fce8e6;
color: #db4437;
border-left: 4px solid #db4437;
}
#debugInfo {
margin-top: 20px;
border: 1px solid #ccc;
border-radius: 5px;
padding: 10px;
font-family: monospace;
font-size: 12px;
max-height: 200px;
overflow-y: auto;
display: none;
}
#showDebug {
margin-top: 10px;
background-color: #f5f5f5;
color: #444;
font-size: 12px;
}
#audioMeter {
margin-top: 10px;
height: 20px;
background-color: #eee;
border-radius: 4px;
overflow: hidden;
display: none;
}
#audioLevel {
height: 100%;
width: 0%;
background-color: #4285f4;
transition: width 0.1s;
}
.conversation {
max-height: 300px;
overflow-y: auto;
border: 1px solid #ddd;
border-radius: 5px;
padding: 10px;
background-color: white;
margin-top: 10px;
}
.message {
margin-bottom: 10px;
padding: 8px 12px;
border-radius: 8px;
max-width: 80%;
}
.user {
background-color: #e2f2ff;
margin-left: auto;
margin-right: 10px;
text-align: right;
}
.server {
background-color: #f0f0f0;
margin-right: auto;
margin-left: 10px;
}
#serverUrl {
flex-grow: 1;
padding: 8px;
border: 1px solid #ddd;
border-radius: 5px;
width: 60%;
margin-right: 10px;
}
#messageInput {
flex-grow: 1;
padding: 8px;
border: 1px solid #ddd;
border-radius: 5px;
width: 70%;
margin-right: 10px;
}
.section {
margin-bottom: 15px;
}
</style>
</head>
<body>
<div class="container">
<h2>小智语音服务测试</h2>
<div id="scriptStatus" class="info">正在加载Opus库...</div>
<div class="section">
<h3>WebSocket连接 <span id="connectionStatus">未连接</span></h3>
<div style="display: flex; align-items: center; margin-bottom: 10px;">
<input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/" placeholder="WebSocket服务器地址">
<button id="connectButton">连接</button>
</div>
</div>
<div class="section">
<h3>录音测试</h3>
<button id="initAudio" style="background-color: #34a853;">初始化音频</button>
<button id="testMic" style="background-color: #fbbc05;">测试麦克风</button>
<p></p>
<button id="start" disabled>开始录音</button>
<button id="stop" disabled>停止录音</button>
<button id="play" disabled>播放录音</button>
<p>录音状态: <span id="status">待机，正在初始化...</span></p>
<div id="audioMeter">
<div id="audioLevel"></div>
</div>
</div>
<div class="section">
<h3>文本消息</h3>
<div style="display: flex; align-items: center;">
<input type="text" id="messageInput" placeholder="输入消息..." disabled>
<button id="sendTextButton" disabled>发送</button>
</div>
</div>
<div class="section">
<h3>会话记录</h3>
<div id="conversation" class="conversation"></div>
</div>
<button id="showDebug">显示/隐藏调试信息</button>
<div id="debugInfo"></div>
</div>
<script>
// 定义全局变量以跟踪库加载状态
window.opusLoaded = false;
window.startButton = document.getElementById("start");
window.stopButton = document.getElementById("stop");
window.playButton = document.getElementById("play");
window.statusLabel = document.getElementById("status");
window.debugInfo = document.getElementById("debugInfo");
window.audioContextReady = false;
window.testMicActive = false;
// 显示/隐藏调试信息
document.getElementById("showDebug").addEventListener("click", function() {
if (debugInfo.style.display === "none" || !debugInfo.style.display) {
debugInfo.style.display = "block";
this.textContent = "隐藏调试信息";
} else {
debugInfo.style.display = "none";
this.textContent = "显示调试信息";
}
});
// 添加初始化音频按钮事件
document.getElementById("initAudio").addEventListener("click", function() {
initializeAudioSystem();
});
// 添加测试麦克风按钮事件
document.getElementById("testMic").addEventListener("click", function() {
if (window.testMicActive) {
stopMicTest();
} else {
startMicTest();
}
});
// 初始化音频系统
function initializeAudioSystem() {
try {
log("初始化音频系统...");
// 创建临时AudioContext来触发用户授权
const tempContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 16000,
latencyHint: 'interactive'
});
// 创建振荡器并播放短促的声音
const oscillator = tempContext.createOscillator();
const gain = tempContext.createGain();
gain.gain.value = 0.1; // 很小的音量
oscillator.connect(gain);
gain.connect(tempContext.destination);
oscillator.frequency.value = 440; // A4
oscillator.start();
// 0.2秒后停止
setTimeout(() => {
oscillator.stop();
// 关闭上下文
tempContext.close().then(() => {
log("音频系统初始化成功", "success");
updateScriptStatus("音频系统已激活", "success");
document.getElementById("initAudio").disabled = true;
document.getElementById("initAudio").textContent = "音频已初始化";
window.audioContextReady = true;
// 如果Opus已加载，启用开始录音按钮
if (window.opusLoaded) {
startButton.disabled = false;
}
});
}, 200);
} catch (err) {
log("初始化音频系统失败: " + err.message, "error");
updateScriptStatus("初始化音频失败: " + err.message, "error");
}
}
// 测试麦克风
function startMicTest() {
const audioMeter = document.getElementById("audioMeter");
const audioLevel = document.getElementById("audioLevel");
const testMicBtn = document.getElementById("testMic");
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
log("浏览器不支持麦克风访问", "error");
return;
}
log("开始麦克风测试...");
audioMeter.style.display = "block";
testMicBtn.textContent = "停止测试";
testMicBtn.style.backgroundColor = "#ea4335";
window.testMicActive = true;
// 创建音频上下文
const testContext = new (window.AudioContext || window.webkitAudioContext)();
window.testContext = testContext;
// 获取麦克风权限
navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true
}
})
.then(stream => {
log("已获取麦克风访问权限", "success");
// 保存流以便稍后关闭
window.testStream = stream;
// 创建音频分析器
const source = testContext.createMediaStreamSource(stream);
const analyser = testContext.createAnalyser();
analyser.fftSize = 256;
source.connect(analyser);
// 创建音量显示更新函数
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
function updateMeter() {
if (!window.testMicActive) return;
analyser.getByteFrequencyData(dataArray);
// 计算音量级别 (0-100)
let sum = 0;
for (let i = 0; i < bufferLength; i++) {
sum += dataArray[i];
}
const average = sum / bufferLength;
const level = Math.min(100, Math.max(0, average * 2));
// 更新音量计
audioLevel.style.width = level + "%";
// 如果有声音，记录日志
if (level > 10) {
log(`检测到声音: ${level.toFixed(1)}%`);
}
// 循环更新
window.testMicAnimationFrame = requestAnimationFrame(updateMeter);
}
// 开始更新
updateMeter();
})
.catch(err => {
log("麦克风测试失败: " + err.message, "error");
window.testMicActive = false;
testMicBtn.textContent = "测试麦克风";
testMicBtn.style.backgroundColor = "#fbbc05";
audioMeter.style.display = "none";
});
}
// 停止麦克风测试
function stopMicTest() {
const audioMeter = document.getElementById("audioMeter");
const testMicBtn = document.getElementById("testMic");
log("停止麦克风测试");
window.testMicActive = false;
testMicBtn.textContent = "测试麦克风";
testMicBtn.style.backgroundColor = "#fbbc05";
// 停止分析器动画
if (window.testMicAnimationFrame) {
cancelAnimationFrame(window.testMicAnimationFrame);
}
// 停止麦克风流
if (window.testStream) {
window.testStream.getTracks().forEach(track => track.stop());
}
// 关闭测试上下文
if (window.testContext) {
window.testContext.close();
}
// 隐藏音量计
audioMeter.style.display = "none";
}
// 添加调试日志
function log(message, type = "info") {
console.log(message);
const time = new Date().toLocaleTimeString();
const entry = document.createElement("div");
entry.textContent = `[${time}] ${message}`;
if (type === "error") {
entry.style.color = "#db4437";
} else if (type === "success") {
entry.style.color = "#0f9d58";
}
debugInfo.appendChild(entry);
debugInfo.scrollTop = debugInfo.scrollHeight;
}
// 更新脚本状态显示
function updateScriptStatus(message, type) {
const statusElement = document.getElementById('scriptStatus');
if (statusElement) {
statusElement.textContent = message;
statusElement.className = type;
statusElement.style.display = 'block';
}
log(message, type);
}
// 检查Opus库是否已加载
function checkOpusLoaded() {
try {
// 检查Module是否存在（本地库导出的全局变量）
if (typeof Module === 'undefined') {
log("Module对象不存在", "error");
throw new Error('Opus库未加载，Module对象不存在');
}
// 记录Module对象结构以便调试
log("Module对象结构: " + Object.keys(Module).join(", "));
// 尝试使用全局Module函数
if (typeof Module._opus_decoder_get_size === 'function') {
window.ModuleInstance = Module;
log('Opus库加载成功（使用全局Module）', "success");
updateScriptStatus('Opus库加载成功', 'success');
// 启用开始录音按钮
startButton.disabled = false;
statusLabel.textContent = "待机";
// 3秒后隐藏状态
setTimeout(() => {
const statusElement = document.getElementById('scriptStatus');
if (statusElement) statusElement.style.display = 'none';
}, 3000);
window.opusLoaded = true;
return true;
}
// 尝试使用Module.instance
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
window.ModuleInstance = Module.instance;
log('Opus库加载成功（使用Module.instance）', "success");
updateScriptStatus('Opus库加载成功', 'success');
// 启用开始录音按钮
startButton.disabled = false;
statusLabel.textContent = "待机";
// 3秒后隐藏状态
setTimeout(() => {
const statusElement = document.getElementById('scriptStatus');
if (statusElement) statusElement.style.display = 'none';
}, 3000);
window.opusLoaded = true;
return true;
}
// 检查是否有其他导出方式
log("Module上可用方法: " + Object.getOwnPropertyNames(Module).filter(prop => typeof Module[prop] === 'function').join(", "));
if (typeof Module.onRuntimeInitialized === 'function' || typeof Module.onRuntimeInitialized === 'undefined') {
log("Module.onRuntimeInitialized 尚未执行，等待模块初始化完成...");
// Module可能未完成初始化，注册回调
Module.onRuntimeInitialized = function() {
log("Opus库运行时初始化完成，重新检查");
checkOpusLoaded();
};
return false;
}
throw new Error('Opus解码函数未找到，可能Module结构不正确');
} catch (err) {
log(`Opus库加载失败: ${err.message}`, "error");
updateScriptStatus(`Opus库加载失败: ${err.message}`, 'error');
statusLabel.textContent = "错误：Opus库加载失败";
return false;
}
}
// 页面加载完成后检查浏览器能力和Opus库
window.addEventListener('load', function() {
log("页面加载完成，开始初始化...");
updateScriptStatus('正在初始化录音环境...', 'info');
// 延迟一小段时间再检查，确保库有时间加载
setTimeout(function checkAndRetry() {
if (!checkOpusLoaded()) {
// 如果加载失败，5秒后重试
log("Opus库加载失败，5秒后重试...");
updateScriptStatus('Opus库加载失败，正在重试...', 'error');
setTimeout(checkAndRetry, 5000);
}
}, 1000);
});
// 防止按钮误操作
document.getElementById("stop").disabled = true;
document.getElementById("play").disabled = true;
</script>
<script src="./../libopus.js"></script>
<script src="app.js"></script>
</body>
</html>
main/xiaozhi-server/test/opus_test/app.js
+670
View File
@@ -0,0 +1,670 @@
const SAMPLE_RATE = 16000;
const CHANNELS = 1;
const FRAME_SIZE = 960; // 对应于60ms帧大小 (16000Hz * 0.06s = 960 samples)
const OPUS_APPLICATION = 2049; // OPUS_APPLICATION_AUDIO
const BUFFER_SIZE = 4096;
let audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE });
let mediaStream, mediaSource, audioProcessor;
let recordedPcmData = []; // 存储原始PCM数据
let recordedOpusData = []; // 存储Opus编码后的数据
let opusEncoder, opusDecoder;
let isRecording = false;
const startButton = document.getElementById("start");
const stopButton = document.getElementById("stop");
const playButton = document.getElementById("play");
const statusLabel = document.getElementById("status");
startButton.addEventListener("click", startRecording);
stopButton.addEventListener("click", stopRecording);
playButton.addEventListener("click", playRecording);
// 初始化Opus编码器与解码器
async function initOpus() {
if (typeof window.ModuleInstance === 'undefined') {
if (typeof Module !== 'undefined') {
// 尝试使用全局Module
window.ModuleInstance = Module;
console.log('使用全局Module作为ModuleInstance');
} else {
console.error("Opus库未加载，ModuleInstance和Module对象都不存在");
return false;
}
}
try {
const mod = window.ModuleInstance;
// 创建编码器
opusEncoder = {
channels: CHANNELS,
sampleRate: SAMPLE_RATE,
frameSize: FRAME_SIZE,
maxPacketSize: 4000,
module: mod,
// 初始化编码器
init: function() {
// 获取编码器大小
const encoderSize = mod._opus_encoder_get_size(this.channels);
console.log(`Opus编码器大小: ${encoderSize}字节`);
// 分配内存
this.encoderPtr = mod._malloc(encoderSize);
if (!this.encoderPtr) {
throw new Error("无法分配编码器内存");
}
// 初始化编码器
const err = mod._opus_encoder_init(
this.encoderPtr,
this.sampleRate,
this.channels,
OPUS_APPLICATION
);
if (err < 0) {
throw new Error(`Opus编码器初始化失败: ${err}`);
}
return true;
},
// 编码方法
encode: function(pcmData) {
const mod = this.module;
// 为PCM数据分配内存
const pcmPtr = mod._malloc(pcmData.length * 2); // Int16 = 2字节
// 将数据复制到WASM内存
for (let i = 0; i < pcmData.length; i++) {
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
}
// 为Opus编码数据分配内存
const maxEncodedSize = this.maxPacketSize;
const encodedPtr = mod._malloc(maxEncodedSize);
// 编码
const encodedBytes = mod._opus_encode(
this.encoderPtr,
pcmPtr,
this.frameSize,
encodedPtr,
maxEncodedSize
);
if (encodedBytes < 0) {
mod._free(pcmPtr);
mod._free(encodedPtr);
throw new Error(`Opus编码失败: ${encodedBytes}`);
}
// 复制编码后的数据
const encodedData = new Uint8Array(encodedBytes);
for (let i = 0; i < encodedBytes; i++) {
encodedData[i] = mod.HEAPU8[encodedPtr + i];
}
// 释放内存
mod._free(pcmPtr);
mod._free(encodedPtr);
return encodedData;
},
// 销毁方法
destroy: function() {
if (this.encoderPtr) {
this.module._free(this.encoderPtr);
this.encoderPtr = null;
}
}
};
// 创建解码器
opusDecoder = {
channels: CHANNELS,
rate: SAMPLE_RATE,
frameSize: FRAME_SIZE,
module: mod,
// 初始化解码器
init: function() {
// 获取解码器大小
const decoderSize = mod._opus_decoder_get_size(this.channels);
console.log(`Opus解码器大小: ${decoderSize}字节`);
// 分配内存
this.decoderPtr = mod._malloc(decoderSize);
if (!this.decoderPtr) {
throw new Error("无法分配解码器内存");
}
// 初始化解码器
const err = mod._opus_decoder_init(
this.decoderPtr,
this.rate,
this.channels
);
if (err < 0) {
throw new Error(`Opus解码器初始化失败: ${err}`);
}
return true;
},
// 解码方法
decode: function(opusData) {
const mod = this.module;
// 为Opus数据分配内存
const opusPtr = mod._malloc(opusData.length);
mod.HEAPU8.set(opusData, opusPtr);
// 为PCM输出分配内存
const pcmPtr = mod._malloc(this.frameSize * 2); // Int16 = 2字节
// 解码
const decodedSamples = mod._opus_decode(
this.decoderPtr,
opusPtr,
opusData.length,
pcmPtr,
this.frameSize,
0 // 不使用FEC
);
if (decodedSamples < 0) {
mod._free(opusPtr);
mod._free(pcmPtr);
throw new Error(`Opus解码失败: ${decodedSamples}`);
}
// 复制解码后的数据
const decodedData = new Int16Array(decodedSamples);
for (let i = 0; i < decodedSamples; i++) {
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
}
// 释放内存
mod._free(opusPtr);
mod._free(pcmPtr);
return decodedData;
},
// 销毁方法
destroy: function() {
if (this.decoderPtr) {
this.module._free(this.decoderPtr);
this.decoderPtr = null;
}
}
};
// 初始化编码器和解码器
if (opusEncoder.init() && opusDecoder.init()) {
console.log("Opus 编码器和解码器初始化成功。");
return true;
} else {
console.error("Opus 初始化失败");
return false;
}
} catch (error) {
console.error("Opus 初始化失败:", error);
return false;
}
}
// 将Float32音频数据转换为Int16音频数据
function convertFloat32ToInt16(float32Data) {
const int16Data = new Int16Array(float32Data.length);
for (let i = 0; i < float32Data.length; i++) {
// 将[-1,1]范围转换为[-32768,32767]
const s = Math.max(-1, Math.min(1, float32Data[i]));
int16Data[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
}
return int16Data;
}
// 将Int16音频数据转换为Float32音频数据
function convertInt16ToFloat32(int16Data) {
const float32Data = new Float32Array(int16Data.length);
for (let i = 0; i < int16Data.length; i++) {
// 将[-32768,32767]范围转换为[-1,1]
float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
}
return float32Data;
}
function startRecording() {
if (isRecording) return;
// 确保有权限并且AudioContext是活跃的
if (audioContext.state === 'suspended') {
audioContext.resume().then(() => {
console.log("AudioContext已恢复");
continueStartRecording();
}).catch(err => {
console.error("恢复AudioContext失败:", err);
statusLabel.textContent = "无法激活音频上下文，请再次点击";
});
} else {
continueStartRecording();
}
}
// 实际开始录音的逻辑
function continueStartRecording() {
// 重置录音数据
recordedPcmData = [];
recordedOpusData = [];
window.audioDataBuffer = new Int16Array(0); // 重置缓冲区
// 初始化Opus
initOpus().then(success => {
if (!success) {
statusLabel.textContent = "Opus初始化失败";
return;
}
console.log("开始录音，参数：", {
sampleRate: SAMPLE_RATE,
channels: CHANNELS,
frameSize: FRAME_SIZE,
bufferSize: BUFFER_SIZE
});
// 请求麦克风权限
navigator.mediaDevices.getUserMedia({
audio: {
sampleRate: SAMPLE_RATE,
channelCount: CHANNELS,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true
}
})
.then(stream => {
console.log("获取到麦克风流，实际参数：", stream.getAudioTracks()[0].getSettings());
// 检查流是否有效
if (!stream || !stream.getAudioTracks().length || !stream.getAudioTracks()[0].enabled) {
throw new Error("获取到的音频流无效");
}
mediaStream = stream;
mediaSource = audioContext.createMediaStreamSource(stream);
// 创建ScriptProcessor(虽然已弃用，但兼容性好)
// 在降级到ScriptProcessor之前尝试使用AudioWorklet
createAudioProcessor().then(processor => {
if (processor) {
console.log("使用AudioWorklet处理音频");
audioProcessor = processor;
// 连接音频处理链
mediaSource.connect(audioProcessor);
audioProcessor.connect(audioContext.destination);
} else {
console.log("回退到ScriptProcessor");
// 创建ScriptProcessor节点
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, CHANNELS, CHANNELS);
// 处理音频数据
audioProcessor.onaudioprocess = processAudioData;
// 连接音频处理链
mediaSource.connect(audioProcessor);
audioProcessor.connect(audioContext.destination);
}
// 更新UI
isRecording = true;
statusLabel.textContent = "录音中...";
startButton.disabled = true;
stopButton.disabled = false;
playButton.disabled = true;
}).catch(error => {
console.error("创建音频处理器失败:", error);
statusLabel.textContent = "创建音频处理器失败";
});
})
.catch(error => {
console.error("获取麦克风失败:", error);
statusLabel.textContent = "获取麦克风失败: " + error.message;
});
});
}
// 创建AudioWorklet处理器
async function createAudioProcessor() {
try {
// 尝试使用更现代的AudioWorklet API
if ('AudioWorklet' in window && 'AudioWorkletNode' in window) {
// 定义AudioWorklet处理器代码
const workletCode = `
class OpusRecorderProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.buffers = [];
this.frameSize = ${FRAME_SIZE};
this.buffer = new Float32Array(this.frameSize);
this.bufferIndex = 0;
this.isRecording = false;
this.port.onmessage = (event) => {
if (event.data.command === 'start') {
this.isRecording = true;
} else if (event.data.command === 'stop') {
this.isRecording = false;
// 发送最后的缓冲区
if (this.bufferIndex > 0) {
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
this.port.postMessage({ buffer: finalBuffer });
}
}
};
}
process(inputs, outputs) {
if (!this.isRecording) return true;
// 获取输入数据
const input = inputs[0][0]; // mono channel
if (!input || input.length === 0) return true;
// 将输入数据添加到缓冲区
for (let i = 0; i < input.length; i++) {
this.buffer[this.bufferIndex++] = input[i];
// 当缓冲区填满时，发送给主线程
if (this.bufferIndex >= this.frameSize) {
this.port.postMessage({ buffer: this.buffer.slice() });
this.bufferIndex = 0;
}
}
return true;
}
}
registerProcessor('opus-recorder-processor', OpusRecorderProcessor);
`;
// 创建Blob URL
const blob = new Blob([workletCode], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
// 加载AudioWorklet模块
await audioContext.audioWorklet.addModule(url);
// 创建AudioWorkletNode
const workletNode = new AudioWorkletNode(audioContext, 'opus-recorder-processor');
// 处理从AudioWorklet接收的消息
workletNode.port.onmessage = (event) => {
if (event.data.buffer) {
// 使用与ScriptProcessor相同的处理逻辑
processAudioData({
inputBuffer: {
getChannelData: () => event.data.buffer
}
});
}
};
// 启动录音
workletNode.port.postMessage({ command: 'start' });
// 保存停止函数
workletNode.stopRecording = () => {
workletNode.port.postMessage({ command: 'stop' });
};
console.log("AudioWorklet 音频处理器创建成功");
return workletNode;
}
} catch (error) {
console.error("创建AudioWorklet失败，将使用ScriptProcessor:", error);
}
// 如果AudioWorklet不可用或失败，返回null以便回退到ScriptProcessor
return null;
}
// 处理音频数据
function processAudioData(e) {
// 获取输入缓冲区
const inputBuffer = e.inputBuffer;
// 获取第一个通道的Float32数据
const inputData = inputBuffer.getChannelData(0);
// 添加调试信息
const nonZeroCount = Array.from(inputData).filter(x => Math.abs(x) > 0.001).length;
console.log(`接收到音频数据: ${inputData.length} 个样本, 非零样本数: ${nonZeroCount}`);
// 如果全是0，可能是麦克风没有正确获取声音
if (nonZeroCount < 5) {
console.warn("警告: 检测到大量静音样本，请检查麦克风是否正常工作");
// 继续处理，以防有些样本确实是静音
}
// 存储PCM数据用于调试
recordedPcmData.push(new Float32Array(inputData));
// 转换为Int16数据供Opus编码
const int16Data = convertFloat32ToInt16(inputData);
// 如果收集到的数据不是FRAME_SIZE的整数倍，需要进行处理
// 创建静态缓冲区来存储不足一帧的数据
if (!window.audioDataBuffer) {
window.audioDataBuffer = new Int16Array(0);
}
// 合并之前缓存的数据和新数据
const combinedData = new Int16Array(window.audioDataBuffer.length + int16Data.length);
combinedData.set(window.audioDataBuffer);
combinedData.set(int16Data, window.audioDataBuffer.length);
// 处理完整帧
const frameCount = Math.floor(combinedData.length / FRAME_SIZE);
console.log(`可编码的完整帧数: ${frameCount}, 缓冲区总大小: ${combinedData.length}`);
for (let i = 0; i < frameCount; i++) {
const frameData = combinedData.subarray(i * FRAME_SIZE, (i + 1) * FRAME_SIZE);
try {
console.log(`编码第 ${i+1}/${frameCount} 帧, 帧大小: ${frameData.length}`);
const encodedData = opusEncoder.encode(frameData);
if (encodedData) {
console.log(`编码成功: ${encodedData.length} 字节`);
recordedOpusData.push(encodedData);
}
} catch (error) {
console.error(`Opus编码帧 ${i+1} 失败:`, error);
}
}
// 保存剩余不足一帧的数据
const remainingSamples = combinedData.length % FRAME_SIZE;
if (remainingSamples > 0) {
window.audioDataBuffer = combinedData.subarray(frameCount * FRAME_SIZE);
console.log(`保留 ${remainingSamples} 个样本到下一次处理`);
} else {
window.audioDataBuffer = new Int16Array(0);
}
}
function stopRecording() {
if (!isRecording) return;
// 处理剩余的缓冲数据
if (window.audioDataBuffer && window.audioDataBuffer.length > 0) {
console.log(`停止录音，处理剩余的 ${window.audioDataBuffer.length} 个样本`);
// 如果剩余数据不足一帧，可以通过补零的方式凑成一帧
if (window.audioDataBuffer.length < FRAME_SIZE) {
const paddedFrame = new Int16Array(FRAME_SIZE);
paddedFrame.set(window.audioDataBuffer);
// 剩余部分填充为0
for (let i = window.audioDataBuffer.length; i < FRAME_SIZE; i++) {
paddedFrame[i] = 0;
}
try {
console.log(`编码最后一帧(补零): ${paddedFrame.length} 样本`);
const encodedData = opusEncoder.encode(paddedFrame);
if (encodedData) {
recordedOpusData.push(encodedData);
}
} catch (error) {
console.error("最后一帧Opus编码失败:", error);
}
} else {
// 如果数据超过一帧，按正常流程处理
processAudioData({
inputBuffer: {
getChannelData: () => convertInt16ToFloat32(window.audioDataBuffer)
}
});
}
window.audioDataBuffer = null;
}
// 如果使用的是AudioWorklet，调用其特定的停止方法
if (audioProcessor && typeof audioProcessor.stopRecording === 'function') {
audioProcessor.stopRecording();
}
// 停止麦克风
if (mediaStream) {
mediaStream.getTracks().forEach(track => track.stop());
}
// 断开音频处理链
if (audioProcessor) {
try {
audioProcessor.disconnect();
if (mediaSource) mediaSource.disconnect();
} catch (error) {
console.warn("断开音频处理链时出错:", error);
}
}
// 更新UI
isRecording = false;
statusLabel.textContent = "已停止录音，收集了 " + recordedOpusData.length + " 帧Opus数据";
startButton.disabled = false;
stopButton.disabled = true;
playButton.disabled = recordedOpusData.length === 0;
console.log("录制完成:",
"PCM帧数:", recordedPcmData.length,
"Opus帧数:", recordedOpusData.length);
}
function playRecording() {
if (!recordedOpusData.length) {
statusLabel.textContent = "没有可播放的录音";
return;
}
// 将所有Opus数据解码为PCM
let allDecodedData = [];
for (const opusData of recordedOpusData) {
try {
// 解码为Int16数据
const decodedData = opusDecoder.decode(opusData);
if (decodedData && decodedData.length > 0) {
// 将Int16数据转换为Float32
const float32Data = convertInt16ToFloat32(decodedData);
// 添加到总解码数据中
allDecodedData.push(...float32Data);
}
} catch (error) {
console.error("Opus解码失败:", error);
}
}
// 如果没有解码出数据，返回
if (allDecodedData.length === 0) {
statusLabel.textContent = "解码失败，无法播放";
return;
}
// 创建音频缓冲区
const audioBuffer = audioContext.createBuffer(CHANNELS, allDecodedData.length, SAMPLE_RATE);
audioBuffer.copyToChannel(new Float32Array(allDecodedData), 0);
// 创建音频源并播放
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioContext.destination);
source.start();
// 更新UI
statusLabel.textContent = "正在播放...";
playButton.disabled = true;
// 播放结束后恢复UI
source.onended = () => {
statusLabel.textContent = "播放完毕";
playButton.disabled = false;
};
}
// 模拟服务端返回的Opus数据进行解码播放
function playOpusFromServer(opusData) {
// 这个函数展示如何处理服务端返回的opus数据
// opusData应该是一个包含opus帧的数组
if (!opusDecoder) {
initOpus().then(success => {
if (success) {
decodeAndPlayOpusData(opusData);
} else {
statusLabel.textContent = "Opus解码器初始化失败";
}
});
} else {
decodeAndPlayOpusData(opusData);
}
}
function decodeAndPlayOpusData(opusData) {
let allDecodedData = [];
for (const frame of opusData) {
try {
const decodedData = opusDecoder.decode(frame);
if (decodedData && decodedData.length > 0) {
const float32Data = convertInt16ToFloat32(decodedData);
allDecodedData.push(...float32Data);
}
} catch (error) {
console.error("服务端Opus数据解码失败:", error);
}
}
if (allDecodedData.length === 0) {
statusLabel.textContent = "服务端数据解码失败";
return;
}
const audioBuffer = audioContext.createBuffer(CHANNELS, allDecodedData.length, SAMPLE_RATE);
audioBuffer.copyToChannel(new Float32Array(allDecodedData), 0);
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioContext.destination);
source.start();
statusLabel.textContent = "正在播放服务端数据...";
source.onended = () => statusLabel.textContent = "服务端数据播放完毕";
}
main/xiaozhi-server/test/opus_test/test.html
+464
View File
@@ -0,0 +1,464 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Opus 编解码测试</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 20px;
}
.container {
background-color: #f5f5f5;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
}
button {
padding: 8px 15px;
margin-right: 10px;
border: none;
border-radius: 5px;
background-color: #4285f4;
color: white;
cursor: pointer;
transition: background-color 0.2s;
}
button:hover {
background-color: #3367d6;
}
button:disabled {
background-color: #cccccc;
cursor: not-allowed;
}
#status {
font-weight: bold;
}
#scriptStatus {
position: fixed;
top: 10px;
left: 50%;
transform: translateX(-50%);
padding: 15px;
border-radius: 5px;
display: block;
z-index: 100;
transition: all 0.3s ease;
}
#scriptStatus.info {
background-color: #e8f0fe;
color: #4285f4;
border-left: 4px solid #4285f4;
}
#scriptStatus.success {
background-color: #e6f4ea;
color: #0f9d58;
border-left: 4px solid #0f9d58;
}
#scriptStatus.error {
background-color: #fce8e6;
color: #db4437;
border-left: 4px solid #db4437;
}
#debugInfo {
margin-top: 20px;
border: 1px solid #ccc;
border-radius: 5px;
padding: 10px;
font-family: monospace;
font-size: 12px;
max-height: 200px;
overflow-y: auto;
display: none;
}
#showDebug {
margin-top: 10px;
background-color: #f5f5f5;
color: #444;
font-size: 12px;
}
</style>
</head>
<body>
<div class="container">
<h2>Opus 编解码录音播放测试</h2>
<div id="scriptStatus" class="info">正在加载Opus库...</div>
<div>
<button id="initAudio" style="background-color: #34a853;">初始化音频</button>
<button id="testMic" style="background-color: #fbbc05;">测试麦克风</button>
<p></p>
<button id="start" disabled>开始录音</button>
<button id="stop" disabled>停止录音</button>
<button id="play" disabled>播放录音</button>
<p>录音状态: <span id="status">待机，正在初始化...</span></p>
<button id="showDebug">显示/隐藏调试信息</button>
<div id="debugInfo"></div>
<div id="audioMeter" style="margin-top: 10px; height: 20px; background-color: #eee; border-radius: 4px; overflow: hidden; display: none;">
<div id="audioLevel" style="height: 100%; width: 0%; background-color: #4285f4; transition: width 0.1s;"></div>
</div>
</div>
</div>
<!-- Opus解码库 - 这个库会设置一个全局Module变量 -->
<script>
// 定义全局变量以跟踪库加载状态
window.opusLoaded = false;
window.startButton = document.getElementById("start");
window.stopButton = document.getElementById("stop");
window.playButton = document.getElementById("play");
window.statusLabel = document.getElementById("status");
window.debugInfo = document.getElementById("debugInfo");
window.audioContextReady = false;
window.testMicActive = false;
// 显示/隐藏调试信息
document.getElementById("showDebug").addEventListener("click", function() {
if (debugInfo.style.display === "none" || !debugInfo.style.display) {
debugInfo.style.display = "block";
this.textContent = "隐藏调试信息";
} else {
debugInfo.style.display = "none";
this.textContent = "显示调试信息";
}
});
// 添加初始化音频按钮事件
document.getElementById("initAudio").addEventListener("click", function() {
initializeAudioSystem();
});
// 添加测试麦克风按钮事件
document.getElementById("testMic").addEventListener("click", function() {
if (window.testMicActive) {
stopMicTest();
} else {
startMicTest();
}
});
// 初始化音频系统
function initializeAudioSystem() {
try {
log("初始化音频系统...");
// 创建临时AudioContext来触发用户授权
const tempContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 16000,
latencyHint: 'interactive'
});
// 创建振荡器并播放短促的声音
const oscillator = tempContext.createOscillator();
const gain = tempContext.createGain();
gain.gain.value = 0.1; // 很小的音量
oscillator.connect(gain);
gain.connect(tempContext.destination);
oscillator.frequency.value = 440; // A4
oscillator.start();
// 0.2秒后停止
setTimeout(() => {
oscillator.stop();
// 关闭上下文
tempContext.close().then(() => {
log("音频系统初始化成功", "success");
updateScriptStatus("音频系统已激活", "success");
document.getElementById("initAudio").disabled = true;
document.getElementById("initAudio").textContent = "音频已初始化";
window.audioContextReady = true;
// 如果Opus已加载，启用开始录音按钮
if (window.opusLoaded) {
startButton.disabled = false;
}
});
}, 200);
} catch (err) {
log("初始化音频系统失败: " + err.message, "error");
updateScriptStatus("初始化音频失败: " + err.message, "error");
}
}
// 测试麦克风
function startMicTest() {
const audioMeter = document.getElementById("audioMeter");
const audioLevel = document.getElementById("audioLevel");
const testMicBtn = document.getElementById("testMic");
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
log("浏览器不支持麦克风访问", "error");
return;
}
log("开始麦克风测试...");
audioMeter.style.display = "block";
testMicBtn.textContent = "停止测试";
testMicBtn.style.backgroundColor = "#ea4335";
window.testMicActive = true;
// 创建音频上下文
const testContext = new (window.AudioContext || window.webkitAudioContext)();
window.testContext = testContext;
// 获取麦克风权限
navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true
}
})
.then(stream => {
log("已获取麦克风访问权限", "success");
// 保存流以便稍后关闭
window.testStream = stream;
// 创建音频分析器
const source = testContext.createMediaStreamSource(stream);
const analyser = testContext.createAnalyser();
analyser.fftSize = 256;
source.connect(analyser);
// 创建音量显示更新函数
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
function updateMeter() {
if (!window.testMicActive) return;
analyser.getByteFrequencyData(dataArray);
// 计算音量级别 (0-100)
let sum = 0;
for (let i = 0; i < bufferLength; i++) {
sum += dataArray[i];
}
const average = sum / bufferLength;
const level = Math.min(100, Math.max(0, average * 2));
// 更新音量计
audioLevel.style.width = level + "%";
// 如果有声音，记录日志
if (level > 10) {
log(`检测到声音: ${level.toFixed(1)}%`);
}
// 循环更新
window.testMicAnimationFrame = requestAnimationFrame(updateMeter);
}
// 开始更新
updateMeter();
})
.catch(err => {
log("麦克风测试失败: " + err.message, "error");
window.testMicActive = false;
testMicBtn.textContent = "测试麦克风";
testMicBtn.style.backgroundColor = "#fbbc05";
audioMeter.style.display = "none";
});
}
// 停止麦克风测试
function stopMicTest() {
const audioMeter = document.getElementById("audioMeter");
const testMicBtn = document.getElementById("testMic");
log("停止麦克风测试");
window.testMicActive = false;
testMicBtn.textContent = "测试麦克风";
testMicBtn.style.backgroundColor = "#fbbc05";
// 停止分析器动画
if (window.testMicAnimationFrame) {
cancelAnimationFrame(window.testMicAnimationFrame);
}
// 停止麦克风流
if (window.testStream) {
window.testStream.getTracks().forEach(track => track.stop());
}
// 关闭测试上下文
if (window.testContext) {
window.testContext.close();
}
// 隐藏音量计
audioMeter.style.display = "none";
}
// 初始化AudioContext
function initAudioContext() {
try {
window.AudioContext = window.AudioContext || window.webkitAudioContext;
if (!window.AudioContext) {
throw new Error("浏览器不支持AudioContext");
}
// 我们在app.js中会创建AudioContext，这里只检查兼容性
log("AudioContext API 可用");
return true;
} catch (err) {
log("AudioContext初始化失败: " + err.message, "error");
updateScriptStatus("AudioContext初始化失败，录音功能不可用", "error");
return false;
}
}
// 检查麦克风权限
function checkMicrophonePermission() {
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
log("浏览器不支持getUserMedia API", "error");
updateScriptStatus("浏览器不支持录音功能", "error");
return false;
}
log("getUserMedia API 可用");
return true;
}
// 添加调试日志
function log(message, type = "info") {
console.log(message);
const time = new Date().toLocaleTimeString();
const entry = document.createElement("div");
entry.textContent = `[${time}] ${message}`;
if (type === "error") {
entry.style.color = "#db4437";
} else if (type === "success") {
entry.style.color = "#0f9d58";
}
debugInfo.appendChild(entry);
debugInfo.scrollTop = debugInfo.scrollHeight;
}
// 更新脚本状态显示
function updateScriptStatus(message, type) {
const statusElement = document.getElementById('scriptStatus');
if (statusElement) {
statusElement.textContent = message;
statusElement.className = type;
statusElement.style.display = 'block';
}
log(message, type);
}
// 检查Opus库是否已加载
function checkOpusLoaded() {
try {
// 检查Module是否存在（本地库导出的全局变量）
if (typeof Module === 'undefined') {
log("Module对象不存在", "error");
throw new Error('Opus库未加载，Module对象不存在');
}
// 记录Module对象结构以便调试
log("Module对象结构: " + Object.keys(Module).join(", "));
// 尝试使用全局Module函数
if (typeof Module._opus_decoder_get_size === 'function') {
window.ModuleInstance = Module;
log('Opus库加载成功（使用全局Module）', "success");
updateScriptStatus('Opus库加载成功', 'success');
// 启用开始录音按钮
startButton.disabled = false;
statusLabel.textContent = "待机";
// 3秒后隐藏状态
setTimeout(() => {
const statusElement = document.getElementById('scriptStatus');
if (statusElement) statusElement.style.display = 'none';
}, 3000);
window.opusLoaded = true;
return true;
}
// 尝试使用Module.instance
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
window.ModuleInstance = Module.instance;
log('Opus库加载成功（使用Module.instance）', "success");
updateScriptStatus('Opus库加载成功', 'success');
// 启用开始录音按钮
startButton.disabled = false;
statusLabel.textContent = "待机";
// 3秒后隐藏状态
setTimeout(() => {
const statusElement = document.getElementById('scriptStatus');
if (statusElement) statusElement.style.display = 'none';
}, 3000);
window.opusLoaded = true;
return true;
}
// 检查是否有其他导出方式
log("Module上可用方法: " + Object.getOwnPropertyNames(Module).filter(prop => typeof Module[prop] === 'function').join(", "));
if (typeof Module.onRuntimeInitialized === 'function' || typeof Module.onRuntimeInitialized === 'undefined') {
log("Module.onRuntimeInitialized 尚未执行，等待模块初始化完成...");
// Module可能未完成初始化，注册回调
Module.onRuntimeInitialized = function() {
log("Opus库运行时初始化完成，重新检查");
checkOpusLoaded();
};
return false;
}
throw new Error('Opus解码函数未找到，可能Module结构不正确');
} catch (err) {
log(`Opus库加载失败: ${err.message}`, "error");
updateScriptStatus(`Opus库加载失败: ${err.message}`, 'error');
statusLabel.textContent = "错误：Opus库加载失败";
return false;
}
}
// 页面加载完成后检查浏览器能力和Opus库
window.addEventListener('load', function() {
log("页面加载完成，开始初始化...");
updateScriptStatus('正在初始化录音环境...', 'info');
// 检查浏览器能力
const audioContextSupported = initAudioContext();
const microphoneSupported = checkMicrophonePermission();
if (!audioContextSupported || !microphoneSupported) {
log("浏览器不支持必要API，无法进行录音", "error");
updateScriptStatus('浏览器不支持录音功能', 'error');
return;
}
log("检查环境完成，开始加载Opus库");
updateScriptStatus('正在加载Opus库...', 'info');
// 延迟一小段时间再检查，确保库有时间加载
setTimeout(function checkAndRetry() {
if (!checkOpusLoaded()) {
// 如果加载失败，5秒后重试
log("Opus库加载失败，5秒后重试...");
updateScriptStatus('Opus库加载失败，正在重试...', 'error');
setTimeout(checkAndRetry, 5000);
}
}, 1000);
});
// 防止按钮误操作
document.getElementById("stop").disabled = true;
document.getElementById("play").disabled = true;
</script>
<script src="./../libopus.js"></script>
<script src="app.js"></script>
</body>
</html>
main/xiaozhi-server/test/test_page.html
+381 -211
View File
@@ -452,6 +452,16 @@
let audioBuffers = []; // 用于存储接收到的所有音频数据
let totalAudioSize = 0; // 跟踪累积的音频大小
let audioBufferQueue = []; // 存储接收到的音频包
let isAudioBuffering = false; // 是否正在缓冲音频
let isAudioPlaying = false; // 是否正在播放音频
const BUFFER_THRESHOLD = 3; // 缓冲包数量阈值，至少累积3个包再开始播放
const MIN_AUDIO_DURATION = 0.1; // 最小音频长度(秒)，小于这个长度的音频会被合并
let streamingContext = null; // 音频流上下文
const SAMPLE_RATE = 16000; // 采样率
const CHANNELS = 1; // 声道数
const FRAME_SIZE = 960; // 帧大小
// DOM元素
const connectButton = document.getElementById('connectButton');
const serverUrlInput = document.getElementById('serverUrl');
@@ -526,221 +536,360 @@
conversationDiv.scrollTop = conversationDiv.scrollHeight;
}
// 播放Opus音频数据
function playOpusData(opusBuffers) {
if (!opusBuffers || opusBuffers.length === 0) {
log('无效的Opus数据，无法播放', 'warning');
return;
}
if (isPlaying) {
// 已经在播放，加入队列
audioQueue.push(opusBuffers);
log(`音频添加到队列，当前队列长度: ${audioQueue.length}`, 'debug');
return;
}
isPlaying = true;
try {
// 确保音频上下文存在
if (!audioContext) {
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 16000
});
log('创建音频上下文，采样率: 16000Hz', 'debug');
// 开始音频缓冲过程
function startAudioBuffering() {
if (isAudioBuffering || isAudioPlaying) return;
isAudioBuffering = true;
log("开始音频缓冲...", 'info');
// 先尝试初始化解码器，以便在播放时已准备好
initOpusDecoder().catch(error => {
log(`预初始化Opus解码器失败: ${error.message}`, 'warning');
// 继续缓冲，我们会在播放时再次尝试初始化
});
// 设置超时，如果在一定时间内没有收集到足够的音频包，就开始播放
setTimeout(() => {
if (isAudioBuffering && audioBufferQueue.length > 0) {
log(`缓冲超时，当前缓冲包数: ${audioBufferQueue.length}，开始播放`, 'info');
playBufferedAudio();
}
// 确保解码器已初始化
if (!opusDecoder) {
try {
// 检查ModuleInstance是否存在（本地库导出的全局变量）
if (typeof window.ModuleInstance === 'undefined') {
if (typeof Module !== 'undefined') {
// 尝试使用全局Module
window.ModuleInstance = Module;
log('使用全局Module作为ModuleInstance', 'info');
} else {
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
}
}
// 初始化Opus解码器 - 16kHz采样率，单声道
opusDecoder = {
channels: 1,
rate: 16000,
frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
frameDuration: 60, // 60ms帧时长
module: window.ModuleInstance, // 保存模块实例的引用
// 使用ModuleInstance提供的函数来解码
decode_float: function (opusData) {
try {
const mod = this.module; // 简写引用
// 打印输入数据的大小
log(`正在解码Opus数据，大小: ${opusData.length}字节`, 'debug');
// 为输入数据分配内存
const inputPtr = mod._malloc(opusData.length);
mod.HEAPU8.set(opusData, inputPtr);
// 为输出PCM数据分配内存 (每个样本为float)
const pcmSamples = this.frameSize;
const pcmPtr = mod._malloc(pcmSamples * 4); // 4字节/float
// 创建解码器实例
const decoderSize = mod._opus_decoder_get_size(this.channels);
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
const decoderPtr = mod._malloc(decoderSize);
const err = mod._opus_decoder_init(decoderPtr, this.rate, this.channels);
if (err < 0) {
throw new Error(`Opus解码器初始化失败: ${err}`);
}
// 解码音频帧
const samplesDecoded = mod._opus_decode_float(
decoderPtr,
inputPtr,
opusData.length,
pcmPtr,
this.frameSize,
0 // 不使用FEC
);
if (samplesDecoded < 0) {
throw new Error(`Opus解码失败，错误码: ${samplesDecoded}`);
}
log(`解码成功，获得${samplesDecoded}个PCM样本`, 'debug');
// 从内存中复制PCM数据
const pcmData = new Float32Array(samplesDecoded);
for (let i = 0; i < samplesDecoded; i++) {
pcmData[i] = mod.HEAPF32[pcmPtr / 4 + i];
}
// 释放内存
mod._free(inputPtr);
mod._free(pcmPtr);
mod._free(decoderPtr);
return pcmData;
} catch (err) {
log(`解码过程中出错: ${err.message}`, 'error');
return new Float32Array(0);
}
}
};
log('Opus解码器初始化成功', 'success');
} catch (err) {
log(`Opus解码器初始化失败: ${err.message}，尝试使用简单播放`, 'error');
// 回退到简单播放
simpleFallbackPlay(opusBuffers);
isPlaying = false;
return;
}
}
// 解码所有Opus帧
let pcmData = [];
let successCount = 0;
for (let i = 0; i < opusBuffers.length; i++) {
try {
const opusBuffer = opusBuffers[i];
// 确保这是Uint8Array
const opusArray = opusBuffer instanceof Uint8Array ?
opusBuffer : new Uint8Array(opusBuffer);
// 解码Opus帧为PCM数据
const decoded = opusDecoder.decode_float(opusArray);
if (decoded && decoded.length > 0) {
pcmData.push(decoded);
successCount++;
}
} catch (error) {
log(`Opus帧 #${i + 1} 解码错误: ${error.message}`, 'error');
}
}
log(`解码完成，成功: ${successCount}/${opusBuffers.length} 帧`, 'debug');
if (pcmData.length === 0) {
log('没有成功解码的帧，尝试使用简单播放', 'warning');
simpleFallbackPlay(opusBuffers);
isPlaying = false;
}, 300); // 300ms超时
// 监控缓冲进度
const bufferCheckInterval = setInterval(() => {
if (!isAudioBuffering) {
clearInterval(bufferCheckInterval);
return;
}
// 连接所有PCM数据
const totalSamples = pcmData.reduce((acc, arr) => acc + arr.length, 0);
log(`总共解码出 ${totalSamples} 个PCM样本`, 'debug');
const combinedPCM = new Float32Array(totalSamples);
let offset = 0;
for (const buffer of pcmData) {
combinedPCM.set(buffer, offset);
offset += buffer.length;
// 当累积了足够的音频包，开始播放
if (audioBufferQueue.length >= BUFFER_THRESHOLD) {
clearInterval(bufferCheckInterval);
log(`已缓冲 ${audioBufferQueue.length} 个音频包，开始播放`, 'info');
playBufferedAudio();
}
// 使用Web Audio API播放PCM数据
const audioBuffer = audioContext.createBuffer(1, combinedPCM.length, 16000);
audioBuffer.getChannelData(0).set(combinedPCM);
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioContext.destination);
source.onended = () => {
isPlaying = false;
log('音频播放完成', 'debug');
// 播放队列中的下一个
if (audioQueue.length > 0) {
const nextBuffers = audioQueue.shift();
playOpusData(nextBuffers);
}
};
source.start();
log(`开始播放PCM音频，时长: ${(combinedPCM.length / 16000).toFixed(2)}秒`, 'debug');
} catch (error) {
log(`播放Opus音频错误: ${error.message}，尝试使用简单播放`, 'error');
simpleFallbackPlay(opusBuffers);
isPlaying = false;
}
}, 50);
}
// 简单播放回退方案
function simpleFallbackPlay(opusBuffers) {
try {
log('使用简单播放回退方案', 'warning');
// 创建一个简单的"滴"声作为反馈
if (audioContext) {
const oscillator = audioContext.createOscillator();
oscillator.type = 'sine';
oscillator.frequency.setValueAtTime(440, audioContext.currentTime); // A4音
const gainNode = audioContext.createGain();
gainNode.gain.setValueAtTime(0.1, audioContext.currentTime); // 设置音量为0.1
oscillator.connect(gainNode);
gainNode.connect(audioContext.destination);
oscillator.start();
oscillator.stop(audioContext.currentTime + 0.2); // 播放0.2秒
log('播放提示音作为音频播放失败的反馈', 'info');
// 播放已缓冲的音频
function playBufferedAudio() {
if (isAudioPlaying || audioBufferQueue.length === 0) return;
isAudioPlaying = true;
isAudioBuffering = false;
// 确保Opus解码器已初始化
const initDecoderAndPlay = async () => {
try {
// 确保音频上下文存在
if (!audioContext) {
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: SAMPLE_RATE
});
log('创建音频上下文，采样率: ' + SAMPLE_RATE + 'Hz', 'debug');
}
// 确保解码器已初始化
if (!opusDecoder) {
log('初始化Opus解码器...', 'info');
try {
opusDecoder = await initOpusDecoder();
if (!opusDecoder) {
throw new Error('解码器初始化失败');
}
log('Opus解码器初始化成功', 'success');
} catch (error) {
log('Opus解码器初始化失败: ' + error.message, 'error');
isAudioPlaying = false;
return;
}
}
// 创建流式播放上下文
if (!streamingContext) {
streamingContext = {
queue: [], // 已解码的PCM队列
playing: false, // 是否正在播放
endOfStream: false, // 是否收到结束信号
source: null, // 当前音频源
totalSamples: 0, // 累积的总样本数
lastPlayTime: 0, // 上次播放的时间戳
// 将Opus数据解码为PCM
decodeOpusFrames: async function(opusFrames) {
if (!opusDecoder) {
log('Opus解码器未初始化，无法解码', 'error');
return;
}
let decodedSamples = [];
for (const frame of opusFrames) {
try {
// 使用Opus解码器解码
const frameData = opusDecoder.decode(frame);
if (frameData && frameData.length > 0) {
// 转换为Float32
const floatData = convertInt16ToFloat32(frameData);
decodedSamples.push(...floatData);
}
} catch (error) {
log("Opus解码失败: " + error.message, 'error');
}
}
if (decodedSamples.length > 0) {
// 添加到解码队列
this.queue.push(...decodedSamples);
this.totalSamples += decodedSamples.length;
// 如果累积了至少0.2秒的音频，开始播放
const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION;
if (!this.playing && this.queue.length >= minSamples) {
this.startPlaying();
}
} else {
log('没有成功解码的样本', 'warning');
}
},
// 开始播放音频
startPlaying: function() {
if (this.playing || this.queue.length === 0) return;
this.playing = true;
// 创建新的音频缓冲区
const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE); // 最多播放1秒
const currentSamples = this.queue.splice(0, minPlaySamples);
const audioBuffer = audioContext.createBuffer(CHANNELS, currentSamples.length, SAMPLE_RATE);
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
// 创建音频源
this.source = audioContext.createBufferSource();
this.source.buffer = audioBuffer;
// 创建增益节点用于平滑过渡
const gainNode = audioContext.createGain();
// 应用淡入淡出效果避免爆音
const fadeDuration = 0.02; // 20毫秒
gainNode.gain.setValueAtTime(0, audioContext.currentTime);
gainNode.gain.linearRampToValueAtTime(1, audioContext.currentTime + fadeDuration);
const duration = audioBuffer.duration;
if (duration > fadeDuration * 2) {
gainNode.gain.setValueAtTime(1, audioContext.currentTime + duration - fadeDuration);
gainNode.gain.linearRampToValueAtTime(0, audioContext.currentTime + duration);
}
// 连接节点并开始播放
this.source.connect(gainNode);
gainNode.connect(audioContext.destination);
this.lastPlayTime = audioContext.currentTime;
log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info');
// 播放结束后的处理
this.source.onended = () => {
this.source = null;
this.playing = false;
// 如果队列中还有数据或者缓冲区有新数据，继续播放
if (this.queue.length > 0) {
setTimeout(() => this.startPlaying(), 10);
} else if (audioBufferQueue.length > 0) {
// 缓冲区有新数据，进行解码
const frames = [...audioBufferQueue];
audioBufferQueue = [];
this.decodeOpusFrames(frames);
} else if (this.endOfStream) {
// 流已结束且没有更多数据
log("音频播放完成", 'info');
isAudioPlaying = false;
streamingContext = null;
} else {
// 等待更多数据
setTimeout(() => {
// 如果仍然没有新数据，但有更多的包到达
if (this.queue.length === 0 && audioBufferQueue.length > 0) {
const frames = [...audioBufferQueue];
audioBufferQueue = [];
this.decodeOpusFrames(frames);
} else if (this.queue.length === 0 && audioBufferQueue.length === 0) {
// 真的没有更多数据了
log("音频播放完成 (超时)", 'info');
isAudioPlaying = false;
streamingContext = null;
}
}, 500); // 500ms超时
}
};
this.source.start();
}
};
}
// 开始处理缓冲的数据
const frames = [...audioBufferQueue];
audioBufferQueue = []; // 清空缓冲队列
// 解码并播放
await streamingContext.decodeOpusFrames(frames);
} catch (error) {
log(`播放已缓冲的音频出错: ${error.message}`, 'error');
isAudioPlaying = false;
streamingContext = null;
}
};
// 执行初始化和播放
initDecoderAndPlay();
}
// 将Int16音频数据转换为Float32音频数据
function convertInt16ToFloat32(int16Data) {
const float32Data = new Float32Array(int16Data.length);
for (let i = 0; i < int16Data.length; i++) {
// 将[-32768,32767]范围转换为[-1,1]
float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
}
return float32Data;
}
// 初始化Opus解码器 - 确保完全初始化完成后才返回
async function initOpusDecoder() {
if (opusDecoder) return opusDecoder; // 已经初始化
try {
// 检查ModuleInstance是否存在
if (typeof window.ModuleInstance === 'undefined') {
if (typeof Module !== 'undefined') {
// 使用全局Module作为ModuleInstance
window.ModuleInstance = Module;
log('使用全局Module作为ModuleInstance', 'info');
} else {
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
}
}
const mod = window.ModuleInstance;
// 创建解码器对象
opusDecoder = {
channels: CHANNELS,
rate: SAMPLE_RATE,
frameSize: FRAME_SIZE,
module: mod,
decoderPtr: null, // 初始为null
// 初始化解码器
init: function() {
if (this.decoderPtr) return true; // 已经初始化
// 获取解码器大小
const decoderSize = mod._opus_decoder_get_size(this.channels);
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
// 分配内存
this.decoderPtr = mod._malloc(decoderSize);
if (!this.decoderPtr) {
throw new Error("无法分配解码器内存");
}
// 初始化解码器
const err = mod._opus_decoder_init(
this.decoderPtr,
this.rate,
this.channels
);
if (err < 0) {
this.destroy(); // 清理资源
throw new Error(`Opus解码器初始化失败: ${err}`);
}
log("Opus解码器初始化成功", 'success');
return true;
},
// 解码方法
decode: function(opusData) {
if (!this.decoderPtr) {
if (!this.init()) {
throw new Error("解码器未初始化且无法初始化");
}
}
try {
const mod = this.module;
// 为Opus数据分配内存
const opusPtr = mod._malloc(opusData.length);
mod.HEAPU8.set(opusData, opusPtr);
// 为PCM输出分配内存
const pcmPtr = mod._malloc(this.frameSize * 2); // Int16 = 2字节
// 解码
const decodedSamples = mod._opus_decode(
this.decoderPtr,
opusPtr,
opusData.length,
pcmPtr,
this.frameSize,
0 // 不使用FEC
);
if (decodedSamples < 0) {
mod._free(opusPtr);
mod._free(pcmPtr);
throw new Error(`Opus解码失败: ${decodedSamples}`);
}
// 复制解码后的数据
const decodedData = new Int16Array(decodedSamples);
for (let i = 0; i < decodedSamples; i++) {
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
}
// 释放内存
mod._free(opusPtr);
mod._free(pcmPtr);
return decodedData;
} catch (error) {
log(`Opus解码错误: ${error.message}`, 'error');
return new Int16Array(0);
}
},
// 销毁方法
destroy: function() {
if (this.decoderPtr) {
this.module._free(this.decoderPtr);
this.decoderPtr = null;
}
}
};
// 初始化解码器
if (!opusDecoder.init()) {
throw new Error("Opus解码器初始化失败");
}
return opusDecoder;
} catch (error) {
log(`简单播放回退方案也失败: ${error.message}`, 'error');
log(`Opus解码器初始化失败: ${error.message}`, 'error');
opusDecoder = null; // 重置为null，以便下次重试
throw error;
}
}
@@ -1481,6 +1630,14 @@
// 初始化Opus编码器
initOpusEncoder();
// 预加载Opus解码器
log('预加载Opus解码器...', 'info');
initOpusDecoder().then(() => {
log('Opus解码器预加载成功', 'success');
}).catch(error => {
log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning');
});
}
// PCM录音处理器代码 - 会被注入到AudioWorklet中
@@ -1647,7 +1804,6 @@
let audioProcessor = null;
let audioProcessorType = null;
let audioSource = null;
let pcmBuffers = [];
// 处理PCM缓冲数据
let pcmDataBuffer = new Int16Array();
@@ -1895,7 +2051,6 @@
}
}
// 处理二进制消息
async function handleBinaryMessage(data) {
try {
let arrayBuffer;
@@ -1917,10 +2072,25 @@
const opusData = new Uint8Array(arrayBuffer);
if (opusData.length > 0) {
// 播放Opus音频
playOpusData([opusData]);
// 将数据添加到缓冲队列
audioBufferQueue.push(opusData);
// 如果收到的是第一个音频包，开始缓冲过程
if (audioBufferQueue.length === 1 && !isAudioBuffering && !isAudioPlaying) {
startAudioBuffering();
}
} else {
log('收到空音频数据帧', 'warning');
log('收到空音频数据帧，可能是结束标志', 'warning');
// 如果缓冲队列中有数据且没有在播放，立即开始播放
if (audioBufferQueue.length > 0 && !isAudioPlaying) {
playBufferedAudio();
}
// 如果正在播放，发送结束信号
if (isAudioPlaying && streamingContext) {
streamingContext.endOfStream = true;
}
}
} catch (error) {
log(`处理二进制消息出错: ${error.message}`, 'error');