Files
xiaozhi-esp32-server/main/xiaozhi-server/test/test_page.html
T

1934 lines
76 KiB
HTML
Raw Normal View History

<!DOCTYPE html>
<html lang="zh-CN">
2025-03-31 21:04:11 +08:00
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>小智服务器测试页面</title>
<style>
body {
font-family: 'PingFang SC', 'Microsoft YaHei', sans-serif;
margin: 0;
padding: 20px;
background-color: #f5f5f5;
}
2025-03-31 21:04:11 +08:00
.container {
max-width: 800px;
margin: 0 auto;
background-color: white;
border-radius: 10px;
2025-03-31 21:04:11 +08:00
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
padding: 10px 20px 10px 20px;
}
2025-03-31 21:04:11 +08:00
h1 {
color: #333;
text-align: center;
margin-bottom: 30px;
}
2025-03-31 21:04:11 +08:00
.section {
margin-bottom: 20px;
padding: 15px;
border-radius: 8px;
background-color: #f9f9f9;
}
2025-03-31 21:04:11 +08:00
.section h2 {
margin-top: 0;
color: #444;
font-size: 18px;
2025-03-31 21:04:11 +08:00
display: flex;
align-items: center;
gap: 10px;
}
2025-03-31 21:04:11 +08:00
.control-panel {
display: flex;
flex-wrap: wrap;
gap: 10px;
2025-03-31 21:04:11 +08:00
margin-top: 10px;
}
2025-03-31 21:04:11 +08:00
button {
padding: 8px 15px;
border: none;
border-radius: 5px;
background-color: #4285f4;
color: white;
cursor: pointer;
transition: background-color 0.2s;
}
2025-03-31 21:04:11 +08:00
button:hover {
background-color: #3367d6;
}
2025-03-31 21:04:11 +08:00
button:disabled {
background-color: #cccccc;
cursor: not-allowed;
}
2025-03-31 21:04:11 +08:00
#serverUrl {
flex-grow: 1;
padding: 8px;
border: 1px solid #ddd;
border-radius: 5px;
}
2025-03-31 21:04:11 +08:00
.message-input {
display: flex;
gap: 10px;
margin-bottom: 20px;
}
2025-03-31 21:04:11 +08:00
#messageInput {
flex-grow: 1;
padding: 8px;
border: 1px solid #ddd;
border-radius: 5px;
}
2025-03-31 21:04:11 +08:00
#nfcCardId {
flex-grow: 1;
padding: 8px;
border: 1px solid #ddd;
border-radius: 5px;
}
2025-03-31 21:04:11 +08:00
.conversation {
max-height: 300px;
overflow-y: auto;
border: 1px solid #ddd;
border-radius: 5px;
padding: 10px;
background-color: white;
2025-03-31 21:04:11 +08:00
flex: 1;
margin-right: 10px;
}
2025-03-31 21:04:11 +08:00
.message {
margin-bottom: 10px;
padding: 8px 12px;
border-radius: 8px;
max-width: 80%;
}
2025-03-31 21:04:11 +08:00
.user {
background-color: #e2f2ff;
margin-left: auto;
margin-right: 10px;
text-align: right;
}
2025-03-31 21:04:11 +08:00
.server {
background-color: #f0f0f0;
margin-right: auto;
margin-left: 10px;
}
2025-03-31 21:04:11 +08:00
.status {
color: #666;
font-style: italic;
2025-03-31 21:04:11 +08:00
font-size: 14px;
margin: 0;
padding: 0;
}
2025-03-31 21:04:11 +08:00
.audio-controls {
display: flex;
justify-content: center;
gap: 10px;
margin-top: 20px;
}
2025-03-31 21:04:11 +08:00
.audio-visualizer {
height: 60px;
width: 100%;
margin-top: 10px;
border: 1px solid #ddd;
border-radius: 5px;
background-color: #fafafa;
}
2025-03-31 21:04:11 +08:00
.record-button {
background-color: #db4437;
}
2025-03-31 21:04:11 +08:00
.record-button:hover {
background-color: #c53929;
}
2025-03-31 21:04:11 +08:00
.record-button.recording {
animation: pulse 1.5s infinite;
}
2025-03-31 21:04:11 +08:00
@keyframes pulse {
2025-03-31 21:04:11 +08:00
0% {
background-color: #db4437;
}
50% {
background-color: #ff6659;
}
100% {
background-color: #db4437;
}
}
2025-03-31 21:04:11 +08:00
#logContainer {
2025-03-31 21:04:11 +08:00
margin-top: 0;
padding: 10px;
background-color: #f0f0f0;
border-radius: 5px;
font-family: monospace;
2025-03-31 21:04:11 +08:00
height: 300px;
overflow-y: auto;
2025-03-31 21:04:11 +08:00
flex: 1;
margin-left: 10px;
}
2025-03-31 21:04:11 +08:00
.log-entry {
margin: 5px 0;
font-size: 12px;
}
2025-03-31 21:04:11 +08:00
.log-info {
color: #333;
}
2025-03-31 21:04:11 +08:00
.log-error {
color: #db4437;
}
.log-success {
color: #0f9d58;
}
@keyframes fadeIn {
from {
opacity: 0;
transform: translate(-50%, -60%);
}
to {
opacity: 1;
transform: translate(-50%, -50%);
}
}
.script-status {
display: inline-block;
width: 10px;
height: 10px;
border-radius: 50%;
margin-right: 5px;
}
2025-03-31 21:04:11 +08:00
.script-loaded {
background-color: #0f9d58;
}
.script-loading {
background-color: #f4b400;
}
.script-error {
background-color: #db4437;
}
.script-list {
margin: 10px 0;
padding: 10px;
background-color: #f9f9f9;
border-radius: 5px;
font-family: monospace;
font-size: 11px;
}
2025-03-31 21:04:11 +08:00
#scriptStatus.success {
background-color: #e6f4ea;
color: #0f9d58;
border-left: 4px solid #0f9d58;
}
2025-03-31 21:04:11 +08:00
#scriptStatus.error {
background-color: #fce8e6;
color: #db4437;
border-left: 4px solid #db4437;
}
2025-03-31 21:04:11 +08:00
#scriptStatus.warning {
background-color: #fef7e0;
color: #f4b400;
border-left: 4px solid #f4b400;
}
2025-03-31 21:04:11 +08:00
/* 标签页样式 */
.tabs {
display: flex;
margin-bottom: 20px;
border-bottom: 2px solid #e0e0e0;
}
.tab {
padding: 10px 20px;
cursor: pointer;
border: none;
background: none;
font-size: 16px;
color: #666;
position: relative;
transition: all 0.3s ease;
}
.tab:hover {
color: #4285f4;
}
.tab.active {
color: #4285f4;
font-weight: bold;
}
.tab.active::after {
content: '';
position: absolute;
bottom: -2px;
left: 0;
width: 100%;
height: 2px;
background-color: #4285f4;
}
.tab-content {
display: none;
}
.tab-content.active {
display: block;
}
.flex-container {
display: flex;
gap: 20px;
margin-top: 10px;
}
</style>
</head>
2025-03-31 21:04:11 +08:00
<body>
<div class="container">
<h1>小智服务器测试页面</h1>
2025-03-31 21:04:11 +08:00
<div id="scriptStatus" style="position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%);">
正在加载Opus库...</div>
<div class="section">
2025-03-31 21:04:11 +08:00
<h2>WebSocket连接 <span id="connectionStatus" class="status">未连接</span></h2>
<div class="control-panel">
2025-03-31 21:04:11 +08:00
<input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/" placeholder="WebSocket服务器地址">
<button id="connectButton">连接</button>
<button id="authTestButton">测试认证</button>
</div>
</div>
2025-03-31 21:04:11 +08:00
<div class="section">
2025-03-31 21:04:11 +08:00
<h2>消息发送</h2>
<div class="tabs">
<button class="tab active" data-tab="text">文本消息</button>
<button class="tab" data-tab="voice">语音消息</button>
</div>
<div class="tab-content active" id="textTab">
<div class="message-input">
<input type="text" id="messageInput" placeholder="输入消息..." disabled>
<button id="sendTextButton" disabled>发送</button>
</div>
</div>
<div class="tab-content" id="voiceTab">
<div class="audio-controls">
<button id="recordButton" class="record-button" disabled>开始录音</button>
</div>
<canvas id="audioVisualizer" class="audio-visualizer"></canvas>
</div>
</div>
2025-03-31 21:04:11 +08:00
<div class="section">
<h2>会话记录</h2>
2025-03-31 21:04:11 +08:00
<div class="flex-container">
<div id="conversation" class="conversation"></div>
<div id="logContainer">
<div class="log-entry log-info">准备就绪,请连接服务器开始测试...</div>
</div>
</div>
</div>
</div>
<!-- Opus解码库 -->
<script src="libopus.js"></script>
2025-03-31 21:04:11 +08:00
<script>
// 需要加载的脚本列表 - 移除Opus依赖
const scriptFiles = [];
2025-03-31 21:04:11 +08:00
// 脚本加载状态
const scriptStatus = {
loading: 0,
loaded: 0,
failed: 0,
total: scriptFiles.length
};
2025-03-31 21:04:11 +08:00
// 检查Opus库是否已加载
function checkOpusLoaded() {
2025-03-31 21:04:11 +08:00
try {
// 检查Module是否存在(本地库导出的全局变量)
if (typeof Module === 'undefined') {
throw new Error('Opus库未加载,Module对象不存在');
}
2025-03-31 21:04:11 +08:00
// 尝试先使用Module.instancelibopus.js最后一行导出方式)
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
// 使用Module.instance对象替换全局Module对象
window.ModuleInstance = Module.instance;
log('Opus库加载成功(使用Module.instance', 'success');
updateScriptStatus('Opus库加载成功', 'success');
// 3秒后隐藏状态
const statusElement = document.getElementById('scriptStatus');
if (statusElement) statusElement.style.display = 'none';
return;
}
// 如果没有Module.instance,检查全局Module函数
if (typeof Module._opus_decoder_get_size === 'function') {
window.ModuleInstance = Module;
log('Opus库加载成功(使用全局Module', 'success');
updateScriptStatus('Opus库加载成功', 'success');
// 3秒后隐藏状态
const statusElement = document.getElementById('scriptStatus');
if (statusElement) statusElement.style.display = 'none';
return;
}
throw new Error('Opus解码函数未找到,可能Module结构不正确');
} catch (err) {
log(`Opus库加载失败,请检查libopus.js文件是否存在且正确: ${err.message}`, 'error');
updateScriptStatus('Opus库加载失败,请检查libopus.js文件是否存在且正确', 'error');
}
}
2025-03-31 21:04:11 +08:00
// 更新脚本状态显示
function updateScriptStatus(message, type) {
const statusElement = document.getElementById('scriptStatus');
if (statusElement) {
statusElement.textContent = message;
statusElement.className = `script-status ${type}`;
statusElement.style.display = 'block';
statusElement.style.width = 'auto';
}
}
2025-03-31 21:04:11 +08:00
// 全局变量
let websocket = null;
let mediaRecorder = null;
let audioContext = null;
let analyser = null;
let audioChunks = [];
let isRecording = false;
let visualizerCanvas = document.getElementById('audioVisualizer');
let visualizerContext = visualizerCanvas.getContext('2d');
let audioQueue = [];
let isPlaying = false;
let opusDecoder = null; // Opus解码器
let visualizationRequest = null; // 动画帧请求ID
2025-03-31 21:04:11 +08:00
// 音频流缓冲相关
let audioBuffers = []; // 用于存储接收到的所有音频数据
let totalAudioSize = 0; // 跟踪累积的音频大小
2025-03-31 21:04:11 +08:00
// DOM元素
const connectButton = document.getElementById('connectButton');
const serverUrlInput = document.getElementById('serverUrl');
const connectionStatus = document.getElementById('connectionStatus');
const messageInput = document.getElementById('messageInput');
const sendTextButton = document.getElementById('sendTextButton');
const recordButton = document.getElementById('recordButton');
const stopButton = document.getElementById('stopButton');
const conversationDiv = document.getElementById('conversation');
const logContainer = document.getElementById('logContainer');
2025-03-31 21:04:11 +08:00
// 日志函数
function log(message, type = 'info') {
const logEntry = document.createElement('div');
logEntry.className = `log-entry log-${type}`;
logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
if (type === 'error') {
logEntry.style.color = 'red';
} else if (type === 'debug') {
logEntry.style.color = 'gray';
return;
} else if (type === 'warning') {
logEntry.style.color = 'orange';
} else if (type === 'success') {
logEntry.style.color = 'green';
} else {
logEntry.style.color = 'black';
}
logContainer.appendChild(logEntry);
logContainer.scrollTop = logContainer.scrollHeight;
}
2025-03-31 21:04:11 +08:00
// 初始化可视化器
function initVisualizer() {
visualizerCanvas.width = visualizerCanvas.clientWidth;
visualizerCanvas.height = visualizerCanvas.clientHeight;
visualizerContext.fillStyle = '#fafafa';
visualizerContext.fillRect(0, 0, visualizerCanvas.width, visualizerCanvas.height);
}
2025-03-31 21:04:11 +08:00
// 绘制音频可视化效果
function drawVisualizer(dataArray) {
visualizationRequest = requestAnimationFrame(() => drawVisualizer(dataArray));
2025-03-31 21:04:11 +08:00
if (!isRecording) return;
2025-03-31 21:04:11 +08:00
analyser.getByteFrequencyData(dataArray);
2025-03-31 21:04:11 +08:00
visualizerContext.fillStyle = '#fafafa';
visualizerContext.fillRect(0, 0, visualizerCanvas.width, visualizerCanvas.height);
2025-03-31 21:04:11 +08:00
const barWidth = (visualizerCanvas.width / dataArray.length) * 2.5;
let barHeight;
let x = 0;
2025-03-31 21:04:11 +08:00
for (let i = 0; i < dataArray.length; i++) {
barHeight = dataArray[i] / 2;
2025-03-31 21:04:11 +08:00
visualizerContext.fillStyle = `rgb(${barHeight + 100}, 50, 50)`;
visualizerContext.fillRect(x, visualizerCanvas.height - barHeight, barWidth, barHeight);
2025-03-31 21:04:11 +08:00
x += barWidth + 1;
}
}
2025-03-31 21:04:11 +08:00
// 添加消息到会话记录
function addMessage(text, isUser = false) {
const messageDiv = document.createElement('div');
messageDiv.className = `message ${isUser ? 'user' : 'server'}`;
messageDiv.textContent = text;
conversationDiv.appendChild(messageDiv);
conversationDiv.scrollTop = conversationDiv.scrollHeight;
}
2025-03-31 21:04:11 +08:00
// 播放Opus音频数据
function playOpusData(opusBuffers) {
if (!opusBuffers || opusBuffers.length === 0) {
log('无效的Opus数据,无法播放', 'warning');
return;
}
2025-03-31 21:04:11 +08:00
if (isPlaying) {
// 已经在播放,加入队列
audioQueue.push(opusBuffers);
log(`音频添加到队列,当前队列长度: ${audioQueue.length}`, 'debug');
return;
}
2025-03-31 21:04:11 +08:00
isPlaying = true;
2025-03-31 21:04:11 +08:00
try {
// 确保音频上下文存在
if (!audioContext) {
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 16000
});
log('创建音频上下文,采样率: 16000Hz', 'debug');
}
2025-03-31 21:04:11 +08:00
// 确保解码器已初始化
if (!opusDecoder) {
try {
// 检查ModuleInstance是否存在(本地库导出的全局变量)
if (typeof window.ModuleInstance === 'undefined') {
if (typeof Module !== 'undefined') {
// 尝试使用全局Module
window.ModuleInstance = Module;
log('使用全局Module作为ModuleInstance', 'info');
} else {
throw new Error('Opus库未加载,ModuleInstance和Module对象都不存在');
}
}
2025-03-31 21:04:11 +08:00
// 初始化Opus解码器 - 16kHz采样率,单声道
opusDecoder = {
channels: 1,
rate: 16000,
frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
frameDuration: 60, // 60ms帧时长
module: window.ModuleInstance, // 保存模块实例的引用
2025-03-31 21:04:11 +08:00
// 使用ModuleInstance提供的函数来解码
2025-03-31 21:04:11 +08:00
decode_float: function (opusData) {
try {
const mod = this.module; // 简写引用
2025-03-31 21:04:11 +08:00
// 打印输入数据的大小
log(`正在解码Opus数据,大小: ${opusData.length}字节`, 'debug');
2025-03-31 21:04:11 +08:00
// 为输入数据分配内存
const inputPtr = mod._malloc(opusData.length);
mod.HEAPU8.set(opusData, inputPtr);
2025-03-31 21:04:11 +08:00
// 为输出PCM数据分配内存 (每个样本为float)
const pcmSamples = this.frameSize;
const pcmPtr = mod._malloc(pcmSamples * 4); // 4字节/float
2025-03-31 21:04:11 +08:00
// 创建解码器实例
const decoderSize = mod._opus_decoder_get_size(this.channels);
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
2025-03-31 21:04:11 +08:00
const decoderPtr = mod._malloc(decoderSize);
const err = mod._opus_decoder_init(decoderPtr, this.rate, this.channels);
2025-03-31 21:04:11 +08:00
if (err < 0) {
throw new Error(`Opus解码器初始化失败: ${err}`);
}
2025-03-31 21:04:11 +08:00
// 解码音频帧
const samplesDecoded = mod._opus_decode_float(
decoderPtr,
inputPtr,
opusData.length,
pcmPtr,
this.frameSize,
0 // 不使用FEC
);
2025-03-31 21:04:11 +08:00
if (samplesDecoded < 0) {
throw new Error(`Opus解码失败,错误码: ${samplesDecoded}`);
}
2025-03-31 21:04:11 +08:00
log(`解码成功,获得${samplesDecoded}个PCM样本`, 'debug');
2025-03-31 21:04:11 +08:00
// 从内存中复制PCM数据
const pcmData = new Float32Array(samplesDecoded);
for (let i = 0; i < samplesDecoded; i++) {
2025-03-31 21:04:11 +08:00
pcmData[i] = mod.HEAPF32[pcmPtr / 4 + i];
}
2025-03-31 21:04:11 +08:00
// 释放内存
mod._free(inputPtr);
mod._free(pcmPtr);
mod._free(decoderPtr);
2025-03-31 21:04:11 +08:00
return pcmData;
} catch (err) {
log(`解码过程中出错: ${err.message}`, 'error');
return new Float32Array(0);
}
}
};
2025-03-31 21:04:11 +08:00
log('Opus解码器初始化成功', 'success');
} catch (err) {
log(`Opus解码器初始化失败: ${err.message},尝试使用简单播放`, 'error');
// 回退到简单播放
simpleFallbackPlay(opusBuffers);
isPlaying = false;
return;
}
}
2025-03-31 21:04:11 +08:00
// 解码所有Opus帧
let pcmData = [];
let successCount = 0;
2025-03-31 21:04:11 +08:00
for (let i = 0; i < opusBuffers.length; i++) {
try {
const opusBuffer = opusBuffers[i];
// 确保这是Uint8Array
2025-03-31 21:04:11 +08:00
const opusArray = opusBuffer instanceof Uint8Array ?
opusBuffer : new Uint8Array(opusBuffer);
// 解码Opus帧为PCM数据
const decoded = opusDecoder.decode_float(opusArray);
if (decoded && decoded.length > 0) {
pcmData.push(decoded);
successCount++;
}
} catch (error) {
2025-03-31 21:04:11 +08:00
log(`Opus帧 #${i + 1} 解码错误: ${error.message}`, 'error');
}
}
2025-03-31 21:04:11 +08:00
log(`解码完成,成功: ${successCount}/${opusBuffers.length} 帧`, 'debug');
2025-03-31 21:04:11 +08:00
if (pcmData.length === 0) {
log('没有成功解码的帧,尝试使用简单播放', 'warning');
simpleFallbackPlay(opusBuffers);
isPlaying = false;
return;
}
2025-03-31 21:04:11 +08:00
// 连接所有PCM数据
const totalSamples = pcmData.reduce((acc, arr) => acc + arr.length, 0);
log(`总共解码出 ${totalSamples} 个PCM样本`, 'debug');
2025-03-31 21:04:11 +08:00
const combinedPCM = new Float32Array(totalSamples);
2025-03-31 21:04:11 +08:00
let offset = 0;
for (const buffer of pcmData) {
combinedPCM.set(buffer, offset);
offset += buffer.length;
}
2025-03-31 21:04:11 +08:00
// 使用Web Audio API播放PCM数据
const audioBuffer = audioContext.createBuffer(1, combinedPCM.length, 16000);
audioBuffer.getChannelData(0).set(combinedPCM);
2025-03-31 21:04:11 +08:00
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioContext.destination);
2025-03-31 21:04:11 +08:00
source.onended = () => {
isPlaying = false;
log('音频播放完成', 'debug');
2025-03-31 21:04:11 +08:00
// 播放队列中的下一个
if (audioQueue.length > 0) {
const nextBuffers = audioQueue.shift();
playOpusData(nextBuffers);
}
};
2025-03-31 21:04:11 +08:00
source.start();
2025-03-31 21:04:11 +08:00
log(`开始播放PCM音频,时长: ${(combinedPCM.length / 16000).toFixed(2)}秒`, 'debug');
} catch (error) {
log(`播放Opus音频错误: ${error.message},尝试使用简单播放`, 'error');
simpleFallbackPlay(opusBuffers);
isPlaying = false;
}
}
2025-03-31 21:04:11 +08:00
// 简单播放回退方案
function simpleFallbackPlay(opusBuffers) {
try {
log('使用简单播放回退方案', 'warning');
2025-03-31 21:04:11 +08:00
// 创建一个简单的"滴"声作为反馈
if (audioContext) {
const oscillator = audioContext.createOscillator();
oscillator.type = 'sine';
oscillator.frequency.setValueAtTime(440, audioContext.currentTime); // A4音
2025-03-31 21:04:11 +08:00
const gainNode = audioContext.createGain();
gainNode.gain.setValueAtTime(0.1, audioContext.currentTime); // 设置音量为0.1
2025-03-31 21:04:11 +08:00
oscillator.connect(gainNode);
gainNode.connect(audioContext.destination);
2025-03-31 21:04:11 +08:00
oscillator.start();
oscillator.stop(audioContext.currentTime + 0.2); // 播放0.2秒
2025-03-31 21:04:11 +08:00
log('播放提示音作为音频播放失败的反馈', 'info');
}
} catch (error) {
log(`简单播放回退方案也失败: ${error.message}`, 'error');
}
}
2025-03-31 21:04:11 +08:00
// 初始化音频录制和处理
async function initAudio() {
try {
// 请求麦克风权限
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
sampleRate: 16000, // 确保16kHz采样率
channelCount: 1 // 确保单声道
}
});
log('已获取麦克风访问权限', 'success');
2025-03-31 21:04:11 +08:00
// 创建音频上下文
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 16000, // 确保采样率与服务器期望的一致
latencyHint: 'interactive'
});
const source = audioContext.createMediaStreamSource(stream);
2025-03-31 21:04:11 +08:00
// 获取实际音频轨道设置
const audioTracks = stream.getAudioTracks();
if (audioTracks.length > 0) {
const track = audioTracks[0];
const settings = track.getSettings();
log(`实际麦克风设置 - 采样率: ${settings.sampleRate || '未知'}Hz, 声道数: ${settings.channelCount || '未知'}`, 'info');
}
2025-03-31 21:04:11 +08:00
// 创建分析器用于可视化
analyser = audioContext.createAnalyser();
analyser.fftSize = 2048;
source.connect(analyser);
2025-03-31 21:04:11 +08:00
// 尝试初始化MediaRecorder,按优先级尝试不同编码选项
try {
// 优先尝试使用Opus编码
2025-03-31 21:04:11 +08:00
mediaRecorder = new MediaRecorder(stream, {
mimeType: 'audio/webm;codecs=opus',
audioBitsPerSecond: 16000
});
log('已初始化MediaRecorder (使用Opus编码)', 'success');
log(`选择的编码格式: ${mediaRecorder.mimeType}`, 'info');
} catch (e1) {
try {
// 如果Opus不支持,尝试MP3
2025-03-31 21:04:11 +08:00
mediaRecorder = new MediaRecorder(stream, {
mimeType: 'audio/webm',
audioBitsPerSecond: 16000
});
log('已初始化MediaRecorder (使用WebM标准编码,Opus不支持)', 'warning');
log(`选择的编码格式: ${mediaRecorder.mimeType}`, 'info');
} catch (e2) {
try {
// 尝试其他备选格式
2025-03-31 21:04:11 +08:00
mediaRecorder = new MediaRecorder(stream, {
mimeType: 'audio/ogg;codecs=opus',
audioBitsPerSecond: 16000
});
log('已初始化MediaRecorder (使用OGG+Opus编码)', 'warning');
log(`选择的编码格式: ${mediaRecorder.mimeType}`, 'info');
} catch (e3) {
// 最后使用默认编码
mediaRecorder = new MediaRecorder(stream);
log(`已初始化MediaRecorder (使用默认编码: ${mediaRecorder.mimeType})`, 'warning');
}
}
}
2025-03-31 21:04:11 +08:00
// 处理录制的数据
mediaRecorder.ondataavailable = (event) => {
if (event.data.size > 0) {
audioChunks.push(event.data);
}
};
2025-03-31 21:04:11 +08:00
// 录制结束后处理数据
mediaRecorder.onstop = async () => {
// 停止可视化
if (visualizationRequest) {
cancelAnimationFrame(visualizationRequest);
visualizationRequest = null;
}
2025-03-31 21:04:11 +08:00
log(`录音结束,已收集的音频块数量: ${audioChunks.length}`, 'info');
if (audioChunks.length === 0) {
log('警告:没有收集到任何音频数据,请检查麦克风是否工作正常', 'error');
return;
}
2025-03-31 21:04:11 +08:00
// 创建完整的录音blob
const blob = new Blob(audioChunks, { type: audioChunks[0].type });
log(`已创建音频BlobMIME类型: ${audioChunks[0].type},大小: ${(blob.size / 1024).toFixed(2)} KB`, 'info');
2025-03-31 21:04:11 +08:00
// 保存原始块,以防清空后需要调试
const chunks = [...audioChunks];
audioChunks = [];
2025-03-31 21:04:11 +08:00
try {
// 将blob转换为ArrayBuffer
const arrayBuffer = await blob.arrayBuffer();
const uint8Array = new Uint8Array(arrayBuffer);
2025-03-31 21:04:11 +08:00
log(`已转换为Uint8Array,准备发送,大小: ${(arrayBuffer.byteLength / 1024).toFixed(2)} KB`, 'info');
2025-03-31 21:04:11 +08:00
// 检查WebSocket状态
if (!websocket) {
log('错误:WebSocket连接不存在', 'error');
return;
}
2025-03-31 21:04:11 +08:00
if (websocket.readyState !== WebSocket.OPEN) {
log(`错误:WebSocket连接未打开,当前状态: ${websocket.readyState}`, 'error');
return;
}
2025-03-31 21:04:11 +08:00
// 直接发送二进制音频数据 - 这是最简单有效的方式
try {
// 注意:开始和结束消息已在录音开始和结束时发送
// 这里只需要发送音频数据
await new Promise(resolve => setTimeout(resolve, 50));
2025-03-31 21:04:11 +08:00
// 处理WebM容器格式,提取纯Opus数据
// 服务器使用opuslib_next.Decoder,需要纯Opus帧
log('正在处理音频数据,提取纯Opus帧...', 'info');
const opusData = extractOpusFrames(uint8Array);
2025-03-31 21:04:11 +08:00
// 记录Opus数据大小
log(`已提取Opus数据,大小: ${(opusData.byteLength / 1024).toFixed(2)} KB`, 'info');
2025-03-31 21:04:11 +08:00
// 发送音频消息第二步:二进制音频数据
websocket.send(opusData);
log(`已发送Opus音频数据: ${(opusData.byteLength / 1024).toFixed(2)} KB`, 'success');
} catch (error) {
log(`音频数据发送失败: ${error.message}`, 'error');
2025-03-31 21:04:11 +08:00
// 尝试使用base64编码作为备选方案
try {
log('尝试使用base64编码方式发送...', 'info');
const base64Data = arrayBufferToBase64(arrayBuffer);
const audioDataMessage = {
type: 'audio',
action: 'data',
format: 'opus',
sample_rate: 16000,
channels: 1,
mime_type: chunks[0].type,
encoding: 'base64',
data: base64Data
};
websocket.send(JSON.stringify(audioDataMessage));
log(`已使用base64编码发送音频数据: ${(arrayBuffer.byteLength / 1024).toFixed(2)} KB`, 'warning');
} catch (base64Error) {
log(`所有数据发送方式均失败: ${base64Error.message}`, 'error');
}
}
} catch (error) {
log(`处理录音数据错误: ${error.message}`, 'error');
}
};
2025-03-31 21:04:11 +08:00
// 尝试初始化Opus解码器
try {
// 检查ModuleInstance是否存在(本地库导出的全局变量)
if (typeof window.ModuleInstance === 'undefined') {
throw new Error('Opus库未加载,ModuleInstance对象不存在');
}
2025-03-31 21:04:11 +08:00
// 简单测试ModuleInstance是否可用
if (typeof window.ModuleInstance._opus_decoder_get_size === 'function') {
const testSize = window.ModuleInstance._opus_decoder_get_size(1);
log(`Opus解码器测试成功,解码器大小: ${testSize} 字节`, 'success');
} else {
throw new Error('Opus解码函数未找到');
}
} catch (err) {
log(`Opus解码器初始化警告: ${err.message},将在需要时重试`, 'warning');
}
2025-03-31 21:04:11 +08:00
log('音频系统初始化完成', 'success');
return true;
} catch (error) {
log(`音频初始化错误: ${error.message}`, 'error');
return false;
}
}
2025-03-31 21:04:11 +08:00
// 开始录音
function startRecording() {
if (isRecording) return;
2025-03-31 21:04:11 +08:00
try {
// 最小录音时长提示
log('请至少录制1-2秒钟的音频,确保采集到足够数据', 'info');
2025-03-31 21:04:11 +08:00
// 获取服务器类型 - 从URL判断
const serverUrl = serverUrlInput.value.trim();
let isXiaozhiNative = false;
2025-03-31 21:04:11 +08:00
// 检查是否是小智原生服务器 (根据URL特征判断)
if (serverUrl.includes('xiaozhi') || serverUrl.includes('localhost') || serverUrl.includes('127.0.0.1')) {
isXiaozhiNative = true;
log('检测到小智原生服务器,使用标准listen协议', 'info');
}
2025-03-31 21:04:11 +08:00
// 使用直接PCM录音和libopus编码的方式
startDirectRecording();
} catch (error) {
log(`录音启动错误: ${error.message}`, 'error');
}
}
2025-03-31 21:04:11 +08:00
// 停止录音
function stopRecording() {
if (!isRecording) return;
2025-03-31 21:04:11 +08:00
try {
// 使用直接PCM录音停止
stopDirectRecording();
} catch (error) {
log(`停止录音错误: ${error.message}`, 'error');
}
}
2025-03-31 21:04:11 +08:00
// 连接WebSocket服务器
function connectToServer() {
const url = serverUrlInput.value.trim();
if (url === '') return;
2025-03-31 21:04:11 +08:00
try {
// 检查URL格式
if (!url.startsWith('ws://') && !url.startsWith('wss://')) {
log('URL格式错误,必须以ws://或wss://开头', 'error');
return;
}
2025-03-31 21:04:11 +08:00
// 使用自定义WebSocket实现以添加认证头信息
// 注意:浏览器原生WebSocket不支持自定义头,需要通过服务器添加一个代理层
// 但我们可以通过URL参数模拟认证信息
let connUrl = new URL(url);
2025-03-31 21:04:11 +08:00
// 添加认证参数
connUrl.searchParams.append('device_id', 'web_test_device');
connUrl.searchParams.append('device_mac', '00:11:22:33:44:55');
2025-03-31 21:04:11 +08:00
log(`正在连接: ${connUrl.toString()}`, 'info');
websocket = new WebSocket(connUrl.toString());
2025-03-31 21:04:11 +08:00
// 设置接收二进制数据的类型为ArrayBuffer
websocket.binaryType = 'arraybuffer';
2025-03-31 21:04:11 +08:00
websocket.onopen = async () => {
log(`已连接到服务器: ${url}`, 'success');
connectionStatus.textContent = '已连接';
connectionStatus.style.color = 'green';
2025-03-31 21:04:11 +08:00
// 连接成功后发送hello消息
await sendHelloMessage();
2025-03-31 21:04:11 +08:00
connectButton.textContent = '断开';
connectButton.onclick = disconnectFromServer;
messageInput.disabled = false;
sendTextButton.disabled = false;
2025-03-31 21:04:11 +08:00
const audioInitialized = await initAudio();
if (audioInitialized) {
recordButton.disabled = false;
}
};
2025-03-31 21:04:11 +08:00
websocket.onclose = () => {
log('已断开连接', 'info');
connectionStatus.textContent = '已断开';
connectionStatus.style.color = 'red';
2025-03-31 21:04:11 +08:00
connectButton.textContent = '连接';
connectButton.onclick = connectToServer;
messageInput.disabled = true;
sendTextButton.disabled = true;
recordButton.disabled = true;
stopButton.disabled = true;
};
2025-03-31 21:04:11 +08:00
websocket.onerror = (error) => {
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
connectionStatus.textContent = '连接错误';
connectionStatus.style.color = 'red';
};
2025-03-31 21:04:11 +08:00
websocket.onmessage = function (event) {
try {
// 检查是否为文本消息
if (typeof event.data === 'string') {
const message = JSON.parse(event.data);
2025-03-31 21:04:11 +08:00
if (message.type === 'hello') {
log(`服务器回应:${message.message}`, 'info');
} else if (message.type === 'tts') {
// TTS状态消息
if (message.state === 'start') {
log('服务器开始发送语音', 'info');
} else if (message.state === 'sentence_start') {
log(`服务器发送语音段: ${message.text}`, 'info');
// 添加文本到会话记录
if (message.text) {
addMessage(message.text);
}
} else if (message.state === 'sentence_end') {
log(`语音段结束: ${message.text}`, 'info');
} else if (message.state === 'stop') {
log('服务器语音传输结束', 'info');
// 结束后更新UI状态
if (recordButton.disabled) {
recordButton.disabled = false;
recordButton.textContent = '开始录音';
recordButton.classList.remove('recording');
}
}
} else if (message.type === 'audio') {
// 音频控制消息
log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
} else if (message.type === 'stt') {
// 语音识别结果
log(`识别结果: ${message.text}`, 'info');
// 添加识别结果到会话记录
addMessage(`[语音识别] ${message.text}`, true);
} else if (message.type === 'llm') {
// 大模型回复
log(`大模型回复: ${message.text}`, 'info');
// 添加大模型回复到会话记录
if (message.text && message.text !== '😊') {
addMessage(message.text);
}
} else {
// 未知消息类型
log(`未知消息类型: ${message.type}`, 'info');
addMessage(JSON.stringify(message, null, 2));
}
} else {
// 处理二进制数据 - 兼容多种二进制格式
handleBinaryMessage(event.data);
}
} catch (error) {
log(`WebSocket消息处理错误: ${error.message}`, 'error');
// 非JSON格式文本消息直接显示
if (typeof event.data === 'string') {
addMessage(event.data);
}
}
};
2025-03-31 21:04:11 +08:00
connectionStatus.textContent = '正在连接...';
connectionStatus.style.color = 'orange';
} catch (error) {
log(`连接错误: ${error.message}`, 'error');
connectionStatus.textContent = '连接失败';
}
}
2025-03-31 21:04:11 +08:00
// 发送hello握手消息
async function sendHelloMessage() {
if (!websocket || websocket.readyState !== WebSocket.OPEN) return;
2025-03-31 21:04:11 +08:00
try {
// 设置设备信息
const helloMessage = {
type: 'hello',
device_id: 'web_test_device',
device_name: 'Web测试设备',
device_mac: '00:11:22:33:44:55',
token: 'your-token1' // 使用config.yaml中配置的token
};
2025-03-31 21:04:11 +08:00
log('发送hello握手消息', 'info');
websocket.send(JSON.stringify(helloMessage));
2025-03-31 21:04:11 +08:00
// 等待服务器响应
return new Promise(resolve => {
// 5秒超时
const timeout = setTimeout(() => {
log('等待hello响应超时', 'error');
log('提示: 请尝试点击"测试认证"按钮进行连接排查', 'info');
resolve(false);
}, 5000);
2025-03-31 21:04:11 +08:00
// 临时监听一次消息,接收hello响应
const onMessageHandler = (event) => {
try {
const response = JSON.parse(event.data);
if (response.type === 'hello' && response.session_id) {
log(`服务器握手成功,会话ID: ${response.session_id}`, 'success');
clearTimeout(timeout);
websocket.removeEventListener('message', onMessageHandler);
resolve(true);
}
} catch (e) {
// 忽略非JSON消息
}
};
2025-03-31 21:04:11 +08:00
websocket.addEventListener('message', onMessageHandler);
});
} catch (error) {
log(`发送hello消息错误: ${error.message}`, 'error');
return false;
}
}
2025-03-31 21:04:11 +08:00
// 断开WebSocket连接
function disconnectFromServer() {
if (!websocket) return;
2025-03-31 21:04:11 +08:00
websocket.close();
stopRecording();
}
2025-03-31 21:04:11 +08:00
// 发送文本消息
function sendTextMessage() {
const message = messageInput.value.trim();
if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return;
2025-03-31 21:04:11 +08:00
try {
// 直接发送listen消息,不需要重复发送hello
const listenMessage = {
type: 'listen',
mode: 'manual',
state: 'detect',
text: message
};
2025-03-31 21:04:11 +08:00
websocket.send(JSON.stringify(listenMessage));
addMessage(message, true);
log(`发送文本消息: ${message}`, 'info');
2025-03-31 21:04:11 +08:00
messageInput.value = '';
} catch (error) {
log(`发送消息错误: ${error.message}`, 'error');
}
}
2025-03-31 21:04:11 +08:00
// 初始化事件监听器
function initEventListeners() {
connectButton.addEventListener('click', connectToServer);
document.getElementById('authTestButton').addEventListener('click', testAuthentication);
2025-03-31 21:04:11 +08:00
// 标签页切换
const tabs = document.querySelectorAll('.tab');
tabs.forEach(tab => {
tab.addEventListener('click', () => {
// 移除所有标签页的active类
tabs.forEach(t => t.classList.remove('active'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active'));
// 添加当前标签页的active类
tab.classList.add('active');
document.getElementById(`${tab.dataset.tab}Tab`).classList.add('active');
});
});
sendTextButton.addEventListener('click', sendTextMessage);
messageInput.addEventListener('keypress', (e) => {
if (e.key === 'Enter') sendTextMessage();
});
2025-03-31 21:04:11 +08:00
recordButton.addEventListener('click', () => {
if (isRecording) {
stopRecording();
} else {
startRecording();
}
});
window.addEventListener('resize', initVisualizer);
}
2025-03-31 21:04:11 +08:00
// 测试认证
async function testAuthentication() {
log('开始测试认证...', 'info');
2025-03-31 21:04:11 +08:00
// 显示服务器配置
log('-------- 服务器认证配置检查 --------', 'info');
log('请确认config.yaml中的auth配置:', 'info');
log('1. server.auth.enabled 为 false 或服务器已正确配置认证', 'info');
log('2. 如果启用了认证,请确认使用了正确的token', 'info');
log('3. 或者在allowed_devices中添加了测试设备MAC00:11:22:33:44:55', 'info');
2025-03-31 21:04:11 +08:00
const serverUrl = serverUrlInput.value.trim();
if (!serverUrl) {
log('请输入服务器地址', 'error');
return;
}
2025-03-31 21:04:11 +08:00
// 测试连接
log('尝试不同认证参数的连接:', 'info');
2025-03-31 21:04:11 +08:00
// 测试1: 无参数连接
try {
log('测试1: 尝试无参数连接...', 'info');
const ws1 = new WebSocket(serverUrl);
2025-03-31 21:04:11 +08:00
ws1.onopen = () => {
log('测试1成功: 无参数可连接,服务器可能没有启用认证', 'success');
ws1.close();
};
2025-03-31 21:04:11 +08:00
ws1.onerror = (error) => {
log('测试1失败: 无参数连接被拒绝,服务器可能启用了认证', 'error');
};
2025-03-31 21:04:11 +08:00
// 5秒后关闭测试连接
setTimeout(() => {
if (ws1.readyState === WebSocket.CONNECTING || ws1.readyState === WebSocket.OPEN) {
ws1.close();
}
}, 5000);
} catch (error) {
log(`测试1出错: ${error.message}`, 'error');
}
2025-03-31 21:04:11 +08:00
// 测试2: 带参数连接
setTimeout(async () => {
try {
log('测试2: 尝试带token参数连接...', 'info');
2025-03-31 21:04:11 +08:00
let url = new URL(serverUrl);
url.searchParams.append('token', 'your-token1');
url.searchParams.append('device_id', 'web_test_device');
url.searchParams.append('device_mac', '00:11:22:33:44:55');
2025-03-31 21:04:11 +08:00
const ws2 = new WebSocket(url.toString());
2025-03-31 21:04:11 +08:00
ws2.onopen = () => {
log('测试2成功: 带token参数可连接', 'success');
2025-03-31 21:04:11 +08:00
// 尝试发送hello消息
const helloMsg = {
type: 'hello',
device_id: 'web_test_device',
device_mac: '00:11:22:33:44:55',
token: 'your-token1'
};
2025-03-31 21:04:11 +08:00
ws2.send(JSON.stringify(helloMsg));
log('已发送hello测试消息', 'info');
2025-03-31 21:04:11 +08:00
// 监听响应
ws2.onmessage = (event) => {
try {
const response = JSON.parse(event.data);
if (response.type === 'hello' && response.session_id) {
log(`测试完全成功! 收到hello响应,会话ID: ${response.session_id}`, 'success');
ws2.close();
}
} catch (e) {
log(`收到非JSON响应: ${event.data}`, 'info');
}
};
2025-03-31 21:04:11 +08:00
// 5秒后关闭
setTimeout(() => ws2.close(), 5000);
};
2025-03-31 21:04:11 +08:00
ws2.onerror = (error) => {
log('测试2失败: 带token参数连接被拒绝', 'error');
log('请检查token是否正确,或服务器是否接受URL参数认证', 'error');
};
} catch (error) {
log(`测试2出错: ${error.message}`, 'error');
}
}, 6000);
2025-03-31 21:04:11 +08:00
log('认证测试已启动,请查看测试结果...', 'info');
}
2025-03-31 21:04:11 +08:00
// 帮助函数:ArrayBuffer转Base64
function arrayBufferToBase64(buffer) {
let binary = '';
const bytes = new Uint8Array(buffer);
const len = bytes.byteLength;
for (let i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
}
2025-03-31 21:04:11 +08:00
// 使用libopus创建一个Opus编码器
let opusEncoder = null;
function initOpusEncoder() {
try {
if (opusEncoder) {
return true; // 已经初始化过
}
2025-03-31 21:04:11 +08:00
if (!window.ModuleInstance) {
log('无法创建Opus编码器:ModuleInstance不可用', 'error');
return false;
}
2025-03-31 21:04:11 +08:00
// 初始化一个Opus编码器
const mod = window.ModuleInstance;
const sampleRate = 16000; // 16kHz采样率
const channels = 1; // 单声道
const application = 2048; // OPUS_APPLICATION_VOIP = 2048
2025-03-31 21:04:11 +08:00
// 创建编码器
opusEncoder = {
channels: channels,
sampleRate: sampleRate,
frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
maxPacketSize: 4000, // 最大包大小
module: mod,
2025-03-31 21:04:11 +08:00
// 初始化编码器
2025-03-31 21:04:11 +08:00
init: function () {
try {
// 获取编码器大小
const encoderSize = mod._opus_encoder_get_size(this.channels);
log(`Opus编码器大小: ${encoderSize}字节`, 'info');
2025-03-31 21:04:11 +08:00
// 分配内存
this.encoderPtr = mod._malloc(encoderSize);
if (!this.encoderPtr) {
throw new Error("无法分配编码器内存");
}
2025-03-31 21:04:11 +08:00
// 初始化编码器
const err = mod._opus_encoder_init(
this.encoderPtr,
this.sampleRate,
this.channels,
application
);
2025-03-31 21:04:11 +08:00
if (err < 0) {
throw new Error(`Opus编码器初始化失败: ${err}`);
}
2025-03-31 21:04:11 +08:00
// 设置位率 (16kbps)
mod._opus_encoder_ctl(this.encoderPtr, 4002, 16000); // OPUS_SET_BITRATE
2025-03-31 21:04:11 +08:00
// 设置复杂度 (0-10, 越高质量越好但CPU使用越多)
mod._opus_encoder_ctl(this.encoderPtr, 4010, 5); // OPUS_SET_COMPLEXITY
2025-03-31 21:04:11 +08:00
// 设置使用DTX (不传输静音帧)
mod._opus_encoder_ctl(this.encoderPtr, 4016, 1); // OPUS_SET_DTX
2025-03-31 21:04:11 +08:00
log("Opus编码器初始化成功", 'success');
return true;
} catch (error) {
if (this.encoderPtr) {
mod._free(this.encoderPtr);
this.encoderPtr = null;
}
log(`Opus编码器初始化失败: ${error.message}`, 'error');
return false;
}
},
2025-03-31 21:04:11 +08:00
// 编码PCM数据为Opus
2025-03-31 21:04:11 +08:00
encode: function (pcmData) {
if (!this.encoderPtr) {
if (!this.init()) {
return null;
}
}
2025-03-31 21:04:11 +08:00
try {
const mod = this.module;
2025-03-31 21:04:11 +08:00
// 为PCM数据分配内存
const pcmPtr = mod._malloc(pcmData.length * 2); // 2字节/int16
2025-03-31 21:04:11 +08:00
// 将PCM数据复制到HEAP
for (let i = 0; i < pcmData.length; i++) {
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
}
2025-03-31 21:04:11 +08:00
// 为输出分配内存
const outPtr = mod._malloc(this.maxPacketSize);
2025-03-31 21:04:11 +08:00
// 进行编码
const encodedLen = mod._opus_encode(
this.encoderPtr,
pcmPtr,
this.frameSize,
outPtr,
this.maxPacketSize
);
2025-03-31 21:04:11 +08:00
if (encodedLen < 0) {
throw new Error(`Opus编码失败: ${encodedLen}`);
}
2025-03-31 21:04:11 +08:00
// 复制编码后的数据
const opusData = new Uint8Array(encodedLen);
for (let i = 0; i < encodedLen; i++) {
opusData[i] = mod.HEAPU8[outPtr + i];
}
2025-03-31 21:04:11 +08:00
// 释放内存
mod._free(pcmPtr);
mod._free(outPtr);
2025-03-31 21:04:11 +08:00
return opusData;
} catch (error) {
log(`Opus编码出错: ${error.message}`, 'error');
return null;
}
},
2025-03-31 21:04:11 +08:00
// 销毁编码器
2025-03-31 21:04:11 +08:00
destroy: function () {
if (this.encoderPtr) {
this.module._free(this.encoderPtr);
this.encoderPtr = null;
}
}
};
2025-03-31 21:04:11 +08:00
const result = opusEncoder.init();
return result;
} catch (error) {
log(`创建Opus编码器失败: ${error.message}`, 'error');
return false;
}
}
2025-03-31 21:04:11 +08:00
// 初始化应用
function initApp() {
initVisualizer();
initEventListeners();
2025-03-31 21:04:11 +08:00
// 检查libopus.js是否正确加载
checkOpusLoaded();
2025-03-31 21:04:11 +08:00
// 初始化Opus编码器
initOpusEncoder();
}
2025-03-31 21:04:11 +08:00
// PCM录音处理器代码 - 会被注入到AudioWorklet中
const audioProcessorCode = `
class AudioRecorderProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.buffers = [];
this.frameSize = 960; // 60ms @ 16kHz = 960 samples
this.buffer = new Int16Array(this.frameSize);
this.bufferIndex = 0;
this.isRecording = false;
// 监听来自主线程的消息
this.port.onmessage = (event) => {
if (event.data.command === 'start') {
this.isRecording = true;
this.port.postMessage({ type: 'status', status: 'started' });
} else if (event.data.command === 'stop') {
this.isRecording = false;
// 发送剩余的缓冲区
if (this.bufferIndex > 0) {
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
this.port.postMessage({
type: 'buffer',
buffer: finalBuffer
});
this.bufferIndex = 0;
}
this.port.postMessage({ type: 'status', status: 'stopped' });
}
};
}
process(inputs, outputs, parameters) {
if (!this.isRecording) return true;
const input = inputs[0][0]; // 获取第一个输入通道
if (!input) return true;
// 将浮点采样转换为16位整数并存储
for (let i = 0; i < input.length; i++) {
if (this.bufferIndex >= this.frameSize) {
// 缓冲区已满,发送给主线程并重置
this.port.postMessage({
type: 'buffer',
buffer: this.buffer.slice(0)
});
this.bufferIndex = 0;
}
// 转换为16位整数 (-32768到32767)
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
return true;
}
}
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
`;
2025-03-31 21:04:11 +08:00
// 创建音频处理器
async function createAudioProcessor() {
if (!audioContext) {
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 16000,
latencyHint: 'interactive'
});
}
2025-03-31 21:04:11 +08:00
try {
// 检查是否支持AudioWorklet
if (audioContext.audioWorklet) {
// 注册音频处理器
const blob = new Blob([audioProcessorCode], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
await audioContext.audioWorklet.addModule(url);
URL.revokeObjectURL(url);
2025-03-31 21:04:11 +08:00
// 创建音频处理节点
const audioProcessor = new AudioWorkletNode(audioContext, 'audio-recorder-processor');
2025-03-31 21:04:11 +08:00
// 设置音频处理消息处理
audioProcessor.port.onmessage = (event) => {
if (event.data.type === 'buffer') {
// 收到PCM缓冲区数据
processPCMBuffer(event.data.buffer);
}
};
2025-03-31 21:04:11 +08:00
log('使用AudioWorklet处理音频', 'success');
return { node: audioProcessor, type: 'worklet' };
} else {
// 使用旧版ScriptProcessorNode作为回退方案
log('AudioWorklet不可用,使用ScriptProcessorNode作为回退方案', 'warning');
2025-03-31 21:04:11 +08:00
const frameSize = 4096; // ScriptProcessorNode缓冲区大小
const scriptProcessor = audioContext.createScriptProcessor(frameSize, 1, 1);
2025-03-31 21:04:11 +08:00
// 将audioProcess事件设置为处理音频数据
scriptProcessor.onaudioprocess = (event) => {
if (!isRecording) return;
2025-03-31 21:04:11 +08:00
const input = event.inputBuffer.getChannelData(0);
const buffer = new Int16Array(input.length);
2025-03-31 21:04:11 +08:00
// 将浮点数据转换为16位整数
for (let i = 0; i < input.length; i++) {
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
2025-03-31 21:04:11 +08:00
// 处理PCM数据
processPCMBuffer(buffer);
};
2025-03-31 21:04:11 +08:00
// 需要连接输出,否则不会触发处理
// 我们创建一个静音通道
const silent = audioContext.createGain();
silent.gain.value = 0;
scriptProcessor.connect(silent);
silent.connect(audioContext.destination);
2025-03-31 21:04:11 +08:00
return { node: scriptProcessor, type: 'processor' };
}
} catch (error) {
log(`创建音频处理器失败: ${error.message},尝试回退方案`, 'error');
2025-03-31 21:04:11 +08:00
// 最后回退方案:使用ScriptProcessorNode
try {
const frameSize = 4096; // ScriptProcessorNode缓冲区大小
const scriptProcessor = audioContext.createScriptProcessor(frameSize, 1, 1);
2025-03-31 21:04:11 +08:00
scriptProcessor.onaudioprocess = (event) => {
if (!isRecording) return;
2025-03-31 21:04:11 +08:00
const input = event.inputBuffer.getChannelData(0);
const buffer = new Int16Array(input.length);
2025-03-31 21:04:11 +08:00
for (let i = 0; i < input.length; i++) {
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
2025-03-31 21:04:11 +08:00
processPCMBuffer(buffer);
};
2025-03-31 21:04:11 +08:00
const silent = audioContext.createGain();
silent.gain.value = 0;
scriptProcessor.connect(silent);
silent.connect(audioContext.destination);
2025-03-31 21:04:11 +08:00
log('使用ScriptProcessorNode作为回退方案成功', 'warning');
return { node: scriptProcessor, type: 'processor' };
} catch (fallbackError) {
log(`回退方案也失败: ${fallbackError.message}`, 'error');
return null;
}
}
}
2025-03-31 21:04:11 +08:00
// 初始化直接从PCM数据录音的系统
let audioProcessor = null;
let audioProcessorType = null;
let audioSource = null;
let pcmBuffers = [];
2025-03-31 21:04:11 +08:00
// 处理PCM缓冲数据
let pcmDataBuffer = new Int16Array();
function processPCMBuffer(buffer) {
if (!isRecording) return;
2025-03-31 21:04:11 +08:00
// 将新的PCM数据追加到缓冲区
const newBuffer = new Int16Array(pcmDataBuffer.length + buffer.length);
newBuffer.set(pcmDataBuffer);
newBuffer.set(buffer, pcmDataBuffer.length);
pcmDataBuffer = newBuffer;
2025-03-31 21:04:11 +08:00
// 检查是否有足够的数据进行Opus编码(16000Hz, 60ms = 960个采样点)
const samplesPerFrame = 960; // 60ms @ 16kHz
2025-03-31 21:04:11 +08:00
while (pcmDataBuffer.length >= samplesPerFrame) {
// 从缓冲区取出一帧数据
const frameData = pcmDataBuffer.slice(0, samplesPerFrame);
pcmDataBuffer = pcmDataBuffer.slice(samplesPerFrame);
2025-03-31 21:04:11 +08:00
// 编码为Opus
encodeAndSendOpus(frameData);
}
}
2025-03-31 21:04:11 +08:00
// 编码并发送Opus数据
function encodeAndSendOpus(pcmData = null) {
if (!opusEncoder) {
log('Opus编码器未初始化', 'error');
return;
}
2025-03-31 21:04:11 +08:00
try {
// 如果提供了PCM数据,则编码该数据
if (pcmData) {
// 使用已初始化的Opus编码器编码
const opusData = opusEncoder.encode(pcmData);
2025-03-31 21:04:11 +08:00
if (opusData && opusData.length > 0) {
// 存储音频帧
audioBuffers.push(opusData.buffer);
totalAudioSize += opusData.length;
2025-03-31 21:04:11 +08:00
// 如果WebSocket已连接,则发送数据
if (websocket && websocket.readyState === WebSocket.OPEN) {
try {
// 服务端期望接收原始Opus数据,不需要任何额外包装
websocket.send(opusData.buffer);
log(`发送Opus帧,大小:${opusData.length}字节`, 'debug');
} catch (error) {
log(`WebSocket发送错误: ${error.message}`, 'error');
}
}
} else {
log('Opus编码失败,无有效数据返回', 'error');
}
} else {
// 处理剩余的PCM数据
if (pcmDataBuffer.length > 0) {
// 如果剩余的采样点不足一帧,用静音填充
const samplesPerFrame = 960;
if (pcmDataBuffer.length < samplesPerFrame) {
const paddedBuffer = new Int16Array(samplesPerFrame);
paddedBuffer.set(pcmDataBuffer);
// 剩余部分为0(静音)
encodeAndSendOpus(paddedBuffer);
} else {
encodeAndSendOpus(pcmDataBuffer.slice(0, samplesPerFrame));
}
pcmDataBuffer = new Int16Array(0);
}
}
} catch (error) {
log(`Opus编码错误: ${error.message}`, 'error');
}
}
2025-03-31 21:04:11 +08:00
// 开始直接从PCM数据录音
async function startDirectRecording() {
if (isRecording) return;
2025-03-31 21:04:11 +08:00
try {
// 初始化Opus编码器
if (!initOpusEncoder()) {
log('无法启动录音: Opus编码器初始化失败', 'error');
return;
}
2025-03-31 21:04:11 +08:00
// 请求麦克风权限
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
sampleRate: 16000,
channelCount: 1
}
});
2025-03-31 21:04:11 +08:00
// 创建音频上下文和分析器
if (!audioContext) {
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 16000,
latencyHint: 'interactive'
});
}
2025-03-31 21:04:11 +08:00
// 创建音频处理器
const processorResult = await createAudioProcessor();
if (!processorResult) {
log('无法创建音频处理器', 'error');
return;
}
2025-03-31 21:04:11 +08:00
audioProcessor = processorResult.node;
audioProcessorType = processorResult.type;
2025-03-31 21:04:11 +08:00
// 连接音频处理链
audioSource = audioContext.createMediaStreamSource(stream);
analyser = audioContext.createAnalyser();
analyser.fftSize = 2048;
2025-03-31 21:04:11 +08:00
audioSource.connect(analyser);
audioSource.connect(audioProcessor);
2025-03-31 21:04:11 +08:00
// 启动录音
pcmDataBuffer = new Int16Array();
audioBuffers = [];
totalAudioSize = 0;
isRecording = true;
2025-03-31 21:04:11 +08:00
// 启动音频处理器的录音 - 只有AudioWorklet才需要发送消息
if (audioProcessorType === 'worklet' && audioProcessor.port) {
audioProcessor.port.postMessage({ command: 'start' });
}
2025-03-31 21:04:11 +08:00
// 发送监听开始消息
if (websocket && websocket.readyState === WebSocket.OPEN) {
// 使用与服务端期望的listen消息格式
const listenMessage = {
type: 'listen',
mode: 'manual', // 使用手动模式,由我们控制开始/停止
state: 'start' // 表示开始录音
};
2025-03-31 21:04:11 +08:00
log(`发送录音开始消息: ${JSON.stringify(listenMessage)}`, 'info');
websocket.send(JSON.stringify(listenMessage));
} else {
log('WebSocket未连接,无法发送开始消息', 'error');
return false;
}
2025-03-31 21:04:11 +08:00
// 开始音频可视化
const dataArray = new Uint8Array(analyser.frequencyBinCount);
drawVisualizer(dataArray);
2025-03-31 21:04:11 +08:00
// 在UI上显示录音计时器
let recordingSeconds = 0;
const recordingTimer = setInterval(() => {
recordingSeconds += 0.1;
2025-03-31 21:04:11 +08:00
recordButton.textContent = `停止录音 ${recordingSeconds.toFixed(1)}秒`;
}, 100);
2025-03-31 21:04:11 +08:00
// 保存计时器,以便在停止时清除
window.recordingTimer = recordingTimer;
2025-03-31 21:04:11 +08:00
recordButton.classList.add('recording');
2025-03-31 21:04:11 +08:00
recordButton.disabled = false;
log('开始PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音启动错误: ${error.message}`, 'error');
isRecording = false;
return false;
}
}
2025-03-31 21:04:11 +08:00
// 停止直接从PCM数据录音
function stopDirectRecording() {
if (!isRecording) return;
2025-03-31 21:04:11 +08:00
try {
// 停止录音
isRecording = false;
2025-03-31 21:04:11 +08:00
// 停止音频处理器的录音
if (audioProcessor) {
// 只有AudioWorklet才需要发送停止消息
if (audioProcessorType === 'worklet' && audioProcessor.port) {
audioProcessor.port.postMessage({ command: 'stop' });
}
2025-03-31 21:04:11 +08:00
audioProcessor.disconnect();
audioProcessor = null;
}
2025-03-31 21:04:11 +08:00
// 断开音频连接
if (audioSource) {
audioSource.disconnect();
audioSource = null;
}
2025-03-31 21:04:11 +08:00
// 停止可视化
if (visualizationRequest) {
cancelAnimationFrame(visualizationRequest);
visualizationRequest = null;
}
2025-03-31 21:04:11 +08:00
// 清除录音计时器
if (window.recordingTimer) {
clearInterval(window.recordingTimer);
window.recordingTimer = null;
}
2025-03-31 21:04:11 +08:00
// 编码并发送剩余的数据
encodeAndSendOpus();
2025-03-31 21:04:11 +08:00
// 发送一个空的消息作为结束标志(模拟接收到空音频数据的情况)
if (websocket && websocket.readyState === WebSocket.OPEN) {
// 使用空的Uint8Array发送最后一个空帧
const emptyOpusFrame = new Uint8Array(0);
websocket.send(emptyOpusFrame);
2025-03-31 21:04:11 +08:00
// 发送监听结束消息
const stopMessage = {
type: 'listen',
mode: 'manual',
state: 'stop'
};
2025-03-31 21:04:11 +08:00
websocket.send(JSON.stringify(stopMessage));
log('已发送录音停止信号', 'info');
}
2025-03-31 21:04:11 +08:00
// 重置UI
recordButton.textContent = '开始录音';
recordButton.classList.remove('recording');
recordButton.disabled = false;
2025-03-31 21:04:11 +08:00
log('停止PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音停止错误: ${error.message}`, 'error');
return false;
}
}
2025-03-31 21:04:11 +08:00
// 处理二进制消息
async function handleBinaryMessage(data) {
try {
let arrayBuffer;
2025-03-31 21:04:11 +08:00
// 根据数据类型进行处理
if (data instanceof ArrayBuffer) {
arrayBuffer = data;
log(`收到ArrayBuffer音频数据,大小: ${data.byteLength}字节`, 'debug');
} else if (data instanceof Blob) {
// 如果是Blob类型,转换为ArrayBuffer
arrayBuffer = await data.arrayBuffer();
log(`收到Blob音频数据,大小: ${arrayBuffer.byteLength}字节`, 'debug');
} else {
log(`收到未知类型的二进制数据: ${typeof data}`, 'warning');
return;
}
2025-03-31 21:04:11 +08:00
// 创建Uint8Array用于处理
const opusData = new Uint8Array(arrayBuffer);
2025-03-31 21:04:11 +08:00
if (opusData.length > 0) {
// 播放Opus音频
playOpusData([opusData]);
} else {
log('收到空音频数据帧', 'warning');
}
} catch (error) {
log(`处理二进制消息出错: ${error.message}`, 'error');
}
}
2025-03-31 21:04:11 +08:00
initApp();
</script>
</body>
2025-03-31 21:04:11 +08:00
</html>