Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 15:13:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into py_wakeup_audio

Browse Source
This commit is contained in:
hrz
2025-08-30 18:15:09 +08:00
committed by GitHub
parent 41b8fac3aa 3e1765a26d
commit 76a7ac04c9
17 changed files with 285 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -94,7 +94,7 @@ Spearheaded by Professor Siyuan Liu's Team (South China University of Technology
</tr>
<tr>
<td>
<a href="https://www.bilibili.com/video/BV1Vy96YCE3R" target="_blank">
<a href="https://www.bilibili.com/video/BV1vchQzaEse" target="_blank">
<picture>
<img alt="自定义音色" src="docs/images/demo6.png" />
</picture>
README_en.md
+1 -1
View File
@@ -93,7 +93,7 @@ Want to see the usage effects? Click the videos below 🎥
</tr>
<tr>
<td>
<a href="https://www.bilibili.com/video/BV1Vy96YCE3R" target="_blank">
<a href="https://www.bilibili.com/video/BV1vchQzaEse" target="_blank">
<picture>
<img alt="Custom voice timbre" src="docs/images/demo6.png" />
</picture>
docs/images/demo6.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 306 KiB

After

Width:  |  Height:  |  Size: 111 KiB

docs/paddlespeech-deploy.md
+1 -1
View File
@@ -75,7 +75,7 @@ TTS:
sample_rate: 24000 # 采样率 [websocket默认24000，http默认0 自动选择]
speed: 1.0 # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢
volume: 1.0 # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小
save_path: ./streaming_tts.wav # 服务器生成的语音文件保存路径
save_path: # 保存路径
```
### 3.启动xiaozhi服务
```py
main/manager-web/src/components/ModelEditDialog.vue
+6 -1
View File
@@ -3,7 +3,7 @@
class="center-dialog" >
<div style="margin: 0 18px; text-align: left; padding: 10px; border-radius: 10px;">
<div style="font-size: 30px; color: #3d4566; margin-top: -10px; margin-bottom: 10px; text-align: center;">
修改模型
{{ modelData.duplicateMode ? '创建副本' : '修改模型' }}
</div>
<button class="custom-close-btn" @click="dialogVisible = false">
@@ -190,6 +190,11 @@ export default {
Api.model.getModelConfig(this.modelData.id, ({ data }) => {
if (data.code === 0 && data.data) {
const model = data.data;
if (this.modelData.duplicateMode) {
model.modelName = this.modelData.modelName + '_副本';
model.modelCode = this.modelData.modelCode + '_副本';
}
this.pendingProviderType = model.configJson.type;
this.pendingModelData = model;
main/manager-web/src/views/ModelConfig.vue
+30 -7
View File
@@ -79,11 +79,14 @@
</el-button>
</template>
</el-table-column>
<el-table-column label="操作" align="center" width="150px">
<el-table-column label="操作" align="center" width="180px">
<template slot-scope="scope">
<el-button type="text" size="mini" @click="editModel(scope.row)" class="edit-btn">
修改
</el-button>
<el-button type="text" size="mini" @click="duplicateModel(scope.row)" class="edit-btn">
创建副本
</el-button>
<el-button type="text" size="mini" @click="deleteModel(scope.row)" class="delete-btn">
删除
</el-button>
@@ -277,6 +280,11 @@ export default {
this.editModelData = JSON.parse(JSON.stringify(model));
this.editDialogVisible = true;
},
duplicateModel(model) {
this.editModelData = JSON.parse(JSON.stringify(model));
this.editModelData.duplicateMode = true;
this.editDialogVisible = true;
},
// 删除单个模型
deleteModel(model) {
this.$confirm('确定要删除该模型吗?', '提示', {
@@ -313,19 +321,34 @@ export default {
const modelType = this.activeTab;
const id = formData.id;
Api.model.updateModel(
{ modelType, provideCode, id, formData },
if (this.editModelData.duplicateMode) {
Api.model.addModel({modelType, provideCode, formData},
({ data }) => {
if (data.code === 0) {
this.$message.success('保存成功');
this.$message.success('创建副本成功');
this.loadData();
this.editDialogVisible = false;
} else {
this.$message.error(data.msg || '保存失败');
this.$message.error(data.msg || '创建副本失败');
}
done && done(); // 调用done回调关闭加载状态
}
);
})
}
else {
Api.model.updateModel(
{ modelType, provideCode, id, formData },
({ data }) => {
if (data.code === 0) {
this.$message.success('保存成功');
this.loadData();
this.editDialogVisible = false;
} else {
this.$message.error(data.msg || '保存失败');
}
done && done(); // 调用done回调关闭加载状态
}
);
}
},
selectAll() {
if (this.isAllSelected) {
main/xiaozhi-server/config.yaml
+1 -3
View File
@@ -59,8 +59,6 @@ log:
delete_audio: true
# 没有语音输入多久后断开连接(秒)，默认2分钟，即120秒
close_connection_no_voice_time: 120
# TTS请求超时时间(秒)
tts_timeout: 10
# 开场是否回复唤醒词
enable_greeting: true
# 说完话是否开启提示音
@@ -899,7 +897,7 @@ TTS:
sample_rate: 24000 # 采样率 [websocket默认24000，http默认0 自动选择]
speed: 1.0 # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢
volume: 1.0 # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小
save_path: ./streaming_tts.wav # 服务器生成的语音文件保存路径
save_path: # 保存路径
IndexStreamTTS:
# 基于Index-TTS-vLLM项目的TTS接口服务
# 参照教程：https://github.com/Ksuriuri/index-tts-vllm/blob/master/README.md
main/xiaozhi-server/core/connection.py
-4
View File
@@ -140,10 +140,6 @@ class ConnectionHandler:
self.func_handler = None
self.cmd_exit = self.config["exit_commands"]
self.max_cmd_length = 0
for cmd in self.cmd_exit:
if len(cmd) > self.max_cmd_length:
self.max_cmd_length = len(cmd)
# 是否在聊天结束后关闭连接
self.close_after_chat = False
main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py
+57 -14
View File
@@ -1,8 +1,10 @@
from config.logger import setup_logging
from http import HTTPStatus
import dashscope
from dashscope import Application
from core.providers.llm.base import LLMProviderBase
from core.utils.util import check_model_key
import time
TAG = __name__
logger = setup_logging()
@@ -15,6 +17,7 @@ class LLMProvider(LLMProviderBase):
self.base_url = config.get("base_url")
self.is_No_prompt = config.get("is_no_prompt")
self.memory_id = config.get("ali_memory_id")
self.streaming_chunk_size = config.get("streaming_chunk_size", 3) # 每次流式返回的字符数
check_model_key("AliBLLLM", self.api_key)
def response(self, session_id, dialogue):
@@ -32,6 +35,8 @@ class LLMProvider(LLMProviderBase):
"app_id": self.app_id,
"session_id": session_id,
"messages": dialogue,
# 开启SDK原生流式
"stream": True,
}
if self.memory_id != False:
# 百练memory需要prompt参数
@@ -42,25 +47,63 @@ class LLMProvider(LLMProviderBase):
f"【阿里百练API服务】处理后的prompt: {prompt}"
)
# 可选地设置自定义API基地址（若配置为兼容模式URL则忽略）
if self.base_url and ("/api/" in self.base_url):
dashscope.base_http_api_url = self.base_url
responses = Application.call(**call_params)
if responses.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={responses.status_code}, "
f"message={responses.message}, "
f"请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
yield "【阿里百练API服务响应异常】"
else:
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】构造参数: {call_params}"
)
yield responses.output.text
# 流式处理（SDK在stream=True时返回可迭代对象；否则返回单次响应对象）
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】构造参数: {dict(call_params, api_key='***')}"
)
last_text = ""
try:
for resp in responses:
if resp.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={resp.status_code}, message={resp.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
continue
current_text = getattr(getattr(resp, "output", None), "text", None)
if current_text is None:
continue
# SDK流式为增量覆盖，计算差量输出
if len(current_text) >= len(last_text):
delta = current_text[len(last_text):]
else:
# 避免偶发回退
delta = current_text
if delta:
yield delta
last_text = current_text
except TypeError:
# 非流式回落（一次性返回）
if responses.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={responses.status_code}, message={responses.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
yield "【阿里百练API服务响应异常】"
else:
full_text = getattr(getattr(responses, "output", None), "text", "")
logger.bind(tag=TAG).info(
f"【阿里百练API服务】完整响应长度: {len(full_text)}"
)
for i in range(0, len(full_text), self.streaming_chunk_size):
chunk = full_text[i:i + self.streaming_chunk_size]
if chunk:
yield chunk
except Exception as e:
logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
yield "【LLM服务响应异常】"
def response_with_functions(self, session_id, dialogue, functions=None):
logger.bind(tag=TAG).error(
f"阿里百练暂未实现完整的工具调用（function call），建议使用其他意图识别"
# 阿里百练当前未支持原生的 function call。为保持兼容，这里回退到普通文本流式输出。
# 上层会按 (content, tool_calls) 的形式消费，这里始终返回 (token, None)
logger.bind(tag=TAG).warning(
"阿里百练未实现原生 function call，已回退为纯文本流式输出"
)
for token in self.response(session_id, dialogue):
yield token, None
main/xiaozhi-server/core/providers/tools/server_mcp/mcp_executor.py
+1 -1
View File
@@ -18,8 +18,8 @@ class ServerMCPExecutor(ToolExecutor):
"""初始化MCP管理器"""
if not self._initialized:
self.mcp_manager = ServerMCPManager(self.conn)
await self.mcp_manager.initialize_servers()
self._initialized = True
await self.mcp_manager.initialize_servers()
async def execute(
self, conn, tool_name: str, arguments: Dict[str, Any]
main/xiaozhi-server/core/providers/tools/server_mcp/mcp_manager.py
+3
View File
@@ -68,6 +68,9 @@ class ServerMCPManager:
# 输出当前支持的服务端MCP工具列表
if hasattr(self.conn, "func_handler") and self.conn.func_handler:
# 刷新工具缓存以确保服务端MCP工具被正确加载
if hasattr(self.conn.func_handler, "tool_manager"):
self.conn.func_handler.tool_manager.refresh_tools()
self.conn.func_handler.current_support_functions()
def get_all_tools(self) -> List[Dict[str, Any]]:
main/xiaozhi-server/core/providers/tts/base.py
-2
View File
@@ -33,7 +33,6 @@ class TTSProviderBase(ABC):
def __init__(self, config, delete_audio_file):
self.interface_type = InterfaceType.NON_STREAM
self.conn = None
self.tts_timeout = 10
self.delete_audio_file = delete_audio_file
self.audio_file_type = "wav"
self.output_file = config.get("output_dir", "tmp/")
@@ -192,7 +191,6 @@ class TTSProviderBase(ABC):
async def open_audio_channels(self, conn):
self.conn = conn
self.tts_timeout = conn.config.get("tts_timeout", 10)
# tts 消化线程
self.tts_priority_thread = threading.Thread(
target=self.tts_text_priority_thread, daemon=True
main/xiaozhi-server/core/providers/tts/paddle_speech.py
+26 -4
View File
@@ -8,6 +8,7 @@ import wave
import websockets
from core.providers.tts.base import TTSProviderBase
from config.logger import setup_logging
from datetime import datetime
TAG = __name__
logger = setup_logging()
@@ -18,11 +19,12 @@ class TTSProvider(TTSProviderBase):
super().__init__(config, delete_audio_file)
self.url = config.get("url", "ws://192.168.1.10:8092/paddlespeech/tts/streaming")
self.protocol = config.get("protocol", "websocket")
if config.get("private_voice"):
self.spk_id = int(config.get("private_voice"))
else:
self.spk_id = int(config.get("spk_id", "0"))
self.spk_id = int(config.get("spk_id", "0"))
sample_rate = config.get("sample_rate", 24000)
self.sample_rate = float(sample_rate) if sample_rate else 24000
@@ -32,7 +34,21 @@ class TTSProvider(TTSProviderBase):
volume = config.get("volume", 1.0)
self.volume = float(volume) if volume else 1.0
self.save_path = config.get("save_path", "./streaming_tts.wav")
self.delete_audio_file = config.get("delete_audio", True)
if not self.delete_audio_file:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
save_path = config.get("save_path")
if save_path:
if not save_path.endswith('.wav'):
save_path = f"{save_path}_{timestamp}.wav"
else:
other_path = save_path[:-4]
save_path = f"{other_path}_{timestamp}.wav"
self.save_path = save_path
else:
self.save_path = f"./streaming_tts_{timestamp}.wav"
else:
self.save_path = None
async def pcm_to_wav(self, pcm_data: bytes, sample_rate: int = 24000, num_channels: int = 1,
bits_per_sample: int = 16) -> bytes:
@@ -151,6 +167,12 @@ class TTSProvider(TTSProviderBase):
# 接收结束响应避免服务抛出异常
await ws.recv()
# 根据配置决定是否保存文件
if not self.delete_audio_file and self.save_path:
with open(self.save_path, "wb") as f:
f.write(wav_data)
logger.bind(tag=TAG).info(f"音频文件已保存到: {self.save_path}")
# 返回或保存音频数据
if output_file:
with open(output_file, "wb") as file_to_save:
@@ -159,4 +181,4 @@ class TTSProvider(TTSProviderBase):
return wav_data
except Exception as e:
raise Exception(f"Error during TTS WebSocket request: {e} while processing text: {text}")
raise Exception(f"Error during TTS WebSocket request: {e} while processing text: {text}")
main/xiaozhi-server/plugins_func/functions/hass_get_state.py
+3 -8
View File
@@ -29,12 +29,7 @@ hass_get_state_function_desc = {
@register_function("hass_get_state", hass_get_state_function_desc, ToolType.SYSTEM_CTL)
def hass_get_state(conn, entity_id=""):
try:
future = asyncio.run_coroutine_threadsafe(
handle_hass_get_state(conn, entity_id), conn.loop
)
# 添加10秒超时
ha_response = future.result(timeout=10)
ha_response = handle_hass_get_state(conn, entity_id)
return ActionResponse(Action.REQLLM, ha_response, None)
except asyncio.TimeoutError:
logger.bind(tag=TAG).error("获取Home Assistant状态超时")
@@ -45,13 +40,13 @@ def hass_get_state(conn, entity_id=""):
return ActionResponse(Action.ERROR, error_msg, None)
async def handle_hass_get_state(conn, entity_id):
def handle_hass_get_state(conn, entity_id):
ha_config = initialize_hass_handler(conn)
api_key = ha_config.get("api_key")
base_url = ha_config.get("base_url")
url = f"{base_url}/api/states/{entity_id}"
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
response = requests.get(url, headers=headers)
response = requests.get(url, headers=headers, timeout=5)
if response.status_code == 200:
responsetext = "设备状态:" + response.json()["state"] + " "
logger.bind(tag=TAG).info(f"api返回内容: {response.json()}")
main/xiaozhi-server/plugins_func/functions/hass_set_state.py
+3 -7
View File
@@ -54,11 +54,7 @@ def hass_set_state(conn, entity_id="", state=None):
if state is None:
state = {}
try:
future = asyncio.run_coroutine_threadsafe(
handle_hass_set_state(conn, entity_id, state), conn.loop
)
# 添加10秒超时
ha_response = future.result(timeout=10)
ha_response = handle_hass_set_state(conn, entity_id, state)
return ActionResponse(Action.REQLLM, ha_response, None)
except asyncio.TimeoutError:
logger.bind(tag=TAG).error("设置Home Assistant状态超时")
@@ -69,7 +65,7 @@ def hass_set_state(conn, entity_id="", state=None):
return ActionResponse(Action.ERROR, error_msg, None)
async def handle_hass_set_state(conn, entity_id, state):
def handle_hass_set_state(conn, entity_id, state):
ha_config = initialize_hass_handler(conn)
api_key = ha_config.get("api_key")
base_url = ha_config.get("base_url")
@@ -169,7 +165,7 @@ async def handle_hass_set_state(conn, entity_id, state):
data = {"entity_id": entity_id, arg: value}
url = f"{base_url}/api/services/{domain}/{action}"
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
response = requests.post(url, headers=headers, json=data)
response = requests.post(url, headers=headers, json=data, timeout=5) # 设置5秒超时
logger.bind(tag=TAG).info(
f"设置状态:{description},url:{url},return_code:{response.status_code}"
)
main/xiaozhi-server/test/js/StreamingContext.js
+149
View File
@@ -0,0 +1,149 @@
import BlockingQueue from './utils/BlockingQueue.js';
import { log } from './utils/logger.js';
// 音频流播放上下文类
export class StreamingContext {
constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
this.opusDecoder = opusDecoder;
this.audioContext = audioContext;
// 音频参数
this.sampleRate = sampleRate;
this.channels = channels;
this.minAudioDuration = minAudioDuration;
// 初始化队列和状态
this.queue = []; // 已解码的PCM队列。正在播放
this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放
this.pendingAudioBufferQueue = []; // 待处理的缓存队列
this.audioBufferQueue = new BlockingQueue(); // 缓存队列
this.playing = false; // 是否正在播放
this.endOfStream = false; // 是否收到结束信号
this.source = null; // 当前音频源
this.totalSamples = 0; // 累积的总样本数
this.lastPlayTime = 0; // 上次播放的时间戳
}
// 缓存音频数组
pushAudioBuffer(item) {
this.audioBufferQueue.enqueue(...item);
}
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
async getPendingAudioBufferQueue() {
// 原子交换 + 清空
[this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()];
}
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
async getQueue(minSamples) {
let TepArray = [];
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
// 原子交换 + 清空
[TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()];
this.queue.push(...TepArray);
}
// 将Int16音频数据转换为Float32音频数据
convertInt16ToFloat32(int16Data) {
const float32Data = new Float32Array(int16Data.length);
for (let i = 0; i < int16Data.length; i++) {
// 将[-32768,32767]范围转换为[-1,1]
float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
}
return float32Data;
}
// 将Opus数据解码为PCM
async decodeOpusFrames() {
if (!this.opusDecoder) {
log('Opus解码器未初始化，无法解码', 'error');
return;
} else {
log('Opus解码器启动', 'info');
}
while (true) {
let decodedSamples = [];
for (const frame of this.pendingAudioBufferQueue) {
try {
// 使用Opus解码器解码
const frameData = this.opusDecoder.decode(frame);
if (frameData && frameData.length > 0) {
// 转换为Float32
const floatData = this.convertInt16ToFloat32(frameData);
// 使用循环替代展开运算符
for (let i = 0; i < floatData.length; i++) {
decodedSamples.push(floatData[i]);
}
}
} catch (error) {
log("Opus解码失败: " + error.message, 'error');
}
}
if (decodedSamples.length > 0) {
// 使用循环替代展开运算符
for (let i = 0; i < decodedSamples.length; i++) {
this.activeQueue.enqueue(decodedSamples[i]);
}
this.totalSamples += decodedSamples.length;
} else {
log('没有成功解码的样本', 'warning');
}
await this.getPendingAudioBufferQueue();
}
}
// 开始播放音频
async startPlaying() {
while (true) {
// 如果累积了至少0.3秒的音频，开始播放
const minSamples = this.sampleRate * this.minAudioDuration * 3;
if (!this.playing && this.queue.length < minSamples) {
await this.getQueue(minSamples);
}
this.playing = true;
while (this.playing && this.queue.length) {
// 创建新的音频缓冲区
const minPlaySamples = Math.min(this.queue.length, this.sampleRate);
const currentSamples = this.queue.splice(0, minPlaySamples);
const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate);
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
// 创建音频源
this.source = this.audioContext.createBufferSource();
this.source.buffer = audioBuffer;
// 创建增益节点用于平滑过渡
const gainNode = this.audioContext.createGain();
// 应用淡入淡出效果避免爆音
const fadeDuration = 0.02; // 20毫秒
gainNode.gain.setValueAtTime(0, this.audioContext.currentTime);
gainNode.gain.linearRampToValueAtTime(1, this.audioContext.currentTime + fadeDuration);
const duration = audioBuffer.duration;
if (duration > fadeDuration * 2) {
gainNode.gain.setValueAtTime(1, this.audioContext.currentTime + duration - fadeDuration);
gainNode.gain.linearRampToValueAtTime(0, this.audioContext.currentTime + duration);
}
// 连接节点并开始播放
this.source.connect(gainNode);
gainNode.connect(this.audioContext.destination);
this.lastPlayTime = this.audioContext.currentTime;
log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'info');
this.source.start();
}
await this.getQueue(minSamples);
}
}
}
// 创建streamingContext实例的工厂函数
export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration);
}
main/xiaozhi-server/test/test_page.html
+3 -128
View File
@@ -181,6 +181,7 @@
import { checkOpusLoaded, initOpusEncoder } from './js/opus.js';
import { addMessage } from './js/document.js'
import BlockingQueue from './js/utils/BlockingQueue.js'
import { createStreamingContext } from './js/StreamingContext.js'
// 需要加载的脚本列表 - 移除Opus依赖
const scriptFiles = [];
@@ -336,125 +337,7 @@
// 创建流式播放上下文
if (!streamingContext) {
streamingContext = {
queue: [], // 已解码的PCM队列。正在播放
activeQueue: new BlockingQueue(), // 已解码的PCM队列。准备播放
pendingAudioBufferQueue: [], // 待处理的缓存队列
audioBufferQueue: new BlockingQueue(), // 缓存队列
playing: false, // 是否正在播放
endOfStream: false, // 是否收到结束信号
source: null, // 当前音频源
totalSamples: 0, // 累积的总样本数
lastPlayTime: 0, // 上次播放的时间戳
// 缓存音频数组
pushAudioBuffer: function (item) {
this.audioBufferQueue.enqueue(...item)
},
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
getPendingAudioBufferQueue: async function () {
// 原子交换 + 清空
[this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()];
},
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
getQueue: async function (minSamples) {
let TepArray = []
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
// 原子交换 + 清空
[TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()];
this.queue.push(...TepArray)
},
// 将Opus数据解码为PCM
decodeOpusFrames: async function () {
if (!opusDecoder) {
log('Opus解码器未初始化，无法解码', 'error');
return;
} else {
log('Opus解码器启动', 'info');
}
while (true) {
let decodedSamples = [];
for (const frame of this.pendingAudioBufferQueue) {
try {
// 使用Opus解码器解码
const frameData = opusDecoder.decode(frame);
if (frameData && frameData.length > 0) {
// 转换为Float32
const floatData = convertInt16ToFloat32(frameData);
// 使用循环替代展开运算符
for (let i = 0; i < floatData.length; i++) {
decodedSamples.push(floatData[i]);
}
}
} catch (error) {
log("Opus解码失败: " + error.message, 'error');
}
}
if (decodedSamples.length > 0) {
// 使用循环替代展开运算符
for (let i = 0; i < decodedSamples.length; i++) {
this.activeQueue.enqueue(decodedSamples[i]);
}
this.totalSamples += decodedSamples.length;
} else {
log('没有成功解码的样本', 'warning');
}
await this.getPendingAudioBufferQueue();
}
},
// 开始播放音频
startPlaying: async function () {
while (true) {
// 如果累积了至少0.3秒的音频，开始播放
const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION * 3;
if (!this.playing && this.queue.length < minSamples) {
await this.getQueue(minSamples)
}
this.playing = true;
while (this.playing && this.queue.length) {
// 创建新的音频缓冲区
const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE);
const currentSamples = this.queue.splice(0, minPlaySamples);
const audioBuffer = audioContext.createBuffer(CHANNELS, currentSamples.length, SAMPLE_RATE);
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
// 创建音频源
this.source = audioContext.createBufferSource();
this.source.buffer = audioBuffer;
// 创建增益节点用于平滑过渡
const gainNode = audioContext.createGain();
// 应用淡入淡出效果避免爆音
const fadeDuration = 0.02; // 20毫秒
gainNode.gain.setValueAtTime(0, audioContext.currentTime);
gainNode.gain.linearRampToValueAtTime(1, audioContext.currentTime + fadeDuration);
const duration = audioBuffer.duration;
if (duration > fadeDuration * 2) {
gainNode.gain.setValueAtTime(1, audioContext.currentTime + duration - fadeDuration);
gainNode.gain.linearRampToValueAtTime(0, audioContext.currentTime + duration);
}
// 连接节点并开始播放
this.source.connect(gainNode);
gainNode.connect(audioContext.destination);
this.lastPlayTime = audioContext.currentTime;
log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info');
this.source.start();
}
await this.getQueue(minSamples)
}
}
};
streamingContext = createStreamingContext(opusDecoder, audioContext, SAMPLE_RATE, CHANNELS, MIN_AUDIO_DURATION);
}
streamingContext.decodeOpusFrames();
@@ -467,15 +350,7 @@
}
}
// 将Int16音频数据转换为Float32音频数据
function convertInt16ToFloat32(int16Data) {
const float32Data = new Float32Array(int16Data.length);
for (let i = 0; i < int16Data.length; i++) {
// 将[-32768,32767]范围转换为[-1,1]
float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
}
return float32Data;
}
// 初始化Opus解码器 - 确保完全初始化完成后才返回
async function initOpusDecoder() {