Hass/open-xiaoai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(xiaozhi): 修复语音识别丢失前面几个字的问题

Browse Source
This commit is contained in:
Del Wang
2025-05-28 23:01:20 +08:00
parent c06bbf6c51
commit 948364dcc9
9 changed files with 103 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/xiaozhi/README.md
-4
View File
@@ -134,10 +134,6 @@ APP_CONFIG = {
文字识别结果取决于你的小智 AI 服务器端的语音识别方案，与本项目无关。
不过需要注意的是，在唤醒后或连续对话时，由于 VAD 激活阈值的关系，开始的一小段语音可能会被丢弃识别不到，使用时需要多注意。
比如：“你知道我是谁吗” 有可能会被识别为“知道我是谁吗”（丢掉了最前面一个字）
### Q：唤醒词一直没有反应？
由于小爱音箱远场拾音音量较小，有时可能会识别不清，你可以调大 `config.py` 配置文件里的 `boost` 参数，然后重启应用 / Docker 试试看。
examples/xiaozhi/xiaozhi/event.py
+7 -9
View File
@@ -8,8 +8,10 @@ from xiaozhi.ref import (
get_vad,
get_xiaoai,
get_xiaozhi,
set_speech_frames,
)
from xiaozhi.services.protocols.typing import AbortReason, DeviceState, ListeningMode
from xiaozhi.utils.base import get_env
class Step:
@@ -29,7 +31,7 @@ class __EventManager:
self.next_step_future = None
def update_step(self, step: Step, step_data=None):
if get_xiaoai().mode == "xiaozhi":
if not get_env("CLI"):
return
self.current_step = step
@@ -39,7 +41,7 @@ class __EventManager:
)
self.next_step_future = None
async def wait_next_step(self, timeout=None) -> Step | None:
async def wait_next_step(self, timeout=None):
current_session = self.session_id
self.next_step_future = get_xiaoai().async_loop.create_future()
@@ -104,7 +106,7 @@ class __EventManager:
)
async def __start_session(self):
if get_xiaoai().mode == "xiaozhi":
if not get_env("CLI"):
return
vad = get_vad()
@@ -120,11 +122,6 @@ class __EventManager:
if self.current_step == Step.on_interrupt:
return
# 尝试打开音频通道
if not xiaozhi.protocol.is_audio_channel_opened():
xiaozhi.set_device_state(DeviceState.CONNECTING)
await xiaozhi.protocol.open_audio_channel()
# 等待 TTS 余音结束
if self.current_step in [Step.on_tts_end]:
vad.resume("silence")
@@ -148,8 +145,9 @@ class __EventManager:
return
# 开始说话
set_speech_frames(speech_buffer)
codec.input_stream.start_stream() # 开启录音
await xiaozhi.protocol.send_start_listening(ListeningMode.MANUAL)
codec.input_stream.input(speech_buffer) # 追加音频输入片段
xiaozhi.set_device_state(DeviceState.LISTENING)
# 等待说话结束
examples/xiaozhi/xiaozhi/ref.py
+8
View File
@@ -49,3 +49,11 @@ def set_kws(kws: Any):
def get_kws() -> Any:
return GLOBAL_STATES.get("kws")
def set_speech_frames(speech_frames: Any):
GLOBAL_STATES["speech_frames"] = speech_frames
def get_speech_frames() -> Any:
return GLOBAL_STATES.get("speech_frames")
examples/xiaozhi/xiaozhi/services/audio/codec.py
+44 -15
View File
@@ -1,8 +1,14 @@
import opuslib_next as opuslib
from xiaozhi.ref import set_audio_codec
from xiaozhi.ref import (
get_speech_frames,
get_xiaozhi,
set_audio_codec,
set_speech_frames,
)
from xiaozhi.services.audio.stream import MyAudio
from xiaozhi.services.protocols.typing import AudioConfig
from xiaozhi.utils.base import get_env
class AudioCodec:
@@ -36,11 +42,11 @@ class AudioCodec:
# 初始化音频输出流
self.output_stream = self.audio.open(
output=True,
format=AudioConfig.FORMAT,
channels=AudioConfig.CHANNELS,
rate=AudioConfig.SAMPLE_RATE,
output=True,
frames_per_buffer=AudioConfig.FRAME_SIZE,
rate=get_xiaozhi().protocol.server_sample_rate,
frames_per_buffer=get_xiaozhi().protocol.server_frame_size,
output_device_index=MyAudio.get_output_device_index(self.audio),
)
@@ -53,18 +59,36 @@ class AudioCodec:
# 初始化Opus解码器
self.opus_decoder = opuslib.Decoder(
fs=AudioConfig.SAMPLE_RATE, channels=AudioConfig.CHANNELS
fs=get_xiaozhi().protocol.server_sample_rate,
channels=AudioConfig.CHANNELS,
)
self.temp_frames = bytes([])
def read_audio(self):
"""读取音频输入数据并编码"""
try:
speech_frames = get_speech_frames()
# 加入语音片段
if speech_frames:
self.temp_frames = speech_frames
set_speech_frames([])
# 读取音频输入数据
data = self.input_stream.read(
AudioConfig.FRAME_SIZE, exception_on_overflow=False
num_frames=None if get_env("CLI") else AudioConfig.FRAME_SIZE,
exception_on_overflow=False,
)
if not data:
return None
return self.opus_encoder.encode(data, AudioConfig.FRAME_SIZE)
self.temp_frames += data
if len(self.temp_frames) < AudioConfig.FRAME_SIZE * 2:
return None
opus_frames, remain_frames = self.encode_audio(self.temp_frames)
self.temp_frames = remain_frames
return opus_frames
except Exception:
return None
@@ -76,24 +100,29 @@ class AudioCodec:
except Exception:
pass
def decode_audio(self, opus_data, frame_size=AudioConfig.FRAME_SIZE):
def decode_audio(self, opus_data):
"""解码音频数据"""
return self.opus_decoder.decode(opus_data, frame_size, decode_fec=False)
return self.opus_decoder.decode(
opus_data,
frame_size=get_xiaozhi().protocol.server_frame_size,
decode_fec=False,
)
def encode_audio(self, buffer, frame_size=AudioConfig.FRAME_SIZE):
def encode_audio(self, buffer: bytes, frame_size=AudioConfig.FRAME_SIZE):
"""编码音频数据"""
opus_frames = []
remain_frames = bytes([])
try:
opus_frames = []
for i in range(0, len(buffer), frame_size * 2):
chunk = buffer[i : i + frame_size * 2]
if len(chunk) < frame_size * 2:
# 如果 buffer 长度不是 FRAME_SIZE 的 2 倍，需要补齐
chunk += b"\x00" * (frame_size * 2 - len(chunk))
remain_frames = chunk
break
opus_frame = self.opus_encoder.encode(chunk, frame_size)
opus_frames.append(opus_frame)
return opus_frames
return opus_frames, remain_frames
except Exception:
return None
return None, remain_frames
def start_streams(self):
"""启动音频流"""
examples/xiaozhi/xiaozhi/services/audio/stream.py
+2 -2
View File
@@ -74,7 +74,7 @@ class MyStream:
self._is_active = False
if self._is_input:
GlobalStream.unregister_reader(self)
self.input_bytes = []
self.input_bytes.clear()
def write(self, frames: bytes) -> None:
# 发送输出音频流到扬声器
@@ -96,7 +96,7 @@ class MyStream:
def read(self, num_frames=None, exception_on_overflow=False) -> bytes:
if num_frames is None:
data = bytes(self.input_bytes)
self.input_bytes = []
self.input_bytes.clear()
return data
num_frames = num_frames * 2
examples/xiaozhi/xiaozhi/services/audio/vad/__init__.py
+15 -12
View File
@@ -79,9 +79,10 @@ class _VAD:
self.speech_count += len(frames)
self.silence_count = 0
if not self.speech_frames:
# 加入静音片段（潜在的语音片段）
self.speech_frames.extend(self.silence_frames)
if self.target == "speech":
if not self.speech_frames:
# 加入静音片段（潜在的语音片段）
self.speech_frames.extend(self.silence_frames)
# 加入语音片段
self.speech_frames.extend(frames)
@@ -93,7 +94,6 @@ class _VAD:
and self.speech_count > self.min_speech_duration * self.sample_rate / 1000
):
self.pause()
# !FIXME: 需要保证音频流的连续性（在发消息期间）
EventManager.on_speech(speech_bytes)
def _handle_silence_frame(self, frames):
@@ -101,14 +101,17 @@ class _VAD:
self.silence_count += len(frames)
self.speech_count = 0
if not self.speech_frames:
# 如果之前没有语音片段，则将当前帧加入静音片段
self.silence_frames.extend(frames)
# 确保静音片段长度不超过 3s
self.silence_frames = self.silence_frames[: 3 * self.sample_rate]
else:
# 如果之前有语音片段，则将当前帧加入语音片段
self.speech_frames.extend(frames)
if self.target == "speech":
if not self.speech_frames:
# 如果之前没有语音片段，则将当前帧加入静音片段
self.silence_frames.extend(frames)
# 确保静音片段长度不超过 1s
self.silence_frames = self.silence_frames[
-1 * 1 * 2 * self.sample_rate :
]
else:
# 如果之前有语音片段，则将当前帧加入语音片段
self.speech_frames.extend(frames)
if (
self.target == "silence"
examples/xiaozhi/xiaozhi/services/protocols/typing.py
+1 -1
View File
@@ -24,7 +24,7 @@ class EventType:
class AudioConfig:
"""音频配置"""
FORMAT = 8
SAMPLE_RATE = 24000
SAMPLE_RATE = 16000
CHANNELS = 1
FRAME_DURATION = 60 # ms
FRAME_SIZE = int(SAMPLE_RATE * (FRAME_DURATION / 1000))
examples/xiaozhi/xiaozhi/services/protocols/websocket_protocol.py
+14 -3
View File
@@ -13,7 +13,11 @@ class WebsocketProtocol(Protocol):
# 获取配置管理器实例
self.config = ConfigManager.instance()
self.websocket = None
self.server_sample_rate = 16000
self.server_sample_rate = 24000
self.server_frame_duration = 60
self.server_frame_size = int(
self.server_sample_rate * (self.server_frame_duration / 1000)
)
self.connected = False
self.hello_received = None # 初始化时先设为 None
self.WEBSOCKET_URL = self.config.get_config("NETWORK.WEBSOCKET_URL")
@@ -100,13 +104,14 @@ class WebsocketProtocol(Protocol):
if self.on_network_error:
self.on_network_error(f"连接错误: {str(e)}")
async def send_audio(self, data: bytes):
async def send_audio(self, frames: list[bytes]):
"""发送音频数据"""
if not self.is_audio_channel_opened(): # 使用已有的 is_connected 方法
return
try:
await self.websocket.send(data)
for frame in frames:
await self.websocket.send(frame)
except Exception as e:
if self.on_network_error:
self.on_network_error(f"发送音频失败: {str(e)}")
@@ -157,6 +162,12 @@ class WebsocketProtocol(Protocol):
sample_rate = audio_params.get("sample_rate")
if sample_rate:
self.server_sample_rate = sample_rate
frame_duration = audio_params.get("frame_duration")
if frame_duration:
self.server_frame_duration = frame_duration
self.server_frame_size = int(
self.server_sample_rate * (self.server_frame_duration / 1000)
)
# 设置 hello 接收事件
self.hello_received.set()
examples/xiaozhi/xiaozhi/xiaozhi.py
+12 -95
View File
@@ -10,7 +10,6 @@ from xiaozhi.services.audio.kws import KWS
from xiaozhi.services.audio.vad import VAD
from xiaozhi.services.protocols.typing import (
AbortReason,
AudioConfig,
DeviceState,
EventType,
ListeningMode,
@@ -88,9 +87,9 @@ class XiaoZhi:
# 等待事件循环准备就绪
time.sleep(0.1)
# 初始化应用程序（移除自动连接）
# 初始化应用程序
asyncio.run_coroutine_threadsafe(XiaoAI.init_xiaoai(), self.loop)
asyncio.run_coroutine_threadsafe(self._initialize_without_connect(), self.loop)
asyncio.run_coroutine_threadsafe(self._initialize_xiaozhi(), self.loop)
# 启动主循环线程
main_loop_thread = threading.Thread(target=self._main_loop)
@@ -109,8 +108,8 @@ class XiaoZhi:
asyncio.set_event_loop(self.loop)
self.loop.run_forever()
async def _initialize_without_connect(self):
"""初始化应用程序组件（不建立连接）"""
async def _initialize_xiaozhi(self):
"""初始化应用程序组件"""
# 初始化音频编解码器
self._initialize_audio()
@@ -122,8 +121,9 @@ class XiaoZhi:
self.protocol.on_audio_channel_opened = self._on_audio_channel_opened
self.protocol.on_audio_channel_closed = self._on_audio_channel_closed
# 设置设备状态为待命
self.set_device_state(DeviceState.IDLE)
# 打开音频通道
self.device_state = DeviceState.CONNECTING
await self.protocol.open_audio_channel()
def _initialize_audio(self):
"""初始化音频设备和编解码器"""
@@ -220,50 +220,6 @@ class XiaoZhi:
self.protocol.close_audio_channel(), self.loop
)
def _attempt_reconnect(self):
"""尝试重新连接服务器"""
if self.device_state != DeviceState.CONNECTING:
self.set_device_state(DeviceState.CONNECTING)
# 关闭现有连接
if self.protocol:
asyncio.run_coroutine_threadsafe(
self.protocol.close_audio_channel(), self.loop
)
# 延迟一秒后尝试重新连接
def delayed_reconnect():
time.sleep(1)
asyncio.run_coroutine_threadsafe(self._reconnect(), self.loop)
threading.Thread(target=delayed_reconnect, daemon=True).start()
async def _reconnect(self):
"""重新连接到服务器"""
# 设置协议回调
self.protocol.on_network_error = self._on_network_error
self.protocol.on_incoming_audio = self._on_incoming_audio
self.protocol.on_incoming_json = self._on_incoming_json
self.protocol.on_audio_channel_opened = self._on_audio_channel_opened
self.protocol.on_audio_channel_closed = self._on_audio_channel_closed
# 连接到服务器
retry_count = 0
max_retries = 3
while retry_count < max_retries:
if await self.protocol.connect():
self.set_device_state(DeviceState.IDLE)
return True
retry_count += 1
await asyncio.sleep(2) # 等待2秒后重试
self.schedule(lambda: self.alert("连接错误", "无法重新连接到服务器"))
self.set_device_state(DeviceState.IDLE)
return False
def _on_incoming_audio(self, data):
"""接收音频数据回调"""
if self.device_state == DeviceState.SPEAKING:
@@ -343,31 +299,8 @@ class XiaoZhi:
async def _on_audio_channel_opened(self):
"""音频通道打开回调"""
self.schedule(lambda: self._start_audio_streams())
def _start_audio_streams(self):
"""启动音频流"""
try:
# 确保流已关闭后再重新打开
if self.audio_codec.input_stream.is_active():
self.audio_codec.input_stream.stop_stream()
# 重新打开流
self.audio_codec.input_stream.start_stream()
if self.audio_codec.output_stream.is_active():
self.audio_codec.output_stream.stop_stream()
# 重新打开流
self.audio_codec.output_stream.start_stream()
# 设置事件触发器
threading.Thread(
target=self._audio_input_event_trigger, daemon=True
).start()
except Exception:
pass
self.set_device_state(DeviceState.IDLE)
threading.Thread(target=self._audio_input_event_trigger, daemon=True).start()
def _audio_input_event_trigger(self):
"""音频输入事件触发器"""
@@ -381,7 +314,7 @@ class XiaoZhi:
except Exception:
pass
time.sleep(AudioConfig.FRAME_DURATION / 1000) # 按帧时长触发
time.sleep(0.01)
async def _on_audio_channel_closed(self):
"""音频通道关闭回调"""
@@ -494,26 +427,10 @@ class XiaoZhi:
self.protocol.send_abort_speaking(AbortReason.ABORT),
self.loop,
)
# 尝试打开音频通道
if not self.protocol.is_audio_channel_opened():
self.set_device_state(DeviceState.CONNECTING) # 设置设备状态为连接中
try:
# 等待异步操作完成
future = asyncio.run_coroutine_threadsafe(
self.protocol.open_audio_channel(), self.loop
)
# 等待操作完成并获取结果
assert future.result(timeout=10.0) # 添加超时时间
except Exception as e:
self.alert("错误", f"打开音频通道失败: {str(e)}")
self.set_device_state(DeviceState.IDLE)
return
asyncio.run_coroutine_threadsafe(
self.protocol.send_start_listening(ListeningMode.MANUAL), self.loop
)
self.set_device_state(DeviceState.LISTENING) # 设置设备状态为监听中
self.set_device_state(DeviceState.LISTENING)
def stop_listening(self):
"""停止监听"""
@@ -521,8 +438,8 @@ class XiaoZhi:
def _stop_listening_impl(self):
"""停止监听的实现"""
self.set_device_state(DeviceState.IDLE)
asyncio.run_coroutine_threadsafe(self.protocol.send_stop_listening(), self.loop)
self.set_device_state(DeviceState.IDLE)
def abort_speaking(self, reason):
"""中止语音输出"""