From 948364dcc9437f22aa72fef20aa0a66742df50da Mon Sep 17 00:00:00 2001 From: Del Wang Date: Wed, 28 May 2025 23:01:20 +0800 Subject: [PATCH] =?UTF-8?q?fix(xiaozhi):=20=E4=BF=AE=E5=A4=8D=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E8=AF=86=E5=88=AB=E4=B8=A2=E5=A4=B1=E5=89=8D=E9=9D=A2?= =?UTF-8?q?=E5=87=A0=E4=B8=AA=E5=AD=97=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/xiaozhi/README.md | 4 - examples/xiaozhi/xiaozhi/event.py | 16 ++- examples/xiaozhi/xiaozhi/ref.py | 8 ++ .../xiaozhi/xiaozhi/services/audio/codec.py | 59 +++++++--- .../xiaozhi/xiaozhi/services/audio/stream.py | 4 +- .../xiaozhi/services/audio/vad/__init__.py | 27 +++-- .../xiaozhi/services/protocols/typing.py | 2 +- .../services/protocols/websocket_protocol.py | 17 ++- examples/xiaozhi/xiaozhi/xiaozhi.py | 107 ++---------------- 9 files changed, 103 insertions(+), 141 deletions(-) diff --git a/examples/xiaozhi/README.md b/examples/xiaozhi/README.md index 45a1723..a0e5ae1 100644 --- a/examples/xiaozhi/README.md +++ b/examples/xiaozhi/README.md @@ -134,10 +134,6 @@ APP_CONFIG = { 文字识别结果取决于你的小智 AI 服务器端的语音识别方案，与本项目无关。 -不过需要注意的是，在唤醒后或连续对话时，由于 VAD 激活阈值的关系，开始的一小段语音可能会被丢弃识别不到，使用时需要多注意。 - -比如：“你知道我是谁吗” 有可能会被识别为“知道我是谁吗”（丢掉了最前面一个字） - ### Q：唤醒词一直没有反应？ 由于小爱音箱远场拾音音量较小，有时可能会识别不清，你可以调大 `config.py` 配置文件里的 `boost` 参数，然后重启应用 / Docker 试试看。 diff --git a/examples/xiaozhi/xiaozhi/event.py b/examples/xiaozhi/xiaozhi/event.py index 88ea0e2..a971332 100644 --- a/examples/xiaozhi/xiaozhi/event.py +++ b/examples/xiaozhi/xiaozhi/event.py @@ -8,8 +8,10 @@ from xiaozhi.ref import ( get_vad, get_xiaoai, get_xiaozhi, + set_speech_frames, ) from xiaozhi.services.protocols.typing import AbortReason, DeviceState, ListeningMode +from xiaozhi.utils.base import get_env class Step: @@ -29,7 +31,7 @@ class __EventManager: self.next_step_future = None def update_step(self, step: Step, step_data=None): - if get_xiaoai().mode == "xiaozhi": + if not get_env("CLI"): return self.current_step = step @@ -39,7 +41,7 @@ class __EventManager: ) self.next_step_future = None - async def wait_next_step(self, timeout=None) -> Step | None: + async def wait_next_step(self, timeout=None): current_session = self.session_id self.next_step_future = get_xiaoai().async_loop.create_future() @@ -104,7 +106,7 @@ class __EventManager: ) async def __start_session(self): - if get_xiaoai().mode == "xiaozhi": + if not get_env("CLI"): return vad = get_vad() @@ -120,11 +122,6 @@ class __EventManager: if self.current_step == Step.on_interrupt: return - # 尝试打开音频通道 - if not xiaozhi.protocol.is_audio_channel_opened(): - xiaozhi.set_device_state(DeviceState.CONNECTING) - await xiaozhi.protocol.open_audio_channel() - # 等待 TTS 余音结束 if self.current_step in [Step.on_tts_end]: vad.resume("silence") @@ -148,8 +145,9 @@ class __EventManager: return # 开始说话 + set_speech_frames(speech_buffer) + codec.input_stream.start_stream() # 开启录音 await xiaozhi.protocol.send_start_listening(ListeningMode.MANUAL) - codec.input_stream.input(speech_buffer) # 追加音频输入片段 xiaozhi.set_device_state(DeviceState.LISTENING) # 等待说话结束 diff --git a/examples/xiaozhi/xiaozhi/ref.py b/examples/xiaozhi/xiaozhi/ref.py index 8f4e088..de86a32 100644 --- a/examples/xiaozhi/xiaozhi/ref.py +++ b/examples/xiaozhi/xiaozhi/ref.py @@ -49,3 +49,11 @@ def set_kws(kws: Any): def get_kws() -> Any: return GLOBAL_STATES.get("kws") + + +def set_speech_frames(speech_frames: Any): + GLOBAL_STATES["speech_frames"] = speech_frames + + +def get_speech_frames() -> Any: + return GLOBAL_STATES.get("speech_frames") diff --git a/examples/xiaozhi/xiaozhi/services/audio/codec.py b/examples/xiaozhi/xiaozhi/services/audio/codec.py index de611d8..26ee6a1 100644 --- a/examples/xiaozhi/xiaozhi/services/audio/codec.py +++ b/examples/xiaozhi/xiaozhi/services/audio/codec.py @@ -1,8 +1,14 @@ import opuslib_next as opuslib -from xiaozhi.ref import set_audio_codec +from xiaozhi.ref import ( + get_speech_frames, + get_xiaozhi, + set_audio_codec, + set_speech_frames, +) from xiaozhi.services.audio.stream import MyAudio from xiaozhi.services.protocols.typing import AudioConfig +from xiaozhi.utils.base import get_env class AudioCodec: @@ -36,11 +42,11 @@ class AudioCodec: # 初始化音频输出流 self.output_stream = self.audio.open( + output=True, format=AudioConfig.FORMAT, channels=AudioConfig.CHANNELS, - rate=AudioConfig.SAMPLE_RATE, - output=True, - frames_per_buffer=AudioConfig.FRAME_SIZE, + rate=get_xiaozhi().protocol.server_sample_rate, + frames_per_buffer=get_xiaozhi().protocol.server_frame_size, output_device_index=MyAudio.get_output_device_index(self.audio), ) @@ -53,18 +59,36 @@ class AudioCodec: # 初始化Opus解码器 self.opus_decoder = opuslib.Decoder( - fs=AudioConfig.SAMPLE_RATE, channels=AudioConfig.CHANNELS + fs=get_xiaozhi().protocol.server_sample_rate, + channels=AudioConfig.CHANNELS, ) + self.temp_frames = bytes([]) def read_audio(self): """读取音频输入数据并编码""" try: + speech_frames = get_speech_frames() + + # 加入语音片段 + if speech_frames: + self.temp_frames = speech_frames + set_speech_frames([]) + + # 读取音频输入数据 data = self.input_stream.read( - AudioConfig.FRAME_SIZE, exception_on_overflow=False + num_frames=None if get_env("CLI") else AudioConfig.FRAME_SIZE, + exception_on_overflow=False, ) if not data: return None - return self.opus_encoder.encode(data, AudioConfig.FRAME_SIZE) + + self.temp_frames += data + if len(self.temp_frames) < AudioConfig.FRAME_SIZE * 2: + return None + + opus_frames, remain_frames = self.encode_audio(self.temp_frames) + self.temp_frames = remain_frames + return opus_frames except Exception: return None @@ -76,24 +100,29 @@ class AudioCodec: except Exception: pass - def decode_audio(self, opus_data, frame_size=AudioConfig.FRAME_SIZE): + def decode_audio(self, opus_data): """解码音频数据""" - return self.opus_decoder.decode(opus_data, frame_size, decode_fec=False) + return self.opus_decoder.decode( + opus_data, + frame_size=get_xiaozhi().protocol.server_frame_size, + decode_fec=False, + ) - def encode_audio(self, buffer, frame_size=AudioConfig.FRAME_SIZE): + def encode_audio(self, buffer: bytes, frame_size=AudioConfig.FRAME_SIZE): """编码音频数据""" + opus_frames = [] + remain_frames = bytes([]) try: - opus_frames = [] for i in range(0, len(buffer), frame_size * 2): chunk = buffer[i : i + frame_size * 2] if len(chunk) < frame_size * 2: - # 如果 buffer 长度不是 FRAME_SIZE 的 2 倍，需要补齐 - chunk += b"\x00" * (frame_size * 2 - len(chunk)) + remain_frames = chunk + break opus_frame = self.opus_encoder.encode(chunk, frame_size) opus_frames.append(opus_frame) - return opus_frames + return opus_frames, remain_frames except Exception: - return None + return None, remain_frames def start_streams(self): """启动音频流""" diff --git a/examples/xiaozhi/xiaozhi/services/audio/stream.py b/examples/xiaozhi/xiaozhi/services/audio/stream.py index a17f651..ca11d4e 100644 --- a/examples/xiaozhi/xiaozhi/services/audio/stream.py +++ b/examples/xiaozhi/xiaozhi/services/audio/stream.py @@ -74,7 +74,7 @@ class MyStream: self._is_active = False if self._is_input: GlobalStream.unregister_reader(self) - self.input_bytes = [] + self.input_bytes.clear() def write(self, frames: bytes) -> None: # 发送输出音频流到扬声器 @@ -96,7 +96,7 @@ class MyStream: def read(self, num_frames=None, exception_on_overflow=False) -> bytes: if num_frames is None: data = bytes(self.input_bytes) - self.input_bytes = [] + self.input_bytes.clear() return data num_frames = num_frames * 2 diff --git a/examples/xiaozhi/xiaozhi/services/audio/vad/__init__.py b/examples/xiaozhi/xiaozhi/services/audio/vad/__init__.py index 4613cff..dcc80ed 100644 --- a/examples/xiaozhi/xiaozhi/services/audio/vad/__init__.py +++ b/examples/xiaozhi/xiaozhi/services/audio/vad/__init__.py @@ -79,9 +79,10 @@ class _VAD: self.speech_count += len(frames) self.silence_count = 0 - if not self.speech_frames: - # 加入静音片段（潜在的语音片段） - self.speech_frames.extend(self.silence_frames) + if self.target == "speech": + if not self.speech_frames: + # 加入静音片段（潜在的语音片段） + self.speech_frames.extend(self.silence_frames) # 加入语音片段 self.speech_frames.extend(frames) @@ -93,7 +94,6 @@ class _VAD: and self.speech_count > self.min_speech_duration * self.sample_rate / 1000 ): self.pause() - # !FIXME: 需要保证音频流的连续性（在发消息期间） EventManager.on_speech(speech_bytes) def _handle_silence_frame(self, frames): @@ -101,14 +101,17 @@ class _VAD: self.silence_count += len(frames) self.speech_count = 0 - if not self.speech_frames: - # 如果之前没有语音片段，则将当前帧加入静音片段 - self.silence_frames.extend(frames) - # 确保静音片段长度不超过 3s - self.silence_frames = self.silence_frames[: 3 * self.sample_rate] - else: - # 如果之前有语音片段，则将当前帧加入语音片段 - self.speech_frames.extend(frames) + if self.target == "speech": + if not self.speech_frames: + # 如果之前没有语音片段，则将当前帧加入静音片段 + self.silence_frames.extend(frames) + # 确保静音片段长度不超过 1s + self.silence_frames = self.silence_frames[ + -1 * 1 * 2 * self.sample_rate : + ] + else: + # 如果之前有语音片段，则将当前帧加入语音片段 + self.speech_frames.extend(frames) if ( self.target == "silence" diff --git a/examples/xiaozhi/xiaozhi/services/protocols/typing.py b/examples/xiaozhi/xiaozhi/services/protocols/typing.py index 63de7f3..499288d 100644 --- a/examples/xiaozhi/xiaozhi/services/protocols/typing.py +++ b/examples/xiaozhi/xiaozhi/services/protocols/typing.py @@ -24,7 +24,7 @@ class EventType: class AudioConfig: """音频配置""" FORMAT = 8 - SAMPLE_RATE = 24000 + SAMPLE_RATE = 16000 CHANNELS = 1 FRAME_DURATION = 60 # ms FRAME_SIZE = int(SAMPLE_RATE * (FRAME_DURATION / 1000)) diff --git a/examples/xiaozhi/xiaozhi/services/protocols/websocket_protocol.py b/examples/xiaozhi/xiaozhi/services/protocols/websocket_protocol.py index 7c567ec..71bb052 100644 --- a/examples/xiaozhi/xiaozhi/services/protocols/websocket_protocol.py +++ b/examples/xiaozhi/xiaozhi/services/protocols/websocket_protocol.py @@ -13,7 +13,11 @@ class WebsocketProtocol(Protocol): # 获取配置管理器实例 self.config = ConfigManager.instance() self.websocket = None - self.server_sample_rate = 16000 + self.server_sample_rate = 24000 + self.server_frame_duration = 60 + self.server_frame_size = int( + self.server_sample_rate * (self.server_frame_duration / 1000) + ) self.connected = False self.hello_received = None # 初始化时先设为 None self.WEBSOCKET_URL = self.config.get_config("NETWORK.WEBSOCKET_URL") @@ -100,13 +104,14 @@ class WebsocketProtocol(Protocol): if self.on_network_error: self.on_network_error(f"连接错误: {str(e)}") - async def send_audio(self, data: bytes): + async def send_audio(self, frames: list[bytes]): """发送音频数据""" if not self.is_audio_channel_opened(): # 使用已有的 is_connected 方法 return try: - await self.websocket.send(data) + for frame in frames: + await self.websocket.send(frame) except Exception as e: if self.on_network_error: self.on_network_error(f"发送音频失败: {str(e)}") @@ -157,6 +162,12 @@ class WebsocketProtocol(Protocol): sample_rate = audio_params.get("sample_rate") if sample_rate: self.server_sample_rate = sample_rate + frame_duration = audio_params.get("frame_duration") + if frame_duration: + self.server_frame_duration = frame_duration + self.server_frame_size = int( + self.server_sample_rate * (self.server_frame_duration / 1000) + ) # 设置 hello 接收事件 self.hello_received.set() diff --git a/examples/xiaozhi/xiaozhi/xiaozhi.py b/examples/xiaozhi/xiaozhi/xiaozhi.py index ce2476c..54eba14 100644 --- a/examples/xiaozhi/xiaozhi/xiaozhi.py +++ b/examples/xiaozhi/xiaozhi/xiaozhi.py @@ -10,7 +10,6 @@ from xiaozhi.services.audio.kws import KWS from xiaozhi.services.audio.vad import VAD from xiaozhi.services.protocols.typing import ( AbortReason, - AudioConfig, DeviceState, EventType, ListeningMode, @@ -88,9 +87,9 @@ class XiaoZhi: # 等待事件循环准备就绪 time.sleep(0.1) - # 初始化应用程序（移除自动连接） + # 初始化应用程序 asyncio.run_coroutine_threadsafe(XiaoAI.init_xiaoai(), self.loop) - asyncio.run_coroutine_threadsafe(self._initialize_without_connect(), self.loop) + asyncio.run_coroutine_threadsafe(self._initialize_xiaozhi(), self.loop) # 启动主循环线程 main_loop_thread = threading.Thread(target=self._main_loop) @@ -109,8 +108,8 @@ class XiaoZhi: asyncio.set_event_loop(self.loop) self.loop.run_forever() - async def _initialize_without_connect(self): - """初始化应用程序组件（不建立连接）""" + async def _initialize_xiaozhi(self): + """初始化应用程序组件""" # 初始化音频编解码器 self._initialize_audio() @@ -122,8 +121,9 @@ class XiaoZhi: self.protocol.on_audio_channel_opened = self._on_audio_channel_opened self.protocol.on_audio_channel_closed = self._on_audio_channel_closed - # 设置设备状态为待命 - self.set_device_state(DeviceState.IDLE) + # 打开音频通道 + self.device_state = DeviceState.CONNECTING + await self.protocol.open_audio_channel() def _initialize_audio(self): """初始化音频设备和编解码器""" @@ -220,50 +220,6 @@ class XiaoZhi: self.protocol.close_audio_channel(), self.loop ) - def _attempt_reconnect(self): - """尝试重新连接服务器""" - if self.device_state != DeviceState.CONNECTING: - self.set_device_state(DeviceState.CONNECTING) - - # 关闭现有连接 - if self.protocol: - asyncio.run_coroutine_threadsafe( - self.protocol.close_audio_channel(), self.loop - ) - - # 延迟一秒后尝试重新连接 - def delayed_reconnect(): - time.sleep(1) - asyncio.run_coroutine_threadsafe(self._reconnect(), self.loop) - - threading.Thread(target=delayed_reconnect, daemon=True).start() - - async def _reconnect(self): - """重新连接到服务器""" - - # 设置协议回调 - self.protocol.on_network_error = self._on_network_error - self.protocol.on_incoming_audio = self._on_incoming_audio - self.protocol.on_incoming_json = self._on_incoming_json - self.protocol.on_audio_channel_opened = self._on_audio_channel_opened - self.protocol.on_audio_channel_closed = self._on_audio_channel_closed - - # 连接到服务器 - retry_count = 0 - max_retries = 3 - - while retry_count < max_retries: - if await self.protocol.connect(): - self.set_device_state(DeviceState.IDLE) - return True - - retry_count += 1 - await asyncio.sleep(2) # 等待2秒后重试 - - self.schedule(lambda: self.alert("连接错误", "无法重新连接到服务器")) - self.set_device_state(DeviceState.IDLE) - return False - def _on_incoming_audio(self, data): """接收音频数据回调""" if self.device_state == DeviceState.SPEAKING: @@ -343,31 +299,8 @@ class XiaoZhi: async def _on_audio_channel_opened(self): """音频通道打开回调""" - self.schedule(lambda: self._start_audio_streams()) - - def _start_audio_streams(self): - """启动音频流""" - try: - # 确保流已关闭后再重新打开 - if self.audio_codec.input_stream.is_active(): - self.audio_codec.input_stream.stop_stream() - - # 重新打开流 - self.audio_codec.input_stream.start_stream() - - if self.audio_codec.output_stream.is_active(): - self.audio_codec.output_stream.stop_stream() - - # 重新打开流 - self.audio_codec.output_stream.start_stream() - - # 设置事件触发器 - threading.Thread( - target=self._audio_input_event_trigger, daemon=True - ).start() - - except Exception: - pass + self.set_device_state(DeviceState.IDLE) + threading.Thread(target=self._audio_input_event_trigger, daemon=True).start() def _audio_input_event_trigger(self): """音频输入事件触发器""" @@ -381,7 +314,7 @@ class XiaoZhi: except Exception: pass - time.sleep(AudioConfig.FRAME_DURATION / 1000) # 按帧时长触发 + time.sleep(0.01) async def _on_audio_channel_closed(self): """音频通道关闭回调""" @@ -494,26 +427,10 @@ class XiaoZhi: self.protocol.send_abort_speaking(AbortReason.ABORT), self.loop, ) - - # 尝试打开音频通道 - if not self.protocol.is_audio_channel_opened(): - self.set_device_state(DeviceState.CONNECTING) # 设置设备状态为连接中 - try: - # 等待异步操作完成 - future = asyncio.run_coroutine_threadsafe( - self.protocol.open_audio_channel(), self.loop - ) - # 等待操作完成并获取结果 - assert future.result(timeout=10.0) # 添加超时时间 - except Exception as e: - self.alert("错误", f"打开音频通道失败: {str(e)}") - self.set_device_state(DeviceState.IDLE) - return - asyncio.run_coroutine_threadsafe( self.protocol.send_start_listening(ListeningMode.MANUAL), self.loop ) - self.set_device_state(DeviceState.LISTENING) # 设置设备状态为监听中 + self.set_device_state(DeviceState.LISTENING) def stop_listening(self): """停止监听""" @@ -521,8 +438,8 @@ class XiaoZhi: def _stop_listening_impl(self): """停止监听的实现""" - self.set_device_state(DeviceState.IDLE) asyncio.run_coroutine_threadsafe(self.protocol.send_stop_listening(), self.loop) + self.set_device_state(DeviceState.IDLE) def abort_speaking(self, reason): """中止语音输出"""