diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 7d46685b..e5b96be2 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -1,5 +1,4 @@ import time -import asyncio import json from core.handle.sendAudioHandle import send_stt_message from core.handle.intentHandler import handle_user_intent @@ -14,14 +13,6 @@ TAG = __name__ async def handleAudioMessage(conn, audio): # 当前片段是否有人说话 have_voice = conn.vad.is_vad(conn, audio) - # 如果设备刚刚被唤醒，短暂忽略VAD检测 - if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up: - have_voice = False - # 设置一个短暂延迟后恢复VAD检测 - conn.asr_audio.clear() - if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done(): - conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn)) - return if have_voice: if conn.client_is_speaking: @@ -31,13 +22,6 @@ async def handleAudioMessage(conn, audio): # 接收音频 await conn.asr.receive_audio(conn, audio, have_voice) - -async def resume_vad_detection(conn): - # 等待2秒后恢复VAD检测 - await asyncio.sleep(1) - conn.just_woken_up = False - - async def startToChat(conn, text): # 检查输入是否是JSON格式（包含说话人信息） speaker_name = None diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index c3c50fbc..bb85c007 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -1,4 +1,5 @@ import json +import asyncio from core.providers.tts.dto.dto import SentenceType from core.utils import textUtils @@ -52,8 +53,12 @@ async def send_tts_message(conn, state, text=None): stop_tts_notify_voice = conn.config.get( "stop_tts_notify_voice", "config/assets/tts_notify.mp3" ) - audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice) - await sendAudio(conn, audios) + conn.tts.audio_to_opus_data_stream( + stop_tts_notify_voice, + callback=lambda audio_data: asyncio.run_coroutine_threadsafe( + sendAudio(conn, audio_data), conn.loop + ), + ) # 清除服务端讲话状态 conn.clearSpeakStatus() diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 972818d4..f4803d31 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -12,7 +12,7 @@ import time import concurrent.futures from abc import ABC, abstractmethod from config.logger import setup_logging -from typing import Optional, Tuple, List, Dict, Any +from typing import Optional, Tuple, List from core.handle.receiveAudioHandle import startToChat from core.handle.reportHandle import enqueue_asr_report from core.utils.util import remove_punctuation_and_length @@ -132,8 +132,6 @@ class ASRProviderBase(ABC): return None # 使用线程池执行器并行运行 - parallel_start_time = time.monotonic() - with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor: asr_future = thread_executor.submit(run_asr) diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 0b942862..35d80fb6 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -11,7 +11,7 @@ from datetime import datetime from core.utils import textUtils from abc import ABC, abstractmethod from config.logger import setup_logging -from core.utils.audio_flow_control import FlowControlConfig +from core.utils.audio_flow_control import FlowControlConfig, simulate_device_consumption from core.utils.util import audio_bytes_to_data_stream, audio_to_data_stream from core.utils.tts import MarkdownCleaner from core.utils.output_counter import add_device_output @@ -356,9 +356,8 @@ class TTSProviderBase(ABC): # 模拟设备消费（实际应用中应该从设备获取反馈）防止音字不同步 if isinstance(audio_datas, bytes): - # 模拟设备播放延迟（60ms per frame）, 实际情况可以低一点（50ms），增加使用体验 - await asyncio.sleep(0.06) - self.flow_controller.update_device_consumption(1) + frame_count = 1 + asyncio.create_task(simulate_device_consumption(self.flow_controller, frame_count)) # 在类中添加流控制器重置方法 def reset_flow_controller(self): diff --git a/main/xiaozhi-server/core/providers/vad/silero.py b/main/xiaozhi-server/core/providers/vad/silero.py index b516d8fb..95ab8ff3 100644 --- a/main/xiaozhi-server/core/providers/vad/silero.py +++ b/main/xiaozhi-server/core/providers/vad/silero.py @@ -33,8 +33,8 @@ class VADProvider(VADProviderBase): int(min_silence_duration_ms) if min_silence_duration_ms else 1000 ) - # 至少要多少帧才算有语音 - self.frame_window_threshold = 3 + # 至少要多少帧才算有语音,增加灵敏度 + self.frame_window_threshold = 1 def is_vad(self, conn, opus_packet): try: diff --git a/main/xiaozhi-server/core/utils/audio_flow_control.py b/main/xiaozhi-server/core/utils/audio_flow_control.py index 98c29676..958e68a7 100644 --- a/main/xiaozhi-server/core/utils/audio_flow_control.py +++ b/main/xiaozhi-server/core/utils/audio_flow_control.py @@ -151,6 +151,21 @@ class AudioFlowController: ) +async def simulate_device_consumption( + flow_controller: AudioFlowController, frame_count: int +): + """ + 模拟设备消费音频帧的过程 + 实际应用中应该根据设备反馈来更新消费情况 + Args: + flow_controller: 流控制器实例 + frame_count: 消费的帧数 + """ + # 模拟设备播放延迟（60ms per frame） + await asyncio.sleep(frame_count * 0.06) + flow_controller.update_device_consumption(frame_count) + + # 流控配置常量 class FlowControlConfig: """流控配置常量""" @@ -183,4 +198,4 @@ class FlowControlConfig: return AudioFlowController( max_device_buffer=max_buffer or cls.DEFAULT_MAX_DEVICE_BUFFER, refill_rate=refill_rate or cls.DEFAULT_REFILL_RATE - ) \ No newline at end of file + )