Merge pull request #2084 from xinnan-tech/py_fix_audio

Py fix audio

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -1,5 +1,4 @@
 import time
-import asyncio
 import json
 from core.handle.sendAudioHandle import send_stt_message
 from core.handle.intentHandler import handle_user_intent
@@ -14,14 +13,6 @@ TAG = __name__
 async def handleAudioMessage(conn, audio):
     # 当前片段是否有人说话
     have_voice = conn.vad.is_vad(conn, audio)
-    # 如果设备刚刚被唤醒，短暂忽略VAD检测
-    if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up:
-        have_voice = False
-        # 设置一个短暂延迟后恢复VAD检测
-        conn.asr_audio.clear()
-        if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
-            conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
-        return
 
     if have_voice:
         if conn.client_is_speaking:
@@ -31,13 +22,6 @@ async def handleAudioMessage(conn, audio):
     # 接收音频
     await conn.asr.receive_audio(conn, audio, have_voice)
 
-
-async def resume_vad_detection(conn):
-    # 等待2秒后恢复VAD检测
-    await asyncio.sleep(1)
-    conn.just_woken_up = False
-
-
 async def startToChat(conn, text):
     # 检查输入是否是JSON格式（包含说话人信息）
     speaker_name = None

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -1,4 +1,5 @@
 import json
+import asyncio
 from core.providers.tts.dto.dto import SentenceType
 from core.utils import textUtils
 
@@ -52,8 +53,12 @@ async def send_tts_message(conn, state, text=None):
             stop_tts_notify_voice = conn.config.get(
                 "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
             )
-            audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
-            await sendAudio(conn, audios)
+            conn.tts.audio_to_opus_data_stream(
+                stop_tts_notify_voice,
+                callback=lambda audio_data: asyncio.run_coroutine_threadsafe(
+                    sendAudio(conn, audio_data), conn.loop
+                ),
+            )
         # 清除服务端讲话状态
         conn.clearSpeakStatus()
 

main/xiaozhi-server/core/providers/asr/base.py

@@ -12,7 +12,7 @@ import time
 import concurrent.futures
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
-from typing import Optional, Tuple, List, Dict, Any
+from typing import Optional, Tuple, List
 from core.handle.receiveAudioHandle import startToChat
 from core.handle.reportHandle import enqueue_asr_report
 from core.utils.util import remove_punctuation_and_length
@@ -132,8 +132,6 @@ class ASRProviderBase(ABC):
                     return None
             
             # 使用线程池执行器并行运行
-            parallel_start_time = time.monotonic()
-            
             with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
                 asr_future = thread_executor.submit(run_asr)
                 

main/xiaozhi-server/core/providers/tts/base.py

@@ -11,7 +11,7 @@ from datetime import datetime
 from core.utils import textUtils
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
-from core.utils.audio_flow_control import FlowControlConfig
+from core.utils.audio_flow_control import FlowControlConfig, simulate_device_consumption
 from core.utils.util import audio_bytes_to_data_stream, audio_to_data_stream
 from core.utils.tts import MarkdownCleaner
 from core.utils.output_counter import add_device_output
@@ -356,9 +356,8 @@ class TTSProviderBase(ABC):
 
         # 模拟设备消费（实际应用中应该从设备获取反馈）防止音字不同步
         if isinstance(audio_datas, bytes):
-            # 模拟设备播放延迟（60ms per frame）, 实际情况可以低一点（50ms），增加使用体验
-            await asyncio.sleep(0.06)
-            self.flow_controller.update_device_consumption(1)
+            frame_count = 1
+            asyncio.create_task(simulate_device_consumption(self.flow_controller, frame_count))
 
     # 在类中添加流控制器重置方法
     def reset_flow_controller(self):

main/xiaozhi-server/core/providers/vad/silero.py

@@ -33,8 +33,8 @@ class VADProvider(VADProviderBase):
             int(min_silence_duration_ms) if min_silence_duration_ms else 1000
         )
 
-        # 至少要多少帧才算有语音
-        self.frame_window_threshold = 3
+        # 至少要多少帧才算有语音,增加灵敏度
+        self.frame_window_threshold = 1
 
     def is_vad(self, conn, opus_packet):
         try:

main/xiaozhi-server/core/utils/audio_flow_control.py

@@ -151,6 +151,21 @@ class AudioFlowController:
             )
 
 
+async def simulate_device_consumption(
+    flow_controller: AudioFlowController, frame_count: int
+):
+    """
+    模拟设备消费音频帧的过程
+    实际应用中应该根据设备反馈来更新消费情况
+    Args:
+        flow_controller: 流控制器实例
+        frame_count: 消费的帧数
+    """
+    # 模拟设备播放延迟（60ms per frame）
+    await asyncio.sleep(frame_count * 0.06)
+    flow_controller.update_device_consumption(frame_count)
+
+
 # 流控配置常量
 class FlowControlConfig:
     """流控配置常量"""
@@ -183,4 +198,4 @@ class FlowControlConfig:
         return AudioFlowController(
             max_device_buffer=max_buffer or cls.DEFAULT_MAX_DEVICE_BUFFER,
             refill_rate=refill_rate or cls.DEFAULT_REFILL_RATE
-        )
+        )