update: 音频后台队列稳定发送

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -16,7 +16,14 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
         await send_tts_message(conn, "start", None)
 
     if sentenceType == SentenceType.FIRST:
-        await send_tts_message(conn, "sentence_start", text)
+        # 同一句子的后续消息加入流控队列，其他情况立即发送
+        if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller and getattr(conn, "audio_flow_control", {}).get("sentence_id") == conn.sentence_id:
+            conn.audio_rate_controller.add_message(
+                lambda: send_tts_message(conn, "sentence_start", text)
+            )
+        else:
+            # 新句子或流控器未初始化，立即发送
+            await send_tts_message(conn, "sentence_start", text)
 
     await sendAudio(conn, audios)
     # 发送句子开始消息
@@ -31,6 +38,22 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
             await conn.close()
 
 
+async def _wait_for_audio_completion(conn):
+    """
+    等待音频队列清空
+
+    Args:
+        conn: 连接对象
+    """
+    if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller:
+        rate_controller = conn.audio_rate_controller
+        conn.logger.bind(tag=TAG).debug(
+            f"等待音频发送完成，队列中还有 {len(rate_controller.queue)} 个包"
+        )
+        await rate_controller.queue_empty_event.wait()
+        conn.logger.bind(tag=TAG).debug("音频发送完成")
+
+
 async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence):
     """
     发送带16字节头部的opus数据包给mqtt_gateway
@@ -53,7 +76,6 @@ async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence):
     await conn.websocket.send(complete_packet)
 
 
-# 播放音频 - 使用 AudioRateController 进行精确流控
 async def sendAudio(conn, audios, frame_duration=60):
     """
     发送音频包，使用 AudioRateController 进行精确的流量控制
@@ -62,130 +84,121 @@ async def sendAudio(conn, audios, frame_duration=60):
         conn: 连接对象
         audios: 单个opus包(bytes) 或 opus包列表
         frame_duration: 帧时长（毫秒），默认60ms
-
-    改进点：
-    1. 使用单一时间基准，避免累积误差
-    2. 每次检查队列时重新计算 elapsed_ms，更精准
-    3. 支持高并发而不产生时间偏差
     """
     if audios is None or len(audios) == 0:
         return
 
-    # 获取发送延迟配置
     send_delay = conn.config.get("tts_audio_send_delay", -1) / 1000.0
+    is_single_packet = isinstance(audios, bytes)
 
-    if isinstance(audios, bytes):
-        # 单个 opus 包处理
-        await _sendAudio_single(conn, audios, send_delay, frame_duration)
-    else:
-        # 音频列表处理（如文件型音频）
-        await _sendAudio_list(conn, audios, send_delay, frame_duration)
+    # 初始化或获取 RateController
+    rate_controller, flow_control = _get_or_create_rate_controller(
+        conn, frame_duration, is_single_packet
+    )
+
+    # 统一转换为列表处理
+    audio_list = [audios] if is_single_packet else audios
+
+    # 发送音频包
+    await _send_audio_with_rate_control(
+        conn, audio_list, rate_controller, flow_control, send_delay
+    )
 
 
-async def _sendAudio_single(conn, opus_packet, send_delay, frame_duration=60):
+def _get_or_create_rate_controller(conn, frame_duration, is_single_packet):
     """
-    发送单个 opus 包
-    使用 AudioRateController 进行流控
+    获取或创建 RateController 和 flow_control
+
+    Args:
+        conn: 连接对象
+        frame_duration: 帧时长
+        is_single_packet: 是否单包模式（True: TTS流式单包, False: 批量包）
+
+    Returns:
+        (rate_controller, flow_control)
     """
-    # 重置流控状态，第一次读取和会话发生转变时
-    if not hasattr(conn, "audio_rate_controller") or conn.audio_flow_control.get("sentence_id") != conn.sentence_id:
-        if hasattr(conn, "audio_rate_controller"):
-            conn.audio_rate_controller.reset()
-        else:
+    # 判断是否需要重置：单包模式且 sentence_id 变化，或者控制器不存在
+    need_reset = (
+        is_single_packet
+        and getattr(conn, "audio_flow_control", {}).get("sentence_id") != conn.sentence_id
+    ) or not hasattr(conn, "audio_rate_controller")
+
+    if need_reset:
+        # 创建或获取 rate_controller
+        if not hasattr(conn, "audio_rate_controller"):
             conn.audio_rate_controller = AudioRateController(frame_duration)
+        else:
             conn.audio_rate_controller.reset()
 
+        # 初始化 flow_control
         conn.audio_flow_control = {
             "packet_count": 0,
             "sequence": 0,
             "sentence_id": conn.sentence_id,
         }
 
-    if conn.client_abort:
-        return
+        # 启动后台发送循环
+        _start_background_sender(conn, conn.audio_rate_controller, conn.audio_flow_control)
 
-    conn.last_activity_time = time.time() * 1000
+    return conn.audio_rate_controller, conn.audio_flow_control
 
-    rate_controller = conn.audio_rate_controller
-    flow_control = conn.audio_flow_control
-    packet_count = flow_control["packet_count"]
 
-    # 预缓冲：前5个包直接发送，不做延迟
+def _start_background_sender(conn, rate_controller, flow_control):
+    """
+    启动后台发送循环任务
+
+    Args:
+        conn: 连接对象
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+    """
+    async def send_callback(packet):
+        # 检查是否应该中止
+        if conn.client_abort:
+            raise asyncio.CancelledError("客户端已中止")
+
+        conn.last_activity_time = time.time() * 1000
+        await _do_send_audio(conn, packet, flow_control)
+        conn.client_is_speaking = True
+
+    # 使用 start_sending 启动后台循环
+    rate_controller.start_sending(send_callback)
+
+
+async def _send_audio_with_rate_control(conn, audio_list, rate_controller, flow_control, send_delay):
+    """
+    使用 rate_controller 发送音频包
+
+    Args:
+        conn: 连接对象
+        audio_list: 音频包列表
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+        send_delay: 固定延迟（秒），-1表示使用动态流控
+    """
     pre_buffer_count = 5
 
-    if packet_count < pre_buffer_count or send_delay > 0:
-        # 预缓冲阶段或固定延迟模式，直接发送
-        await _do_send_audio(conn, opus_packet, flow_control, frame_duration)
-        conn.client_is_speaking = True
-
-        if send_delay > 0 and packet_count >= pre_buffer_count:
-            await asyncio.sleep(send_delay)
-    else:
-        # 使用流控器进行精确的速率控制
-        rate_controller.add_audio(opus_packet)
-
-        async def send_callback(packet):
-            await _do_send_audio(conn, packet, flow_control, frame_duration)
-
-        await rate_controller.check_queue(send_callback)
-        conn.client_is_speaking = True
-
-    # 更新流控状态
-    flow_control["packet_count"] += 1
-    flow_control["sequence"] += 1
-
-
-async def _sendAudio_list(conn, audios, send_delay, frame_duration=60):
-    """
-    发送音频列表（如文件型音频）
-    """
-    if not audios:
-        return
-
-    rate_controller = AudioRateController(frame_duration)
-    rate_controller.reset()
-
-    flow_control = {
-        "packet_count": 0,
-        "sequence": 0,
-    }
-
-    # 预缓冲：前5个包直接发送
-    pre_buffer_frames = min(5, len(audios))
-    for i in range(pre_buffer_frames):
+    for packet in audio_list:
         if conn.client_abort:
             return
-        await _do_send_audio(conn, audios[i], flow_control, frame_duration)
-        conn.client_is_speaking = True
-
-    remaining_audios = audios[pre_buffer_frames:]
-
-    # 处理剩余音频帧
-    for i, opus_packet in enumerate(remaining_audios):
-        if conn.client_abort:
-            break
 
         conn.last_activity_time = time.time() * 1000
 
-        if send_delay > 0:
+        # 预缓冲：前5个包直接发送
+        if flow_control["packet_count"] < pre_buffer_count:
+            await _do_send_audio(conn, packet, flow_control)
+            conn.client_is_speaking = True
+        elif send_delay > 0:
             # 固定延迟模式
             await asyncio.sleep(send_delay)
-        else:
-            # 使用流控器进行精确延迟
-            rate_controller.add_audio(opus_packet)
-
-            async def send_callback(packet):
-                await _do_send_audio(conn, packet, flow_control, frame_duration)
-
-            await rate_controller.check_queue(send_callback)
+            await _do_send_audio(conn, packet, flow_control)
             conn.client_is_speaking = True
-            continue
-
-        await _do_send_audio(conn, opus_packet, flow_control, frame_duration)
-        conn.client_is_speaking = True
+        else:
+            # 动态流控模式：仅添加到队列，由后台循环负责发送
+            rate_controller.add_audio(packet)
 
 
-async def _do_send_audio(conn, opus_packet, flow_control, frame_duration=60):
+async def _do_send_audio(conn, opus_packet, flow_control):
     """
     执行实际的音频发送
     """
@@ -224,6 +237,8 @@ async def send_tts_message(conn, state, text=None):
             )
             audios = audio_to_data(stop_tts_notify_voice, is_opus=True)
             await sendAudio(conn, audios)
+        # 等待所有音频包发送完成
+        await _wait_for_audio_completion(conn)
         # 清除服务端讲话状态
         conn.clearSpeakStatus()