diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 248c5f73..efe66b59 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -101,7 +101,7 @@ async def checkWakeupWords(conn, text): } # 获取音频数据 - opus_packets = audio_to_data(response.get("file_path")) + opus_packets = await audio_to_data(response.get("file_path")) # 播放唤醒词回复 conn.client_abort = False diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 4eaf9ab1..8879564f 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -123,7 +123,7 @@ async def max_out_size(conn): text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！" await send_stt_message(conn, text) file_path = "config/assets/max_output_size.wav" - opus_packets = audio_to_data(file_path) + opus_packets = await audio_to_data(file_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text)) conn.close_after_chat = True @@ -142,7 +142,7 @@ async def check_bind_device(conn): # 播放提示音 music_path = "config/assets/bind_code.wav" - opus_packets = audio_to_data(music_path) + opus_packets = await audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text)) # 逐个播放数字 @@ -150,7 +150,7 @@ async def check_bind_device(conn): try: digit = conn.bind_code[i] num_path = f"config/assets/bind_code/{digit}.wav" - num_packets = audio_to_data(num_path) + num_packets = await audio_to_data(num_path) conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None)) except Exception as e: conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") @@ -162,5 +162,5 @@ async def check_bind_device(conn): text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。" await send_stt_message(conn, text) music_path = "config/assets/bind_not_found.wav" - opus_packets = audio_to_data(music_path) + opus_packets = await audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text)) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index f662b6c6..d0167498 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -17,7 +17,12 @@ async def sendAudioMessage(conn, sentenceType, audios, text): if sentenceType == SentenceType.FIRST: # 同一句子的后续消息加入流控队列，其他情况立即发送 - if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller and getattr(conn, "audio_flow_control", {}).get("sentence_id") == conn.sentence_id: + if ( + hasattr(conn, "audio_rate_controller") + and conn.audio_rate_controller + and getattr(conn, "audio_flow_control", {}).get("sentence_id") + == conn.sentence_id + ): conn.audio_rate_controller.add_message( lambda: send_tts_message(conn, "sentence_start", text) ) @@ -120,7 +125,8 @@ def _get_or_create_rate_controller(conn, frame_duration, is_single_packet): # 判断是否需要重置：单包模式且 sentence_id 变化，或者控制器不存在 need_reset = ( is_single_packet - and getattr(conn, "audio_flow_control", {}).get("sentence_id") != conn.sentence_id + and getattr(conn, "audio_flow_control", {}).get("sentence_id") + != conn.sentence_id ) or not hasattr(conn, "audio_rate_controller") if need_reset: @@ -138,7 +144,9 @@ def _get_or_create_rate_controller(conn, frame_duration, is_single_packet): } # 启动后台发送循环 - _start_background_sender(conn, conn.audio_rate_controller, conn.audio_flow_control) + _start_background_sender( + conn, conn.audio_rate_controller, conn.audio_flow_control + ) return conn.audio_rate_controller, conn.audio_flow_control @@ -152,6 +160,7 @@ def _start_background_sender(conn, rate_controller, flow_control): rate_controller: 速率控制器 flow_control: 流控状态 """ + async def send_callback(packet): # 检查是否应该中止 if conn.client_abort: @@ -165,7 +174,9 @@ def _start_background_sender(conn, rate_controller, flow_control): rate_controller.start_sending(send_callback) -async def _send_audio_with_rate_control(conn, audio_list, rate_controller, flow_control, send_delay): +async def _send_audio_with_rate_control( + conn, audio_list, rate_controller, flow_control, send_delay +): """ 使用 rate_controller 发送音频包 @@ -235,7 +246,7 @@ async def send_tts_message(conn, state, text=None): stop_tts_notify_voice = conn.config.get( "stop_tts_notify_voice", "config/assets/tts_notify.mp3" ) - audios = audio_to_data(stop_tts_notify_voice, is_opus=True) + audios = await audio_to_data(stop_tts_notify_voice, is_opus=True) await sendAudio(conn, audios) # 等待所有音频包发送完成 await _wait_for_audio_completion(conn) diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index f72dda2c..2baedf68 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -4,6 +4,7 @@ import json import copy import wave import socket +import asyncio import requests import subprocess import numpy as np @@ -268,56 +269,62 @@ def audio_to_data_stream( pcm_to_data_stream(raw_data, is_opus, callback) -def audio_to_data(audio_file_path: str, is_opus: bool = True) -> list[bytes]: +async def audio_to_data(audio_file_path: str, is_opus: bool = True) -> list[bytes]: """ 将音频文件转换为Opus/PCM编码的帧列表 Args: audio_file_path: 音频文件路径 is_opus: 是否进行Opus编码 """ - # 获取文件后缀名 - file_type = os.path.splitext(audio_file_path)[1] - if file_type: - file_type = file_type.lstrip(".") - # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞 - audio = AudioSegment.from_file( - audio_file_path, format=file_type, parameters=["-nostdin"] - ) - # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配） - audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + def _sync_audio_to_data(): + # 获取文件后缀名 + file_type = os.path.splitext(audio_file_path)[1] + if file_type: + file_type = file_type.lstrip(".") + # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞 + audio = AudioSegment.from_file( + audio_file_path, format=file_type, parameters=["-nostdin"] + ) - # 获取原始PCM数据（16位小端） - raw_data = audio.raw_data + # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配） + audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) - # 初始化Opus编码器 - encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) + # 获取原始PCM数据（16位小端） + raw_data = audio.raw_data - # 编码参数 - frame_duration = 60 # 60ms per frame - frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame + # 初始化Opus编码器 + encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) - datas = [] - # 按帧处理所有音频数据（包括最后一帧可能补零） - for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample - # 获取当前帧的二进制数据 - chunk = raw_data[i : i + frame_size * 2] + # 编码参数 + frame_duration = 60 # 60ms per frame + frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame - # 如果最后一帧不足，补零 - if len(chunk) < frame_size * 2: - chunk += b"\x00" * (frame_size * 2 - len(chunk)) + datas = [] + # 按帧处理所有音频数据（包括最后一帧可能补零） + for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample + # 获取当前帧的二进制数据 + chunk = raw_data[i : i + frame_size * 2] - if is_opus: - # 转换为numpy数组处理 - np_frame = np.frombuffer(chunk, dtype=np.int16) - # 编码Opus数据 - frame_data = encoder.encode(np_frame.tobytes(), frame_size) - else: - frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) + # 如果最后一帧不足，补零 + if len(chunk) < frame_size * 2: + chunk += b"\x00" * (frame_size * 2 - len(chunk)) - datas.append(frame_data) + if is_opus: + # 转换为numpy数组处理 + np_frame = np.frombuffer(chunk, dtype=np.int16) + # 编码Opus数据 + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + else: + frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) - return datas + datas.append(frame_data) + + return datas + + loop = asyncio.get_running_loop() + # 在单独的线程中执行同步的音频处理操作 + return await loop.run_in_executor(None, _sync_audio_to_data) def audio_bytes_to_data_stream(