diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 641e2655..088d8962 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -5,6 +5,8 @@ import re import os import wave from io import BytesIO +from typing import Callable, Any + from core.utils import p3 import numpy as np import requests @@ -245,6 +247,23 @@ def audio_bytes_to_data(audio_bytes, file_type, is_opus=True): raw_data = audio.raw_data return pcm_to_data(raw_data, is_opus) +def audio_bytes_to_data_stream(audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]): + """ + 直接用音频二进制数据转为opus/pcm数据,支持wav、mp3、p3 + """ + if file_type == "p3": + # 直接用p3解码 + return p3.decode_opus_from_bytes_stream(audio_bytes, callback) + else: + # 其他格式用pydub + audio = AudioSegment.from_file( + BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"] + ) + audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + raw_data = audio.raw_data + pcm_to_data_stream(raw_data, is_opus, callback) + return None + def pcm_to_data(raw_data, is_opus=True): # 初始化Opus编码器 @@ -276,6 +295,32 @@ def pcm_to_data(raw_data, is_opus=True): return datas +def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any]=None): + # 初始化Opus编码器 + encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) + + # 编码参数 + frame_duration = 60 # 60ms per frame + frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame + + # 按帧处理所有音频数据(包括最后一帧可能补零) + for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample + # 获取当前帧的二进制数据 + chunk = raw_data[i : i + frame_size * 2] + + # 如果最后一帧不足,补零 + if len(chunk) < frame_size * 2: + chunk += b"\x00" * (frame_size * 2 - len(chunk)) + + if is_opus: + # 转换为numpy数组处理 + np_frame = np.frombuffer(chunk, dtype=np.int16) + # 编码Opus数据 + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + callback(frame_data) + else: + frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) + callback(frame_data) def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1): """