From 16e77b02c21efb6ed74f90e4559f5bd6f9e67f7b Mon Sep 17 00:00:00 2001 From: MakerZorky <1053714527zhq@gmail.com> Date: Tue, 29 Apr 2025 01:28:06 +0800 Subject: [PATCH] =?UTF-8?q?update:=E6=89=80=E6=9C=89ASR=E5=9D=87=E6=94=B9?= =?UTF-8?q?=E4=B8=BAPCM=E7=9B=B4=E6=8E=A5=E8=AF=86=E5=88=AB=EF=BC=8C?= =?UTF-8?q?=E9=80=9A=E8=BF=87delete=5Faudio=E5=8F=82=E6=95=B0=E5=88=A4?= =?UTF-8?q?=E6=96=AD=E8=A6=81=E4=B8=8D=E8=A6=81=E5=82=A8=E5=AD=98wav?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/providers/asr/base.py | 4 +-- .../core/providers/asr/doubao.py | 21 ++++++------ .../core/providers/asr/fun_local.py | 32 +++++++++++++++---- .../core/providers/asr/sherpa_onnx_local.py | 24 ++++++++------ .../core/providers/asr/tencent.py | 18 ++++------- 5 files changed, 59 insertions(+), 40 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index d8db22a8..6d9c2c90 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -9,8 +9,8 @@ logger = setup_logging() class ASRProviderBase(ABC): @abstractmethod - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """解码Opus数据并保存为WAV文件""" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" pass @abstractmethod diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index b19ceb97..6fba472c 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -89,6 +89,7 @@ class ASRProvider(ASRProviderBase): self.cluster = config.get("cluster") self.access_token = config.get("access_token") self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file self.host = "openspeech.bytedance.com" self.ws_url = f"wss://{self.host}/api/v2/asr" @@ -98,21 +99,11 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -274,6 +265,12 @@ class ASRProvider(ASRProviderBase): pcm_data = self.decode_opus(opus_data, session_id) combined_pcm_data = b"".join(pcm_data) + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) + # 直接使用PCM数据 # 计算分段大小 (单声道, 16bit, 16kHz采样率) size_per_sec = 1 * 2 * 16000 # nchannels * sampwidth * framerate diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 82d09ab5..a678780b 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -51,7 +51,7 @@ class ASRProvider(ASRProviderBase): ) def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + """PCM数据保存为WAV文件""" file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) @@ -72,20 +72,40 @@ class ASRProvider(ASRProviderBase): wf.writeframes(b"".join(pcm_data)) return file_path + + @staticmethod + def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: + + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return pcm_data async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: - # 保存音频文件 - start_time = time.time() - file_path = self.save_audio_to_file(opus_data, session_id) - logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + # 合并所有opus数据包 + pcm_data = self.decode_opus(opus_data, session_id) + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) # 语音识别 start_time = time.time() result = self.model.generate( - input=file_path, + input=combined_pcm_data, cache={}, language="auto", use_itn=True, diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 3dfa3162..b086022b 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -84,10 +84,21 @@ class ASRProvider(ASRProviderBase): ) def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + """PCM数据保存为WAV文件""" file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path + + @staticmethod + def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 pcm_data = [] @@ -98,13 +109,7 @@ class ASRProvider(ASRProviderBase): except opuslib_next.OpusError as e: logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) # 2 bytes = 16-bit - wf.setframerate(16000) - wf.writeframes(b"".join(pcm_data)) - - return file_path + return pcm_data def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]: """ @@ -136,7 +141,8 @@ class ASRProvider(ASRProviderBase): try: # 保存音频文件 start_time = time.time() - file_path = self.save_audio_to_file(opus_data, session_id) + pcm_data = self.decode_opus(opus_data, session_id) + file_path = self.save_audio_to_file(pcm_data, session_id) logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") # 语音识别 diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index 8221f9df..0ac297a6 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -31,21 +31,11 @@ class ASRProvider(ASRProviderBase): os.makedirs(self.output_dir, exist_ok=True) def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + """PCM数据保存为WAV文件""" file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -85,6 +75,12 @@ class ASRProvider(ASRProviderBase): # 将Opus音频数据解码为PCM pcm_data = self.decode_opus(opus_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) # 将音频数据转换为Base64编码 base64_audio = base64.b64encode(pcm_data).decode('utf-8')