diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index 36f41745..8824d17b 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -213,36 +213,24 @@ class ASRProvider(ASRProviderBase): return None async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if self._is_token_expired(): logger.warning("Token已过期，正在自动刷新...") self._refresh_token() - file_path = None try: - # 解码Opus为PCM - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - combined_pcm_data = b"".join(pcm_data) - - # 判断是否保存为WAV文件 - if self.delete_audio_file: - pass - else: - file_path = self.save_audio_to_file(pcm_data, session_id) - + if artifacts is None: + return "", None # 发送请求并获取文本 - text = await self._send_request(combined_pcm_data) + text = await self._send_request(artifacts.pcm_bytes) if text: - return text, file_path + return text, artifacts.file_path - return "", file_path + return "", artifacts.file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", file_path + return "", None diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py index bf62e343..f65d2b4e 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -324,7 +324,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug("ASR会话清理完成") - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): """获取识别结果""" result = self.text self.text = "" diff --git a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py index d7e71fce..c9aa7c27 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py @@ -308,7 +308,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug("ASR会话清理完成") - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): """获取识别结果""" result = self.text self.text = "" diff --git a/main/xiaozhi-server/core/providers/asr/baidu.py b/main/xiaozhi-server/core/providers/asr/baidu.py index aa8ea901..aa14f443 100644 --- a/main/xiaozhi-server/core/providers/asr/baidu.py +++ b/main/xiaozhi-server/core/providers/asr/baidu.py @@ -30,37 +30,26 @@ class ASRProvider(ASRProviderBase): os.makedirs(self.output_dir, exist_ok=True) async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: logger.bind(tag=TAG).warning("音频数据为空！") return None, None - file_path = None try: # 检查配置是否已设置 if not self.app_id or not self.api_key or not self.secret_key: logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别") - return None, file_path + return None, None - # 将Opus音频数据解码为PCM - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - combined_pcm_data = b"".join(pcm_data) - - # 判断是否保存为WAV文件 - if self.delete_audio_file: - pass - else: - self.save_audio_to_file(pcm_data, session_id) + if artifacts is None: + return "", None start_time = time.time() # 识别本地文件 result = self.client.asr( - combined_pcm_data, + artifacts.pcm_bytes, "pcm", 16000, { @@ -73,13 +62,13 @@ class ASRProvider(ASRProviderBase): f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}" ) result = result["result"][0] - return result, file_path + return result, artifacts.file_path else: raise Exception( f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}" ) - return None, file_path + return None, artifacts.file_path except Exception as e: logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True) - return None, file_path + return None, None diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 3e2c66e5..74ea0c2d 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -8,15 +8,18 @@ import queue import asyncio import traceback import threading +import shutil import opuslib_next + from abc import ABC, abstractmethod from config.logger import setup_logging -from typing import Optional, Tuple, List +from typing import Optional, Tuple, List, NamedTuple from core.providers.asr.dto.dto import InterfaceType from core.handle.receiveAudioHandle import startToChat from core.handle.reportHandle import enqueue_asr_report from core.utils.util import remove_punctuation_and_length from core.handle.receiveAudioHandle import handleAudioMessage +import tempfile TAG = __name__ logger = setup_logging() @@ -93,10 +96,14 @@ class ASRProviderBase(ABC): wav_data = self._pcm_to_wav(combined_pcm_data) # 定义ASR任务 - asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format) + asr_task = self.speech_to_text_wrapper( + asr_audio_task, conn.session_id, conn.audio_format + ) if conn.voiceprint_provider and wav_data: - voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id) + voiceprint_task = conn.voiceprint_provider.identify_speaker( + wav_data, conn.session_id + ) # 并发等待两个结果 asr_result, voiceprint_result = await asyncio.gather( asr_task, voiceprint_task, return_exceptions=True @@ -161,19 +168,18 @@ class ASRProviderBase(ABC): await startToChat(conn, enhanced_text) audio_snapshot = asr_audio_task.copy() enqueue_asr_report(conn, enhanced_text, audio_snapshot) - except Exception as e: logger.bind(tag=TAG).error(f"处理语音停止失败: {e}") import traceback + logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}") def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str: """构建包含说话人信息的文本（仅用于纯文本ASR）""" if speaker_name and speaker_name.strip(): - return json.dumps({ - "speaker": speaker_name, - "content": text - }, ensure_ascii=False) + return json.dumps( + {"speaker": speaker_name, "content": text}, ensure_ascii=False + ) else: return text @@ -182,23 +188,23 @@ class ASRProviderBase(ABC): if len(pcm_data) == 0: logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV") return b"" - + # 确保数据长度是偶数（16位音频） if len(pcm_data) % 2 != 0: pcm_data = pcm_data[:-1] - + # 创建WAV文件头 wav_buffer = io.BytesIO() try: - with wave.open(wav_buffer, 'wb') as wav_file: - wav_file.setnchannels(1) # 单声道 - wav_file.setsampwidth(2) # 16位 + with wave.open(wav_buffer, "wb") as wav_file: + wav_file.setnchannels(1) # 单声道 + wav_file.setsampwidth(2) # 16位 wav_file.setframerate(16000) # 16kHz采样率 wav_file.writeframes(pcm_data) - + wav_buffer.seek(0) wav_data = wav_buffer.read() - + return wav_data except Exception as e: logger.bind(tag=TAG).error(f"WAV转换失败: {e}") @@ -210,6 +216,41 @@ class ASRProviderBase(ABC): async def close(self): pass + class AudioArtifacts(NamedTuple): + pcm_frames: List[bytes] + """PCM音频帧列表""" + pcm_bytes: bytes + """合并后的PCM音频字节数据""" + file_path: Optional[str] + """WAV文件路径""" + temp_path: Optional[str] + """临时WAV文件路径""" + + def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]: + return self._current_artifacts + + def requires_file(self) -> bool: + """是否需要文件输入""" + return False + + def prefers_temp_file(self) -> bool: + """是否优先使用临时文件""" + return False + + def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]: + try: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: + temp_path = temp_file.name + with wave.open(temp_path, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(pcm_bytes) + return temp_path + except Exception as e: + logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}") + return None + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" module_name = __name__.split(".")[-1] @@ -224,11 +265,80 @@ class ASRProviderBase(ABC): return file_path - @abstractmethod - async def speech_to_text( + async def speech_to_text_wrapper( self, opus_data: List[bytes], session_id: str, audio_format="opus" ) -> Tuple[Optional[str], Optional[str]]: - """将语音数据转换为文本""" + file_path = None + temp_path = None + try: + if audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) + + free_space = shutil.disk_usage(self.output_dir).free + if free_space < len(combined_pcm_data) * 2: + raise OSError("磁盘空间不足") + + if self.requires_file() and self.prefers_temp_file(): + temp_path = self.build_temp_file(combined_pcm_data) + + if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or ( + self.requires_file() and not self.prefers_temp_file() + ): + file_path = self.save_audio_to_file(pcm_data, session_id) + + if len(combined_pcm_data) == 0: + artifacts = None + else: + artifacts = ASRProviderBase.AudioArtifacts( + pcm_frames=pcm_data, + pcm_bytes=combined_pcm_data, + file_path=file_path, + temp_path=temp_path, + ) + + text, _ = await self.speech_to_text( + opus_data, session_id, audio_format, artifacts + ) + return text, file_path + except OSError as e: + logger.bind(tag=TAG).error(f"文件操作错误: {e}") + return None, None + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}") + return None, None + finally: + try: + if temp_path and os.path.exists(temp_path): + os.unlink(temp_path) + if ( + hasattr(self, "delete_audio_file") + and self.delete_audio_file + and file_path + and os.path.exists(file_path) + ): + os.remove(file_path) + except Exception as e: + logger.bind(tag=TAG).error(f"文件清理失败: {e}") + + @abstractmethod + async def speech_to_text( + self, + opus_data: List[bytes], + session_id: str, + audio_format="opus", + artifacts: Optional[AudioArtifacts] = None, + ) -> Tuple[Optional[str], Optional[str]]: + """将语音数据转换为文本 + + :param opus_data: 输入的Opus音频数据 + :param session_id: 会话ID + :param audio_format: 音频格式，默认"opus" + :param artifacts: 音频工件，包含PCM数据、文件路径等 + :return: 识别结果文本和文件路径（如果有） + """ pass @staticmethod @@ -239,23 +349,23 @@ class ASRProviderBase(ABC): decoder = opuslib_next.Decoder(16000, 1) pcm_data = [] buffer_size = 960 # 每次处理960个采样点 (60ms at 16kHz) - + for i, opus_packet in enumerate(opus_data): try: if not opus_packet or len(opus_packet) == 0: continue - + pcm_frame = decoder.decode(opus_packet, buffer_size) if pcm_frame and len(pcm_frame) > 0: pcm_data.append(pcm_frame) - + except opuslib_next.OpusError as e: logger.bind(tag=TAG).warning(f"Opus解码错误，跳过数据包 {i}: {e}") except Exception as e: logger.bind(tag=TAG).error(f"音频处理错误，数据包 {i}: {e}") - + return pcm_data - + except Exception as e: logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}") return [] diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index f6cd79c6..016ac0fd 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -232,24 +232,13 @@ class ASRProvider(ASRProviderBase): yield data[offset:data_len], True async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" - file_path = None try: - # 合并所有opus数据包 - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - combined_pcm_data = b"".join(pcm_data) - - # 判断是否保存为WAV文件 - if self.delete_audio_file: - pass - else: - file_path = self.save_audio_to_file(pcm_data, session_id) + if artifacts is None: + return "", None # 直接使用PCM数据 # 计算分段大小 (单声道, 16bit, 16kHz采样率) @@ -258,14 +247,14 @@ class ASRProvider(ASRProviderBase): # 语音识别 start_time = time.time() - text = await self._send_request(combined_pcm_data, segment_size) + text = await self._send_request(artifacts.pcm_bytes, segment_size) if text: logger.bind(tag=TAG).debug( f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" ) - return text, file_path - return "", file_path + return text, artifacts.file_path + return "", artifacts.file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", file_path + return "", None diff --git a/main/xiaozhi-server/core/providers/asr/doubao_stream.py b/main/xiaozhi-server/core/providers/asr/doubao_stream.py index db9cbc90..3bb9997a 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao_stream.py +++ b/main/xiaozhi-server/core/providers/asr/doubao_stream.py @@ -388,7 +388,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"原始响应数据: {res.hex()}") raise - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): result = self.text self.text = "" # 清空text return result, None diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index bba53f8a..62ae99e7 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -64,39 +64,21 @@ class ASRProvider(ASRProviderBase): ) async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" - file_path = None retry_count = 0 - + while retry_count < MAX_RETRIES: try: - # 合并所有opus数据包 - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - - combined_pcm_data = b"".join(pcm_data) - - # 检查磁盘空间 - if not self.delete_audio_file: - free_space = shutil.disk_usage(self.output_dir).free - if free_space < len(combined_pcm_data) * 2: # 预留2倍空间 - raise OSError("磁盘空间不足") - - # 判断是否保存为WAV文件 - if self.delete_audio_file: - pass - else: - file_path = self.save_audio_to_file(pcm_data, session_id) + if artifacts is None: + return "", None # 语音识别 - 使用线程池避免阻塞事件循环 start_time = time.time() result = await asyncio.to_thread( self.model.generate, - input=combined_pcm_data, + input=artifacts.pcm_bytes, cache={}, language="auto", use_itn=True, @@ -107,7 +89,7 @@ class ASRProvider(ASRProviderBase): f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}" ) - return text, file_path + return text, artifacts.file_path except OSError as e: retry_count += 1 @@ -115,7 +97,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error( f"语音识别失败（已重试{retry_count}次）: {e}", exc_info=True ) - return "", file_path + return "", None logger.bind(tag=TAG).warning( f"语音识别失败，正在重试（{retry_count}/{MAX_RETRIES}）: {e}" ) @@ -123,15 +105,4 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", file_path - - finally: - # 文件清理逻辑 - if self.delete_audio_file and file_path and os.path.exists(file_path): - try: - os.remove(file_path) - logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") - except Exception as e: - logger.bind(tag=TAG).error( - f"文件删除失败: {file_path} | 错误: {e}" - ) + return "", None diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index c6674df9..19b5022d 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -101,7 +101,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """ Convert speech data to text using FunASR. @@ -109,18 +109,9 @@ class ASRProvider(ASRProviderBase): :param session_id: Unique session identifier. :return: Tuple containing recognized text and optional timestamp. """ - file_path = None - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - combined_pcm_data = b"".join(pcm_data) - - # 判断是否保存为WAV文件 - if self.delete_audio_file: - pass - else: - file_path = self.save_audio_to_file(pcm_data, session_id) + + if artifacts is None: + return "", None auth_header = {"Authorization": "Bearer; {}".format(self.api_key)} async with websockets.connect( self.uri, @@ -132,7 +123,7 @@ class ASRProvider(ASRProviderBase): try: # Use asyncio to handle WebSocket communication send_task = asyncio.create_task( - self._send_data(ws, combined_pcm_data, session_id) + self._send_data(ws, artifacts.pcm_bytes, session_id) ) receive_task = asyncio.create_task(self._receive_responses(ws)) @@ -161,14 +152,14 @@ class ASRProvider(ASRProviderBase): result = lang_tag_filter(result) return ( result, - file_path, + artifacts.file_path, ) # Return the recognized text and timestamp (if any) except websockets.exceptions.ConnectionClosed as e: logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") - return "", file_path + return "", artifacts.file_path except Exception as e: logger.bind(tag=TAG).error( f"Error during speech-to-text conversion: {e}", exc_info=True ) - return "", file_path + return "", artifacts.file_path diff --git a/main/xiaozhi-server/core/providers/asr/openai.py b/main/xiaozhi-server/core/providers/asr/openai.py index 5c6e849c..7b215589 100644 --- a/main/xiaozhi-server/core/providers/asr/openai.py +++ b/main/xiaozhi-server/core/providers/asr/openai.py @@ -21,20 +21,16 @@ class ASRProvider(ASRProviderBase): os.makedirs(self.output_dir, exist_ok=True) - async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]: + def requires_file(self) -> bool: + return True + + async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None) -> Tuple[Optional[str], Optional[str]]: file_path = None try: - start_time = time.time() - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - file_path = self.save_audio_to_file(pcm_data, session_id) - - logger.bind(tag=TAG).debug( - f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" - ) - + if artifacts is None: + return "", None + file_path = artifacts.file_path + logger.bind(tag=TAG).info(f"file path: {file_path}") headers = { "Authorization": f"Bearer {self.api_key}", @@ -71,12 +67,4 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}") return "", None - finally: - # 文件清理逻辑 - if self.delete_audio_file and file_path and os.path.exists(file_path): - try: - os.remove(file_path) - logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") - except Exception as e: - logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") diff --git a/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py b/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py index 51c84a37..21438c1c 100644 --- a/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py +++ b/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py @@ -1,5 +1,4 @@ import os -import tempfile from typing import Optional, Tuple, List import dashscope from config.logger import setup_logging @@ -35,56 +34,25 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def _prepare_audio_file(self, pcm_data: bytes) -> str: - """将PCM数据转换为WAV文件并返回文件路径""" - try: - import wave - - # 创建临时WAV文件 - with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file: - temp_path = temp_file.name - - # 写入WAV格式 - with wave.open(temp_path, 'wb') as wav_file: - wav_file.setnchannels(1) # 单声道 - wav_file.setsampwidth(2) # 16位 - wav_file.setframerate(16000) # 16kHz采样率 - wav_file.writeframes(pcm_data) - - return temp_path - - except Exception as e: - logger.bind(tag=tag).error(f"音频文件准备失败: {e}") - return None + def prefers_temp_file(self) -> bool: + return True + + def requires_file(self) -> bool: + return True async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" temp_file_path = None file_path = None - try: - # 解码音频数据 - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - - combined_pcm_data = b"".join(pcm_data) - if len(combined_pcm_data) == 0: - logger.bind(tag=tag).warning("音频数据为空") + if artifacts is None: return "", None - - # 准备音频文件 - temp_file_path = self._prepare_audio_file(combined_pcm_data) + temp_file_path = artifacts.temp_path + file_path = artifacts.file_path if not temp_file_path: - return "", None - - # 保存音频文件（如果需要） - if not self.delete_audio_file: - file_path = self.save_audio_to_file(pcm_data, session_id) - + return "", file_path # 构造请求消息 messages = [ { @@ -141,11 +109,3 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=tag).error(f"语音识别失败: {e}") return "", file_path - - finally: - # 清理临时文件 - if temp_file_path and os.path.exists(temp_file_path): - try: - os.unlink(temp_file_path) - except Exception as e: - logger.bind(tag=tag).warning(f"清理临时文件失败: {e}") \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index c34909b2..742c3567 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -120,24 +120,19 @@ class ASRProvider(ASRProviderBase): samples_float32 = samples_float32 / 32768 return samples_float32, f.getframerate() + def requires_file(self) -> bool: + return True + async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: - # 保存音频文件 - start_time = time.time() - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - file_path = self.save_audio_to_file(pcm_data, session_id) - logger.bind(tag=TAG).debug( - f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" - ) + if artifacts is None: + return "", None + file_path = artifacts.file_path - # 语音识别 start_time = time.time() s = self.model.create_stream() samples, sample_rate = self.read_wave(file_path) @@ -153,11 +148,3 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) return "", file_path - finally: - # 文件清理逻辑 - if self.delete_audio_file and file_path and os.path.exists(file_path): - try: - os.remove(file_path) - logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") - except Exception as e: - logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index 1fd1c6f4..d873bd19 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -32,35 +32,24 @@ class ASRProvider(ASRProviderBase): os.makedirs(self.output_dir, exist_ok=True) async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: logger.bind(tag=TAG).warning("音频数据为空！") return None, None - file_path = None try: # 检查配置是否已设置 if not self.secret_id or not self.secret_key: logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别") - return None, file_path + return None, None - # 将Opus音频数据解码为PCM - if audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) - combined_pcm_data = b"".join(pcm_data) - - # 判断是否保存为WAV文件 - if self.delete_audio_file: - pass - else: - self.save_audio_to_file(pcm_data, session_id) + if artifacts is None: + return "", None # 将音频数据转换为Base64编码 - base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8") + base64_audio = base64.b64encode(artifacts.pcm_bytes).decode("utf-8") # 构建请求体 request_body = self._build_request_body(base64_audio) @@ -77,11 +66,11 @@ class ASRProvider(ASRProviderBase): f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}" ) - return result, file_path + return result, artifacts.file_path except Exception as e: logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True) - return None, file_path + return None, None def _build_request_body(self, base64_audio: str) -> str: """构建请求体""" diff --git a/main/xiaozhi-server/core/providers/asr/vosk.py b/main/xiaozhi-server/core/providers/asr/vosk.py index 19d31822..77cbf986 100644 --- a/main/xiaozhi-server/core/providers/asr/vosk.py +++ b/main/xiaozhi-server/core/providers/asr/vosk.py @@ -44,36 +44,21 @@ class ASRProvider(ASRProviderBase): raise async def speech_to_text( - self, audio_data: List[bytes], session_id: str, audio_format: str = "opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" - file_path = None try: # 检查模型是否加载成功 if not self.model: logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别") return "", None - - # 解码音频（如果原始格式是Opus） - if audio_format == "pcm": - pcm_data = audio_data - else: - pcm_data = self.decode_opus(audio_data) - - if not pcm_data: - logger.bind(tag=TAG).warning("解码后的PCM数据为空，无法进行识别") + + if artifacts is None: return "", None - - # 合并PCM数据 - combined_pcm_data = b"".join(pcm_data) - if len(combined_pcm_data) == 0: + if not artifacts.pcm_bytes: logger.bind(tag=TAG).warning("合并后的PCM数据为空") return "", None - # 判断是否保存为WAV文件 - if not self.delete_audio_file: - file_path = self.save_audio_to_file(pcm_data, session_id) - start_time = time.time() @@ -81,8 +66,8 @@ class ASRProvider(ASRProviderBase): chunk_size = 2000 text_result = "" - for i in range(0, len(combined_pcm_data), chunk_size): - chunk = combined_pcm_data[i:i+chunk_size] + for i in range(0, len(artifacts.pcm_bytes), chunk_size): + chunk = artifacts.pcm_bytes[i:i+chunk_size] if self.recognizer.AcceptWaveform(chunk): result = json.loads(self.recognizer.Result()) text = result.get('text', '') @@ -99,16 +84,8 @@ class ASRProvider(ASRProviderBase): f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}" ) - return text_result.strip(), file_path + return text_result.strip(), artifacts.file_path except Exception as e: logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}") return "", None - finally: - # 文件清理逻辑 - if self.delete_audio_file and file_path and os.path.exists(file_path): - try: - os.remove(file_path) - logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") - except Exception as e: - logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") diff --git a/main/xiaozhi-server/core/providers/asr/xunfei_stream.py b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py index 30516063..31087542 100644 --- a/main/xiaozhi-server/core/providers/asr/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py @@ -318,7 +318,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug("ASR会话清理完成") - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): """获取识别结果""" result = self.text self.text = ""