From 614b662e3dc0a6d5e4ccd1826ad441eadae142f1 Mon Sep 17 00:00:00 2001 From: huozaimengli Date: Thu, 29 Jan 2026 16:39:35 +0800 Subject: [PATCH] =?UTF-8?q?refactor(asr):=20=E9=87=8D=E6=9E=84speech=5Fto?= =?UTF-8?q?=5Ftext=E6=96=B9=E6=B3=95=E4=BB=A5=E6=8E=A5=E6=94=B6artifacts?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除各ASR提供者中重复的get_current_artifacts调用，改为通过参数传递artifacts 修改base类中process_audio方法，根据combined_pcm_data长度决定是否创建artifacts 更新所有speech_to_text方法签名，添加artifacts可选参数并更新文档字符串 --- .../core/providers/asr/aliyun.py | 3 +- .../core/providers/asr/aliyun_stream.py | 2 +- .../core/providers/asr/aliyunbl_stream.py | 2 +- .../core/providers/asr/baidu.py | 3 +- .../xiaozhi-server/core/providers/asr/base.py | 41 ++++++++++++++----- .../core/providers/asr/doubao.py | 3 +- .../core/providers/asr/doubao_stream.py | 2 +- .../core/providers/asr/fun_local.py | 3 +- .../core/providers/asr/fun_server.py | 3 +- .../core/providers/asr/openai.py | 3 +- .../core/providers/asr/qwen3_asr_flash.py | 3 +- .../core/providers/asr/sherpa_onnx_local.py | 3 +- .../core/providers/asr/tencent.py | 3 +- .../xiaozhi-server/core/providers/asr/vosk.py | 3 +- .../core/providers/asr/xunfei_stream.py | 2 +- 15 files changed, 44 insertions(+), 35 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index 60b0815a..8824d17b 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -213,7 +213,7 @@ class ASRProvider(ASRProviderBase): return None async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if self._is_token_expired(): @@ -221,7 +221,6 @@ class ASRProvider(ASRProviderBase): self._refresh_token() try: - artifacts = self.get_current_artifacts() if artifacts is None: return "", None # 发送请求并获取文本 diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py index 4ce588a7..af0e7e2f 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -341,7 +341,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug("ASR会话清理完成") - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): """获取识别结果""" result = self.text self.text = "" diff --git a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py index a321ce9b..722ab7d5 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py @@ -325,7 +325,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug("ASR会话清理完成") - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): """获取识别结果""" result = self.text self.text = "" diff --git a/main/xiaozhi-server/core/providers/asr/baidu.py b/main/xiaozhi-server/core/providers/asr/baidu.py index a808e4d5..aa14f443 100644 --- a/main/xiaozhi-server/core/providers/asr/baidu.py +++ b/main/xiaozhi-server/core/providers/asr/baidu.py @@ -30,7 +30,7 @@ class ASRProvider(ASRProviderBase): os.makedirs(self.output_dir, exist_ok=True) async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: @@ -43,7 +43,6 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别") return None, None - artifacts = self.get_current_artifacts() if artifacts is None: return "", None diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 65922f81..18c9532d 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -25,8 +25,7 @@ logger = setup_logging() class ASRProviderBase(ABC): def __init__(self): - self._current_artifacts: Optional[ASRProviderBase.AudioArtifacts] = None - """当前正在处理的音频 artifact""" + pass # 打开音频通道 async def open_audio_channels(self, conn): @@ -215,9 +214,13 @@ class ASRProviderBase(ABC): class AudioArtifacts(NamedTuple): pcm_frames: List[bytes] + """PCM音频帧列表""" pcm_bytes: bytes + """合并后的PCM音频字节数据""" file_path: Optional[str] + """WAV文件路径""" temp_path: Optional[str] + """临时WAV文件路径""" def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]: return self._current_artifacts @@ -282,14 +285,19 @@ class ASRProviderBase(ABC): ): file_path = self.save_audio_to_file(pcm_data, session_id) - self._current_artifacts = ASRProviderBase.AudioArtifacts( - pcm_frames=pcm_data, - pcm_bytes=combined_pcm_data, - file_path=file_path, - temp_path=temp_path, - ) + if len(combined_pcm_data) == 0: + artifacts = None + else: + artifacts = ASRProviderBase.AudioArtifacts( + pcm_frames=pcm_data, + pcm_bytes=combined_pcm_data, + file_path=file_path, + temp_path=temp_path, + ) - text, _ = await self.speech_to_text(opus_data, session_id, audio_format) + text, _ = await self.speech_to_text( + opus_data, session_id, audio_format, artifacts + ) return text, file_path except OSError as e: logger.bind(tag=TAG).error(f"文件操作错误: {e}") @@ -313,9 +321,20 @@ class ASRProviderBase(ABC): @abstractmethod async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, + opus_data: List[bytes], + session_id: str, + audio_format="opus", + artifacts: Optional[AudioArtifacts] = None, ) -> Tuple[Optional[str], Optional[str]]: - """将语音数据转换为文本""" + """将语音数据转换为文本 + + :param opus_data: 输入的Opus音频数据 + :param session_id: 会话ID + :param audio_format: 音频格式，默认"opus" + :param artifacts: 音频工件，包含PCM数据、文件路径等 + :return: 识别结果文本和文件路径（如果有） + """ pass @staticmethod diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index a3ce9e35..016ac0fd 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -232,12 +232,11 @@ class ASRProvider(ASRProviderBase): yield data[offset:data_len], True async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" try: - artifacts = self.get_current_artifacts() if artifacts is None: return "", None diff --git a/main/xiaozhi-server/core/providers/asr/doubao_stream.py b/main/xiaozhi-server/core/providers/asr/doubao_stream.py index 3a726b08..2ffb5b24 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao_stream.py +++ b/main/xiaozhi-server/core/providers/asr/doubao_stream.py @@ -408,7 +408,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"原始响应数据: {res.hex()}") raise - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): result = self.text self.text = "" # 清空text return result, None diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index ee364d51..62ae99e7 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -64,14 +64,13 @@ class ASRProvider(ASRProviderBase): ) async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" retry_count = 0 while retry_count < MAX_RETRIES: try: - artifacts = self.get_current_artifacts() if artifacts is None: return "", None diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index cf169ef1..19b5022d 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -101,7 +101,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """ Convert speech data to text using FunASR. @@ -109,7 +109,6 @@ class ASRProvider(ASRProviderBase): :param session_id: Unique session identifier. :return: Tuple containing recognized text and optional timestamp. """ - artifacts = self.get_current_artifacts() if artifacts is None: return "", None diff --git a/main/xiaozhi-server/core/providers/asr/openai.py b/main/xiaozhi-server/core/providers/asr/openai.py index 7e9ca7f3..7b215589 100644 --- a/main/xiaozhi-server/core/providers/asr/openai.py +++ b/main/xiaozhi-server/core/providers/asr/openai.py @@ -24,10 +24,9 @@ class ASRProvider(ASRProviderBase): def requires_file(self) -> bool: return True - async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None) -> Tuple[Optional[str], Optional[str]]: file_path = None try: - artifacts = self.get_current_artifacts() if artifacts is None: return "", None file_path = artifacts.file_path diff --git a/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py b/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py index aeb74703..21438c1c 100644 --- a/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py +++ b/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py @@ -41,13 +41,12 @@ class ASRProvider(ASRProviderBase): return True async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" temp_file_path = None file_path = None try: - artifacts = self.get_current_artifacts() if artifacts is None: return "", None temp_file_path = artifacts.temp_path diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 44435773..742c3567 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -124,12 +124,11 @@ class ASRProvider(ASRProviderBase): return True async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: - artifacts = self.get_current_artifacts() if artifacts is None: return "", None file_path = artifacts.file_path diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index e3554737..d873bd19 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -32,7 +32,7 @@ class ASRProvider(ASRProviderBase): os.makedirs(self.output_dir, exist_ok=True) async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: @@ -45,7 +45,6 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别") return None, None - artifacts = self.get_current_artifacts() if artifacts is None: return "", None diff --git a/main/xiaozhi-server/core/providers/asr/vosk.py b/main/xiaozhi-server/core/providers/asr/vosk.py index 0f935c5d..77cbf986 100644 --- a/main/xiaozhi-server/core/providers/asr/vosk.py +++ b/main/xiaozhi-server/core/providers/asr/vosk.py @@ -44,7 +44,7 @@ class ASRProvider(ASRProviderBase): raise async def speech_to_text( - self, opus_data: List[bytes], session_id: str, audio_format="opus" + self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" try: @@ -53,7 +53,6 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别") return "", None - artifacts = self.get_current_artifacts() if artifacts is None: return "", None if not artifacts.pcm_bytes: diff --git a/main/xiaozhi-server/core/providers/asr/xunfei_stream.py b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py index b7e7886e..a907472d 100644 --- a/main/xiaozhi-server/core/providers/asr/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py @@ -334,7 +334,7 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug("ASR会话清理完成") - async def speech_to_text(self, opus_data, session_id, audio_format): + async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None): """获取识别结果""" result = self.text self.text = ""