规范asr部分的代码

main/xiaozhi-server/config.yaml

@@ -221,6 +221,8 @@ ASR:
     type: fun_server
     host: 127.0.0.1
     port: 10096
+    is_ssl: true
+    output_dir: tmp/
   SherpaASR:
     type: sherpa_onnx_local
     model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
@@ -253,7 +255,7 @@ ASR:
     type: aliyun
     appkey: 你的阿里云智能语音交互服务项目Appkey
     token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
-    access_key_id: 的阿里云账号access_key_id
+    access_key_id: 你的阿里云账号access_key_id
     access_key_secret: 你的阿里云账号access_key_secret
     output_dir: tmp/
 

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -25,14 +25,14 @@ class AccessToken:
     def _encode_text(text):
         encoded_text = parse.quote_plus(text)
         return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
-    
+
     @staticmethod
     def _encode_dict(dic):
         keys = dic.keys()
         dic_sorted = [(key, dic[key]) for key in sorted(keys)]
         encoded_text = parse.urlencode(dic_sorted)
         return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
-    
+
     @staticmethod
     def create_token(access_key_id, access_key_secret):
         parameters = {'AccessKeyId': access_key_id,
@@ -98,7 +98,7 @@ class ASRProvider(ASRProviderBase):
             # 直接使用预生成的长期token
             self.token = config.get("token")
             self.expire_time = None
-        
+
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
@@ -107,7 +107,7 @@ class ASRProvider(ASRProviderBase):
         """刷新Token并记录过期时间"""
         if self.access_key_id and self.access_key_secret:
             self.token, expire_time_str = AccessToken.create_token(
-                self.access_key_id, 
+                self.access_key_id,
                 self.access_key_secret
             )
             if not expire_time_str:
@@ -121,16 +121,16 @@ class ASRProvider(ASRProviderBase):
                     expire_time = datetime.fromtimestamp(int(expire_str))
                 else:
                     expire_time = datetime.strptime(
-                        expire_str, 
+                        expire_str,
                         "%Y-%m-%dT%H:%M:%SZ"
                     )
                 self.expire_time = expire_time.timestamp() - 60
             except Exception as e:
                 raise ValueError(f"无效的过期时间格式: {expire_str}") from e
-        
+
         else:
             self.expire_time = None
-            
+
         if not self.token:
             raise ValueError("无法获取有效的访问Token")
 
@@ -173,13 +173,12 @@ class ASRProvider(ASRProviderBase):
 
         return pcm_data
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
+        """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        pcm_data = self.decode_opus(opus_data, session_id)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)  # 单声道
             wf.setsampwidth(2)  # 16-bit
@@ -202,7 +201,7 @@ class ASRProvider(ASRProviderBase):
             # 创建连接并发送请求
             conn = http.client.HTTPSConnection(self.host)
             request_url = self._construct_request_url()
-            
+
             loop = asyncio.get_event_loop()
             await loop.run_in_executor(None, lambda: conn.request(
                 method='POST',
@@ -220,7 +219,7 @@ class ASRProvider(ASRProviderBase):
             try:
                 body_json = json.loads(body)
                 status = body_json.get('status')
-                
+
                 if status == 20000000:
                     result = body_json.get('result', '')
                     logger.bind(tag=TAG).debug(f"ASR结果: {result}")
@@ -228,7 +227,7 @@ class ASRProvider(ASRProviderBase):
                 else:
                     logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
                     return None
-                    
+
             except ValueError:
                 logger.bind(tag=TAG).error("响应不是JSON格式")
                 return None
@@ -242,24 +241,27 @@ class ASRProvider(ASRProviderBase):
         if self._is_token_expired():
             logger.warning("Token已过期，正在自动刷新...")
             self._refresh_token()
-            
+
+        file_path = None
         try:
             # 解码Opus为PCM
-            pcm_data_list = self.decode_opus(opus_data, session_id)
+            pcm_data = self.decode_opus(opus_data, session_id)
-            combined_pcm_data = b''.join(pcm_data_list)
+            combined_pcm_data = b''.join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 发送请求并获取文本
             text = await self._send_request(combined_pcm_data)
-            
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            if self.delete_audio_file:
-                os.remove(file_path)
-                logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}")
 
             if text:
-                return text, None
+                return text, file_path
-            return "", None
+
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -103,7 +103,8 @@ class ASRProvider(ASRProviderBase):
 
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
         with wave.open(file_path, "wb") as wf:
@@ -139,8 +140,8 @@ class ASRProvider(ASRProviderBase):
                 "uid": str(uuid.uuid4()),
             },
             "request": {
-                "reqid": reqid, 
+                "reqid": reqid,
-                "show_utterances": False, 
+                "show_utterances": False,
                 "sequence": 1,
                 "boosting_table_name": self.boosting_table_name,
                 "correct_table_name": self.correct_table_name,
@@ -260,6 +261,8 @@ class ASRProvider(ASRProviderBase):
         self, opus_data: List[bytes], session_id: str
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
+
+        file_path = None
         try:
             # 合并所有opus数据包
             pcm_data = self.decode_opus(opus_data, session_id)
@@ -269,7 +272,7 @@ class ASRProvider(ASRProviderBase):
             if self.delete_audio_file:
                 pass
             else:
-                self.save_audio_to_file(pcm_data, session_id)
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 直接使用PCM数据
             # 计算分段大小 (单声道, 16bit, 16kHz采样率)
@@ -283,9 +286,9 @@ class ASRProvider(ASRProviderBase):
                 logger.bind(tag=TAG).debug(
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
                 )
-                return text, None
+                return text, file_path
-            return "", None
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -50,21 +50,12 @@ class ASRProvider(ASRProviderBase):
                 # device="cuda:0",  # 启用GPU加速
             )
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -72,7 +63,7 @@ class ASRProvider(ASRProviderBase):
             wf.writeframes(b"".join(pcm_data))
 
         return file_path
-    
+
     @staticmethod
     def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
 
@@ -100,7 +91,7 @@ class ASRProvider(ASRProviderBase):
             if self.delete_audio_file:
                 pass
             else:
-                self.save_audio_to_file(pcm_data, session_id)
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 语音识别
             start_time = time.time()
@@ -118,13 +109,13 @@ class ASRProvider(ASRProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
 
-        finally:
+        # finally:
-            # 文件清理逻辑
+        #     # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
+        #     if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
+        #         try:
-                    os.remove(file_path)
+        #             os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+        #             logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
+        #         except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
+        #             logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -1,8 +1,11 @@
 from typing import Optional, Tuple, List
 import opuslib_next
 from core.providers.asr.base import ASRProviderBase
-import  ssl
+import os
+import ssl
 import json
+import uuid
+import wave
 import websockets
 from config.logger import setup_logging
 import asyncio
@@ -19,6 +22,7 @@ class ASRProvider(ASRProviderBase):
         self.host = config.get('host', 'localhost')
         self.port = config.get('port', 10095)
         self.is_ssl = config.get('is_ssl', True)
+        self.output_dir = config.get("output_dir")
         self.delete_audio_file = delete_audio_file
         self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}"
         self.ssl_context = ssl.SSLContext() if self.is_ssl else None
@@ -26,25 +30,35 @@ class ASRProvider(ASRProviderBase):
             self.ssl_context.check_hostname = False
             self.ssl_context.verify_mode = ssl.CERT_NONE
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
-        """解码Opus数据并保存为WAV文件"""
+        """PCM数据保存为WAV文件"""
-        pass
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
 
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
-        """将Opus音频数据解码为PCM数据"""        
+        """将Opus音频数据解码为PCM数据"""
         decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
         pcm_data = []
-        
+
         for opus_packet in opus_data:
             try:
                 pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
                 pcm_data.append(pcm_frame)
             except opuslib_next.OpusError as e:
                 logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
+
-        return b"".join(pcm_data)
+        return pcm_data
-    
+
 
 
     async def _receive_responses(self, ws) -> None:
@@ -90,15 +104,15 @@ class ASRProvider(ASRProviderBase):
         })
         await ws.send(config_message)
         logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
-        
+
         # Send PCM data
         await ws.send(pcm_data)
         logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
-        
+
         # Indicate end of speech
         end_message = json.dumps({"is_speaking": False})
         await ws.send(end_message)
-        logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")    
+        logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
 
 
     async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
@@ -108,13 +122,20 @@ class ASRProvider(ASRProviderBase):
         :param session_id: Unique session identifier.
         :return: Tuple containing recognized text and optional timestamp.
         '''
-
+        file_path = None
         pcm_data = self.decode_opus(opus_data)
+        combined_pcm_data = b"".join(pcm_data)
+
+        # 判断是否保存为WAV文件
+        if self.delete_audio_file:
+            pass
+        else:
+            file_path = self.save_audio_to_file(pcm_data, session_id)
 
         async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws:
             try:
                 # Use asyncio to handle WebSocket communication
-                send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id))
+                send_task = asyncio.create_task(self._send_data(ws, combined_pcm_data, session_id))
                 receive_task = asyncio.create_task(self._receive_responses(ws))
 
                 # Gather tasks with error handling
@@ -134,11 +155,11 @@ class ASRProvider(ASRProviderBase):
 
                 # Get the result from the receive task
                 result = receive_task.result()
-                return result, None  # Return the recognized text and timestamp (if any)
+                return result, file_path  # Return the recognized text and timestamp (if any)
 
             except websockets.exceptions.ConnectionClosed as e:
                 logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
-                return "", None
+                return "", file_path
             except Exception as e:
                 logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True)
-                return "", None
+                return "", file_path

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -85,7 +85,8 @@ class ASRProvider(ASRProviderBase):
 
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
         with wave.open(file_path, "wb") as wf:
@@ -164,7 +165,7 @@ class ASRProvider(ASRProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
         finally:
             # 文件清理逻辑
             if self.delete_audio_file and file_path and os.path.exists(file_path):

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -26,40 +26,40 @@ class ASRProvider(ASRProviderBase):
         self.secret_id = config.get("secret_id")
         self.secret_key = config.get("secret_key")
         self.output_dir = config.get("output_dir")
-        
+        self.delete_audio_file = delete_audio_file
+
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-
+        module_name = __name__.split(".")[-1]
-        file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
-        
+
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
             wf.setframerate(16000)
             wf.writeframes(b"".join(pcm_data))
-        
+
         return file_path
 
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
         """将Opus音频数据解码为PCM数据"""
-        import opuslib_next
+
-        
         decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
         pcm_data = []
-        
+
         for opus_packet in opus_data:
             try:
                 pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
                 pcm_data.append(pcm_frame)
             except opuslib_next.OpusError as e:
                 logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
+
-        return b"".join(pcm_data)
+        return pcm_data
 
     async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
@@ -67,23 +67,25 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).warn("音频数据为空！")
             return None, None
 
+        file_path = None
         try:
             # 检查配置是否已设置
             if not self.secret_id or not self.secret_key:
                 logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
-                return None, None
+                return None, file_path
 
             # 将Opus音频数据解码为PCM
             pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件
             if self.delete_audio_file:
                 pass
             else:
                 self.save_audio_to_file(pcm_data, session_id)
-            
+
             # 将音频数据转换为Base64编码
-            base64_audio = base64.b64encode(pcm_data).decode('utf-8')
+            base64_audio = base64.b64encode(combined_pcm_data).decode('utf-8')
 
             # 构建请求体
             request_body = self._build_request_body(base64_audio)
@@ -94,15 +96,15 @@ class ASRProvider(ASRProviderBase):
             # 发送请求
             start_time = time.time()
             result = self._send_request(request_body, timestamp, authorization)
-            
+
             if result:
                 logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
-            
+
-            return result, None
+            return result, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
-            return None, None
+            return None, file_path
 
     def _build_request_body(self, base64_audio: str) -> str:
         """构建请求体"""
@@ -206,26 +208,26 @@ class ASRProvider(ASRProviderBase):
 
         try:
             response = requests.post(self.API_URL, headers=headers, data=request_body)
-            
+
             if not response.ok:
                 raise IOError(f"请求失败: {response.status_code} {response.reason}")
-            
+
             response_json = response.json()
-            
+
             # 检查是否有错误
             if "Response" in response_json and "Error" in response_json["Response"]:
                 error = response_json["Response"]["Error"]
                 error_code = error["Code"]
                 error_message = error["Message"]
                 raise IOError(f"API返回错误: {error_code}: {error_message}")
-            
+
             # 提取识别结果
             if "Response" in response_json and "Result" in response_json["Response"]:
                 return response_json["Response"]["Result"]
             else:
                 logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
                 return ""
-                
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
             return None
@@ -239,7 +241,7 @@ class ASRProvider(ASRProviderBase):
         """计算HMAC-SHA256"""
         if isinstance(key, str):
             key = key.encode('utf-8')
-        
+
         return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest()
 
     def _bytes_to_hex(self, bytes_data: bytes) -> str: