Merge pull request #1091 from xinnan-tech/asr_code_clear

规范asr部分的代码，添加百度asr支持

docs/Deployment_all.md

@@ -213,7 +213,7 @@ CREATE DATABASE xiaozhi_esp32_server CHARACTER SET utf8mb4 COLLATE utf8mb4_unico
 如果还没有MySQL，你可以通过docker安装mysql
 
 ```
-docker run --name xiaozhi-esp32-server-db -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -e MYSQL_DATABASE=xiaozhi_esp32_server -e MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" -d mysql:latest
+docker run --name xiaozhi-esp32-server-db -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -e MYSQL_DATABASE=xiaozhi_esp32_server -e MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" -e TZ=Asia/Shanghai -d mysql:latest
 ```
 
 ## 2.安装redis

main/xiaozhi-server/config.yaml

@@ -221,6 +221,8 @@ ASR:
     type: fun_server
     host: 127.0.0.1
     port: 10096
+    is_ssl: true
+    output_dir: tmp/
   SherpaASR:
     type: sherpa_onnx_local
     model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
@@ -253,9 +255,19 @@ ASR:
     type: aliyun
     appkey: 你的阿里云智能语音交互服务项目Appkey
     token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
-    access_key_id: 的阿里云账号access_key_id
+    access_key_id: 你的阿里云账号access_key_id
     access_key_secret: 你的阿里云账号access_key_secret
     output_dir: tmp/
+  BaiduASR:
+    # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list
+    # 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list
+    type: baidu
+    app_id: 你的百度语音技术AppID
+    api_key: 你的百度语音技术APIKey
+    secret_key: 你的百度语音技术SecretKey
+    # 语言参数，1537为普通话，具体参考：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b
+    dev_pid: 1537
+    output_dir: tmp/
 
 VAD:
   SileroVAD:

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -173,13 +173,12 @@ class ASRProvider(ASRProviderBase):
 
         return pcm_data
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        pcm_data = self.decode_opus(opus_data, session_id)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)  # 单声道
             wf.setsampwidth(2)  # 16-bit
@@ -243,23 +242,26 @@ class ASRProvider(ASRProviderBase):
             logger.warning("Token已过期，正在自动刷新...")
             self._refresh_token()
 
+        file_path = None
         try:
             # 解码Opus为PCM
-            pcm_data_list = self.decode_opus(opus_data, session_id)
-            combined_pcm_data = b''.join(pcm_data_list)
+            pcm_data = self.decode_opus(opus_data, session_id)
+            combined_pcm_data = b''.join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 发送请求并获取文本
             text = await self._send_request(combined_pcm_data)
 
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            if self.delete_audio_file:
-                os.remove(file_path)
-                logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}")
-
             if text:
-                return text, None
-            return "", None
+                return text, file_path
+
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -0,0 +1,118 @@
+import base64
+import hashlib
+import hmac
+import json
+import time
+from datetime import datetime, timezone
+import os
+import uuid
+from typing import Optional, Tuple, List
+import wave
+import opuslib_next
+
+from aip import AipSpeech
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.secret_key = config.get("secret_key")
+
+        dev_pid = config.get("dev_pid", "1537")
+        self.dev_pid = int(dev_pid) if dev_pid else 1537
+
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        self.client = AipSpeech(str(self.app_id), self.api_key, self.secret_key)
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes]) -> bytes:
+        """将Opus音频数据解码为PCM数据"""
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if not opus_data:
+            logger.bind(tag=TAG).warn("音频数据为空！")
+            return None, None
+
+        file_path = None
+        try:
+            # 检查配置是否已设置
+            if not self.app_id or not self.api_key or not self.secret_key:
+                logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
+                return None, file_path
+
+            # 将Opus音频数据解码为PCM
+            pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                self.save_audio_to_file(pcm_data, session_id)
+
+            start_time = time.time()
+            # 识别本地文件
+            result = self.client.asr(
+                combined_pcm_data,
+                "pcm",
+                16000,
+                {
+                    "dev_pid": str(self.dev_pid),
+                },
+            )
+
+            if result and result["err_no"] == 0:
+                logger.bind(tag=TAG).debug(
+                    f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+                result = result["result"][0]
+                return result, file_path
+            else:
+                raise Exception(
+                    f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
+                )
+                return None, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
+            return None, file_path

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -103,7 +103,8 @@ class ASRProvider(ASRProviderBase):
 
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
         with wave.open(file_path, "wb") as wf:
@@ -260,6 +261,8 @@ class ASRProvider(ASRProviderBase):
         self, opus_data: List[bytes], session_id: str
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
+
+        file_path = None
         try:
             # 合并所有opus数据包
             pcm_data = self.decode_opus(opus_data, session_id)
@@ -269,7 +272,7 @@ class ASRProvider(ASRProviderBase):
             if self.delete_audio_file:
                 pass
             else:
-                self.save_audio_to_file(pcm_data, session_id)
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 直接使用PCM数据
             # 计算分段大小 (单声道, 16bit, 16kHz采样率)
@@ -283,9 +286,9 @@ class ASRProvider(ASRProviderBase):
                 logger.bind(tag=TAG).debug(
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
                 )
-                return text, None
-            return "", None
+                return text, file_path
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -50,21 +50,12 @@ class ASRProvider(ASRProviderBase):
                 # device="cuda:0",  # 启用GPU加速
             )
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -100,7 +91,7 @@ class ASRProvider(ASRProviderBase):
             if self.delete_audio_file:
                 pass
             else:
-                self.save_audio_to_file(pcm_data, session_id)
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 语音识别
             start_time = time.time()
@@ -118,13 +109,13 @@ class ASRProvider(ASRProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
 
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
+        # finally:
+        #     # 文件清理逻辑
+        #     if self.delete_audio_file and file_path and os.path.exists(file_path):
+        #         try:
+        #             os.remove(file_path)
+        #             logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+        #         except Exception as e:
+        #             logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -1,8 +1,11 @@
 from typing import Optional, Tuple, List
 import opuslib_next
 from core.providers.asr.base import ASRProviderBase
-import  ssl
+import os
+import ssl
 import json
+import uuid
+import wave
 import websockets
 from config.logger import setup_logging
 import asyncio
@@ -19,6 +22,7 @@ class ASRProvider(ASRProviderBase):
         self.host = config.get('host', 'localhost')
         self.port = config.get('port', 10095)
         self.is_ssl = config.get('is_ssl', True)
+        self.output_dir = config.get("output_dir")
         self.delete_audio_file = delete_audio_file
         self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}"
         self.ssl_context = ssl.SSLContext() if self.is_ssl else None
@@ -26,9 +30,19 @@ class ASRProvider(ASRProviderBase):
             self.ssl_context.check_hostname = False
             self.ssl_context.verify_mode = ssl.CERT_NONE
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """解码Opus数据并保存为WAV文件"""
-        pass
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
 
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
@@ -43,7 +57,7 @@ class ASRProvider(ASRProviderBase):
             except opuslib_next.OpusError as e:
                 logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
 
-        return b"".join(pcm_data)
+        return pcm_data
 
 
 
@@ -108,13 +122,20 @@ class ASRProvider(ASRProviderBase):
         :param session_id: Unique session identifier.
         :return: Tuple containing recognized text and optional timestamp.
         '''
-
+        file_path = None
         pcm_data = self.decode_opus(opus_data)
+        combined_pcm_data = b"".join(pcm_data)
+
+        # 判断是否保存为WAV文件
+        if self.delete_audio_file:
+            pass
+        else:
+            file_path = self.save_audio_to_file(pcm_data, session_id)
 
         async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws:
             try:
                 # Use asyncio to handle WebSocket communication
-                send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id))
+                send_task = asyncio.create_task(self._send_data(ws, combined_pcm_data, session_id))
                 receive_task = asyncio.create_task(self._receive_responses(ws))
 
                 # Gather tasks with error handling
@@ -134,11 +155,11 @@ class ASRProvider(ASRProviderBase):
 
                 # Get the result from the receive task
                 result = receive_task.result()
-                return result, None  # Return the recognized text and timestamp (if any)
+                return result, file_path  # Return the recognized text and timestamp (if any)
 
             except websockets.exceptions.ConnectionClosed as e:
                 logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
-                return "", None
+                return "", file_path
             except Exception as e:
                 logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True)
-                return "", None
+                return "", file_path

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -85,7 +85,8 @@ class ASRProvider(ASRProviderBase):
 
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
         with wave.open(file_path, "wb") as wf:
@@ -164,7 +165,7 @@ class ASRProvider(ASRProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
         finally:
             # 文件清理逻辑
             if self.delete_audio_file and file_path and os.path.exists(file_path):

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -26,14 +26,15 @@ class ASRProvider(ASRProviderBase):
         self.secret_id = config.get("secret_id")
         self.secret_key = config.get("secret_key")
         self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
 
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-
-        file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
         with wave.open(file_path, "wb") as wf:
@@ -47,7 +48,6 @@ class ASRProvider(ASRProviderBase):
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
         """将Opus音频数据解码为PCM数据"""
-        import opuslib_next
 
         decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
         pcm_data = []
@@ -59,7 +59,7 @@ class ASRProvider(ASRProviderBase):
             except opuslib_next.OpusError as e:
                 logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
 
-        return b"".join(pcm_data)
+        return pcm_data
 
     async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
@@ -67,14 +67,16 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).warn("音频数据为空！")
             return None, None
 
+        file_path = None
         try:
             # 检查配置是否已设置
             if not self.secret_id or not self.secret_key:
                 logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
-                return None, None
+                return None, file_path
 
             # 将Opus音频数据解码为PCM
             pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件
             if self.delete_audio_file:
@@ -83,7 +85,7 @@ class ASRProvider(ASRProviderBase):
                 self.save_audio_to_file(pcm_data, session_id)
 
             # 将音频数据转换为Base64编码
-            base64_audio = base64.b64encode(pcm_data).decode('utf-8')
+            base64_audio = base64.b64encode(combined_pcm_data).decode('utf-8')
 
             # 构建请求体
             request_body = self._build_request_body(base64_audio)
@@ -98,11 +100,11 @@ class ASRProvider(ASRProviderBase):
             if result:
                 logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
 
-            return result, None
+            return result, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
-            return None, None
+            return None, file_path
 
     def _build_request_body(self, base64_audio: str) -> str:
         """构建请求体"""

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -51,7 +51,7 @@ class TTSProvider(TTSProviderBase):
         request_json = {
             "app": {
                 "appid": f"{self.appid}",
-                "token": "access_token",
+                "token": self.access_token,
                 "cluster": self.cluster,
             },
             "user": {"uid": "1"},

main/xiaozhi-server/requirements.txt

@@ -26,3 +26,5 @@ mcp==1.4.1
 cnlunar==0.2.0
 PySocks==1.7.1
 dashscope==1.23.1
+baidu-aip==4.16.13
+chardet==5.2.0