Merge pull request #1125 from xinnan-tech/pcm

客户端上传编码为PCM时，服务端下发PCM格式的音频

main/xiaozhi-server/core/connection.py

@@ -136,6 +136,8 @@ class ConnectionHandler:
             int(self.config.get("close_connection_no_voice_time", 120)) + 60
         )  # 在原来第一道关闭的基础上加60秒，进行二道关闭
 
+        self.audio_format = "opus"
+
     async def handle_connection(self, ws):
         try:
             # 获取并验证headers
@@ -803,9 +805,12 @@ class ConnectionHandler:
                             f"TTS生成：文件路径: {tts_file}"
                         )
                         if os.path.exists(tts_file):
-                            opus_datas, _ = self.tts.audio_to_opus_data(tts_file)
+                            if self.audio_format == "pcm":
+                                audio_datas, _ = self.tts.audio_to_pcm_data(tts_file)
+                            else:
+                                audio_datas, _ = self.tts.audio_to_opus_data(tts_file)
                             # 在这里上报TTS数据（使用文件路径）
-                            enqueue_tts_report(self, 2, text, opus_datas)
+                            enqueue_tts_report(self, 2, text, audio_datas)
                         else:
                             self.logger.bind(tag=TAG).error(
                                 f"TTS出错：文件不存在{tts_file}"
@@ -816,7 +821,7 @@ class ConnectionHandler:
                     self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
                 if not self.client_abort:
                     # 如果没有中途打断就发送语音
-                    self.audio_play_queue.put((opus_datas, text, text_index))
+                    self.audio_play_queue.put((audio_datas, text, text_index))
                 if (
                     self.tts.delete_audio_file
                     and tts_file is not None
@@ -847,13 +852,13 @@ class ConnectionHandler:
             text = None
             try:
                 try:
-                    opus_datas, text, text_index = self.audio_play_queue.get(timeout=1)
+                    audio_datas, text, text_index = self.audio_play_queue.get(timeout=1)
                 except queue.Empty:
                     if self.stop_event.is_set():
                         break
                     continue
                 future = asyncio.run_coroutine_threadsafe(
-                    sendAudioMessage(self, opus_datas, text, text_index), self.loop
+                    sendAudioMessage(self, audio_datas, text, text_index), self.loop
                 )
                 future.result()
             except Exception as e:

main/xiaozhi-server/core/handle/helloHandle.py

@@ -1,5 +1,4 @@
 import json
-from config.logger import setup_logging
 from core.handle.sendAudioHandle import send_stt_message
 from core.utils.util import remove_punctuation_and_length
 import shutil
@@ -20,7 +19,16 @@ WAKEUP_CONFIG = {
 }
 
 
-async def handleHelloMessage(conn):
+async def handleHelloMessage(conn, msg_json):
+    """处理hello消息"""
+    audio_params = msg_json.get("audio_params")
+    if audio_params:
+        format = audio_params.get("format")
+        conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
+        conn.audio_format = format
+        conn.asr.set_audio_format(format)
+        conn.welcome_msg["audio_params"] = audio_params
+
     await conn.websocket.send(json.dumps(conn.welcome_msg))
 
 

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -5,7 +5,7 @@ from core.handle.sendAudioHandle import send_stt_message
 from core.handle.intentHandler import handle_user_intent
 from core.utils.output_counter import check_device_output_limit
 from core.handle.ttsReportHandle import enqueue_tts_report
-from core.providers.tts.base import audio_to_opus_data
+from core.utils.util import audio_to_data
 
 TAG = __name__
 
@@ -111,7 +111,7 @@ async def max_out_size(conn):
     conn.tts_last_text_index = 0
     conn.llm_finish_task = True
     file_path = "config/assets/max_output_size.wav"
-    opus_packets, _ = audio_to_opus_data(file_path)
+    opus_packets, _ = audio_to_data(file_path)
     conn.audio_play_queue.put((opus_packets, text, 0))
     conn.close_after_chat = True
 
@@ -133,7 +133,7 @@ async def check_bind_device(conn):
 
         # 播放提示音
         music_path = "config/assets/bind_code.wav"
-        opus_packets, _ = audio_to_opus_data(music_path)
+        opus_packets, _ = audio_to_data(music_path)
         conn.audio_play_queue.put((opus_packets, text, 0))
 
         # 逐个播放数字
@@ -141,7 +141,7 @@ async def check_bind_device(conn):
             try:
                 digit = conn.bind_code[i]
                 num_path = f"config/assets/bind_code/{digit}.wav"
-                num_packets, _ = audio_to_opus_data(num_path)
+                num_packets, _ = audio_to_data(num_path)
                 conn.audio_play_queue.put((num_packets, None, i + 1))
             except Exception as e:
                 conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
@@ -153,5 +153,5 @@ async def check_bind_device(conn):
         conn.tts_last_text_index = 0
         conn.llm_finish_task = True
         music_path = "config/assets/bind_not_found.wav"
-        opus_packets, _ = audio_to_opus_data(music_path)
+        opus_packets, _ = audio_to_data(music_path)
         conn.audio_play_queue.put((opus_packets, text, 0))

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -115,7 +115,7 @@ async def send_tts_message(conn, state, text=None):
             stop_tts_notify_voice = conn.config.get(
                 "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
             )
-            audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
+            audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
             await sendAudio(conn, audios)
         # 清除服务端讲话状态
         conn.clearSpeakStatus()

main/xiaozhi-server/core/handle/textHandle.py

@@ -20,7 +20,7 @@ async def handleTextMessage(conn, message):
             await conn.websocket.send(message)
             return
         if msg_json["type"] == "hello":
-            await handleHelloMessage(conn)
+            await handleHelloMessage(conn, msg_json)
         elif msg_json["type"] == "abort":
             await handleAbortMessage(conn)
         elif msg_json["type"] == "listen":

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -20,62 +20,75 @@ from core.providers.asr.base import ASRProviderBase
 TAG = __name__
 logger = setup_logging()
 
+
 class AccessToken:
     @staticmethod
     def _encode_text(text):
         encoded_text = parse.quote_plus(text)
-        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
 
     @staticmethod
     def _encode_dict(dic):
         keys = dic.keys()
         dic_sorted = [(key, dic[key]) for key in sorted(keys)]
         encoded_text = parse.urlencode(dic_sorted)
-        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
 
     @staticmethod
     def create_token(access_key_id, access_key_secret):
-        parameters = {'AccessKeyId': access_key_id,
+        parameters = {
-                      'Action': 'CreateToken',
+            "AccessKeyId": access_key_id,
-                      'Format': 'JSON',
+            "Action": "CreateToken",
-                      'RegionId': 'cn-shanghai',
+            "Format": "JSON",
-                      'SignatureMethod': 'HMAC-SHA1',
+            "RegionId": "cn-shanghai",
-                      'SignatureNonce': str(uuid.uuid1()),
+            "SignatureMethod": "HMAC-SHA1",
-                      'SignatureVersion': '1.0',
+            "SignatureNonce": str(uuid.uuid1()),
-                      'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "SignatureVersion": "1.0",
-                      'Version': '2019-02-28'}
+            "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "Version": "2019-02-28",
+        }
         # 构造规范化的请求字符串
         query_string = AccessToken._encode_dict(parameters)
         # print('规范化的请求字符串: %s' % query_string)
         # 构造待签名字符串
-        string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string)
+        string_to_sign = (
+            "GET"
+            + "&"
+            + AccessToken._encode_text("/")
+            + "&"
+            + AccessToken._encode_text(query_string)
+        )
         # print('待签名的字符串: %s' % string_to_sign)
         # 计算签名
-        secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'),
+        secreted_string = hmac.new(
-                                   bytes(string_to_sign, encoding='utf-8'),
+            bytes(access_key_secret + "&", encoding="utf-8"),
-                                   hashlib.sha1).digest()
+            bytes(string_to_sign, encoding="utf-8"),
+            hashlib.sha1,
+        ).digest()
         signature = base64.b64encode(secreted_string)
         # print('签名: %s' % signature)
         # 进行URL编码
         signature = AccessToken._encode_text(signature)
         # print('URL编码后的签名: %s' % signature)
         # 调用服务
-        full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string)
+        full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
+            signature,
+            query_string,
+        )
         # print('url: %s' % full_url)
         # 提交HTTP GET请求
         response = requests.get(full_url)
         if response.ok:
             root_obj = response.json()
-            key = 'Token'
+            key = "Token"
             if key in root_obj:
-                token = root_obj[key]['Id']
+                token = root_obj[key]["Id"]
-                expire_time = root_obj[key]['ExpireTime']
+                expire_time = root_obj[key]["ExpireTime"]
                 return token, expire_time
         # print(response.text)
         return None, None
 
 
-
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
         """阿里云ASR初始化"""
@@ -102,28 +115,23 @@ class ASRProvider(ASRProviderBase):
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-
     def _refresh_token(self):
         """刷新Token并记录过期时间"""
         if self.access_key_id and self.access_key_secret:
             self.token, expire_time_str = AccessToken.create_token(
-                self.access_key_id,
+                self.access_key_id, self.access_key_secret
-                self.access_key_secret
             )
             if not expire_time_str:
                 raise ValueError("无法获取有效的Token过期时间")
 
             try:
-                #统一转换为字符串处理
+                # 统一转换为字符串处理
                 expire_str = str(expire_time_str).strip()
 
                 if expire_str.isdigit():
                     expire_time = datetime.fromtimestamp(int(expire_str))
                 else:
-                    expire_time = datetime.strptime(
+                    expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
-                        expire_str,
-                        "%Y-%m-%dT%H:%M:%SZ"
-                    )
                 self.expire_time = expire_time.timestamp() - 60
             except Exception as e:
                 raise ValueError(f"无效的过期时间格式: {expire_str}") from e
@@ -145,9 +153,12 @@ class ASRProvider(ASRProviderBase):
         #              f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
         #              f"剩余 {remaining:.2f}秒")
         return time.time() > self.expire_time
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(
+            self.output_file,
+            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        )
 
     def _construct_request_url(self) -> str:
         """构造请求URL，包含参数"""
@@ -159,20 +170,6 @@ class ASRProvider(ASRProviderBase):
         request += "&enable_voice_detection=false"
         return request
 
-    def decode_opus(self, opus_data: List[bytes], session_id: str) -> List[bytes]:
-        """将Opus数据解码为PCM"""
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
         module_name = __name__.split(".")[-1]
@@ -183,7 +180,7 @@ class ASRProvider(ASRProviderBase):
             wf.setnchannels(1)  # 单声道
             wf.setsampwidth(2)  # 16-bit
             wf.setframerate(self.sample_rate)
-            wf.writeframes(b''.join(pcm_data))
+            wf.writeframes(b"".join(pcm_data))
 
         logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}")
         return file_path
@@ -193,9 +190,9 @@ class ASRProvider(ASRProviderBase):
         try:
             # 设置HTTP头
             headers = {
-                'X-NLS-Token': self.token,
+                "X-NLS-Token": self.token,
-                'Content-type': 'application/octet-stream',
+                "Content-type": "application/octet-stream",
-                'Content-Length': str(len(pcm_data))
+                "Content-Length": str(len(pcm_data)),
             }
 
             # 创建连接并发送请求
@@ -203,12 +200,12 @@ class ASRProvider(ASRProviderBase):
             request_url = self._construct_request_url()
 
             loop = asyncio.get_event_loop()
-            await loop.run_in_executor(None, lambda: conn.request(
+            await loop.run_in_executor(
-                method='POST',
+                None,
-                url=request_url,
+                lambda: conn.request(
-                body=pcm_data,
+                    method="POST", url=request_url, body=pcm_data, headers=headers
-                headers=headers
+                ),
-            ))
+            )
 
             # 获取响应
             response = await loop.run_in_executor(None, conn.getresponse)
@@ -218,10 +215,10 @@ class ASRProvider(ASRProviderBase):
             # 解析响应
             try:
                 body_json = json.loads(body)
-                status = body_json.get('status')
+                status = body_json.get("status")
 
                 if status == 20000000:
-                    result = body_json.get('result', '')
+                    result = body_json.get("result", "")
                     logger.bind(tag=TAG).debug(f"ASR结果: {result}")
                     return result
                 else:
@@ -236,7 +233,9 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True)
             return None
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if self._is_token_expired():
             logger.warning("Token已过期，正在自动刷新...")
@@ -245,8 +244,11 @@ class ASRProvider(ASRProviderBase):
         file_path = None
         try:
             # 解码Opus为PCM
-            pcm_data = self.decode_opus(opus_data, session_id)
+            if self.audio_format == "pcm":
-            combined_pcm_data = b''.join(pcm_data)
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件
             if self.delete_audio_file:
@@ -264,4 +266,4 @@ class ASRProvider(ASRProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", file_path
+            return "", file_path

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -49,22 +49,6 @@ class ASRProvider(ASRProviderBase):
 
         return file_path
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes]) -> bytes:
-        """将Opus音频数据解码为PCM数据"""
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
     async def speech_to_text(
         self, opus_data: List[bytes], session_id: str
     ) -> Tuple[Optional[str], Optional[str]]:
@@ -81,7 +65,10 @@ class ASRProvider(ASRProviderBase):
                 return None, file_path
 
             # 将Opus音频数据解码为PCM
-            pcm_data = self.decode_opus(opus_data)
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
             combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件

main/xiaozhi-server/core/providers/asr/base.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from typing import Optional, Tuple, List
-
+import opuslib_next
 from config.logger import setup_logging
 
 TAG = __name__
@@ -17,3 +17,23 @@ class ASRProviderBase(ABC):
     async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         pass
+    
+    def set_audio_format(self, format: str) -> None:
+        """设置音频格式"""
+        self.audio_format = format
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes]) -> bytes:
+        """将Opus音频数据解码为PCM数据"""
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -226,21 +226,6 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
             return None
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
     @staticmethod
     def slice_data(data: bytes, chunk_size: int) -> (list, bool):
         """
@@ -265,7 +250,10 @@ class ASRProvider(ASRProviderBase):
         file_path = None
         try:
             # 合并所有opus数据包
-            pcm_data = self.decode_opus(opus_data, session_id)
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
             combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -6,9 +6,7 @@ import io
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
 import uuid
-import opuslib_next
 from core.providers.asr.base import ASRProviderBase
-
 from funasr import AutoModel
 from funasr.utils.postprocess_utils import rich_transcription_postprocess
 
@@ -46,7 +44,7 @@ class ASRProvider(ASRProviderBase):
                 model=self.model_dir,
                 vad_kwargs={"max_single_segment_time": 30000},
                 disable_update=True,
-                hub="hf"
+                hub="hf",
                 # device="cuda:0",  # 启用GPU加速
             )
 
@@ -64,27 +62,18 @@ class ASRProvider(ASRProviderBase):
 
         return file_path
 
-    @staticmethod
+    async def speech_to_text(
-    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
+        self, opus_data: List[bytes], session_id: str
-
+    ) -> Tuple[Optional[str], Optional[str]]:
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
         try:
             # 合并所有opus数据包
-            pcm_data = self.decode_opus(opus_data, session_id)
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+
             combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件
@@ -103,7 +92,9 @@ class ASRProvider(ASRProviderBase):
                 batch_size_s=60,
             )
             text = rich_transcription_postprocess(result[0]["text"])
-            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+            logger.bind(tag=TAG).debug(
+                f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+            )
 
             return text, file_path
 

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -44,23 +44,6 @@ class ASRProvider(ASRProviderBase):
 
         return file_path
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes]) -> bytes:
-        """将Opus音频数据解码为PCM数据"""
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
-
-
     async def _receive_responses(self, ws) -> None:
         '''
         Asynchronous generator to receive messages from the WebSocket.
@@ -123,7 +106,10 @@ class ASRProvider(ASRProviderBase):
         :return: Tuple containing recognized text and optional timestamp.
         '''
         file_path = None
-        pcm_data = self.decode_opus(opus_data)
+        if self.audio_format == "pcm":
+            pcm_data = opus_data
+        else:
+            pcm_data = self.decode_opus(opus_data)
         combined_pcm_data = b"".join(pcm_data)
 
         # 判断是否保存为WAV文件

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -97,21 +97,6 @@ class ASRProvider(ASRProviderBase):
 
         return file_path
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
     def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
         """
         Args:
@@ -144,7 +129,10 @@ class ASRProvider(ASRProviderBase):
         try:
             # 保存音频文件
             start_time = time.time()
-            pcm_data = self.decode_opus(opus_data, session_id)
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
             file_path = self.save_audio_to_file(pcm_data, session_id)
             logger.bind(tag=TAG).debug(
                 f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -45,22 +45,6 @@ class ASRProvider(ASRProviderBase):
 
         return file_path
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes]) -> bytes:
-        """将Opus音频数据解码为PCM数据"""
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
     async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
@@ -75,7 +59,10 @@ class ASRProvider(ASRProviderBase):
                 return None, file_path
 
             # 将Opus音频数据解码为PCM
-            pcm_data = self.decode_opus(opus_data)
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
             combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件

main/xiaozhi-server/core/providers/tts/base.py

@@ -3,7 +3,7 @@ from config.logger import setup_logging
 import os
 from abc import ABC, abstractmethod
 from core.utils.tts import MarkdownCleaner
-from core.utils.util import audio_to_opus_data
+from core.utils.util import audio_to_data
 
 TAG = __name__
 logger = setup_logging()
@@ -53,5 +53,10 @@ class TTSProviderBase(ABC):
     async def text_to_speak(self, text, output_file):
         pass
 
+    def audio_to_pcm_data(self, audio_file_path):
+        """音频文件转换为PCM编码"""
+        return audio_to_data(audio_file_path, is_opus=False)
+
     def audio_to_opus_data(self, audio_file_path):
-        return audio_to_opus_data(audio_file_path)
+        """音频文件转换为Opus编码"""
+        return audio_to_data(audio_file_path, is_opus=True)

main/xiaozhi-server/core/utils/util.py

@@ -862,8 +862,7 @@ def analyze_emotion(text):
     return top_emotions[0]  # 如果都不在优先级列表里，返回第一个
 
 
-def audio_to_opus_data(audio_file_path):
+def audio_to_data(audio_file_path, is_opus=True):
-    """音频文件转换为Opus编码"""
     # 获取文件后缀名
     file_type = os.path.splitext(audio_file_path)[1]
     if file_type:
@@ -889,7 +888,7 @@ def audio_to_opus_data(audio_file_path):
     frame_duration = 60  # 60ms per frame
     frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
 
-    opus_datas = []
+    datas = []
     # 按帧处理所有音频数据（包括最后一帧可能补零）
     for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
         # 获取当前帧的二进制数据
@@ -899,14 +898,17 @@ def audio_to_opus_data(audio_file_path):
         if len(chunk) < frame_size * 2:
             chunk += b"\x00" * (frame_size * 2 - len(chunk))
 
-        # 转换为numpy数组处理
+        if is_opus:
-        np_frame = np.frombuffer(chunk, dtype=np.int16)
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+            # 编码Opus数据
+            frame_data = encoder.encode(np_frame.tobytes(), frame_size)
+        else:
+            frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
 
-        # 编码Opus数据
+        datas.append(frame_data)
-        opus_data = encoder.encode(np_frame.tobytes(), frame_size)
-        opus_datas.append(opus_data)
 
-    return opus_datas, duration
+    return datas, duration
 
 
 def check_vad_update(before_config, new_config):