diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index b49917ba..4c355b5b 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -136,6 +136,8 @@ class ConnectionHandler: int(self.config.get("close_connection_no_voice_time", 120)) + 60 ) # 在原来第一道关闭的基础上加60秒，进行二道关闭 + self.audio_format = "opus" + async def handle_connection(self, ws): try: # 获取并验证headers @@ -803,9 +805,12 @@ class ConnectionHandler: f"TTS生成：文件路径: {tts_file}" ) if os.path.exists(tts_file): - opus_datas, _ = self.tts.audio_to_opus_data(tts_file) + if self.audio_format == "pcm": + audio_datas, _ = self.tts.audio_to_pcm_data(tts_file) + else: + audio_datas, _ = self.tts.audio_to_opus_data(tts_file) # 在这里上报TTS数据（使用文件路径） - enqueue_tts_report(self, 2, text, opus_datas) + enqueue_tts_report(self, 2, text, audio_datas) else: self.logger.bind(tag=TAG).error( f"TTS出错：文件不存在{tts_file}" @@ -816,7 +821,7 @@ class ConnectionHandler: self.logger.bind(tag=TAG).error(f"TTS出错: {e}") if not self.client_abort: # 如果没有中途打断就发送语音 - self.audio_play_queue.put((opus_datas, text, text_index)) + self.audio_play_queue.put((audio_datas, text, text_index)) if ( self.tts.delete_audio_file and tts_file is not None @@ -847,13 +852,13 @@ class ConnectionHandler: text = None try: try: - opus_datas, text, text_index = self.audio_play_queue.get(timeout=1) + audio_datas, text, text_index = self.audio_play_queue.get(timeout=1) except queue.Empty: if self.stop_event.is_set(): break continue future = asyncio.run_coroutine_threadsafe( - sendAudioMessage(self, opus_datas, text, text_index), self.loop + sendAudioMessage(self, audio_datas, text, text_index), self.loop ) future.result() except Exception as e: diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index b23c2f0a..f902a64f 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,5 +1,4 @@ import json -from config.logger import setup_logging from core.handle.sendAudioHandle import send_stt_message from core.utils.util import remove_punctuation_and_length import shutil @@ -20,7 +19,16 @@ WAKEUP_CONFIG = { } -async def handleHelloMessage(conn): +async def handleHelloMessage(conn, msg_json): + """处理hello消息""" + audio_params = msg_json.get("audio_params") + if audio_params: + format = audio_params.get("format") + conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}") + conn.audio_format = format + conn.asr.set_audio_format(format) + conn.welcome_msg["audio_params"] = audio_params + await conn.websocket.send(json.dumps(conn.welcome_msg)) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 383616c3..992ae96f 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -5,7 +5,7 @@ from core.handle.sendAudioHandle import send_stt_message from core.handle.intentHandler import handle_user_intent from core.utils.output_counter import check_device_output_limit from core.handle.ttsReportHandle import enqueue_tts_report -from core.providers.tts.base import audio_to_opus_data +from core.utils.util import audio_to_data TAG = __name__ @@ -111,7 +111,7 @@ async def max_out_size(conn): conn.tts_last_text_index = 0 conn.llm_finish_task = True file_path = "config/assets/max_output_size.wav" - opus_packets, _ = audio_to_opus_data(file_path) + opus_packets, _ = audio_to_data(file_path) conn.audio_play_queue.put((opus_packets, text, 0)) conn.close_after_chat = True @@ -133,7 +133,7 @@ async def check_bind_device(conn): # 播放提示音 music_path = "config/assets/bind_code.wav" - opus_packets, _ = audio_to_opus_data(music_path) + opus_packets, _ = audio_to_data(music_path) conn.audio_play_queue.put((opus_packets, text, 0)) # 逐个播放数字 @@ -141,7 +141,7 @@ async def check_bind_device(conn): try: digit = conn.bind_code[i] num_path = f"config/assets/bind_code/{digit}.wav" - num_packets, _ = audio_to_opus_data(num_path) + num_packets, _ = audio_to_data(num_path) conn.audio_play_queue.put((num_packets, None, i + 1)) except Exception as e: conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") @@ -153,5 +153,5 @@ async def check_bind_device(conn): conn.tts_last_text_index = 0 conn.llm_finish_task = True music_path = "config/assets/bind_not_found.wav" - opus_packets, _ = audio_to_opus_data(music_path) + opus_packets, _ = audio_to_data(music_path) conn.audio_play_queue.put((opus_packets, text, 0)) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 78fe743c..89f16426 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -115,7 +115,7 @@ async def send_tts_message(conn, state, text=None): stop_tts_notify_voice = conn.config.get( "stop_tts_notify_voice", "config/assets/tts_notify.mp3" ) - audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice) + audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice) await sendAudio(conn, audios) # 清除服务端讲话状态 conn.clearSpeakStatus() diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index f446edf8..7ed36122 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -20,7 +20,7 @@ async def handleTextMessage(conn, message): await conn.websocket.send(message) return if msg_json["type"] == "hello": - await handleHelloMessage(conn) + await handleHelloMessage(conn, msg_json) elif msg_json["type"] == "abort": await handleAbortMessage(conn) elif msg_json["type"] == "listen": diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index 250d3a66..14421d7d 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -20,62 +20,75 @@ from core.providers.asr.base import ASRProviderBase TAG = __name__ logger = setup_logging() + class AccessToken: @staticmethod def _encode_text(text): encoded_text = parse.quote_plus(text) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") @staticmethod def _encode_dict(dic): keys = dic.keys() dic_sorted = [(key, dic[key]) for key in sorted(keys)] encoded_text = parse.urlencode(dic_sorted) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") @staticmethod def create_token(access_key_id, access_key_secret): - parameters = {'AccessKeyId': access_key_id, - 'Action': 'CreateToken', - 'Format': 'JSON', - 'RegionId': 'cn-shanghai', - 'SignatureMethod': 'HMAC-SHA1', - 'SignatureNonce': str(uuid.uuid1()), - 'SignatureVersion': '1.0', - 'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - 'Version': '2019-02-28'} + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } # 构造规范化的请求字符串 query_string = AccessToken._encode_dict(parameters) # print('规范化的请求字符串: %s' % query_string) # 构造待签名字符串 - string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string) + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) # print('待签名的字符串: %s' % string_to_sign) # 计算签名 - secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'), - bytes(string_to_sign, encoding='utf-8'), - hashlib.sha1).digest() + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() signature = base64.b64encode(secreted_string) # print('签名: %s' % signature) # 进行URL编码 signature = AccessToken._encode_text(signature) # print('URL编码后的签名: %s' % signature) # 调用服务 - full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) # print('url: %s' % full_url) # 提交HTTP GET请求 response = requests.get(full_url) if response.ok: root_obj = response.json() - key = 'Token' + key = "Token" if key in root_obj: - token = root_obj[key]['Id'] - expire_time = root_obj[key]['ExpireTime'] + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] return token, expire_time # print(response.text) return None, None - class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): """阿里云ASR初始化""" @@ -102,28 +115,23 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def _refresh_token(self): """刷新Token并记录过期时间""" if self.access_key_id and self.access_key_secret: self.token, expire_time_str = AccessToken.create_token( - self.access_key_id, - self.access_key_secret + self.access_key_id, self.access_key_secret ) if not expire_time_str: raise ValueError("无法获取有效的Token过期时间") try: - #统一转换为字符串处理 + # 统一转换为字符串处理 expire_str = str(expire_time_str).strip() if expire_str.isdigit(): expire_time = datetime.fromtimestamp(int(expire_str)) else: - expire_time = datetime.strptime( - expire_str, - "%Y-%m-%dT%H:%M:%SZ" - ) + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") self.expire_time = expire_time.timestamp() - 60 except Exception as e: raise ValueError(f"无效的过期时间格式: {expire_str}") from e @@ -145,9 +153,12 @@ class ASRProvider(ASRProviderBase): # f"过期时间 {datetime.fromtimestamp(self.expire_time)} | " # f"剩余 {remaining:.2f}秒") return time.time() > self.expire_time - def generate_filename(self, extension=".wav"): - return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + def generate_filename(self, extension=".wav"): + return os.path.join( + self.output_file, + f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) def _construct_request_url(self) -> str: """构造请求URL，包含参数""" @@ -159,20 +170,6 @@ class ASRProvider(ASRProviderBase): request += "&enable_voice_detection=false" return request - def decode_opus(self, opus_data: List[bytes], session_id: str) -> List[bytes]: - """将Opus数据解码为PCM""" - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" module_name = __name__.split(".")[-1] @@ -183,7 +180,7 @@ class ASRProvider(ASRProviderBase): wf.setnchannels(1) # 单声道 wf.setsampwidth(2) # 16-bit wf.setframerate(self.sample_rate) - wf.writeframes(b''.join(pcm_data)) + wf.writeframes(b"".join(pcm_data)) logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}") return file_path @@ -193,9 +190,9 @@ class ASRProvider(ASRProviderBase): try: # 设置HTTP头 headers = { - 'X-NLS-Token': self.token, - 'Content-type': 'application/octet-stream', - 'Content-Length': str(len(pcm_data)) + "X-NLS-Token": self.token, + "Content-type": "application/octet-stream", + "Content-Length": str(len(pcm_data)), } # 创建连接并发送请求 @@ -203,12 +200,12 @@ class ASRProvider(ASRProviderBase): request_url = self._construct_request_url() loop = asyncio.get_event_loop() - await loop.run_in_executor(None, lambda: conn.request( - method='POST', - url=request_url, - body=pcm_data, - headers=headers - )) + await loop.run_in_executor( + None, + lambda: conn.request( + method="POST", url=request_url, body=pcm_data, headers=headers + ), + ) # 获取响应 response = await loop.run_in_executor(None, conn.getresponse) @@ -218,10 +215,10 @@ class ASRProvider(ASRProviderBase): # 解析响应 try: body_json = json.loads(body) - status = body_json.get('status') + status = body_json.get("status") if status == 20000000: - result = body_json.get('result', '') + result = body_json.get("result", "") logger.bind(tag=TAG).debug(f"ASR结果: {result}") return result else: @@ -236,7 +233,9 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True) return None - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if self._is_token_expired(): logger.warning("Token已过期，正在自动刷新...") @@ -245,8 +244,11 @@ class ASRProvider(ASRProviderBase): file_path = None try: # 解码Opus为PCM - pcm_data = self.decode_opus(opus_data, session_id) - combined_pcm_data = b''.join(pcm_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 if self.delete_audio_file: @@ -264,4 +266,4 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", file_path \ No newline at end of file + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/baidu.py b/main/xiaozhi-server/core/providers/asr/baidu.py index 94996a59..58b26375 100644 --- a/main/xiaozhi-server/core/providers/asr/baidu.py +++ b/main/xiaozhi-server/core/providers/asr/baidu.py @@ -49,22 +49,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - async def speech_to_text( self, opus_data: List[bytes], session_id: str ) -> Tuple[Optional[str], Optional[str]]: @@ -81,7 +65,10 @@ class ASRProvider(ASRProviderBase): return None, file_path # 将Opus音频数据解码为PCM - pcm_data = self.decode_opus(opus_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 6d9c2c90..61c1b754 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from typing import Optional, Tuple, List - +import opuslib_next from config.logger import setup_logging TAG = __name__ @@ -17,3 +17,23 @@ class ASRProviderBase(ABC): async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" pass + + def set_audio_format(self, format: str) -> None: + """设置音频格式""" + self.audio_format = format + + @staticmethod + def decode_opus(opus_data: List[bytes]) -> bytes: + """将Opus音频数据解码为PCM数据""" + + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return pcm_data \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index ce7e05ee..0775c2ab 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -226,21 +226,6 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True) return None - @staticmethod - def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - @staticmethod def slice_data(data: bytes, chunk_size: int) -> (list, bool): """ @@ -265,7 +250,10 @@ class ASRProvider(ASRProviderBase): file_path = None try: # 合并所有opus数据包 - pcm_data = self.decode_opus(opus_data, session_id) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 3724ec2e..107417b9 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -6,9 +6,7 @@ import io from config.logger import setup_logging from typing import Optional, Tuple, List import uuid -import opuslib_next from core.providers.asr.base import ASRProviderBase - from funasr import AutoModel from funasr.utils.postprocess_utils import rich_transcription_postprocess @@ -46,7 +44,7 @@ class ASRProvider(ASRProviderBase): model=self.model_dir, vad_kwargs={"max_single_segment_time": 30000}, disable_update=True, - hub="hf" + hub="hf", # device="cuda:0", # 启用GPU加速 ) @@ -64,27 +62,18 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: # 合并所有opus数据包 - pcm_data = self.decode_opus(opus_data, session_id) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 @@ -103,7 +92,9 @@ class ASRProvider(ASRProviderBase): batch_size_s=60, ) text = rich_transcription_postprocess(result[0]["text"]) - logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) return text, file_path diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index 501d43bd..f20d5be7 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -44,23 +44,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - - - async def _receive_responses(self, ws) -> None: ''' Asynchronous generator to receive messages from the WebSocket. @@ -123,7 +106,10 @@ class ASRProvider(ASRProviderBase): :return: Tuple containing recognized text and optional timestamp. ''' file_path = None - pcm_data = self.decode_opus(opus_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 3e720210..2b86f6d7 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -97,21 +97,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]: """ Args: @@ -144,7 +129,10 @@ class ASRProvider(ASRProviderBase): try: # 保存音频文件 start_time = time.time() - pcm_data = self.decode_opus(opus_data, session_id) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) file_path = self.save_audio_to_file(pcm_data, session_id) logger.bind(tag=TAG).debug( f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index dbdcbe26..2aa7bec5 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -45,22 +45,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: @@ -75,7 +59,10 @@ class ASRProvider(ASRProviderBase): return None, file_path # 将Opus音频数据解码为PCM - pcm_data = self.decode_opus(opus_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 4cf37edb..f2632c02 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -3,7 +3,7 @@ from config.logger import setup_logging import os from abc import ABC, abstractmethod from core.utils.tts import MarkdownCleaner -from core.utils.util import audio_to_opus_data +from core.utils.util import audio_to_data TAG = __name__ logger = setup_logging() @@ -53,5 +53,10 @@ class TTSProviderBase(ABC): async def text_to_speak(self, text, output_file): pass + def audio_to_pcm_data(self, audio_file_path): + """音频文件转换为PCM编码""" + return audio_to_data(audio_file_path, is_opus=False) + def audio_to_opus_data(self, audio_file_path): - return audio_to_opus_data(audio_file_path) + """音频文件转换为Opus编码""" + return audio_to_data(audio_file_path, is_opus=True) diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 2ed67b19..effbf38e 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -862,8 +862,7 @@ def analyze_emotion(text): return top_emotions[0] # 如果都不在优先级列表里，返回第一个 -def audio_to_opus_data(audio_file_path): - """音频文件转换为Opus编码""" +def audio_to_data(audio_file_path, is_opus=True): # 获取文件后缀名 file_type = os.path.splitext(audio_file_path)[1] if file_type: @@ -889,7 +888,7 @@ def audio_to_opus_data(audio_file_path): frame_duration = 60 # 60ms per frame frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame - opus_datas = [] + datas = [] # 按帧处理所有音频数据（包括最后一帧可能补零） for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample # 获取当前帧的二进制数据 @@ -899,14 +898,17 @@ def audio_to_opus_data(audio_file_path): if len(chunk) < frame_size * 2: chunk += b"\x00" * (frame_size * 2 - len(chunk)) - # 转换为numpy数组处理 - np_frame = np.frombuffer(chunk, dtype=np.int16) + if is_opus: + # 转换为numpy数组处理 + np_frame = np.frombuffer(chunk, dtype=np.int16) + # 编码Opus数据 + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + else: + frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) - # 编码Opus数据 - opus_data = encoder.encode(np_frame.tobytes(), frame_size) - opus_datas.append(opus_data) + datas.append(frame_data) - return opus_datas, duration + return datas, duration def check_vad_update(before_config, new_config):