From fd1dd39463474803dc0a4ae29b4daaf8a1b90166 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 14 Jan 2026 17:54:50 +0800 Subject: [PATCH] =?UTF-8?q?update:=20=E8=87=AA=E5=AE=9A=E4=B9=89=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E8=BE=93=E5=87=BA=E9=9F=B3=E9=A2=91=E9=87=87=E6=A0=B7?= =?UTF-8?q?=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202601141645.sql | 31 ++++++++++++ .../db/changelog/db.changelog-master.yaml | 7 +++ main/xiaozhi-server/config.yaml | 9 +--- main/xiaozhi-server/core/connection.py | 5 ++ .../xiaozhi-server/core/handle/helloHandle.py | 3 +- .../core/providers/tts/alibl_stream.py | 11 +---- .../core/providers/tts/aliyun.py | 4 +- .../core/providers/tts/aliyun_stream.py | 13 +---- .../xiaozhi-server/core/providers/tts/base.py | 19 +++++-- .../providers/tts/huoshan_double_stream.py | 7 +-- .../core/providers/tts/index_stream.py | 14 ++++++ .../core/providers/tts/linkerai.py | 11 ++--- .../core/providers/tts/minimax_httpstream.py | 12 +++-- .../core/providers/tts/paddle_speech.py | 5 +- .../core/providers/tts/siliconflow.py | 1 - .../core/providers/tts/xunfei_stream.py | 10 +--- main/xiaozhi-server/core/utils/util.py | 49 +++++++++++++------ 17 files changed, 129 insertions(+), 82 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202601141645.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202601141645.sql b/main/manager-api/src/main/resources/db/changelog/202601141645.sql new file mode 100644 index 00000000..8b0fca88 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202601141645.sql @@ -0,0 +1,31 @@ +-- 批量清理 ai_model_provider 中的 sample_rate 字段定义 +UPDATE `ai_model_provider` ap +JOIN ( + SELECT + id, + JSON_ARRAYAGG( + JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t) + ) AS new_fields + FROM `ai_model_provider`, + JSON_TABLE(`fields`, '$[*]' COLUMNS ( + k VARCHAR(50) PATH '$.key', + l VARCHAR(100) PATH '$.label', + t VARCHAR(20) PATH '$.type' + )) AS jt + WHERE `model_type` = 'TTS' + AND jt.k != 'sample_rate' + GROUP BY id +) filtered ON ap.id = filtered.id +SET ap.fields = filtered.new_fields; + +-- 清理 config_json 顶层的 sample_rate +UPDATE `ai_model_config` +SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate') +WHERE `model_type` = 'TTS' + AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL; + +-- 清理Minimax流式TTS的sample_rate参数（位于audio_setting内部） +UPDATE `ai_model_config` SET +`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate')) +WHERE `id` = 'TTS_MinimaxStreamTTS' +AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 9a8b23cc..131aeb5c 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -487,3 +487,10 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202601051433.sql + - changeSet: + id: 202601141645 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202601141645.sql diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a3c54c17..90d1b3b3 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -89,7 +89,8 @@ xiaozhi: transport: websocket audio_params: format: opus - sample_rate: 16000 + # Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000] + sample_rate: 24000 channels: 1 frame_duration: 60 @@ -837,7 +838,6 @@ TTS: # - "处理/(chu3)(li3)" # - "危险/dangerous" # audio_setting: - # sample_rate: 24000 # bitrate: 128000 # format: "mp3" # channel: 1 @@ -865,7 +865,6 @@ TTS: # 以下可不用设置，使用默认设置 # format: wav - # sample_rate: 16000 # volume: 50 # speech_rate: 0 # pitch_rate: 0 @@ -889,7 +888,6 @@ TTS: host: nls-gateway-cn-beijing.aliyuncs.com # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3 - # sample_rate: 16000 # 采样率：8000、16000、24000 # volume: 50 # 音量：0-100 # speech_rate: 0 # 语速：-500到500 # pitch_rate: 0 # 语调：-500到500 @@ -1004,7 +1002,6 @@ TTS: protocol: websocket # protocol choices = ['websocket', 'http'] url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址，指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming，http默认http://127.0.0.1:8090/paddlespeech/tts] spk_id: 0 # 发音人 ID，0 通常表示默认的发音人 - sample_rate: 24000 # 采样率 [websocket默认24000，http默认0 自动选择] speed: 1.0 # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢 volume: 1.0 # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小 save_path: # 保存路径 @@ -1028,7 +1025,6 @@ TTS: output_dir: tmp/ # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3、opus - # sample_rate: 24000 # 采样率：16000, 24000, 48000 # volume: 50 # 音量：0-100 # rate: 1 # 语速：0.5~2 # pitch: 1 # 语调：0.5~2 @@ -1050,7 +1046,6 @@ TTS: # stop_split: 0 # 关闭服务端拆句 不关闭：0，关闭：1 # remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0 # format: raw # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb - # sample_rate: 24000 # 采样率：16000, 8000, 24000 # volume: 50 # 音量：0-100 # speed: 50 # 语速：0-100 # pitch: 50 # 语调：0-100 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 449fbd2d..356cbb64 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -85,6 +85,7 @@ class ConnectionHandler: self.max_output_size = 0 self.chat_history_conf = 0 self.audio_format = "opus" + self.sample_rate = 16000 # 默认采样率，从客户端 hello 消息中动态更新 # 客户端状态相关 self.client_abort = False @@ -206,6 +207,10 @@ class ConnectionHandler: self.welcome_msg = self.config["xiaozhi"] self.welcome_msg["session_id"] = self.session_id + # 从配置中读取采样率 + self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"] + self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}") + # 在后台初始化配置和组件（完全不阻塞主循环） asyncio.create_task(self._background_initialize()) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 129b0aa7..740d9fe7 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -145,7 +145,8 @@ async def wakeupWordsResponse(conn): # 获取当前音色 voice = getattr(conn.tts, "voice", "default") - wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000) + # 使用链接的sample_rate + wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate) file_path = wakeup_words_config.generate_file_path(voice) with open(file_path, "wb") as f: f.write(wav_bytes) diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index 42658008..c636d0cb 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase): # 音频参数配置 self.format = config.get("format", "pcm") - sample_rate = config.get("sample_rate", "24000") - self.sample_rate = int(sample_rate) if sample_rate else 24000 volume = config.get("volume", "50") self.volume = int(volume) if volume else 50 @@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase): "X-DashScope-DataInspection": "enable", } - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=self.sample_rate, channels=1, frame_size_ms=60 - ) - async def _ensure_connection(self): """确保WebSocket连接可用，支持60秒内连接复用""" try: @@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase): "text_type": "PlainText", "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "rate": self.rate, "pitch": self.pitch, @@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase): "text_type": "PlainText", "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "rate": self.rate, "pitch": self.pitch, diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index 1a4cab88..e97c13e4 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase): self.appkey = config.get("appkey") self.format = config.get("format", "wav") self.audio_file_type = config.get("format", "wav") - sample_rate = config.get("sample_rate", "16000") - self.sample_rate = int(sample_rate) if sample_rate else 16000 if config.get("private_voice"): self.voice = config.get("private_voice") @@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase): "token": self.token, "text": text, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "voice": self.voice, "volume": self.volume, "speech_rate": self.speech_rate, diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 52284466..6026eafa 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase): self.format = config.get("format", "pcm") self.audio_file_type = config.get("format", "pcm") - # 采样率配置 - sample_rate = config.get("sample_rate", "16000") - self.sample_rate = int(sample_rate) if sample_rate else 16000 - # 音色配置 - CosyVoice大模型音色 if config.get("private_voice"): self.voice = config.get("private_voice") @@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase): # 专属tts设置 self.task_id = uuid.uuid4().hex - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, channels=1, frame_size_ms=60 - ) - # Token管理 if self.access_key_id and self.access_key_secret: self._refresh_token() @@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase): "payload": { "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, @@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase): "payload": { "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 018215d5..2ad36e05 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -1,17 +1,18 @@ import os import re -import time import uuid import queue import asyncio import threading import traceback + from core.utils import p3 from datetime import datetime from core.utils import textUtils from typing import Callable, Any from abc import ABC, abstractmethod from config.logger import setup_logging +from core.utils import opus_encoder_utils from core.utils.tts import MarkdownCleaner from core.utils.output_counter import add_device_output from core.handle.reportHandle import enqueue_tts_report @@ -97,6 +98,8 @@ class TTSProviderBase(ABC): file_type=self.audio_file_type, is_opus=True, callback=opus_handler, + sample_rate=self.conn.sample_rate, + opus_encoder=self.opus_encoder, ) break else: @@ -158,7 +161,8 @@ class TTSProviderBase(ABC): audio_bytes, file_type=self.audio_file_type, is_opus=True, - callback=lambda data: audio_datas.append(data) + callback=lambda data: audio_datas.append(data), + sample_rate=self.conn.sample_rate, ) return audio_datas else: @@ -214,13 +218,13 @@ class TTSProviderBase(ABC): self, audio_file_path, callback: Callable[[Any], Any] = None ): """音频文件转换为PCM编码""" - return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback) + return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None) def audio_to_opus_data_stream( self, audio_file_path, callback: Callable[[Any], Any] = None ): """音频文件转换为Opus编码""" - return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback) + return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder) def tts_one_sentence( self, @@ -252,6 +256,13 @@ class TTSProviderBase(ABC): async def open_audio_channels(self, conn): self.conn = conn + + # 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理） + if not hasattr(self, 'opus_encoder') or self.opus_encoder is None: + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=conn.sample_rate, channels=1, frame_size_ms=60 + ) + # tts 消化线程 self.tts_priority_thread = threading.Thread( target=self.tts_text_priority_thread, daemon=True diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index d7829e35..2a501c85 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -171,9 +171,7 @@ class TTSProvider(TTSProviderBase): enable_ws_reuse_value = config.get("enable_ws_reuse", True) self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True self.tts_text = "" - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, channels=1, frame_size_ms=60 - ) + model_key_msg = check_model_key("TTS", self.access_token) if model_key_msg: logger.bind(tag=TAG).error(model_key_msg) @@ -646,11 +644,10 @@ class TTSProvider(TTSProviderBase): text="", speaker="", audio_format="pcm", - audio_sample_rate=16000, ): audio_params = { "format": audio_format, - "sample_rate": audio_sample_rate, + "sample_rate": self.conn.sample_rate, "speech_rate": self.speech_rate, "loudness_rate": self.loudness_rate } diff --git a/main/xiaozhi-server/core/providers/tts/index_stream.py b/main/xiaozhi-server/core/providers/tts/index_stream.py index 186e1fc8..72d2d68d 100644 --- a/main/xiaozhi-server/core/providers/tts/index_stream.py +++ b/main/xiaozhi-server/core/providers/tts/index_stream.py @@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).error(f"TTS请求异常: {e}") self.tts_audio_queue.put((SentenceType.LAST, [], None)) + def audio_to_pcm_data_stream( + self, audio_file_path, callback=None + ): + """音频文件转换为PCM编码，使用24kHz采样率""" + from core.utils.util import audio_to_data_stream + return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None) + + def audio_to_opus_data_stream( + self, audio_file_path, callback=None + ): + """音频文件转换为Opus编码，使用24kHz采样率和自己的编码器""" + from core.utils.util import audio_to_data_stream + return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder) + async def close(self): """资源清理""" await super().close() diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 9b5e946f..7de5f3cd 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase): self.audio_format = "pcm" self.before_stop_play_files = [] - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, channels=1, frame_size_ms=60 - ) - # PCM缓冲区 self.pcm_buffer = bytearray() @@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase): "spk_id": self.voice, "frame_durition": 60, "stream": "true", - "target_sr": 16000, + "target_sr": self.conn.sample_rate, "audio_format": "pcm", "instruct_text": "请生成一段自然流畅的语音", } @@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase): "Content-Type": "application/json", } - # 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920 + # 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B frame_bytes = int( self.opus_encoder.sample_rate * self.opus_encoder.channels # 1 @@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase): "spk_id": self.voice, "frame_duration": 60, "stream": False, - "target_sr": 16000, + "target_sr": self.conn.sample_rate, "audio_format": self.audio_format, "instruct_text": "请生成一段自然流畅的语音", } diff --git a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py index 514d8912..73ce1c1c 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py +++ b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py @@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase): } self.audio_file_type = defult_audio_setting.get("format", "pcm") - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=24000, channels=1, frame_size_ms=60 - ) - # PCM缓冲区 self.pcm_buffer = bytearray() + async def open_audio_channels(self, conn): + """初始化音频通道,并根据conn.sample_rate更新配置""" + # 调用父类方法 + await super().open_audio_channels(conn) + + # 更新audio_setting中的采样率为实际的conn.sample_rate + self.audio_setting["sample_rate"] = conn.sample_rate + def tts_text_priority_thread(self): """流式文本处理线程""" while not self.conn.stop_event.is_set(): diff --git a/main/xiaozhi-server/core/providers/tts/paddle_speech.py b/main/xiaozhi-server/core/providers/tts/paddle_speech.py index 864cf505..48b0bb0c 100644 --- a/main/xiaozhi-server/core/providers/tts/paddle_speech.py +++ b/main/xiaozhi-server/core/providers/tts/paddle_speech.py @@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase): self.spk_id = int(config.get("private_voice")) else: self.spk_id = int(config.get("spk_id", "0")) - - sample_rate = config.get("sample_rate", 24000) - self.sample_rate = float(sample_rate) if sample_rate else 24000 - + speed = config.get("speed", 1.0) self.speed = float(speed) if speed else 1.0 diff --git a/main/xiaozhi-server/core/providers/tts/siliconflow.py b/main/xiaozhi-server/core/providers/tts/siliconflow.py index b85cebf4..ebc3dbb9 100644 --- a/main/xiaozhi-server/core/providers/tts/siliconflow.py +++ b/main/xiaozhi-server/core/providers/tts/siliconflow.py @@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("voice") self.response_format = config.get("response_format", "mp3") self.audio_file_type = config.get("response_format", "mp3") - self.sample_rate = config.get("sample_rate") self.speed = float(config.get("speed", 1.0)) self.gain = config.get("gain") diff --git a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py index dc8d833f..e7d4d4e9 100644 --- a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py @@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase): # 音频编码配置 self.format = config.get("format", "raw") - sample_rate = config.get("sample_rate", "24000") - self.sample_rate = int(sample_rate) if sample_rate else 24000 - # 口语化配置 self.oral_level = config.get("oral_level", "mid") @@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase): # 序列号管理 self.text_seq = 0 - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=self.sample_rate, channels=1, frame_size_ms=60 - ) - # 验证必需参数 if not all([self.app_id, self.api_key, self.api_secret]): raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret") @@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase): "rhy": 0, "audio": { "encoding": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "channels": 1, "bit_depth": 16, "frame_size": 0 diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 9339da7e..97158b13 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -227,7 +227,7 @@ def extract_json_from_string(input_string): def audio_to_data_stream( - audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None + audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None ) -> None: # 获取文件后缀名 file_type = os.path.splitext(audio_file_path)[1] @@ -238,12 +238,12 @@ def audio_to_data_stream( audio_file_path, format=file_type, parameters=["-nostdin"] ) - # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配） - audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + # 转换为单声道/指定采样率/16位小端编码（确保与编码器匹配） + audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2) # 获取原始PCM数据（16位小端） raw_data = audio.raw_data - pcm_to_data_stream(raw_data, is_opus, callback) + pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder) async def audio_to_data( @@ -325,7 +325,7 @@ async def audio_to_data( def audio_bytes_to_data_stream( - audio_bytes, file_type, is_opus, callback: Callable[[Any], Any] + audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None ) -> None: """ 直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3 @@ -338,18 +338,30 @@ def audio_bytes_to_data_stream( audio = AudioSegment.from_file( BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"] ) - audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2) raw_data = audio.raw_data - pcm_to_data_stream(raw_data, is_opus, callback) + pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder) -def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None): - # 初始化Opus编码器 - encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) +def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None): + """ + 将PCM数据流式编码为Opus或直接输出PCM + + Args: + raw_data: PCM原始数据 + is_opus: 是否编码为Opus + callback: 回调函数 + sample_rate: 采样率 + opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续) + """ + using_temp_encoder = False + if is_opus and opus_encoder is None: + encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO) + using_temp_encoder = True # 编码参数 frame_duration = 60 # 60ms per frame - frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame + frame_size = int(sample_rate * frame_duration / 1000) # samples/frame # 按帧处理所有音频数据（包括最后一帧可能补零） for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample @@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = chunk += b"\x00" * (frame_size * 2 - len(chunk)) if is_opus: - # 转换为numpy数组处理 - np_frame = np.frombuffer(chunk, dtype=np.int16) - # 编码Opus数据 - frame_data = encoder.encode(np_frame.tobytes(), frame_size) - callback(frame_data) + if using_temp_encoder: + # 使用临时编码器（仅用于独立音频场景） + np_frame = np.frombuffer(chunk, dtype=np.int16) + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + callback(frame_data) + else: + # 使用外部编码器（TTS流式场景,保持状态连续） + is_last = (i + frame_size * 2 >= len(raw_data)) + opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback) else: + # PCM模式,直接输出 frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) callback(frame_data)