update:优化

main/xiaozhi-server/core/connection.py

@@ -29,6 +29,7 @@ from core.handle.receiveAudioHandle import handleAudioMessage
 from core.handle.functionHandler import FunctionHandler
 from plugins_func.register import Action, ActionResponse
 from core.auth import AuthMiddleware, AuthenticationError
+from core.providers.tts.base import TTSImplementationType
 from core.mcp.manager import MCPManager
 from config.config_loader import get_private_config_from_api
 from config.manage_api_client import DeviceNotFoundException, DeviceBindException
@@ -576,16 +577,23 @@ class ConnectionHandler:
                     if self.client_abort:
                         break
 
-                    end_time = time.time()
-                    # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
-
                     # 处理文本分段和TTS逻辑
                     # 合并当前全部文本并处理未分割部分
                     full_text = "".join(response_message)
                     current_text = full_text[processed_chars:]  # 从未处理的位置开始
 
                     # 查找最后一个有效标点
-                    punctuations = ("。", ".", "？", "?", "！", "!", "；", ";", "：")
+                    punctuations = (
+                        "。",
+                        ".",
+                        "？",
+                        "?",
+                        "！",
+                        "!",
+                        "；",
+                        ";",
+                        "：",
+                    )
                     last_punct_pos = -1
                     number_flag = True
                     for punct in punctuations:
@@ -807,7 +815,7 @@ class ConnectionHandler:
         if content is None or len(content) <= 0:
             self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{content}")
             return None, content, text_index
-        tts_file = self.tts.to_tts(content)
+        tts_file = self.tts.to_tts(content, text_index)
         if tts_file is None:
             self.logger.bind(tag=TAG).error(f"tts转换失败，{content}")
             return None, content, text_index

main/xiaozhi-server/core/handle/helloHandle.py

@@ -91,7 +91,7 @@ async def wakeupWordsResponse(conn):
     result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
     if result is None or result == "":
         return
-    tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
+    tts_file = await asyncio.to_thread(conn.tts.to_tts, result, 0)
 
     if tts_file is not None and os.path.exists(tts_file):
         file_type = os.path.splitext(tts_file)[1]

main/xiaozhi-server/core/providers/tts/base.py

@@ -11,7 +11,6 @@ from core.handle.reportHandle import enqueue_tts_report
 from abc import ABC, abstractmethod
 from core.utils.tts import MarkdownCleaner
 from core.utils.util import audio_to_data
-from core.utils import opus_encoder_utils
 
 TAG = __name__
 logger = setup_logging()
@@ -33,9 +32,6 @@ class TTSProviderBase(ABC):
         self.output_file = config.get("output_dir")
         self.tts_queue = queue.Queue()
         self.audio_play_queue = queue.Queue()
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=16000, channels=1, frame_size_ms=60
-        )
         # 添加实现类型属性，默认为非流式
         self.interface_type = TTSImplementationType.NON_STREAMING
 
@@ -43,9 +39,14 @@ class TTSProviderBase(ABC):
     def generate_filename(self):
         pass
 
-    def to_tts(self, text):
+    def to_tts(self, text, index):
         """如果是流式实现，一般没有文件生成，我们返回枚举值"""
-        if self.interface_type != TTSImplementationType.SINGLE_STREAMING:
+        if self.interface_type != TTSImplementationType.NON_STREAMING:
+            if index == 1:
+                future = asyncio.run_coroutine_threadsafe(
+                    self.start_session(self.conn.session_id), loop=self.conn.loop
+                )
+                future.result()
             asyncio.run(self.text_to_speak(text, None))
             return self.interface_type.value
 
@@ -221,10 +222,6 @@ class TTSProviderBase(ABC):
                     f"audio_play_priority priority_thread: {text} {e}"
                 )
 
-    def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False):
-        opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end)
-        return opus_datas
-
     async def start_session(self, session_id):
         pass
 

main/xiaozhi-server/core/providers/tts/huoshan.py

@@ -4,12 +4,11 @@ import threading
 import traceback
 import uuid
 import json
-from datetime import datetime
-
 import websockets
-
+from datetime import datetime
 from config.logger import setup_logging
 from core.providers.tts.base import TTSProviderBase, TTSImplementationType
+from core.utils.util import pcm_to_data
 
 TAG = __name__
 logger = setup_logging()
@@ -167,7 +166,6 @@ class TTSProvider(TTSProviderBase):
             target=self._start_monitor_tts_response_thread(), daemon=True
         )
         tts_priority.start()
-        await self.start_session(conn.session_id)
 
     def generate_filename(self, extension=".wav"):
         return os.path.join(
@@ -381,7 +379,7 @@ class TTSProvider(TTSProviderBase):
                     and res.header.message_type == AUDIO_ONLY_RESPONSE
                 ):
                     logger.bind(tag=TAG).info(f"推送数据到队列里面～～")
-                    opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
+                    opus_datas = pcm_to_data(res.payload)
                     logger.bind(tag=TAG).info(
                         f"推送数据到队列里面帧数～～{len(opus_datas)}"
                     )
@@ -390,15 +388,15 @@ class TTSProvider(TTSProviderBase):
                     json_data = json.loads(res.payload.decode("utf-8"))
                     self.tts_text = json_data.get("text", "")
                     logger.bind(tag=TAG).info(f"句子开始～～{self.tts_text}")
-                    self.audio_play_queue.put((None, self.tts_text, 0))
+                    self.audio_play_queue.put(([], self.tts_text, 0))
 
                 elif res.optional.event == EVENT_TTSSentenceEnd:
                     logger.bind(tag=TAG).info(f"句子结束～～{self.tts_text}")
-                    self.audio_play_queue.put((None, self.tts_text, 0))
+                    self.audio_play_queue.put(([], self.tts_text, 0))
                 elif res.optional.event == EVENT_SessionFinished:
                     logger.bind(tag=TAG).info(f"会话结束～～,最后一句补零")
-                    # opus_datas = self.wav_to_opus_data_audio_raw(b"", is_end=True)
+                    opus_datas = pcm_to_data(b"")
-                    self.audio_play_queue.put((None, self.tts_text, 0))
+                    self.audio_play_queue.put((opus_datas, self.tts_text, 0))
                 else:
                     continue
             except websockets.ConnectionClosed:

main/xiaozhi-server/core/utils/opus_encoder_utils.py

@@ -1,132 +0,0 @@
-import array
-import logging
-import traceback
-
-import numpy as np
-from typing import List, Optional
-from opuslib_next import Encoder
-from opuslib_next import constants
-
-
-class OpusEncoderUtils:
-    """PCM到Opus的编码器"""
-
-    def __init__(self, sample_rate: int, channels: int, frame_size_ms: int):
-        """
-        初始化Opus编码器
-
-        Args:
-            sample_rate: 采样率 (Hz)
-            channels: 通道数 (1=单声道, 2=立体声)
-            frame_size_ms: 帧大小 (毫秒)
-        """
-        self.sample_rate = sample_rate
-        self.channels = channels
-        self.frame_size_ms = frame_size_ms
-        # 计算每帧样本数 = 采样率 * 帧大小(毫秒) / 1000
-        self.frame_size = (sample_rate * frame_size_ms) // 1000
-        # 总帧大小 = 每帧样本数 * 通道数
-        self.total_frame_size = self.frame_size * channels
-
-        # 比特率和复杂度设置
-        self.bitrate = 24000  # bps
-        self.complexity = 10  # 最高质量
-
-        # 缓冲区初始化为空
-        self.buffer = np.array([], dtype=np.int16)
-
-        try:
-            # 创建Opus编码器
-            self.encoder = Encoder(
-                sample_rate, channels, constants.APPLICATION_AUDIO  # 音频优化模式
-            )
-            self.encoder.bitrate = self.bitrate
-            self.encoder.complexity = self.complexity
-            self.encoder.signal = constants.SIGNAL_VOICE  # 语音信号优化
-        except Exception as e:
-            logging.error(f"初始化Opus编码器失败: {e}")
-            raise RuntimeError("初始化失败") from e
-
-    def reset_state(self):
-        """重置编码器状态"""
-        self.encoder.reset_state()
-        self.buffer = np.array([], dtype=np.int16)
-
-    def encode_pcm_to_opus(self, pcm_data: bytes, end_of_stream: bool) -> List[bytes]:
-        """
-        将PCM数据编码为Opus格式
-
-        Args:
-            pcm_data: PCM字节数据
-            end_of_stream: 是否为流的结束
-
-        Returns:
-            Opus数据包列表
-        """
-        # 将字节数据转换为short数组
-        new_samples = self._convert_bytes_to_shorts(pcm_data)
-
-        # 校验PCM数据
-        self._validate_pcm_data(new_samples)
-
-        # 将新数据追加到缓冲区
-        self.buffer = np.append(self.buffer, new_samples)
-
-        opus_packets = []
-        offset = 0
-
-        # 处理所有完整帧
-        while offset <= len(self.buffer) - self.total_frame_size:
-            frame = self.buffer[offset : offset + self.total_frame_size]
-            output = self._encode(frame)
-            if output:
-                opus_packets.append(output)
-            offset += self.total_frame_size
-
-        # 保留未处理的样本
-        self.buffer = self.buffer[offset:]
-
-        # 流结束时处理剩余数据
-        if end_of_stream and len(self.buffer) > 0:
-            # 创建最后一帧并用0填充
-            last_frame = np.zeros(self.total_frame_size, dtype=np.int16)
-            last_frame[: len(self.buffer)] = self.buffer
-
-            output = self._encode(last_frame)
-            if output:
-                opus_packets.append(output)
-            self.buffer = np.array([], dtype=np.int16)
-
-        return opus_packets
-
-    def _encode(self, frame: np.ndarray) -> Optional[bytes]:
-        """编码一帧音频数据"""
-        try:
-            # 将numpy数组转换为bytes
-            frame_bytes = frame.tobytes()
-            # opuslib要求输入字节数必须是channels*2的倍数
-            encoded = self.encoder.encode(frame_bytes, self.frame_size)
-            return encoded
-        except Exception as e:
-            logging.error(f"Opus编码失败: {e}")
-            traceback.print_exc()
-            return None
-
-    def _convert_bytes_to_shorts(self, bytes_data: bytes) -> np.ndarray:
-        """将字节数组转换为short数组 (16位PCM)"""
-        # 假设输入是小端字节序的16位PCM
-        return np.frombuffer(bytes_data, dtype=np.int16)
-
-    def _validate_pcm_data(self, pcm_shorts: np.ndarray) -> None:
-        """验证PCM数据是否有效"""
-        # 16位PCM数据范围是 -32768 到 32767
-        if np.any((pcm_shorts < -32768) | (pcm_shorts > 32767)):
-            invalid_samples = pcm_shorts[(pcm_shorts < -32768) | (pcm_shorts > 32767)]
-            logging.warning(f"发现无效PCM样本: {invalid_samples[:5]}...")
-            # 在实际应用中可以选择裁剪而不是抛出异常
-            # np.clip(pcm_shorts, -32768, 32767, out=pcm_shorts)
-
-    def close(self):
-        """关闭编码器并释放资源"""
-        # opuslib没有明确的关闭方法，Python的垃圾回收会处理
-        pass

main/xiaozhi-server/core/utils/util.py

@@ -882,7 +882,10 @@ def audio_to_data(audio_file_path, is_opus=True):
 
     # 获取原始PCM数据（16位小端）
     raw_data = audio.raw_data
+    return pcm_to_data(raw_data, is_opus), duration
 
+
+def pcm_to_data(raw_data, is_opus=True):
     # 初始化Opus编码器
     encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
 
@@ -910,7 +913,7 @@ def audio_to_data(audio_file_path, is_opus=True):
 
         datas.append(frame_data)
 
-    return datas, duration
+    return datas
 
 
 def check_vad_update(before_config, new_config):