From 7a598d58398db2b721fd556a8ee02945bdddf1ab Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 24 May 2025 23:43:16 +0800 Subject: [PATCH] =?UTF-8?q?update:=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 18 ++- .../xiaozhi-server/core/handle/helloHandle.py | 2 +- .../xiaozhi-server/core/providers/tts/base.py | 17 +-- .../core/providers/tts/huoshan.py | 16 +-- .../core/utils/opus_encoder_utils.py | 132 ------------------ main/xiaozhi-server/core/utils/util.py | 5 +- 6 files changed, 32 insertions(+), 158 deletions(-) delete mode 100644 main/xiaozhi-server/core/utils/opus_encoder_utils.py diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index abe09d02..ecdc828b 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -29,6 +29,7 @@ from core.handle.receiveAudioHandle import handleAudioMessage from core.handle.functionHandler import FunctionHandler from plugins_func.register import Action, ActionResponse from core.auth import AuthMiddleware, AuthenticationError +from core.providers.tts.base import TTSImplementationType from core.mcp.manager import MCPManager from config.config_loader import get_private_config_from_api from config.manage_api_client import DeviceNotFoundException, DeviceBindException @@ -576,16 +577,23 @@ class ConnectionHandler: if self.client_abort: break - end_time = time.time() - # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") - # 处理文本分段和TTS逻辑 # 合并当前全部文本并处理未分割部分 full_text = "".join(response_message) current_text = full_text[processed_chars:] # 从未处理的位置开始 # 查找最后一个有效标点 - punctuations = ("。", ".", "？", "?", "！", "!", "；", ";", "：") + punctuations = ( + "。", + ".", + "？", + "?", + "！", + "!", + "；", + ";", + "：", + ) last_punct_pos = -1 number_flag = True for punct in punctuations: @@ -807,7 +815,7 @@ class ConnectionHandler: if content is None or len(content) <= 0: self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{content}") return None, content, text_index - tts_file = self.tts.to_tts(content) + tts_file = self.tts.to_tts(content, text_index) if tts_file is None: self.logger.bind(tag=TAG).error(f"tts转换失败，{content}") return None, content, text_index diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index f81a0229..100dde80 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -91,7 +91,7 @@ async def wakeupWordsResponse(conn): result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word) if result is None or result == "": return - tts_file = await asyncio.to_thread(conn.tts.to_tts, result) + tts_file = await asyncio.to_thread(conn.tts.to_tts, result, 0) if tts_file is not None and os.path.exists(tts_file): file_type = os.path.splitext(tts_file)[1] diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index fbcd6355..18b5c73c 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -11,7 +11,6 @@ from core.handle.reportHandle import enqueue_tts_report from abc import ABC, abstractmethod from core.utils.tts import MarkdownCleaner from core.utils.util import audio_to_data -from core.utils import opus_encoder_utils TAG = __name__ logger = setup_logging() @@ -33,9 +32,6 @@ class TTSProviderBase(ABC): self.output_file = config.get("output_dir") self.tts_queue = queue.Queue() self.audio_play_queue = queue.Queue() - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, channels=1, frame_size_ms=60 - ) # 添加实现类型属性，默认为非流式 self.interface_type = TTSImplementationType.NON_STREAMING @@ -43,9 +39,14 @@ class TTSProviderBase(ABC): def generate_filename(self): pass - def to_tts(self, text): + def to_tts(self, text, index): """如果是流式实现，一般没有文件生成，我们返回枚举值""" - if self.interface_type != TTSImplementationType.SINGLE_STREAMING: + if self.interface_type != TTSImplementationType.NON_STREAMING: + if index == 1: + future = asyncio.run_coroutine_threadsafe( + self.start_session(self.conn.session_id), loop=self.conn.loop + ) + future.result() asyncio.run(self.text_to_speak(text, None)) return self.interface_type.value @@ -221,10 +222,6 @@ class TTSProviderBase(ABC): f"audio_play_priority priority_thread: {text} {e}" ) - def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False): - opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end) - return opus_datas - async def start_session(self, session_id): pass diff --git a/main/xiaozhi-server/core/providers/tts/huoshan.py b/main/xiaozhi-server/core/providers/tts/huoshan.py index 54978bea..39d54f49 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan.py @@ -4,12 +4,11 @@ import threading import traceback import uuid import json -from datetime import datetime - import websockets - +from datetime import datetime from config.logger import setup_logging from core.providers.tts.base import TTSProviderBase, TTSImplementationType +from core.utils.util import pcm_to_data TAG = __name__ logger = setup_logging() @@ -167,7 +166,6 @@ class TTSProvider(TTSProviderBase): target=self._start_monitor_tts_response_thread(), daemon=True ) tts_priority.start() - await self.start_session(conn.session_id) def generate_filename(self, extension=".wav"): return os.path.join( @@ -381,7 +379,7 @@ class TTSProvider(TTSProviderBase): and res.header.message_type == AUDIO_ONLY_RESPONSE ): logger.bind(tag=TAG).info(f"推送数据到队列里面～～") - opus_datas = self.wav_to_opus_data_audio_raw(res.payload) + opus_datas = pcm_to_data(res.payload) logger.bind(tag=TAG).info( f"推送数据到队列里面帧数～～{len(opus_datas)}" ) @@ -390,15 +388,15 @@ class TTSProvider(TTSProviderBase): json_data = json.loads(res.payload.decode("utf-8")) self.tts_text = json_data.get("text", "") logger.bind(tag=TAG).info(f"句子开始～～{self.tts_text}") - self.audio_play_queue.put((None, self.tts_text, 0)) + self.audio_play_queue.put(([], self.tts_text, 0)) elif res.optional.event == EVENT_TTSSentenceEnd: logger.bind(tag=TAG).info(f"句子结束～～{self.tts_text}") - self.audio_play_queue.put((None, self.tts_text, 0)) + self.audio_play_queue.put(([], self.tts_text, 0)) elif res.optional.event == EVENT_SessionFinished: logger.bind(tag=TAG).info(f"会话结束～～,最后一句补零") - # opus_datas = self.wav_to_opus_data_audio_raw(b"", is_end=True) - self.audio_play_queue.put((None, self.tts_text, 0)) + opus_datas = pcm_to_data(b"") + self.audio_play_queue.put((opus_datas, self.tts_text, 0)) else: continue except websockets.ConnectionClosed: diff --git a/main/xiaozhi-server/core/utils/opus_encoder_utils.py b/main/xiaozhi-server/core/utils/opus_encoder_utils.py deleted file mode 100644 index 1a34a622..00000000 --- a/main/xiaozhi-server/core/utils/opus_encoder_utils.py +++ /dev/null @@ -1,132 +0,0 @@ -import array -import logging -import traceback - -import numpy as np -from typing import List, Optional -from opuslib_next import Encoder -from opuslib_next import constants - - -class OpusEncoderUtils: - """PCM到Opus的编码器""" - - def __init__(self, sample_rate: int, channels: int, frame_size_ms: int): - """ - 初始化Opus编码器 - - Args: - sample_rate: 采样率 (Hz) - channels: 通道数 (1=单声道, 2=立体声) - frame_size_ms: 帧大小 (毫秒) - """ - self.sample_rate = sample_rate - self.channels = channels - self.frame_size_ms = frame_size_ms - # 计算每帧样本数 = 采样率 * 帧大小(毫秒) / 1000 - self.frame_size = (sample_rate * frame_size_ms) // 1000 - # 总帧大小 = 每帧样本数 * 通道数 - self.total_frame_size = self.frame_size * channels - - # 比特率和复杂度设置 - self.bitrate = 24000 # bps - self.complexity = 10 # 最高质量 - - # 缓冲区初始化为空 - self.buffer = np.array([], dtype=np.int16) - - try: - # 创建Opus编码器 - self.encoder = Encoder( - sample_rate, channels, constants.APPLICATION_AUDIO # 音频优化模式 - ) - self.encoder.bitrate = self.bitrate - self.encoder.complexity = self.complexity - self.encoder.signal = constants.SIGNAL_VOICE # 语音信号优化 - except Exception as e: - logging.error(f"初始化Opus编码器失败: {e}") - raise RuntimeError("初始化失败") from e - - def reset_state(self): - """重置编码器状态""" - self.encoder.reset_state() - self.buffer = np.array([], dtype=np.int16) - - def encode_pcm_to_opus(self, pcm_data: bytes, end_of_stream: bool) -> List[bytes]: - """ - 将PCM数据编码为Opus格式 - - Args: - pcm_data: PCM字节数据 - end_of_stream: 是否为流的结束 - - Returns: - Opus数据包列表 - """ - # 将字节数据转换为short数组 - new_samples = self._convert_bytes_to_shorts(pcm_data) - - # 校验PCM数据 - self._validate_pcm_data(new_samples) - - # 将新数据追加到缓冲区 - self.buffer = np.append(self.buffer, new_samples) - - opus_packets = [] - offset = 0 - - # 处理所有完整帧 - while offset <= len(self.buffer) - self.total_frame_size: - frame = self.buffer[offset : offset + self.total_frame_size] - output = self._encode(frame) - if output: - opus_packets.append(output) - offset += self.total_frame_size - - # 保留未处理的样本 - self.buffer = self.buffer[offset:] - - # 流结束时处理剩余数据 - if end_of_stream and len(self.buffer) > 0: - # 创建最后一帧并用0填充 - last_frame = np.zeros(self.total_frame_size, dtype=np.int16) - last_frame[: len(self.buffer)] = self.buffer - - output = self._encode(last_frame) - if output: - opus_packets.append(output) - self.buffer = np.array([], dtype=np.int16) - - return opus_packets - - def _encode(self, frame: np.ndarray) -> Optional[bytes]: - """编码一帧音频数据""" - try: - # 将numpy数组转换为bytes - frame_bytes = frame.tobytes() - # opuslib要求输入字节数必须是channels*2的倍数 - encoded = self.encoder.encode(frame_bytes, self.frame_size) - return encoded - except Exception as e: - logging.error(f"Opus编码失败: {e}") - traceback.print_exc() - return None - - def _convert_bytes_to_shorts(self, bytes_data: bytes) -> np.ndarray: - """将字节数组转换为short数组 (16位PCM)""" - # 假设输入是小端字节序的16位PCM - return np.frombuffer(bytes_data, dtype=np.int16) - - def _validate_pcm_data(self, pcm_shorts: np.ndarray) -> None: - """验证PCM数据是否有效""" - # 16位PCM数据范围是 -32768 到 32767 - if np.any((pcm_shorts < -32768) | (pcm_shorts > 32767)): - invalid_samples = pcm_shorts[(pcm_shorts < -32768) | (pcm_shorts > 32767)] - logging.warning(f"发现无效PCM样本: {invalid_samples[:5]}...") - # 在实际应用中可以选择裁剪而不是抛出异常 - # np.clip(pcm_shorts, -32768, 32767, out=pcm_shorts) - - def close(self): - """关闭编码器并释放资源""" - # opuslib没有明确的关闭方法，Python的垃圾回收会处理 - pass diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 0b7e46a1..4c026b12 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -882,7 +882,10 @@ def audio_to_data(audio_file_path, is_opus=True): # 获取原始PCM数据（16位小端） raw_data = audio.raw_data + return pcm_to_data(raw_data, is_opus), duration + +def pcm_to_data(raw_data, is_opus=True): # 初始化Opus编码器 encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) @@ -910,7 +913,7 @@ def audio_to_data(audio_file_path, is_opus=True): datas.append(frame_data) - return datas, duration + return datas def check_vad_update(before_config, new_config):