Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 15:13:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

update:优化

Browse Source
This commit is contained in:
hrz
2025-05-24 23:43:16 +08:00
parent 574d34bc2c
commit 7a598d5839
6 changed files with 32 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+13 -5
View File
@@ -29,6 +29,7 @@ from core.handle.receiveAudioHandle import handleAudioMessage
from core.handle.functionHandler import FunctionHandler
from plugins_func.register import Action, ActionResponse
from core.auth import AuthMiddleware, AuthenticationError
from core.providers.tts.base import TTSImplementationType
from core.mcp.manager import MCPManager
from config.config_loader import get_private_config_from_api
from config.manage_api_client import DeviceNotFoundException, DeviceBindException
@@ -576,16 +577,23 @@ class ConnectionHandler:
if self.client_abort:
break
end_time = time.time()
# self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 处理文本分段和TTS逻辑
# 合并当前全部文本并处理未分割部分
full_text = "".join(response_message)
current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点
punctuations = ("。", ".", "？", "?", "！", "!", "；", ";", "：")
punctuations = (
"。",
".",
"？",
"?",
"！",
"!",
"；",
";",
"：",
)
last_punct_pos = -1
number_flag = True
for punct in punctuations:
@@ -807,7 +815,7 @@ class ConnectionHandler:
if content is None or len(content) <= 0:
self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{content}")
return None, content, text_index
tts_file = self.tts.to_tts(content)
tts_file = self.tts.to_tts(content, text_index)
if tts_file is None:
self.logger.bind(tag=TAG).error(f"tts转换失败，{content}")
return None, content, text_index
main/xiaozhi-server/core/handle/helloHandle.py
+1 -1
View File
@@ -91,7 +91,7 @@ async def wakeupWordsResponse(conn):
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
if result is None or result == "":
return
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
tts_file = await asyncio.to_thread(conn.tts.to_tts, result, 0)
if tts_file is not None and os.path.exists(tts_file):
file_type = os.path.splitext(tts_file)[1]
main/xiaozhi-server/core/providers/tts/base.py
+7 -10
View File
@@ -11,7 +11,6 @@ from core.handle.reportHandle import enqueue_tts_report
from abc import ABC, abstractmethod
from core.utils.tts import MarkdownCleaner
from core.utils.util import audio_to_data
from core.utils import opus_encoder_utils
TAG = __name__
logger = setup_logging()
@@ -33,9 +32,6 @@ class TTSProviderBase(ABC):
self.output_file = config.get("output_dir")
self.tts_queue = queue.Queue()
self.audio_play_queue = queue.Queue()
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# 添加实现类型属性，默认为非流式
self.interface_type = TTSImplementationType.NON_STREAMING
@@ -43,9 +39,14 @@ class TTSProviderBase(ABC):
def generate_filename(self):
pass
def to_tts(self, text):
def to_tts(self, text, index):
"""如果是流式实现，一般没有文件生成，我们返回枚举值"""
if self.interface_type != TTSImplementationType.SINGLE_STREAMING:
if self.interface_type != TTSImplementationType.NON_STREAMING:
if index == 1:
future = asyncio.run_coroutine_threadsafe(
self.start_session(self.conn.session_id), loop=self.conn.loop
)
future.result()
asyncio.run(self.text_to_speak(text, None))
return self.interface_type.value
@@ -221,10 +222,6 @@ class TTSProviderBase(ABC):
f"audio_play_priority priority_thread: {text} {e}"
)
def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False):
opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end)
return opus_datas
async def start_session(self, session_id):
pass
main/xiaozhi-server/core/providers/tts/huoshan.py
+7 -9
View File
@@ -4,12 +4,11 @@ import threading
import traceback
import uuid
import json
from datetime import datetime
import websockets
from datetime import datetime
from config.logger import setup_logging
from core.providers.tts.base import TTSProviderBase, TTSImplementationType
from core.utils.util import pcm_to_data
TAG = __name__
logger = setup_logging()
@@ -167,7 +166,6 @@ class TTSProvider(TTSProviderBase):
target=self._start_monitor_tts_response_thread(), daemon=True
)
tts_priority.start()
await self.start_session(conn.session_id)
def generate_filename(self, extension=".wav"):
return os.path.join(
@@ -381,7 +379,7 @@ class TTSProvider(TTSProviderBase):
and res.header.message_type == AUDIO_ONLY_RESPONSE
):
logger.bind(tag=TAG).info(f"推送数据到队列里面～～")
opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
opus_datas = pcm_to_data(res.payload)
logger.bind(tag=TAG).info(
f"推送数据到队列里面帧数～～{len(opus_datas)}"
)
@@ -390,15 +388,15 @@ class TTSProvider(TTSProviderBase):
json_data = json.loads(res.payload.decode("utf-8"))
self.tts_text = json_data.get("text", "")
logger.bind(tag=TAG).info(f"句子开始～～{self.tts_text}")
self.audio_play_queue.put((None, self.tts_text, 0))
self.audio_play_queue.put(([], self.tts_text, 0))
elif res.optional.event == EVENT_TTSSentenceEnd:
logger.bind(tag=TAG).info(f"句子结束～～{self.tts_text}")
self.audio_play_queue.put((None, self.tts_text, 0))
self.audio_play_queue.put(([], self.tts_text, 0))
elif res.optional.event == EVENT_SessionFinished:
logger.bind(tag=TAG).info(f"会话结束～～,最后一句补零")
# opus_datas = self.wav_to_opus_data_audio_raw(b"", is_end=True)
self.audio_play_queue.put((None, self.tts_text, 0))
opus_datas = pcm_to_data(b"")
self.audio_play_queue.put((opus_datas, self.tts_text, 0))
else:
continue
except websockets.ConnectionClosed:
main/xiaozhi-server/core/utils/opus_encoder_utils.py
-132
View File
@@ -1,132 +0,0 @@
import array
import logging
import traceback
import numpy as np
from typing import List, Optional
from opuslib_next import Encoder
from opuslib_next import constants
class OpusEncoderUtils:
"""PCM到Opus的编码器"""
def __init__(self, sample_rate: int, channels: int, frame_size_ms: int):
"""
初始化Opus编码器
Args:
sample_rate: 采样率 (Hz)
channels: 通道数 (1=单声道, 2=立体声)
frame_size_ms: 帧大小 (毫秒)
"""
self.sample_rate = sample_rate
self.channels = channels
self.frame_size_ms = frame_size_ms
# 计算每帧样本数 = 采样率 * 帧大小(毫秒) / 1000
self.frame_size = (sample_rate * frame_size_ms) // 1000
# 总帧大小 = 每帧样本数 * 通道数
self.total_frame_size = self.frame_size * channels
# 比特率和复杂度设置
self.bitrate = 24000 # bps
self.complexity = 10 # 最高质量
# 缓冲区初始化为空
self.buffer = np.array([], dtype=np.int16)
try:
# 创建Opus编码器
self.encoder = Encoder(
sample_rate, channels, constants.APPLICATION_AUDIO # 音频优化模式
)
self.encoder.bitrate = self.bitrate
self.encoder.complexity = self.complexity
self.encoder.signal = constants.SIGNAL_VOICE # 语音信号优化
except Exception as e:
logging.error(f"初始化Opus编码器失败: {e}")
raise RuntimeError("初始化失败") from e
def reset_state(self):
"""重置编码器状态"""
self.encoder.reset_state()
self.buffer = np.array([], dtype=np.int16)
def encode_pcm_to_opus(self, pcm_data: bytes, end_of_stream: bool) -> List[bytes]:
"""
将PCM数据编码为Opus格式
Args:
pcm_data: PCM字节数据
end_of_stream: 是否为流的结束
Returns:
Opus数据包列表
"""
# 将字节数据转换为short数组
new_samples = self._convert_bytes_to_shorts(pcm_data)
# 校验PCM数据
self._validate_pcm_data(new_samples)
# 将新数据追加到缓冲区
self.buffer = np.append(self.buffer, new_samples)
opus_packets = []
offset = 0
# 处理所有完整帧
while offset <= len(self.buffer) - self.total_frame_size:
frame = self.buffer[offset : offset + self.total_frame_size]
output = self._encode(frame)
if output:
opus_packets.append(output)
offset += self.total_frame_size
# 保留未处理的样本
self.buffer = self.buffer[offset:]
# 流结束时处理剩余数据
if end_of_stream and len(self.buffer) > 0:
# 创建最后一帧并用0填充
last_frame = np.zeros(self.total_frame_size, dtype=np.int16)
last_frame[: len(self.buffer)] = self.buffer
output = self._encode(last_frame)
if output:
opus_packets.append(output)
self.buffer = np.array([], dtype=np.int16)
return opus_packets
def _encode(self, frame: np.ndarray) -> Optional[bytes]:
"""编码一帧音频数据"""
try:
# 将numpy数组转换为bytes
frame_bytes = frame.tobytes()
# opuslib要求输入字节数必须是channels*2的倍数
encoded = self.encoder.encode(frame_bytes, self.frame_size)
return encoded
except Exception as e:
logging.error(f"Opus编码失败: {e}")
traceback.print_exc()
return None
def _convert_bytes_to_shorts(self, bytes_data: bytes) -> np.ndarray:
"""将字节数组转换为short数组 (16位PCM)"""
# 假设输入是小端字节序的16位PCM
return np.frombuffer(bytes_data, dtype=np.int16)
def _validate_pcm_data(self, pcm_shorts: np.ndarray) -> None:
"""验证PCM数据是否有效"""
# 16位PCM数据范围是 -32768 到 32767
if np.any((pcm_shorts < -32768) | (pcm_shorts > 32767)):
invalid_samples = pcm_shorts[(pcm_shorts < -32768) | (pcm_shorts > 32767)]
logging.warning(f"发现无效PCM样本: {invalid_samples[:5]}...")
# 在实际应用中可以选择裁剪而不是抛出异常
# np.clip(pcm_shorts, -32768, 32767, out=pcm_shorts)
def close(self):
"""关闭编码器并释放资源"""
# opuslib没有明确的关闭方法，Python的垃圾回收会处理
pass
main/xiaozhi-server/core/utils/util.py
+4 -1
View File
@@ -882,7 +882,10 @@ def audio_to_data(audio_file_path, is_opus=True):
# 获取原始PCM数据（16位小端）
raw_data = audio.raw_data
return pcm_to_data(raw_data, is_opus), duration
def pcm_to_data(raw_data, is_opus=True):
# 初始化Opus编码器
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
@@ -910,7 +913,7 @@ def audio_to_data(audio_file_path, is_opus=True):
datas.append(frame_data)
return datas, duration
return datas
def check_vad_update(before_config, new_config):