Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-29 10:43:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

update: TTS复用链接，VAD双阈值判断 (#1742)

Browse Source 
* update: TTS复用链接，VAD双阈值判断

* fix: 播放音乐时，引导词卡顿

* update:优化引导词

* update: 优化chat函数流程 优化huoshan处理 会话保持一致性

* fix: 等待时可能已经完成（设置为None），后续对None错误访问

* update:增加书名号

* fix: 打断状态未重置 监听未完成时服务端可能还在发送数据 此时复用链接会接收上个语音的残余 需要两者一同关闭

---------

Co-authored-by: hrz <1710360675@qq.com>
This commit is contained in:
欣南科技
2025-07-09 18:34:04 +08:00
committed by GitHub
co-authored by hrz
parent 203ea89c6d 6bfb5f1340
commit f4cfa04954
8 changed files with 137 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/tts/base.py
+4 -18
View File
@@ -161,13 +161,6 @@ class TTSProviderBase(ABC):
else:
sentence_id = str(uuid.uuid4()).replace("-", "")
conn.sentence_id = sentence_id
self.tts_text_queue.put(
TTSMessageDTO(
sentence_id=sentence_id,
sentence_type=SentenceType.FIRST,
content_type=ContentType.ACTION,
)
)
# 对于单句的文本，进行分段处理
segments = re.split(r"([。！？!?；;\n])", content_detail)
for seg in segments:
@@ -180,13 +173,6 @@ class TTSProviderBase(ABC):
content_file=content_file,
)
)
self.tts_text_queue.put(
TTSMessageDTO(
sentence_id=sentence_id,
sentence_type=SentenceType.LAST,
content_type=ContentType.ACTION,
)
)
async def open_audio_channels(self, conn):
self.conn = conn
@@ -209,6 +195,8 @@ class TTSProviderBase(ABC):
while not self.conn.stop_event.is_set():
try:
message = self.tts_text_queue.get(timeout=1)
if message.sentence_type == SentenceType.FIRST:
self.conn.client_abort = False
if self.conn.client_abort:
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
continue
@@ -362,10 +350,8 @@ class TTSProviderBase(ABC):
return audio_datas
def _process_before_stop_play_files(self):
for tts_file, text in self.before_stop_play_files:
if tts_file and os.path.exists(tts_file):
audio_datas = self._process_audio_file(tts_file)
self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text))
for audio_datas, text in self.before_stop_play_files:
self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text))
self.before_stop_play_files.clear()
self.tts_audio_queue.put((SentenceType.LAST, [], None))
main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py
+39 -53
View File
@@ -10,7 +10,6 @@ from config.logger import setup_logging
from core.utils import opus_encoder_utils
from core.utils.util import check_model_key
from core.providers.tts.base import TTSProviderBase
from core.handle.abortHandle import handleAbortMessage
from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
from asyncio import Task
@@ -175,6 +174,9 @@ class TTSProvider(TTSProviderBase):
async def _ensure_connection(self):
"""建立新的WebSocket连接"""
try:
if self.ws:
logger.bind(tag=TAG).info(f"使用已有链接...")
return self.ws
logger.bind(tag=TAG).info("开始建立新连接...")
ws_header = {
"X-Api-App-Key": self.appId,
@@ -200,6 +202,10 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).debug(
f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
)
if message.sentence_type == SentenceType.FIRST:
self.conn.client_abort = False
if self.conn.client_abort:
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
continue
@@ -244,9 +250,12 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).info(
f"添加音频文件到待播放列表: {message.content_file}"
)
self.before_stop_play_files.append(
(message.content_file, message.content_detail)
)
if message.content_file and os.path.exists(message.content_file):
# 先处理文件音频数据
file_audio = self._process_audio_file(message.content_file)
self.before_stop_play_files.append(
(file_audio, message.content_detail)
)
if message.sentence_type == SentenceType.LAST:
try:
@@ -295,25 +304,15 @@ class TTSProvider(TTSProviderBase):
async def start_session(self, session_id):
logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
try:
task = self._monitor_task
# 会话开始时检测上个会话的监听状态
if (
task is not None
and isinstance(task, Task)
and not task.done()
self._monitor_task is not None
and isinstance(self._monitor_task, Task)
and not self._monitor_task.done()
):
logger.bind(tag=TAG).info("等待上一个监听任务结束...")
if self.ws is not None:
logger.bind(tag=TAG).info("强制关闭上一个WebSocket连接以唤醒监听任务...")
try:
await self.ws.close()
except Exception as e:
logger.bind(tag=TAG).warning(f"关闭上一个ws异常: {e}")
self.ws = None
try:
await asyncio.wait_for(task, timeout=8)
except Exception as e:
logger.bind(tag=TAG).warning(f"等待监听任务异常: {e}")
self._monitor_task = None
logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...")
await self.close()
# 建立新连接
await self._ensure_connection()
@@ -336,19 +335,7 @@ class TTSProvider(TTSProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
# 确保清理资源
if hasattr(self, "_monitor_task"):
try:
self._monitor_task.cancel()
await self._monitor_task
except:
pass
self._monitor_task = None
if self.ws:
try:
await self.ws.close()
except:
pass
self.ws = None
await self.close()
raise
async def finish_session(self, session_id):
@@ -368,7 +355,7 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).info("会话结束请求已发送")
# 等待监听任务完成
if hasattr(self, "_monitor_task"):
if self._monitor_task:
try:
await self._monitor_task
except Exception as e:
@@ -378,28 +365,25 @@ class TTSProvider(TTSProviderBase):
finally:
self._monitor_task = None
# 关闭连接
await self.close()
except Exception as e:
logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
# 确保清理资源
if hasattr(self, "_monitor_task"):
try:
self._monitor_task.cancel()
await self._monitor_task
except:
pass
self._monitor_task = None
if self.ws:
try:
await self.ws.close()
except:
pass
self.ws = None
await self.close()
raise
async def close(self):
"""资源清理方法"""
# 取消监听任务
if self._monitor_task:
try:
self._monitor_task.cancel()
await self._monitor_task
except asyncio.CancelledError:
pass
except Exception as e:
logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
self._monitor_task = None
if self.ws:
try:
await self.ws.close()
@@ -413,6 +397,7 @@ class TTSProvider(TTSProviderBase):
is_first_sentence = True
first_sentence_segment_count = 0 # 添加计数器
try:
session_finished = False # 标记会话是否正常结束
while not self.conn.stop_event.is_set():
try:
# 确保 `recv()` 运行在同一个 event loop
@@ -466,6 +451,7 @@ class TTSProvider(TTSProviderBase):
elif res.optional.event == EVENT_SessionFinished:
logger.bind(tag=TAG).debug(f"会话结束～～")
self._process_before_stop_play_files()
session_finished = True
break
except websockets.ConnectionClosed:
logger.bind(tag=TAG).warning("WebSocket连接已关闭")
@@ -476,15 +462,15 @@ class TTSProvider(TTSProviderBase):
)
traceback.print_exc()
break
finally:
# 确保清理资源
if self.ws:
# 仅在连接异常时才关闭
if not session_finished and self.ws:
try:
await self.ws.close()
except:
pass
self.ws = None
# 监听任务退出时清理引用
finally:
self._monitor_task = None
async def send_event(
main/xiaozhi-server/core/providers/tts/linkerai.py
+7 -3
View File
@@ -1,3 +1,4 @@
import os
import queue
import asyncio
import traceback
@@ -63,9 +64,12 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).info(
f"添加音频文件到待播放列表: {message.content_file}"
)
self.before_stop_play_files.append(
(message.content_file, message.content_detail)
)
if message.content_file and os.path.exists(message.content_file):
# 先处理文件音频数据
file_audio = self._process_audio_file(message.content_file)
self.before_stop_play_files.append(
(file_audio, message.content_detail)
)
if message.sentence_type == SentenceType.LAST:
# 处理剩余的文本
main/xiaozhi-server/core/providers/vad/silero.py
+18 -10
View File
@@ -23,22 +23,24 @@ class VADProvider(VADProviderBase):
# 处理空字符串的情况
threshold = config.get("threshold", "0.5")
threshold_low = config.get("threshold_low", "0.2")
min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
self.vad_threshold = float(threshold) if threshold else 0.5
self.vad_threshold_low = float(threshold_low) if threshold_low else 0.2
self.silence_threshold_ms = (
int(min_silence_duration_ms) if min_silence_duration_ms else 1000
)
# 至少要多少帧才算有语音
self.frame_window_threshold = 3
def is_vad(self, conn, opus_packet):
try:
pcm_frame = self.decoder.decode(opus_packet, 960)
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区
# 确保帧计数器存在
if not hasattr(conn, "client_voice_frame_count"):
conn.client_voice_frame_count = 0
# 处理缓冲区中的完整帧（每次处理512采样点）
client_have_voice = False
while len(conn.client_audio_buffer) >= 512 * 2:
@@ -54,15 +56,21 @@ class VADProvider(VADProviderBase):
# 检测语音活动
with torch.no_grad():
speech_prob = self.model(audio_tensor, 16000).item()
is_voice = speech_prob >= self.vad_threshold
if is_voice:
conn.client_voice_frame_count += 1
# 双阈值判断
if speech_prob >= self.vad_threshold:
is_voice = True
elif speech_prob <= self.vad_threshold_low:
is_voice = False
else:
conn.client_voice_frame_count = 0
is_voice = conn.last_is_voice
# 只有连续4帧检测到语音才认为有语音
client_have_voice = conn.client_voice_frame_count >= 4
# 声音没低于最低值则延续前一个状态，判断为有声音
conn.last_is_voice = is_voice
# 更新滑动窗口
conn.client_voice_window.append(is_voice)
client_have_voice = (conn.client_voice_window.count(True) >= self.frame_window_threshold)
# 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话
if conn.client_have_voice and not client_have_voice: