mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 03:53:55 +08:00
86 lines
3.4 KiB
Python
86 lines
3.4 KiB
Python
import time
|
|
import torch
|
|
import numpy as np
|
|
import opuslib_next
|
|
from config.logger import setup_logging
|
|
from core.providers.vad.base import VADProviderBase
|
|
|
|
TAG = __name__
|
|
logger = setup_logging()
|
|
|
|
|
|
class VADProvider(VADProviderBase):
|
|
def __init__(self, config):
|
|
logger.bind(tag=TAG).info("SileroVAD", config)
|
|
self.model, _ = torch.hub.load(
|
|
repo_or_dir=config["model_dir"],
|
|
source="local",
|
|
model="silero_vad",
|
|
force_reload=False,
|
|
)
|
|
|
|
self.decoder = opuslib_next.Decoder(16000, 1)
|
|
|
|
# 处理空字符串的情况
|
|
threshold = config.get("threshold", "0.5")
|
|
threshold_low = config.get("threshold_low", "0.2")
|
|
min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
|
|
|
|
# 双阈值判断参数
|
|
self.vad_threshold = float(threshold) if threshold else 0.5
|
|
self.vad_threshold_low = float(threshold_low) if threshold_low else 0.2
|
|
|
|
# 静默阈值，连续多少帧低于最低值才判断为静音
|
|
self.stop_false_count = int(min_silence_duration_ms) // 32 if min_silence_duration_ms else 31
|
|
|
|
def is_vad(self, conn, opus_packet):
|
|
try:
|
|
pcm_frame = self.decoder.decode(opus_packet, 960)
|
|
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区
|
|
|
|
# 处理缓冲区中的完整帧（每次处理512采样点）
|
|
client_have_voice = False
|
|
while len(conn.client_audio_buffer) >= 512 * 2:
|
|
# 提取前512个采样点（1024字节）
|
|
chunk = conn.client_audio_buffer[: 512 * 2]
|
|
conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :]
|
|
|
|
# 转换为模型需要的张量格式
|
|
audio_int16 = np.frombuffer(chunk, dtype=np.int16)
|
|
audio_float32 = audio_int16.astype(np.float32) / 32768.0
|
|
audio_tensor = torch.from_numpy(audio_float32)
|
|
|
|
# 检测语音活动
|
|
with torch.no_grad():
|
|
speech_prob = self.model(audio_tensor, 16000).item()
|
|
|
|
# 双阈值判断
|
|
if speech_prob >= self.vad_threshold:
|
|
client_have_voice = True
|
|
elif speech_prob <= self.vad_threshold_low:
|
|
client_have_voice = False
|
|
else:
|
|
client_have_voice = conn.last_is_voice
|
|
|
|
if not client_have_voice:
|
|
conn.continuous_false_count += 1
|
|
else:
|
|
conn.continuous_false_count = 0
|
|
|
|
# 声音没低于最低值则延续前一个状态，判断为有声音
|
|
conn.last_is_voice = client_have_voice
|
|
|
|
# 如果之前有声音，且连续为静音帧，达到静默阈值要求帧数，则认为已经说完一句话
|
|
if conn.client_have_voice and conn.continuous_false_count >= self.stop_false_count:
|
|
conn.client_voice_stop = True
|
|
|
|
if client_have_voice:
|
|
conn.client_have_voice = True
|
|
conn.last_activity_time = time.time() * 1000
|
|
|
|
return client_have_voice
|
|
except opuslib_next.OpusError as e:
|
|
logger.bind(tag=TAG).info(f"解码错误: {e}")
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
|