mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-21 22:53:56 +08:00
117 lines
4.5 KiB
Python
117 lines
4.5 KiB
Python
import time
|
|
import os
|
|
import numpy as np
|
|
import onnxruntime
|
|
from config.logger import setup_logging
|
|
from core.providers.vad.base import VADProviderBase
|
|
|
|
TAG = __name__
|
|
logger = setup_logging()
|
|
|
|
|
|
class VADProvider(VADProviderBase):
|
|
def __init__(self, config):
|
|
logger.bind(tag=TAG).info("SileroVAD", config)
|
|
|
|
model_path = os.path.join(
|
|
config["model_dir"], "src", "silero_vad", "data", "silero_vad.onnx"
|
|
)
|
|
opts = onnxruntime.SessionOptions()
|
|
opts.inter_op_num_threads = 1
|
|
opts.intra_op_num_threads = 1
|
|
self.session = onnxruntime.InferenceSession(
|
|
model_path, providers=["CPUExecutionProvider"], sess_options=opts
|
|
)
|
|
|
|
threshold = config.get("threshold", "0.5")
|
|
threshold_low = config.get("threshold_low", "0.2")
|
|
min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
|
|
|
|
self.vad_threshold = float(threshold) if threshold else 0.5
|
|
self.vad_threshold_low = float(threshold_low) if threshold_low else 0.2
|
|
|
|
self.silence_threshold_ms = (
|
|
int(min_silence_duration_ms) if min_silence_duration_ms else 1000
|
|
)
|
|
|
|
self.frame_window_threshold = 3
|
|
|
|
def _init_connection_state(self, conn):
|
|
"""为连接初始化独立的 VAD 状态"""
|
|
if not hasattr(conn, "_vad_state"):
|
|
conn._vad_state = np.zeros((2, 1, 128), dtype=np.float32)
|
|
if not hasattr(conn, "_vad_context"):
|
|
conn._vad_context = np.zeros((1, 64), dtype=np.float32)
|
|
|
|
def release_conn_resources(self, conn):
|
|
"""释放连接的 VAD 资源（连接关闭时调用）"""
|
|
for attr in ("_vad_state", "_vad_context"):
|
|
if hasattr(conn, attr):
|
|
try:
|
|
delattr(conn, attr)
|
|
except Exception:
|
|
pass
|
|
|
|
def is_vad(self, conn, pcm_frame):
|
|
# 手动模式：直接返回True，不进行实时VAD检测，所有音频都缓存
|
|
if conn.client_listen_mode == "manual":
|
|
return True
|
|
|
|
try:
|
|
self._init_connection_state(conn)
|
|
|
|
# pcm_frame已经是处理后的PCM数据
|
|
conn.client_audio_buffer.extend(pcm_frame)
|
|
|
|
client_have_voice = False
|
|
while len(conn.client_audio_buffer) >= 512 * 2:
|
|
chunk = conn.client_audio_buffer[: 512 * 2]
|
|
conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :]
|
|
|
|
audio_int16 = np.frombuffer(chunk, dtype=np.int16)
|
|
audio_float32 = audio_int16.astype(np.float32) / 32768.0
|
|
audio_input = np.concatenate(
|
|
[conn._vad_context, audio_float32.reshape(1, -1)], axis=1
|
|
).astype(np.float32)
|
|
|
|
ort_inputs = {
|
|
"input": audio_input,
|
|
"state": conn._vad_state,
|
|
"sr": np.array(16000, dtype=np.int64),
|
|
}
|
|
out, state = self.session.run(None, ort_inputs)
|
|
|
|
conn._vad_state = state
|
|
conn._vad_context = audio_input[:, -64:]
|
|
speech_prob = out.item()
|
|
|
|
# 双阈值判断
|
|
if speech_prob >= self.vad_threshold:
|
|
is_voice = True
|
|
elif speech_prob <= self.vad_threshold_low:
|
|
is_voice = False
|
|
else:
|
|
is_voice = conn.last_is_voice
|
|
|
|
# 声音没低于最低值则延续前一个状态，判断为有声音
|
|
conn.last_is_voice = is_voice
|
|
|
|
# 更新滑动窗口
|
|
conn.client_voice_window.append(is_voice)
|
|
client_have_voice = (
|
|
conn.client_voice_window.count(True) >= self.frame_window_threshold
|
|
)
|
|
|
|
# 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话
|
|
if conn.client_have_voice and not client_have_voice:
|
|
stop_duration = time.time() * 1000 - conn.vad_last_voice_time
|
|
if stop_duration >= self.silence_threshold_ms:
|
|
conn.client_voice_stop = True
|
|
if client_have_voice:
|
|
conn.client_have_voice = True
|
|
conn.vad_last_voice_time = time.time() * 1000
|
|
|
|
return client_have_voice
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
|