mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-22 15:13:55 +08:00
78 lines
3.0 KiB
Python
78 lines
3.0 KiB
Python
|
from abc import ABC, abstractmethod
|
||
|
|
import logging
|
||
|
|
import opuslib
|
||
|
|
import time
|
||
|
|
import numpy as np
|
||
|
|
import torch
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
class VAD(ABC):
|
||
|
|
@abstractmethod
|
||
|
|
def is_vad(self, conn, data):
|
||
|
|
"""检测音频数据中的语音活动"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
class SileroVAD(VAD):
|
||
|
|
def __init__(self, config):
|
||
|
|
logger.info("SileroVAD", config)
|
||
|
|
self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
|
||
|
|
source='local',
|
||
|
|
model='silero_vad',
|
||
|
|
force_reload=False)
|
||
|
|
(get_speech_timestamps, _, _, _, _) = self.utils
|
||
|
|
|
||
|
|
self.decoder = opuslib.Decoder(16000, 1)
|
||
|
|
self.vad_threshold = config.get("threshold")
|
||
|
|
self.silence_threshold_ms = config.get("min_silence_duration_ms")
|
||
|
|
|
||
|
|
def is_vad(self, conn, opus_packet):
|
||
|
|
try:
|
||
|
|
pcm_frame = self.decoder.decode(opus_packet, 960)
|
||
|
|
conn.client_audio_buffer += pcm_frame # 将新数据加入缓冲区
|
||
|
|
|
||
|
|
# 处理缓冲区中的完整帧（每次处理512采样点）
|
||
|
|
client_have_voice = False
|
||
|
|
while len(conn.client_audio_buffer) >= 512 * 2:
|
||
|
|
# 提取前512个采样点（1024字节）
|
||
|
|
chunk = conn.client_audio_buffer[:512 * 2]
|
||
|
|
conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
|
||
|
|
|
||
|
|
# 转换为模型需要的张量格式
|
||
|
|
audio_int16 = np.frombuffer(chunk, dtype=np.int16)
|
||
|
|
audio_float32 = audio_int16.astype(np.float32) / 32768.0
|
||
|
|
audio_tensor = torch.from_numpy(audio_float32)
|
||
|
|
|
||
|
|
# 检测语音活动
|
||
|
|
speech_prob = self.model(audio_tensor, 16000).item()
|
||
|
|
client_have_voice = speech_prob >= self.vad_threshold
|
||
|
|
|
||
|
|
# 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
|
||
|
|
if conn.client_have_voice and not client_have_voice:
|
||
|
|
stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
|
||
|
|
if stop_duration >= self.silence_threshold_ms:
|
||
|
|
conn.client_voice_stop = True
|
||
|
|
if client_have_voice:
|
||
|
|
conn.client_have_voice = True
|
||
|
|
conn.client_have_voice_last_time = time.time() * 1000
|
||
|
|
|
||
|
|
return client_have_voice
|
||
|
|
except opuslib.OpusError as e:
|
||
|
|
logger.info(f"解码错误: {e}")
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Error processing audio packet: {e}")
|
||
|
|
|
||
|
|
|
||
|
|
def create_instance(class_name, *args, **kwargs) -> VAD:
|
||
|
|
# 获取类对象
|
||
|
|
cls_map = {
|
||
|
|
"SileroVAD": SileroVAD,
|
||
|
|
# 可扩展其他SileroVAD实现
|
||
|
|
}
|
||
|
|
|
||
|
|
if cls := cls_map.get(class_name):
|
||
|
|
return cls(*args, **kwargs)
|
||
|
|
raise ValueError(f"不支持的SileroVAD类型: {class_name}")
|