update:初始化上传

2026-07-25 08:33:53 +08:00 · 2025-02-02 23:01:14 +08:00
parent 08e3d10b1b
commit db34e7c16e
42 changed files with 2067 additions and 238 deletions
@@ -0,0 +1,77 @@
+from abc import ABC, abstractmethod
+import logging
+import opuslib
+import time
+import numpy as np
+import torch
+
+logger = logging.getLogger(__name__)
+
+
+class VAD(ABC):
+    @abstractmethod
+    def is_vad(self, conn, data):
+        """检测音频数据中的语音活动"""
+        pass
+
+
+class SileroVAD(VAD):
+    def __init__(self, config):
+        logger.info("SileroVAD", config)
+        self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
+                                                source='local',
+                                                model='silero_vad',
+                                                force_reload=False)
+        (get_speech_timestamps, _, _, _, _) = self.utils
+
+        self.decoder = opuslib.Decoder(16000, 1)
+        self.vad_threshold = config.get("threshold")
+        self.silence_threshold_ms = config.get("min_silence_duration_ms")
+
+    def is_vad(self, conn, opus_packet):
+        try:
+            pcm_frame = self.decoder.decode(opus_packet, 960)
+            conn.client_audio_buffer += pcm_frame  # 将新数据加入缓冲区
+
+            # 处理缓冲区中的完整帧（每次处理512采样点）
+            client_have_voice = False
+            while len(conn.client_audio_buffer) >= 512 * 2:
+                # 提取前512个采样点（1024字节）
+                chunk = conn.client_audio_buffer[:512 * 2]
+                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
+
+                # 转换为模型需要的张量格式
+                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+                audio_float32 = audio_int16.astype(np.float32) / 32768.0
+                audio_tensor = torch.from_numpy(audio_float32)
+
+                # 检测语音活动
+                speech_prob = self.model(audio_tensor, 16000).item()
+                client_have_voice = speech_prob >= self.vad_threshold
+
+                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
+                if conn.client_have_voice and not client_have_voice:
+                    stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
+                    if stop_duration >= self.silence_threshold_ms:
+                        conn.client_voice_stop = True
+                if client_have_voice:
+                    conn.client_have_voice = True
+                    conn.client_have_voice_last_time = time.time() * 1000
+
+            return client_have_voice
+        except opuslib.OpusError as e:
+            logger.info(f"解码错误: {e}")
+        except Exception as e:
+            logger.error(f"Error processing audio packet: {e}")
+
+
+def create_instance(class_name, *args, **kwargs) -> VAD:
+    # 获取类对象
+    cls_map = {
+        "SileroVAD": SileroVAD,
+        # 可扩展其他SileroVAD实现
+    }
+
+    if cls := cls_map.get(class_name):
+        return cls(*args, **kwargs)
+    raise ValueError(f"不支持的SileroVAD类型: {class_name}")