重新划分目录 (#214)

* 🦄 refactor(web): 修改zhikongtaiweb到web * 🦄 refactor: 重写前端路由守护尚未写完 * 🦄 refactor: 标准化路由 * update:前端重写，保留后端 * update:添加前端代码 * update:pip转成poetry启动 * update:增加mem0ai包依赖 * update:文档增加mem0ai的描述 * feat: play online mp3 (#181) Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com> * 修改前端代码 * update:调整项目目录 * update:优化 * update:配置文件去除8002端口 * update:增加开发说明 * update:更新开发协议 --------- Co-authored-by: kalicyh <34980061+kaliCYH@users.noreply.github.com> Co-authored-by: hrz <1710360675@qq.com> Co-authored-by: freshlife001 <talent@mises.site> Co-authored-by: CGD <3030332422@qq.com>
2026-07-29 02:53:57 +08:00 · 2025-03-05 23:13:24 +08:00
parent 8b151d07c2
commit 0e43748fdc
289 changed files with 18127 additions and 85418 deletions
@@ -0,0 +1,77 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+import opuslib_next
+import time
+import numpy as np
+import torch
+
+TAG = __name__
+logger = setup_logging()
+
+class VAD(ABC):
+    @abstractmethod
+    def is_vad(self, conn, data):
+        """检测音频数据中的语音活动"""
+        pass
+
+
+class SileroVAD(VAD):
+    def __init__(self, config):
+        logger.bind(tag=TAG).info("SileroVAD", config)
+        self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
+                                                source='local',
+                                                model='silero_vad',
+                                                force_reload=False)
+        (get_speech_timestamps, _, _, _, _) = self.utils
+
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.vad_threshold = config.get("threshold")
+        self.silence_threshold_ms = config.get("min_silence_duration_ms")
+
+    def is_vad(self, conn, opus_packet):
+        try:
+            pcm_frame = self.decoder.decode(opus_packet, 960)
+            conn.client_audio_buffer += pcm_frame  # 将新数据加入缓冲区
+
+            # 处理缓冲区中的完整帧（每次处理512采样点）
+            client_have_voice = False
+            while len(conn.client_audio_buffer) >= 512 * 2:
+                # 提取前512个采样点（1024字节）
+                chunk = conn.client_audio_buffer[:512 * 2]
+                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
+
+                # 转换为模型需要的张量格式
+                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+                audio_float32 = audio_int16.astype(np.float32) / 32768.0
+                audio_tensor = torch.from_numpy(audio_float32)
+
+                # 检测语音活动
+                speech_prob = self.model(audio_tensor, 16000).item()
+                client_have_voice = speech_prob >= self.vad_threshold
+
+                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
+                if conn.client_have_voice and not client_have_voice:
+                    stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
+                    if stop_duration >= self.silence_threshold_ms:
+                        conn.client_voice_stop = True
+                if client_have_voice:
+                    conn.client_have_voice = True
+                    conn.client_have_voice_last_time = time.time() * 1000
+
+            return client_have_voice
+        except opuslib_next.OpusError as e:
+            logger.bind(tag=TAG).info(f"解码错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
+
+
+def create_instance(class_name, *args, **kwargs) -> VAD:
+    # 获取类对象
+    cls_map = {
+        "SileroVAD": SileroVAD,
+        # 可扩展其他SileroVAD实现
+    }
+
+    if cls := cls_map.get(class_name):
+        return cls(*args, **kwargs)
+    raise ValueError(f"不支持的SileroVAD类型: {class_name}")