Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-28 01:53:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into update-tts-voice-data

Browse Source
This commit is contained in:
Sakura-RanChen
2026-02-27 11:21:18 +08:00
parent 7b020ba55c 187c2ee977
commit 0d1c18f5dd
27 changed files with 2447 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/.claude/settings.local.json
+2 -1
View File
@@ -2,7 +2,8 @@
"permissions": {
"allow": [
"Bash(tree:*)",
"Bash(find:*)"
"Bash(find:*)",
"Bash(python3:*)"
],
"deny": [],
"ask": []
main/xiaozhi-server/core/connection.py
+8
View File
@@ -1118,6 +1118,14 @@ class ConnectionHandler:
async def close(self, ws=None):
"""资源清理方法"""
try:
# 清理 VAD 连接资源
if (
hasattr(self, "vad")
and self.vad
and hasattr(self.vad, "release_conn_resources")
):
self.vad.release_conn_resources(self)
# 清理音频缓冲区
if hasattr(self, "audio_buffer"):
self.audio_buffer.clear()
main/xiaozhi-server/core/providers/vad/silero.py
+47 -27
View File
@@ -1,7 +1,8 @@
import time
import os
import numpy as np
import torch
import opuslib_next
import onnxruntime
from config.logger import setup_logging
from core.providers.vad.base import VADProviderBase
@@ -12,16 +13,17 @@ logger = setup_logging()
class VADProvider(VADProviderBase):
def __init__(self, config):
logger.bind(tag=TAG).info("SileroVAD", config)
self.model, _ = torch.hub.load(
repo_or_dir=config["model_dir"],
source="local",
model="silero_vad",
force_reload=False,
model_path = os.path.join(
config["model_dir"], "src", "silero_vad", "data", "silero_vad.onnx"
)
opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1
self.session = onnxruntime.InferenceSession(
model_path, providers=["CPUExecutionProvider"], sess_options=opts
)
self.decoder = opuslib_next.Decoder(16000, 1)
# 处理空字符串的情况
threshold = config.get("threshold", "0.5")
threshold_low = config.get("threshold_low", "0.2")
min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
@@ -33,40 +35,58 @@ class VADProvider(VADProviderBase):
int(min_silence_duration_ms) if min_silence_duration_ms else 1000
)
# 至少要多少帧才算有语音
self.frame_window_threshold = 3
def __del__(self):
if hasattr(self, 'decoder') and self.decoder is not None:
try:
del self.decoder
except Exception:
pass
def _init_connection_state(self, conn):
"""为连接初始化独立的 VAD 状态"""
if not hasattr(conn, "_vad_opus_decoder"):
conn._vad_opus_decoder = opuslib_next.Decoder(16000, 1)
if not hasattr(conn, "_vad_state"):
conn._vad_state = np.zeros((2, 1, 128), dtype=np.float32)
if not hasattr(conn, "_vad_context"):
conn._vad_context = np.zeros((1, 64), dtype=np.float32)
def release_conn_resources(self, conn):
"""释放连接的 VAD 资源（连接关闭时调用）"""
for attr in ("_vad_opus_decoder", "_vad_state", "_vad_context"):
if hasattr(conn, attr):
try:
delattr(conn, attr)
except Exception:
pass
def is_vad(self, conn, opus_packet):
# 手动模式：直接返回True，不进行实时VAD检测，所有音频都缓存
if conn.client_listen_mode == "manual":
return True
try:
pcm_frame = self.decoder.decode(opus_packet, 960)
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区
# 处理缓冲区中的完整帧（每次处理512采样点）
try:
self._init_connection_state(conn)
pcm_frame = conn._vad_opus_decoder.decode(opus_packet, 960)
conn.client_audio_buffer.extend(pcm_frame)
client_have_voice = False
while len(conn.client_audio_buffer) >= 512 * 2:
# 提取前512个采样点（1024字节）
chunk = conn.client_audio_buffer[: 512 * 2]
conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :]
# 转换为模型需要的张量格式
audio_int16 = np.frombuffer(chunk, dtype=np.int16)
audio_float32 = audio_int16.astype(np.float32) / 32768.0
audio_tensor = torch.from_numpy(audio_float32)
audio_input = np.concatenate(
[conn._vad_context, audio_float32.reshape(1, -1)], axis=1
).astype(np.float32)
# 检测语音活动
with torch.no_grad():
speech_prob = self.model(audio_tensor, 16000).item()
ort_inputs = {
"input": audio_input,
"state": conn._vad_state,
"sr": np.array(16000, dtype=np.int64),
}
out, state = self.session.run(None, ort_inputs)
conn._vad_state = state
conn._vad_context = audio_input[:, -64:]
speech_prob = out.item()
# 双阈值判断
if speech_prob >= self.vad_threshold:
main/xiaozhi-server/test/css/test_page.css
+120 -64
View File
@@ -1201,6 +1201,113 @@ body {
z-index: 999;
}
/* 滚动条样式 */
.chat-stream::-webkit-scrollbar,
.modal-body::-webkit-scrollbar,
.mcp-tools-list::-webkit-scrollbar {
width: 6px;
}
.chat-stream::-webkit-scrollbar-track,
.modal-body::-webkit-scrollbar-track,
.mcp-tools-list::-webkit-scrollbar-track {
background: #2f3136;
border-radius: 3px;
}
.chat-stream::-webkit-scrollbar-thumb,
.modal-body::-webkit-scrollbar-thumb,
.mcp-tools-list::-webkit-scrollbar-thumb {
background: #40444b;
border-radius: 3px;
}
.chat-stream::-webkit-scrollbar-thumb:hover,
.modal-body::-webkit-scrollbar-thumb:hover,
.mcp-tools-list::-webkit-scrollbar-thumb:hover {
background: #4f545c;
}
/* ==================== 摄像头显示区域样式 ==================== */
.camera-container {
position: fixed;
top: 20px;
left: 20px;
width: 180px;
height: 240px;
background: rgba(0, 0, 0, 0.5);
backdrop-filter: blur(10px);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.2);
z-index: 1000;
display: none;
overflow: hidden;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
cursor: move;
}
.camera-container.active {
display: block;
}
.camera-container.dragging {
cursor: grabbing;
box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4);
border-color: rgba(88, 101, 242, 0.5);
}
@keyframes flipWithPosition {
0% {
transform: var(--original-transform) rotateY(0deg);
}
100% {
transform: var(--original-transform) rotateY(180deg);
}
}
.camera-container.flip {
animation: flipWithPosition 0.5s ease-in-out 1 forwards;
}
#cameraVideo {
width: 100%;
height: 100%;
object-fit: cover;
background: #1a1a1a;
pointer-events: none;
transform: scaleX(-1);
}
.camera-switch-mask {
position: absolute;
top: 0;
right: 0;
bottom: 0;
left: 0;
}
.camera-switch-container {
position: absolute;
bottom: 10px;
right: 10px;
}
.camera-switch {
display: none;
height: 20px;
color: #fff;
}
.camera-switch .btn-icon {
margin-bottom: 0;
}
.camera-switch.active {
display: block;
}
/* ==================== 响应式设计 ==================== */
@media (max-width: 768px) {
.container {
@@ -1264,6 +1371,19 @@ body {
left: 10px;
bottom: 80px;
}
.chat-ipt input {
font-size: 12px;
}
.camera-container {
width: 130px;
height: 180px;
}
.status-indicator {
font-size: 12px;
}
}
@media (max-width: 480px) {
@@ -1343,68 +1463,4 @@ body {
width: 250px;
height: 80px;
}
}
/* 滚动条样式 */
.chat-stream::-webkit-scrollbar,
.modal-body::-webkit-scrollbar,
.mcp-tools-list::-webkit-scrollbar {
width: 6px;
}
.chat-stream::-webkit-scrollbar-track,
.modal-body::-webkit-scrollbar-track,
.mcp-tools-list::-webkit-scrollbar-track {
background: #2f3136;
border-radius: 3px;
}
.chat-stream::-webkit-scrollbar-thumb,
.modal-body::-webkit-scrollbar-thumb,
.mcp-tools-list::-webkit-scrollbar-thumb {
background: #40444b;
border-radius: 3px;
}
.chat-stream::-webkit-scrollbar-thumb:hover,
.modal-body::-webkit-scrollbar-thumb:hover,
.mcp-tools-list::-webkit-scrollbar-thumb:hover {
background: #4f545c;
}
/* ==================== 摄像头显示区域样式 ==================== */
.camera-container {
position: fixed;
top: 20px;
left: 20px;
width: 240px;
height: 180px;
background: rgba(0, 0, 0, 0.5);
backdrop-filter: blur(10px);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.2);
z-index: 1000;
display: none;
overflow: hidden;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
cursor: move;
}
.camera-container.active {
display: block;
}
.camera-container.dragging {
cursor: grabbing;
box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4);
border-color: rgba(88, 101, 242, 0.5);
}
#cameraVideo {
width: 100%;
height: 100%;
object-fit: cover;
background: #1a1a1a;
pointer-events: none;
transform: scaleX(-1);
}
main/xiaozhi-server/test/js/app.js
+44 -1
View File
@@ -25,6 +25,7 @@ class App {
this.audioPlayer = null;
this.live2dManager = null;
this.cameraStream = null;
this.currentFacingMode = 'user';
}
// 初始化应用
@@ -127,6 +128,9 @@ class App {
async initCamera() {
const cameraContainer = document.getElementById('cameraContainer');
const cameraVideo = document.getElementById('cameraVideo');
const cameraSwitch = document.getElementById('cameraSwitch');
const cameraSwitchMask = document.getElementById('cameraSwitchMask');
const dialBtn = document.getElementById('dialBtn');
if (!cameraContainer || !cameraVideo) {
log('摄像头元素未找到，跳过初始化', 'warning');
@@ -188,11 +192,24 @@ class App {
}
log('正在请求摄像头权限...', 'info');
this.cameraStream = await navigator.mediaDevices.getUserMedia({
video: { width: 320, height: 240, facingMode: 'user' },
video: { width: 320, height: 240, facingMode: this.currentFacingMode },
audio: false
});
cameraVideo.srcObject = this.cameraStream;
const devices = await navigator.mediaDevices.enumerateDevices();
const videoDevices = devices.filter(device => device.kind === 'videoinput');
if (videoDevices.length > 1) {
if (cameraSwitch) cameraSwitch.classList.add('active');
}
cameraContainer.classList.add('active');
// 切换时挂断情况
const hasActive = dialBtn.classList.contains('dial-active');
if (!hasActive) {
cameraContainer.classList.remove('active');
cameraSwitch.classList.remove('active');
window.stopCamera();
}
log('摄像头已启动', 'success');
return true;
} catch (error) {
@@ -217,6 +234,32 @@ class App {
}
};
window.switchCamera = async() => {
if (window.switchCameraTimer) return;
if (this.cameraStream) {
const currentTransform = window.getComputedStyle(cameraContainer).transform;
const originalTransform = currentTransform === 'none' ? 'translate(0px, 0px)' : currentTransform;
cameraContainer.style.setProperty('--original-transform', originalTransform);
cameraContainer.classList.add('flip');
if (cameraSwitchMask) cameraSwitchMask.style.opacity = 0;
this.currentFacingMode = this.currentFacingMode === 'user' ? 'environment' : 'user';
window.stopCamera();
window.startCamera();
window.switchCameraTimer = setTimeout(() => {
if (this.currentFacingMode === 'user') {
cameraVideo.style.transform = 'scaleX(-1)';
} else {
cameraVideo.style.transform = 'scaleX(1)';
}
window.switchCameraTimer = null;
cameraContainer.classList.remove('flip');
cameraContainer.style.removeProperty('--original-transform');
if (cameraSwitchMask) cameraSwitchMask.style.opacity = 1;
}, 500);
}
};
window.takePhoto = (question = '描述一下看到的物品') => {
return new Promise(async (resolve) => {
const canvas = document.createElement('canvas');
main/xiaozhi-server/test/js/ui/controller.js
+14
View File
@@ -98,6 +98,18 @@ class UIController {
});
}
// Camera switch button
const cameraSwitch = document.getElementById('cameraSwitch');
const cameraSwitchMask = document.getElementById('cameraSwitchMask');
if (cameraSwitchMask) {
cameraSwitchMask.addEventListener('click', () => {
const isCameraActive = cameraSwitch.classList.contains('active');
if (isCameraActive) {
window.switchCamera();
}
})
}
// Dial button
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
@@ -115,6 +127,7 @@ class UIController {
if (isConnected) {
wsHandler.disconnect();
this.updateDialButton(false);
if (cameraSwitch) cameraSwitch.classList.remove('active');
this.addChatMessage('Disconnected, see you next time~😊', false);
} else {
// Check if OTA URL is filled
@@ -153,6 +166,7 @@ class UIController {
if (isActive) {
// 关闭摄像头
if (typeof window.stopCamera === 'function') {
if (cameraSwitch) cameraSwitch.classList.remove('active');
window.stopCamera();
}
cameraContainer.classList.remove('active');
main/xiaozhi-server/test/test_page.html
+9
View File
@@ -44,9 +44,18 @@
<!-- 主容器 -->
<div class="container">
<!-- 摄像头显示区域（可拖动） -->
<div class="camera-container" id="cameraContainer">
<video id="cameraVideo" autoplay playsinline muted></video>
<div class="camera-switch-mask" id="cameraSwitchMask">
<div class="camera-switch-container">
<!-- 摄像头切换按钮 -->
<div class="camera-switch" id="cameraSwitch">
<svg class="btn-icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="7319" width="32" height="32"><path d="M722.1 530.3c0-44.9-13.8-86.5-37.2-120.9-0.3-0.6-0.6-1.2-0.9-1.7-2.9-4.2-6.1-8.1-9.2-12-0.4-0.5-0.7-1-1.1-1.5-21.4-26.3-48.3-46.7-78.7-59.9-0.8-0.4-1.7-0.8-2.5-1.2-4.9-2-9.8-3.8-14.9-5.5-1.8-0.6-3.6-1.3-5.4-1.8-4.4-1.3-8.8-2.4-13.3-3.5-2.5-0.6-4.9-1.2-7.4-1.7-1.2-0.2-2.4-0.6-3.6-0.8-3.3-0.6-6.7-0.8-10-1.3-2.3-0.3-4.6-0.7-6.9-0.9-5.6-0.5-11.2-0.8-16.7-0.9-1 0-2-0.1-3-0.1-0.2 0-0.3 0.1-0.5 0.1-42.8 0-84.6 13.1-120.4 38.6-11.5 8.1-14.2 24.1-6.2 35.7 8 11.6 23.8 14.4 35.2 6.3 27.7-19.7 60-29.6 93.1-29.3 4.8 0 9.5 0.3 14.1 0.7 1.4 0.2 2.8 0.4 4.3 0.5 3.8 0.5 7.6 1 11.3 1.8 1.6 0.3 3.3 0.8 4.9 1.1 3.7 0.8 7.3 1.7 10.9 2.8 1.1 0.4 2.2 0.8 3.4 1.2 4.1 1.3 8.1 2.8 12 4.5 0.4 0.2 0.8 0.4 1.2 0.5 23.6 10.3 44.3 26.1 60.4 45.9 0.1 0.1 0.2 0.3 0.3 0.4 22.7 28 36.4 63.9 36.4 102.9h-42.2L697 632.9l67.5-102.6h-42.4z m-119 133c-27.9 19.8-60.5 29.7-93.9 29.3-4.4-0.1-8.8-0.3-13.1-0.7-1.8-0.2-3.5-0.5-5.3-0.7-3.4-0.5-6.8-0.9-10.1-1.6-2-0.4-4-0.9-6.1-1.4-3.3-0.8-6.5-1.5-9.7-2.5-1.5-0.5-3-1-4.6-1.5-3.7-1.2-7.3-2.5-10.8-4-0.8-0.3-1.6-0.7-2.4-1.1-4.1-1.9-8.2-3.8-12.2-6l-0.6-0.3c-13.4-7.5-25.7-16.8-36.4-27.7-0.2-0.2-0.3-0.4-0.5-0.6-3.3-3.4-6.5-6.9-9.5-10.7-0.6-0.8-1.3-1.6-1.9-2.5-21.9-27.8-35.1-63-35.1-101.2h42.2l-67.5-102.6-67.5 102.6h42.2c0 45 13.9 86.7 37.3 121.1 0.3 0.5 0.5 1 0.8 1.5 2.4 3.5 5.1 6.8 7.7 10.1 1 1.3 1.9 2.6 2.9 3.8 3.9 4.7 7.9 9.2 12.1 13.5 0.4 0.4 0.8 0.9 1.2 1.3 14.1 14.3 30.1 26.4 47.4 36 0.5 0.3 0.9 0.6 1.4 0.8 5 2.7 10.1 5.2 15.3 7.5l3.9 1.8c4.4 1.9 9 3.5 13.6 5.1 2.2 0.7 4.4 1.5 6.6 2.2 4 1.2 8.1 2.2 12.3 3.2 2.8 0.7 5.5 1.4 8.3 1.9 1.2 0.3 2.3 0.6 3.4 0.8 3.9 0.7 7.8 1.1 11.8 1.6l4.2 0.6c7 0.7 14.1 1.2 21.1 1.2 42.9 0 84.7-13.3 120.6-38.8 11.4-8.2 14.2-24.1 6.2-35.8-8-11.5-23.8-14.3-35.3-6.2z m271.3-421.6H763c-26.5 0-82.9-119.7-112.5-119.7H370.6c-29.6 0-85.9 119.7-112.4 119.7H147.5c-46.3 0-83.9 40.2-83.9 89.7v478.8c0 49.6 37.6 89.8 83.9 89.8h726.9c46.3 0 83.9-40.1 83.9-89.8V331.4c0-49.5-37.6-89.7-83.9-89.7z m28 538.6c0 33.1-25 59.8-55.9 59.8h-671c-30.9 0-55.9-26.8-55.9-59.8V361.4c0-33 25-59.8 55.9-59.8h111.2c25.8 0 82-119.7 111.6-119.7h223.2c29.5 0 85.6 119.7 111.5 119.7h113.5c30.9 0 55.9 26.8 55.9 59.8v418.9z" p-id="7320" fill="#ffffff"></path></svg>
</div>
</div>
</div>
</div>
<!-- 连接状态指示器 - 页面顶部中部 -->