Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into py_test_AudioArtifacts

Browse Source
This commit is contained in:
wengzh
2026-02-02 14:23:01 +08:00
committed by GitHub
parent d02d094219 5aa671d6ff
commit f0b176768d
27 changed files with 404 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-base-image.yml
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
file: Dockerfile-server-base
push: true
tags: ghcr.io/${{ github.repository }}:server-base
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha,scope=server-base
cache-to: type=gha,mode=max,scope=server-base
build-args: |
.github/workflows/docker-image.yml
+2 -2
View File
@@ -66,7 +66,7 @@ jobs:
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:server_{1},ghcr.io/{0}:server_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:server_latest', github.repository) }}
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
@@ -81,7 +81,7 @@ jobs:
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:web_{1},ghcr.io/{0}:web_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:web_latest', github.repository) }}
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
Dockerfile-web
+9 -8
View File
@@ -1,5 +1,5 @@
# 第一阶段：构建Vue前端
FROM node:18 as web-builder
FROM node:18 AS web-builder
WORKDIR /app
COPY main/manager-web/package*.json ./
RUN npm install
@@ -7,7 +7,7 @@ COPY main/manager-web .
RUN npm run build
# 第二阶段：构建Java后端
FROM maven:3.9.4-eclipse-temurin-21 as api-builder
FROM maven:3.9.4-eclipse-temurin-21 AS api-builder
WORKDIR /app
COPY main/manager-api/pom.xml .
COPY main/manager-api/src ./src
@@ -18,18 +18,19 @@ FROM bellsoft/liberica-runtime-container:jre-21-glibc
# 安装Nginx和字体库
RUN apk update && \
apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
apk add --no-cache --no-scripts \
nginx \
bash \
fontconfig \
ttf-dejavu \
msttcorefonts-installer \
&& ACCEPT_EULA=Y apk add --no-cache msttcorefonts-installer \
&& fc-cache -f -v \
&& rm -rf /var/cache/apk/*
&& rm -rf /var/cache/apk/* \
&& mkdir -p /run/nginx /var/log/nginx /var/tmp/nginx /etc/nginx/conf.d
# 复制项目自带的中文字体
COPY main/manager-web/public/generator/static/fonts/*.ttf /usr/share/fonts/
# 更新字体缓存
RUN (printf 'YES\n' | update-ms-fonts || true) && fc-cache -f -v
RUN fc-cache -f -v
# 配置Nginx
COPY docs/docker/nginx.conf /etc/nginx/nginx.conf
main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java
+1 -1
View File
@@ -304,7 +304,7 @@ public interface Constant {
/**
* 版本号
*/
public static final String VERSION = "0.8.11";
public static final String VERSION = "0.9.1";
/**
* 无效固件URL
main/manager-mobile/src/pages/settings/index.vue
+1 -1
View File
@@ -235,7 +235,7 @@ function showAbout() {
title: t('settings.aboutApp', { appName: import.meta.env.VITE_APP_TITLE }),
content: t('settings.aboutContent', {
appName: import.meta.env.VITE_APP_TITLE,
version: '0.8.11'
version: '0.9.1'
}),
showCancel: false,
confirmText: t('common.confirm'),
main/xiaozhi-server/config/logger.py
+1 -1
View File
@@ -5,7 +5,7 @@ from config.config_loader import load_config
from config.settings import check_config_file
from datetime import datetime
SERVER_VERSION = "0.8.11"
SERVER_VERSION = "0.9.1"
_logger_initialized = False
main/xiaozhi-server/core/connection.py
+15 -3
View File
@@ -1166,6 +1166,8 @@ class ConnectionHandler:
if self.tts:
await self.tts.close()
if self.asr:
await self.asr.close()
# 最后关闭线程池（避免阻塞）
if self.executor:
@@ -1214,11 +1216,21 @@ class ConnectionHandler:
f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
)
def reset_vad_states(self):
self.client_audio_buffer = bytearray()
def reset_audio_states(self):
"""
重置所有音频相关状态(VAD + ASR)
"""
# Reset VAD states
self.client_audio_buffer.clear()
self.client_have_voice = False
self.client_voice_stop = False
self.logger.bind(tag=TAG).debug("VAD states reset.")
self.client_voice_window.clear()
self.last_is_voice = False
# Clear ASR buffers
self.asr_audio.clear()
self.logger.bind(tag=TAG).debug("All audio states reset.")
def chat_and_close(self, text):
"""Chat with the user and then close the connection"""
main/xiaozhi-server/core/handle/receiveAudioHandle.py
-1
View File
@@ -17,7 +17,6 @@ async def handleAudioMessage(conn, audio):
if hasattr(conn, "just_woken_up") and conn.just_woken_up:
have_voice = False
# 设置一个短暂延迟后恢复VAD检测
conn.asr_audio.clear()
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
return
main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py
+4 -6
View File
@@ -26,10 +26,9 @@ class ListenTextMessageHandler(TextMessageHandler):
f"客户端拾音模式：{conn.client_listen_mode}"
)
if msg_json["state"] == "start":
conn.client_have_voice = True
conn.client_voice_stop = False
# 设备从播放模式切回录音模式,清除所有音频状态和缓冲区
conn.reset_audio_states()
elif msg_json["state"] == "stop":
conn.client_have_voice = True
conn.client_voice_stop = True
if conn.asr.interface_type == InterfaceType.STREAM:
# 流式模式下，发送结束请求
@@ -38,14 +37,13 @@ class ListenTextMessageHandler(TextMessageHandler):
# 非流式模式：直接触发ASR识别
if len(conn.asr_audio) > 0:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.reset_audio_states()
if len(asr_audio_task) > 0:
await conn.asr.handle_voice_stop(conn, asr_audio_task)
elif msg_json["state"] == "detect":
conn.client_have_voice = False
conn.asr_audio.clear()
conn.reset_audio_states()
if "text" in msg_json:
conn.last_activity_time = time.time() * 1000
original_text = msg_json["text"] # 保留原始文本
main/xiaozhi-server/core/providers/asr/aliyun_stream.py
+7 -24
View File
@@ -126,16 +126,8 @@ class ASRProvider(ASRProviderBase):
await super().open_audio_channels(conn)
async def receive_audio(self, conn, audio, audio_have_voice):
# 初始化音频缓存
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
# 存储音频数据
if audio:
conn.asr_audio_for_voiceprint.append(audio)
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 只在有声音且没有连接时建立连接（排除正在停止的情况）
if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -204,6 +196,8 @@ class ASRProvider(ASRProviderBase):
"""转发识别结果"""
try:
while not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = conn.asr_audio
try:
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
result = json.loads(response)
@@ -257,19 +251,12 @@ class ASRProvider(ASRProviderBase):
# 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
if conn.client_voice_stop:
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
break
else:
# 自动模式下直接覆盖
self.text = text
conn.reset_vad_states()
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
break
@@ -289,11 +276,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接的音频缓存
await self._cleanup()
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
conn.reset_audio_states()
async def _send_stop_request(self):
"""发送停止识别请求（不关闭连接）"""
main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py
+7 -24
View File
@@ -52,16 +52,8 @@ class ASRProvider(ASRProviderBase):
await super().open_audio_channels(conn)
async def receive_audio(self, conn, audio, audio_have_voice):
# 初始化音频缓存
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
# 存储音频数据
if audio:
conn.asr_audio_for_voiceprint.append(audio)
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 只在有声音且没有连接时建立连接
if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -166,6 +158,8 @@ class ASRProvider(ASRProviderBase):
"""转发识别结果"""
try:
while not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = conn.asr_audio
try:
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
result = json.loads(response)
@@ -214,19 +208,12 @@ class ASRProvider(ASRProviderBase):
# 手动模式下,只有在收到stop信号后才触发处理
if conn.client_voice_stop:
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
break
else:
# 自动模式下直接覆盖
self.text = text
conn.reset_vad_states()
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
break
@@ -257,11 +244,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接的音频缓存
await self._cleanup()
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
conn.reset_audio_states()
async def _send_stop_request(self):
"""发送停止请求(用于手动模式停止录音)"""
main/xiaozhi-server/core/providers/asr/base.py
+12 -8
View File
@@ -10,9 +10,11 @@ import traceback
import threading
import shutil
import opuslib_next
from abc import ABC, abstractmethod
from config.logger import setup_logging
from typing import Optional, Tuple, List, NamedTuple
from core.providers.asr.dto.dto import InterfaceType
from core.handle.receiveAudioHandle import startToChat
from core.handle.reportHandle import enqueue_asr_report
from core.utils.util import remove_punctuation_and_length
@@ -59,18 +61,17 @@ class ASRProviderBase(ABC):
conn.asr_audio.append(audio)
else:
# 自动/实时模式：使用VAD检测
have_voice = audio_have_voice
conn.asr_audio.append(audio)
if not have_voice and not conn.client_have_voice:
# 如果没有语音，且之前也没有声音，缓存部分音频
if not audio_have_voice and not conn.client_have_voice:
conn.asr_audio = conn.asr_audio[-10:]
return
# 自动模式下通过VAD检测到语音停止时触发识别
if conn.client_voice_stop:
if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.reset_audio_states()
if len(asr_audio_task) > 15:
await self.handle_voice_stop(conn, asr_audio_task)
@@ -165,8 +166,8 @@ class ASRProviderBase(ABC):
if text_len > 0:
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
audio_snapshot = asr_audio_task.copy()
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
except Exception as e:
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
import traceback
@@ -212,6 +213,9 @@ class ASRProviderBase(ABC):
def stop_ws_connection(self):
pass
async def close(self):
pass
class AudioArtifacts(NamedTuple):
pcm_frames: List[bytes]
"""PCM音频帧列表"""
main/xiaozhi-server/core/providers/asr/doubao_stream.py
+6 -34
View File
@@ -60,17 +60,8 @@ class ASRProvider(ASRProviderBase):
await super().open_audio_channels(conn)
async def receive_audio(self, conn, audio, audio_have_voice):
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 存储音频数据
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
conn.asr_audio_for_voiceprint.append(audio)
# 当没有音频数据时处理完整语音片段
if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
conn.asr_audio_for_voiceprint = []
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 如果本次有声音，且之前没有建立连接
if audio_have_voice and self.asr_ws is None and not self.is_processing:
@@ -164,7 +155,7 @@ class ASRProvider(ASRProviderBase):
try:
while self.asr_ws and not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
audio_data = conn.asr_audio
try:
response = await self.asr_ws.recv()
result = self.parse_response(response)
@@ -189,7 +180,6 @@ class ASRProvider(ASRProviderBase):
):
logger.bind(tag=TAG).error(f"识别文本：空")
self.text = ""
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(conn, audio_data)
break
@@ -200,12 +190,9 @@ class ASRProvider(ASRProviderBase):
if self.enable_multilingual:
continue
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 15:
logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
for utterance in utterances:
@@ -226,14 +213,10 @@ class ASRProvider(ASRProviderBase):
if conn.client_voice_stop and len(audio_data) > 0:
logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
else:
# 自动模式下直接覆盖
self.text = current_text
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(conn, audio_data)
break
@@ -262,11 +245,8 @@ class ASRProvider(ASRProviderBase):
await self.asr_ws.close()
self.asr_ws = None
self.is_processing = False
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
# 重置所有音频相关状态
conn.reset_audio_states()
def stop_ws_connection(self):
if self.asr_ws:
@@ -435,11 +415,3 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("Doubao decoder resources released")
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, '_connections'):
for conn in self._connections.values():
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
main/xiaozhi-server/core/providers/asr/xunfei_stream.py
+3 -26
View File
@@ -101,11 +101,6 @@ class ASRProvider(ASRProviderBase):
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 存储音频数据用于声纹识别
if not hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
conn.asr_audio_for_voiceprint.append(audio)
# 如果本次有声音，且之前没有建立连接
if audio_have_voice and self.asr_ws is None and not self.is_processing:
try:
@@ -232,13 +227,8 @@ class ASRProvider(ASRProviderBase):
if status == 2:
if conn.client_listen_mode == "manual":
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, conn.asr_audio)
break
except asyncio.TimeoutError:
@@ -262,13 +252,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接资源
await self._cleanup()
# 清理连接的音频缓存
if conn:
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
conn.reset_audio_states()
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
"""处理语音停止，发送最后一帧并处理识别结果"""
@@ -363,10 +347,3 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, "_connections"):
for conn in self._connections.values():
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
main/xiaozhi-server/core/providers/tts/base.py
+1 -1
View File
@@ -141,7 +141,7 @@ class TTSProviderBase(ABC):
logger.bind(tag=TAG).error(
f"语音生成失败: {text}，请检查网络或服务是否正常"
)
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
self._process_audio_file_stream(tmp_file, callback=opus_handler)
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
main/xiaozhi-server/test/favicon.ico
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

main/xiaozhi-server/test/images/1.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 142 KiB

After

Width:  |  Height:  |  Size: 136 KiB

main/xiaozhi-server/test/js/app.js
+35 -21
View File
@@ -1,9 +1,10 @@
// 主应用入口
import { log } from './utils/logger.js';
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js';
import { uiController } from './ui/controller.js';
import { getAudioPlayer } from './core/audio/player.js';
import { initMcpTools } from './core/mcp/tools.js';
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js?v=0127';
import { getAudioPlayer } from './core/audio/player.js?v=0127';
import { checkMicrophoneAvailability, isHttpNonLocalhost } from './core/audio/recorder.js?v=0127';
import { initMcpTools } from './core/mcp/tools.js?v=0127';
import { uiController } from './ui/controller.js?v=0127';
import { log } from './utils/logger.js?v=0127';
// 应用类
class App {
@@ -16,30 +17,24 @@ class App {
// 初始化应用
async init() {
log('正在初始化应用...', 'info');
// 初始化UI控制器
this.uiController = uiController;
this.uiController.init();
// 检查Opus库
checkOpusLoaded();
// 初始化Opus编码器
initOpusEncoder();
// 初始化音频播放器
this.audioPlayer = getAudioPlayer();
await this.audioPlayer.start();
// 初始化MCP工具
initMcpTools();
// 检查麦克风可用性
await this.checkMicrophoneAvailability();
// 初始化Live2D
await this.initLive2D();
// 关闭加载loading
this.setModelLoadingStatus(false);
log('应用初始化完成', 'success');
}
@@ -50,21 +45,17 @@ class App {
if (typeof window.Live2DManager === 'undefined') {
throw new Error('Live2DManager未加载，请检查脚本引入顺序');
}
this.live2dManager = new window.Live2DManager();
await this.live2dManager.initializeLive2D();
// 更新UI状态
const live2dStatus = document.getElementById('live2dStatus');
if (live2dStatus) {
live2dStatus.textContent = '● 已加载';
live2dStatus.className = 'status loaded';
}
log('Live2D初始化完成', 'success');
} catch (error) {
log(`Live2D初始化失败: ${error.message}`, 'error');
// 更新UI状态
const live2dStatus = document.getElementById('live2dStatus');
if (live2dStatus) {
@@ -81,18 +72,41 @@ class App {
modelLoading.style.display = isLoading ? 'flex' : 'none';
}
}
/**
* 检查麦克风可用性
* 在应用初始化时调用，检查麦克风是否可用并更新UI状态
*/
async checkMicrophoneAvailability() {
try {
const isAvailable = await checkMicrophoneAvailability();
const isHttp = isHttpNonLocalhost();
// 保存可用性状态到全局变量
window.microphoneAvailable = isAvailable;
window.isHttpNonLocalhost = isHttp;
// 更新UI
if (this.uiController) {
this.uiController.updateMicrophoneAvailability(isAvailable, isHttp);
}
log(`麦克风可用性检查完成: ${isAvailable ? '可用' : '不可用'}`, isAvailable ? 'success' : 'warning');
} catch (error) {
log(`检查麦克风可用性失败: ${error.message}`, 'error');
// 默认设置为不可用
window.microphoneAvailable = false;
window.isHttpNonLocalhost = isHttpNonLocalhost();
if (this.uiController) {
this.uiController.updateMicrophoneAvailability(false, window.isHttpNonLocalhost);
}
}
}
}
// 创建并启动应用
const app = new App();
// 将应用实例暴露到全局，供其他模块访问
window.chatApp = app;
document.addEventListener('DOMContentLoaded', () => {
// 初始化应用
app.init();
});
export default app;
main/xiaozhi-server/test/js/core/audio/opus-codec.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// 检查Opus库是否已加载
main/xiaozhi-server/test/js/core/audio/player.js
+3 -3
View File
@@ -1,7 +1,7 @@
// 音频播放模块
import { log } from '../../utils/logger.js';
import BlockingQueue from '../../utils/blocking-queue.js';
import { createStreamingContext } from './stream-context.js';
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
import { createStreamingContext } from './stream-context.js?v=0127';
// 音频播放器类
export class AudioPlayer {
main/xiaozhi-server/test/js/core/audio/recorder.js
+90 -123
View File
@@ -1,9 +1,9 @@
// 音频录制模块
import { log } from '../../utils/logger.js';
import { initOpusEncoder } from './opus-codec.js';
import { getAudioPlayer } from './player.js';
// Audio recording module
import { log } from '../../utils/logger.js?v=0127';
import { initOpusEncoder } from './opus-codec.js?v=0127';
import { getAudioPlayer } from './player.js?v=0127';
// 音频录制器类
// Audio recorder class
export class AudioRecorder {
constructor() {
this.isRecording = false;
@@ -19,25 +19,23 @@ export class AudioRecorder {
this.visualizationRequest = null;
this.recordingTimer = null;
this.websocket = null;
// 回调函数
// Callback functions
this.onRecordingStart = null;
this.onRecordingStop = null;
this.onVisualizerUpdate = null;
}
// 设置WebSocket实例
// Set WebSocket instance
setWebSocket(ws) {
this.websocket = ws;
}
// 获取AudioContext实例
// Get AudioContext instance
getAudioContext() {
const audioPlayer = getAudioPlayer();
return audioPlayer.getAudioContext();
return getAudioPlayer().getAudioContext();
}
// 初始化编码器
// Initialize encoder
initEncoder() {
if (!this.opusEncoder) {
this.opusEncoder = initOpusEncoder();
@@ -45,7 +43,7 @@ export class AudioRecorder {
return this.opusEncoder;
}
// PCM处理器代码
// PCM processor code
getAudioProcessorCode() {
return `
class AudioRecorderProcessor extends AudioWorkletProcessor {
@@ -56,166 +54,132 @@ export class AudioRecorder {
this.buffer = new Int16Array(this.frameSize);
this.bufferIndex = 0;
this.isRecording = false;
this.port.onmessage = (event) => {
if (event.data.command === 'start') {
this.isRecording = true;
this.port.postMessage({ type: 'status', status: 'started' });
} else if (event.data.command === 'stop') {
this.isRecording = false;
if (this.bufferIndex > 0) {
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
this.port.postMessage({
type: 'buffer',
buffer: finalBuffer
});
this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
this.bufferIndex = 0;
}
this.port.postMessage({ type: 'status', status: 'stopped' });
}
};
}
process(inputs, outputs, parameters) {
if (!this.isRecording) return true;
const input = inputs[0][0];
if (!input) return true;
for (let i = 0; i < input.length; i++) {
if (this.bufferIndex >= this.frameSize) {
this.port.postMessage({
type: 'buffer',
buffer: this.buffer.slice(0)
});
this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
this.bufferIndex = 0;
}
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
return true;
}
}
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
`;
}
// 创建音频处理器
// Create audio processor
async createAudioProcessor() {
this.audioContext = this.getAudioContext();
try {
if (this.audioContext.audioWorklet) {
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
await this.audioContext.audioWorklet.addModule(url);
URL.revokeObjectURL(url);
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
audioProcessor.port.onmessage = (event) => {
if (event.data.type === 'buffer') {
this.processPCMBuffer(event.data.buffer);
}
};
log('使用AudioWorklet处理音频', 'success');
const silent = this.audioContext.createGain();
silent.gain.value = 0;
audioProcessor.connect(silent);
silent.connect(this.audioContext.destination);
return { node: audioProcessor, type: 'worklet' };
} else {
log('AudioWorklet不可用，使用ScriptProcessorNode作为回退方案', 'warning');
log('AudioWorklet不可用，使用ScriptProcessorNode作为后备方案', 'warning');
return this.createScriptProcessor();
}
} catch (error) {
log(`创建音频处理器失败: ${error.message}，尝试回退方案`, 'error');
log(`创建音频处理器失败: ${error.message}，尝试后备方案`, 'error');
return this.createScriptProcessor();
}
}
// 创建ScriptProcessor作为回退
// Create ScriptProcessor as fallback
createScriptProcessor() {
try {
const frameSize = 4096;
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
scriptProcessor.onaudioprocess = (event) => {
if (!this.isRecording) return;
const input = event.inputBuffer.getChannelData(0);
const buffer = new Int16Array(input.length);
for (let i = 0; i < input.length; i++) {
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
this.processPCMBuffer(buffer);
};
const silent = this.audioContext.createGain();
silent.gain.value = 0;
scriptProcessor.connect(silent);
silent.connect(this.audioContext.destination);
log('使用ScriptProcessorNode作为回退方案成功', 'warning');
log('使用ScriptProcessorNode作为后备方案成功', 'warning');
return { node: scriptProcessor, type: 'processor' };
} catch (fallbackError) {
log(`回退方案也失败: ${fallbackError.message}`, 'error');
log(`后备方案也失败: ${fallbackError.message}`, 'error');
return null;
}
}
// 处理PCM缓冲数据
// Process PCM buffer data
processPCMBuffer(buffer) {
if (!this.isRecording) return;
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
newBuffer.set(this.pcmDataBuffer);
newBuffer.set(buffer, this.pcmDataBuffer.length);
this.pcmDataBuffer = newBuffer;
const samplesPerFrame = 960;
while (this.pcmDataBuffer.length >= samplesPerFrame) {
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
this.encodeAndSendOpus(frameData);
}
}
// 编码并发送Opus数据
// Encode and send Opus data
encodeAndSendOpus(pcmData = null) {
if (!this.opusEncoder) {
log('Opus编码器未初始化', 'error');
return;
}
try {
if (pcmData) {
const opusData = this.opusEncoder.encode(pcmData);
if (opusData && opusData.length > 0) {
this.audioBuffers.push(opusData.buffer);
this.totalAudioSize += opusData.length;
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
try {
this.websocket.send(opusData.buffer);
log(`发送Opus帧，大小：${opusData.length}字节`, 'debug');
} catch (error) {
log(`WebSocket发送错误: ${error.message}`, 'error');
}
}
} else {
log('Opus编码失败，无有效数据返回', 'error');
log('Opus编码失败，未返回有效数据', 'error');
}
} else {
if (this.pcmDataBuffer.length > 0) {
@@ -235,96 +199,67 @@ export class AudioRecorder {
}
}
// 开始录音
// Start recording
async start() {
if (this.isRecording) return false;
try {
// 检查是否有WebSocketHandler实例
const { getWebSocketHandler } = await import('../network/websocket.js');
// Check if WebSocketHandler instance exists
const { getWebSocketHandler } = await import('../network/websocket.js?v=0127');
const wsHandler = getWebSocketHandler();
// 如果机器正在说话，发送打断消息
// If machine is speaking, send abort message
if (wsHandler && wsHandler.isRemoteSpeaking && wsHandler.currentSessionId) {
const abortMessage = {
session_id: wsHandler.currentSessionId,
type: 'abort',
reason: 'wake_word_detected'
};
const abortMessage = { session_id: wsHandler.currentSessionId, type: 'abort', reason: 'wake_word_detected' };
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
this.websocket.send(JSON.stringify(abortMessage));
log('发送打断消息', 'info');
log('已发送中止消息', 'info');
}
}
if (!this.initEncoder()) {
log('无法启动录音: Opus编码器初始化失败', 'error');
log('无法开始录音: Opus编码器初始化失败', 'error');
return false;
}
log('请至少录制1-2秒钟的音频，确保采集到足够数据', 'info');
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
sampleRate: 16000,
channelCount: 1
}
});
log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
this.audioContext = this.getAudioContext();
if (this.audioContext.state === 'suspended') {
await this.audioContext.resume();
}
const processorResult = await this.createAudioProcessor();
if (!processorResult) {
log('无法创建音频处理器', 'error');
return false;
}
this.audioProcessor = processorResult.node;
this.audioProcessorType = processorResult.type;
this.audioSource = this.audioContext.createMediaStreamSource(stream);
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 2048;
this.audioSource.connect(this.analyser);
this.audioSource.connect(this.audioProcessor);
this.pcmDataBuffer = new Int16Array();
this.audioBuffers = [];
this.totalAudioSize = 0;
this.isRecording = true;
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
this.audioProcessor.port.postMessage({ command: 'start' });
}
// 发送监听开始消息
// Send listening start message
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
log(`发送录音开始消息`, 'info');
log(`已发送录音开始消息`, 'info');
} else {
log('WebSocket未连接，无法发送开始消息', 'error');
return false;
}
// 开始可视化
// Start visualization
if (this.onVisualizerUpdate) {
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
this.startVisualization(dataArray);
}
// 立即通知录音开始，更新按钮状态
// Immediately notify recording start, update button state
if (this.onRecordingStart) {
this.onRecordingStart(0);
}
// 启动录音计时器
// Start recording timer
let recordingSeconds = 0;
this.recordingTimer = setInterval(() => {
recordingSeconds += 0.1;
@@ -332,8 +267,7 @@ export class AudioRecorder {
this.onRecordingStart(recordingSeconds);
}
}, 100);
log('开始PCM直接录音', 'success');
log('已开始PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音启动错误: ${error.message}`, 'error');
@@ -342,15 +276,12 @@ export class AudioRecorder {
}
}
// 开始可视化
// Start visualization
startVisualization(dataArray) {
const draw = () => {
this.visualizationRequest = requestAnimationFrame(() => draw());
if (!this.isRecording) return;
this.analyser.getByteFrequencyData(dataArray);
if (this.onVisualizerUpdate) {
this.onVisualizerUpdate(dataArray);
}
@@ -358,52 +289,42 @@ export class AudioRecorder {
draw();
}
// 停止录音
// Stop recording
stop() {
if (!this.isRecording) return false;
try {
this.isRecording = false;
if (this.audioProcessor) {
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
this.audioProcessor.port.postMessage({ command: 'stop' });
}
this.audioProcessor.disconnect();
this.audioProcessor = null;
}
if (this.audioSource) {
this.audioSource.disconnect();
this.audioSource = null;
}
if (this.visualizationRequest) {
cancelAnimationFrame(this.visualizationRequest);
this.visualizationRequest = null;
}
if (this.recordingTimer) {
clearInterval(this.recordingTimer);
this.recordingTimer = null;
}
// 编码并发送剩余的数据
// Encode and send remaining data
this.encodeAndSendOpus();
// 发送结束信号
// Send end signal
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
const emptyOpusFrame = new Uint8Array(0);
this.websocket.send(emptyOpusFrame);
log('已发送录音停止信号', 'info');
}
if (this.onRecordingStop) {
this.onRecordingStop();
}
log('停止PCM直接录音', 'success');
log('已停止PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音停止错误: ${error.message}`, 'error');
@@ -411,13 +332,13 @@ export class AudioRecorder {
}
}
// 获取分析器
// Get analyser
getAnalyser() {
return this.analyser;
}
}
// 创建单例
// Create singleton instance
let audioRecorderInstance = null;
export function getAudioRecorder() {
@@ -426,3 +347,49 @@ export function getAudioRecorder() {
}
return audioRecorderInstance;
}
/**
* Check if microphone is available
* @returns {Promise<boolean>} Returns true if available, false if not available
*/
export async function checkMicrophoneAvailability() {
// Check if browser supports getUserMedia API
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
log('浏览器不支持getUserMedia API', 'warning');
return false;
}
try {
// Try to access microphone
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
// Immediately stop all tracks to release microphone
stream.getTracks().forEach(track => track.stop());
log('麦克风可用性检查成功', 'success');
return true;
} catch (error) {
log(`麦克风不可用: ${error.message}`, 'warning');
return false;
}
}
/**
* Check if it is HTTP non-localhost access
* @returns {boolean} Returns true if it is HTTP non-localhost access
*/
export function isHttpNonLocalhost() {
const protocol = window.location.protocol;
const hostname = window.location.hostname;
// Check if it is HTTP protocol
if (protocol !== 'http:') {
return false;
}
// localhost and 127.0.0.1 can use microphone
if (hostname === 'localhost' || hostname === '127.0.0.1') {
return false;
}
// Private IP addresses can also use microphone (browser allows)
if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
return false;
}
// Other HTTP access is considered non-localhost
return true;
}
main/xiaozhi-server/test/js/core/audio/stream-context.js
+2 -2
View File
@@ -1,5 +1,5 @@
import BlockingQueue from '../../utils/blocking-queue.js';
import { log } from '../../utils/logger.js';
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
// 音频流播放上下文类
export class StreamingContext {
main/xiaozhi-server/test/js/core/mcp/tools.js
+23 -76
View File
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// ==========================================
// MCP 工具管理逻辑
@@ -24,7 +24,6 @@ export function setWebSocket(ws) {
export async function initMcpTools() {
// 加载默认工具数据
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
const savedTools = localStorage.getItem('mcpTools');
if (savedTools) {
try {
@@ -36,9 +35,11 @@ export async function initMcpTools() {
} else {
mcpTools = [...defaultMcpTools];
}
renderMcpTools();
setupMcpEventListeners();
// Only setup event listeners if DOM elements exist
if (document.getElementById('toggleMcpTools')) {
setupMcpEventListeners();
}
}
/**
@@ -47,21 +48,20 @@ export async function initMcpTools() {
function renderMcpTools() {
const container = document.getElementById('mcpToolsContainer');
const countSpan = document.getElementById('mcpToolsCount');
if (!container) {
return; // Container not found, skip rendering
}
if (countSpan) {
countSpan.textContent = `${mcpTools.length} 个工具`;
}
if (mcpTools.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
return;
}
container.innerHTML = mcpTools.map((tool, index) => {
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
return `
<div class="mcp-tool-card">
<div class="mcp-tool-header">
@@ -96,12 +96,13 @@ function renderMcpTools() {
*/
function renderMcpProperties() {
const container = document.getElementById('mcpPropertiesContainer');
if (!container) {
return; // Container not found, skip rendering
}
if (mcpProperties.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 20px; color: #999; font-size: 14px;">暂无参数，点击下方按钮添加参数</div>';
return;
}
container.innerHTML = mcpProperties.map((prop, index) => `
<div class="mcp-property-item">
<div class="mcp-property-header">
@@ -161,12 +162,7 @@ function renderMcpProperties() {
* 添加参数
*/
function addMcpProperty() {
mcpProperties.push({
name: `param_${mcpProperties.length + 1}`,
type: 'string',
required: false,
description: ''
});
mcpProperties.push({ name: `param_${mcpProperties.length + 1}`, type: 'string', required: false, description: '' });
renderMcpProperties();
}
@@ -182,9 +178,7 @@ function updateMcpProperty(index, field, value) {
return;
}
}
mcpProperties[index][field] = value;
if (field === 'type' && value !== 'integer' && value !== 'number') {
delete mcpProperties[index].minimum;
delete mcpProperties[index].maximum;
@@ -212,25 +206,24 @@ function setupMcpEventListeners() {
const cancelBtn = document.getElementById('cancelMcpBtn');
const form = document.getElementById('mcpToolForm');
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
// Return early if required elements don't exist (e.g., in test environment)
if (!toggleBtn || !panel || !addBtn || !modal || !closeBtn || !cancelBtn || !form || !addPropertyBtn) {
return;
}
toggleBtn.addEventListener('click', () => {
const isExpanded = panel.classList.contains('expanded');
panel.classList.toggle('expanded');
toggleBtn.textContent = isExpanded ? '展开' : '收起';
toggleBtn.textContent = isExpanded ? '收起' : '展开';
});
// 确保面板默认展开
panel.classList.add('expanded');
addBtn.addEventListener('click', () => openMcpModal());
closeBtn.addEventListener('click', closeMcpModal);
cancelBtn.addEventListener('click', closeMcpModal);
addPropertyBtn.addEventListener('click', addMcpProperty);
modal.addEventListener('click', (e) => {
if (e.target === modal) closeMcpModal();
});
form.addEventListener('submit', handleMcpSubmit);
}
@@ -243,18 +236,15 @@ function openMcpModal(index = null) {
alert('WebSocket 已连接，无法编辑工具');
return;
}
mcpEditingIndex = index;
const errorContainer = document.getElementById('mcpErrorContainer');
errorContainer.innerHTML = '';
if (index !== null) {
document.getElementById('mcpModalTitle').textContent = '编辑工具';
const tool = mcpTools[index];
document.getElementById('mcpToolName').value = tool.name;
document.getElementById('mcpToolDescription').value = tool.description;
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
mcpProperties = [];
const schema = tool.inputSchema;
if (schema.properties) {
@@ -275,7 +265,6 @@ function openMcpModal(index = null) {
document.getElementById('mcpToolForm').reset();
mcpProperties = [];
}
renderMcpProperties();
document.getElementById('mcpToolModal').style.display = 'block';
}
@@ -298,21 +287,15 @@ function handleMcpSubmit(e) {
e.preventDefault();
const errorContainer = document.getElementById('mcpErrorContainer');
errorContainer.innerHTML = '';
const name = document.getElementById('mcpToolName').value.trim();
const description = document.getElementById('mcpToolDescription').value.trim();
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
// 检查名称重复
const isDuplicate = mcpTools.some((tool, index) =>
tool.name === name && index !== mcpEditingIndex
);
const isDuplicate = mcpTools.some((tool, index) => tool.name === name && index !== mcpEditingIndex);
if (isDuplicate) {
showMcpError('工具名称已存在，请使用不同的名称');
return;
}
// 解析模拟返回结果
let mockResponse = null;
if (mockResponseText) {
@@ -323,21 +306,13 @@ function handleMcpSubmit(e) {
return;
}
}
// 构建 inputSchema
const inputSchema = {
type: "object",
properties: {},
required: []
};
const inputSchema = { type: "object", properties: {}, required: [] };
mcpProperties.forEach(prop => {
const propSchema = { type: prop.type };
if (prop.description) {
propSchema.description = prop.description;
}
if ((prop.type === 'integer' || prop.type === 'number')) {
if (prop.minimum !== undefined && prop.minimum !== '') {
propSchema.minimum = prop.minimum;
@@ -346,20 +321,15 @@ function handleMcpSubmit(e) {
propSchema.maximum = prop.maximum;
}
}
inputSchema.properties[prop.name] = propSchema;
if (prop.required) {
inputSchema.required.push(prop.name);
}
});
if (inputSchema.required.length === 0) {
delete inputSchema.required;
}
const tool = { name, description, inputSchema, mockResponse };
if (mcpEditingIndex !== null) {
mcpTools[mcpEditingIndex] = tool;
log(`已更新工具: ${name}`, 'success');
@@ -367,7 +337,6 @@ function handleMcpSubmit(e) {
mcpTools.push(tool);
log(`已添加工具: ${name}`, 'success');
}
saveMcpTools();
renderMcpTools();
closeMcpModal();
@@ -417,11 +386,7 @@ function saveMcpTools() {
* 获取工具列表
*/
export function getMcpTools() {
return mcpTools.map(tool => ({
name: tool.name,
description: tool.description,
inputSchema: tool.inputSchema
}));
return mcpTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
}
/**
@@ -429,20 +394,14 @@ export function getMcpTools() {
*/
export function executeMcpTool(toolName, toolArgs) {
const tool = mcpTools.find(t => t.name === toolName);
if (!tool) {
log(`未找到工具: ${toolName}`, 'error');
return {
success: false,
error: `未知工具: ${toolName}`
};
return { success: false, error: `未知工具: ${toolName}` };
}
// 如果有模拟返回结果，使用它
if (tool.mockResponse) {
// 替换模板变量
let responseStr = JSON.stringify(tool.mockResponse);
// 替换 ${paramName} 格式的变量
if (toolArgs) {
Object.keys(toolArgs).forEach(key => {
@@ -450,7 +409,6 @@ export function executeMcpTool(toolName, toolArgs) {
responseStr = responseStr.replace(regex, toolArgs[key]);
});
}
try {
const response = JSON.parse(responseStr);
log(`工具 ${toolName} 执行成功，返回模拟结果: ${responseStr}`, 'success');
@@ -460,21 +418,10 @@ export function executeMcpTool(toolName, toolArgs) {
return tool.mockResponse;
}
}
// 没有模拟返回结果，返回默认成功消息
log(`工具 ${toolName} 执行成功，返回默认结果`, 'success');
return {
success: true,
message: `工具 ${toolName} 执行成功`,
tool: toolName,
arguments: toolArgs
};
return { success: true, message: `工具 ${toolName} 执行成功`, tool: toolName, arguments: toolArgs };
}
// 暴露全局方法供 HTML 内联事件调用
window.mcpModule = {
updateMcpProperty,
deleteMcpProperty,
editMcpTool,
deleteMcpTool
};
window.mcpModule = { updateMcpProperty, deleteMcpProperty, editMcpTool, deleteMcpTool };
main/xiaozhi-server/test/js/core/network/ota-connector.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// WebSocket 连接
export async function webSocketConnect(otaUrl, config) {
main/xiaozhi-server/test/js/core/network/websocket.js
+8 -9
View File
@@ -1,11 +1,11 @@
// WebSocket消息处理模块
import { getConfig, saveConnectionUrls } from '../../config/manager.js';
import { uiController } from '../../ui/controller.js';
import { log } from '../../utils/logger.js';
import { getAudioPlayer } from '../audio/player.js';
import { getAudioRecorder } from '../audio/recorder.js';
import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js';
import { webSocketConnect } from './ota-connector.js';
import { getConfig, saveConnectionUrls } from '../../config/manager.js?v=0127';
import { uiController } from '../../ui/controller.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
import { getAudioPlayer } from '../audio/player.js?v=0127';
import { getAudioRecorder } from '../audio/recorder.js?v=0127';
import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js?v=0127';
import { webSocketConnect } from './ota-connector.js?v=0127';
// WebSocket处理器类
export class WebSocketHandler {
@@ -304,7 +304,6 @@ export class WebSocketHandler {
let arrayBuffer;
if (data instanceof ArrayBuffer) {
arrayBuffer = data;
log(`收到ArrayBuffer音频数据，大小: ${data.byteLength}字节`, 'debug');
} else if (data instanceof Blob) {
arrayBuffer = await data.arrayBuffer();
log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`, 'debug');
@@ -392,7 +391,7 @@ export class WebSocketHandler {
this.websocket.onerror = (error) => {
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
uiController.addChatMessage(`⚠️ WebSocket错误: ${error.message || '未知错误'}`, false);
if (this.onConnectionStateChange) {
this.onConnectionStateChange(false);
}
main/xiaozhi-server/test/js/ui/controller.js
+161 -110
View File
@@ -1,10 +1,10 @@
// UI控制模块
import { loadConfig, saveConfig } from '../config/manager.js';
import { getAudioRecorder } from '../core/audio/recorder.js';
import { getWebSocketHandler } from '../core/network/websocket.js';
import { getAudioPlayer } from '../core/audio/player.js';
// UI controller module
import { loadConfig, saveConfig } from '../config/manager.js?v=0127';
import { getAudioPlayer } from '../core/audio/player.js?v=0127';
import { getAudioRecorder } from '../core/audio/recorder.js?v=0127';
import { getWebSocketHandler } from '../core/network/websocket.js?v=0127';
// UI控制器类
// UI controller class
class UIController {
constructor() {
this.isEditing = false;
@@ -14,7 +14,7 @@ class UIController {
this.currentBackgroundIndex = 0;
this.backgroundImages = ['1.png', '2.png', '3.png'];
// 绑定方法
// Bind methods
this.init = this.init.bind(this);
this.initEventListeners = this.initEventListeners.bind(this);
this.updateDialButton = this.updateDialButton.bind(this);
@@ -25,7 +25,7 @@ class UIController {
this.switchTab = this.switchTab.bind(this);
}
// 初始化
// Initialize
init() {
console.log('UIController init started');
@@ -35,7 +35,7 @@ class UIController {
this.initVisualizer();
}
// 检查连接按钮在初始化时是否存在
// Check if connect button exists during initialization
const connectBtn = document.getElementById('connectBtn');
console.log('connectBtn during init:', connectBtn);
@@ -43,20 +43,20 @@ class UIController {
this.startAudioStatsMonitor();
loadConfig();
// 设置录音器回调
// Register recording callback
const audioRecorder = getAudioRecorder();
audioRecorder.onRecordingStart = (seconds) => {
this.updateRecordButtonState(true, seconds);
};
// 初始化状态显示
// Initialize status display
this.updateConnectionUI(false);
this.updateDialButton(false);
console.log('UIController init completed');
}
// 初始化可视化器
// Initialize visualizer
initVisualizer() {
if (this.visualizerCanvas) {
this.visualizerCanvas.width = this.visualizerCanvas.clientWidth;
@@ -66,9 +66,9 @@ class UIController {
}
}
// 初始化事件监听器
// Initialize event listeners
initEventListeners() {
// 设置按钮
// Settings button
const settingsBtn = document.getElementById('settingsBtn');
if (settingsBtn) {
settingsBtn.addEventListener('click', () => {
@@ -76,13 +76,13 @@ class UIController {
});
}
// 背景切换按钮
// Background switch button
const backgroundBtn = document.getElementById('backgroundBtn');
if (backgroundBtn) {
backgroundBtn.addEventListener('click', this.switchBackground);
}
// 拨号按钮
// Dial button
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.addEventListener('click', () => {
@@ -92,40 +92,40 @@ class UIController {
if (isConnected) {
wsHandler.disconnect();
this.updateDialButton(false);
this.addChatMessage('已断开连接，期待下次再见~😉', false);
this.addChatMessage('Disconnected, see you next time~😊', false);
} else {
// 检查OTA地址是否已填写
// Check if OTA URL is filled
const otaUrlInput = document.getElementById('otaUrl');
if (!otaUrlInput || !otaUrlInput.value.trim()) {
// 如果OTA地址未填写，显示设置弹窗并切换到设备配置页
// If OTA URL is not filled, show settings modal and switch to device tab
this.showModal('settingsModal');
this.switchTab('device');
this.addChatMessage('请先填写OTA服务器地址', false);
this.addChatMessage('Please fill in OTA server URL', false);
return;
}
// 执行连接操作
// Start connection process
this.handleConnect();
}
});
}
// 录音按钮
// Record button
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.addEventListener('click', () => {
const audioRecorder = getAudioRecorder();
if (audioRecorder.isRecording) {
audioRecorder.stop();
// 停止录音时移除录音样式
// Restore record button to normal state
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
} else {
// 先更新按钮状态为录音中
// Update button state to recording
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
// 延迟开始录音，确保按钮状态已更新
// Start recording, update button state after delay
setTimeout(() => {
audioRecorder.start();
}, 100);
@@ -133,7 +133,7 @@ class UIController {
});
}
// 消息输入框事件
// Chat input event listener
const chatIpt = document.getElementById('chatIpt');
if (chatIpt) {
const wsHandler = getWebSocketHandler();
@@ -148,7 +148,7 @@ class UIController {
});
}
// 关闭按钮
// Close button
const closeButtons = document.querySelectorAll('.close-btn');
closeButtons.forEach(btn => {
btn.addEventListener('click', (e) => {
@@ -163,7 +163,7 @@ class UIController {
});
});
// 设置标签页切换
// Settings tab switch
const tabBtns = document.querySelectorAll('.tab-btn');
tabBtns.forEach(btn => {
btn.addEventListener('click', (e) => {
@@ -171,7 +171,7 @@ class UIController {
});
});
// 点击模态框外部关闭
// Click modal background to close
const modals = document.querySelectorAll('.modal');
modals.forEach(modal => {
modal.addEventListener('click', (e) => {
@@ -184,7 +184,7 @@ class UIController {
});
});
// 添加MCP工具按钮
// Add MCP tool button
const addMCPToolBtn = document.getElementById('addMCPToolBtn');
if (addMCPToolBtn) {
addMCPToolBtn.addEventListener('click', (e) => {
@@ -193,10 +193,10 @@ class UIController {
});
}
// 连接按钮和取消按钮已被移除，功能已集成到拨号按钮中
// Connect button and send button are not removed, can be added to dial button later
}
// 更新连接状态UI
// Update connection status UI
updateConnectionUI(isConnected) {
const connectionStatus = document.getElementById('connectionStatus');
const statusDot = document.querySelector('.status-dot');
@@ -216,7 +216,7 @@ class UIController {
}
}
// 更新拨号按钮状态
// Update dial button state
updateDialButton(isConnected) {
const dialBtn = document.getElementById('dialBtn');
const recordBtn = document.getElementById('recordBtn');
@@ -225,39 +225,44 @@ class UIController {
if (isConnected) {
dialBtn.classList.add('dial-active');
dialBtn.querySelector('.btn-text').textContent = '挂断';
// 更新拨号按钮图标为挂断图标
// Update dial button icon to hang up icon
dialBtn.querySelector('svg').innerHTML = `
<path d="M12,9C10.4,9 9,10.4 9,12C9,13.6 10.4,15 12,15C13.6,15 15,13.6 15,12C15,10.4 13.6,9 12,9M12,17C9.2,17 7,14.8 7,12C7,9.2 9.2,7 12,7C14.8,7 17,9.2 17,12C17,14.8 14.8,17 12,17M12,4.5C7,4.5 2.7,7.6 1,12C2.7,16.4 7,19.5 12,19.5C17,19.5 21.3,16.4 23,12C21.3,7.6 17,4.5 12,4.5Z"/>
`;
} else {
dialBtn.classList.remove('dial-active');
dialBtn.querySelector('.btn-text').textContent = '拨号';
// 恢复拨号按钮图标
// Restore dial button icon
dialBtn.querySelector('svg').innerHTML = `
<path d="M6.62,10.79C8.06,13.62 10.38,15.94 13.21,17.38L15.41,15.18C15.69,14.9 16.08,14.82 16.43,14.93C17.55,15.3 18.75,15.5 20,15.5A1,1 0 0,1 21,16.5V20A1,1 0 0,1 20,21A17,17 0 0,1 3,4A1,1 0 0,1 4,3H7.5A1,1 0 0,1 8.5,4C8.5,5.25 8.7,6.45 9.07,7.57C9.18,7.92 9.1,8.31 8.82,8.59L6.62,10.79Z"/>
`;
}
}
// 更新录音按钮状态
// Update record button state
if (recordBtn) {
if (isConnected) {
const microphoneAvailable = window.microphoneAvailable !== false;
if (isConnected && microphoneAvailable) {
recordBtn.disabled = false;
recordBtn.title = '开始录音';
// 确保录音按钮恢复到正常状态
// Restore record button to normal state
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
} else {
recordBtn.disabled = true;
recordBtn.title = '请先连接服务器';
// 确保录音按钮恢复到正常状态
if (!microphoneAvailable) {
recordBtn.title = window.isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
} else {
recordBtn.title = '请先连接服务器';
}
// Restore record button to normal state
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
}
}
}
// 更新录音按钮状态
// Update record button state
updateRecordButtonState(isRecording, seconds = 0) {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
@@ -268,11 +273,37 @@ class UIController {
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
}
recordBtn.disabled = false;
// Only enable button when microphone is available
recordBtn.disabled = window.microphoneAvailable === false;
}
}
// 添加聊天消息
/**
* Update microphone availability state
* @param {boolean} isAvailable - Whether microphone is available
* @param {boolean} isHttpNonLocalhost - Whether it is HTTP non-localhost access
*/
updateMicrophoneAvailability(isAvailable, isHttpNonLocalhost) {
const recordBtn = document.getElementById('recordBtn');
if (!recordBtn) return;
if (!isAvailable) {
// Disable record button
recordBtn.disabled = true;
// Update button text and title
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.title = isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
} else {
// If connected, enable record button
const wsHandler = getWebSocketHandler();
if (wsHandler && wsHandler.isConnected()) {
recordBtn.disabled = false;
recordBtn.title = '开始录音';
}
}
}
// Add chat message
addChatMessage(content, isUser = false) {
const chatStream = document.getElementById('chatStream');
if (!chatStream) return;
@@ -282,11 +313,11 @@ class UIController {
messageDiv.innerHTML = `<div class="message-bubble">${content}</div>`;
chatStream.appendChild(messageDiv);
// 自动滚动到底部
// Scroll to bottom
chatStream.scrollTop = chatStream.scrollHeight;
}
// 切换背景
// Switch background
switchBackground() {
this.currentBackgroundIndex = (this.currentBackgroundIndex + 1) % this.backgroundImages.length;
const backgroundContainer = document.querySelector('.background-container');
@@ -295,7 +326,7 @@ class UIController {
}
}
// 显示模态框
// Show modal
showModal(modalId) {
const modal = document.getElementById(modalId);
if (modal) {
@@ -303,7 +334,7 @@ class UIController {
}
}
// 隐藏模态框
// Hide modal
hideModal(modalId) {
const modal = document.getElementById(modalId);
if (modal) {
@@ -311,16 +342,16 @@ class UIController {
}
}
// 切换标签页
// Switch tab
switchTab(tabName) {
// 移除所有标签页的active类
// Remove active class from all tabs
const tabBtns = document.querySelectorAll('.tab-btn');
const tabContents = document.querySelectorAll('.tab-content');
tabBtns.forEach(btn => btn.classList.remove('active'));
tabContents.forEach(content => content.classList.remove('active'));
// 激活选中的标签页
// Activate selected tab
const activeTabBtn = document.querySelector(`[data-tab="${tabName}"]`);
const activeTabContent = document.getElementById(`${tabName}Tab`);
@@ -330,24 +361,34 @@ class UIController {
}
}
// 连接成功后开始对话
// Start AI chat session after connection
startAIChatSession() {
this.addChatMessage('连接成功，开始聊天吧~🙂', false);
// 开启录音
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.click();
this.addChatMessage('连接成功，开始聊天吧~😊', false);
// Check microphone availability and show error messages if needed
if (!window.microphoneAvailable) {
if (window.isHttpNonLocalhost) {
this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
} else {
this.addChatMessage('⚠️ 麦克风不可用，请检查权限设置，只能用文字交互', false);
}
}
// Start recording only if microphone is available
if (window.microphoneAvailable) {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.click();
}
}
}
// 处理连接按钮点击
// Handle connect button click
async handleConnect() {
console.log('handleConnect called');
// 确保切换到设备配置标签页
// Switch to device settings tab
this.switchTab('device');
// 等待DOM更新
// Wait for DOM update
await new Promise(resolve => setTimeout(resolve, 50));
const otaUrlInput = document.getElementById('otaUrl');
@@ -362,7 +403,7 @@ class UIController {
const otaUrl = otaUrlInput.value;
console.log('otaUrl value:', otaUrl);
// 更新拨号按钮状态为连接中
// Update dial button state to connecting
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.classList.add('dial-active');
@@ -370,7 +411,7 @@ class UIController {
dialBtn.disabled = true;
}
// 显示连接中消息
// Show connecting message
this.addChatMessage('正在连接服务器...', false);
const chatIpt = document.getElementById('chatIpt');
@@ -380,41 +421,51 @@ class UIController {
try {
// 获取WebSocket处理器
// Get WebSocket handler instance
const wsHandler = getWebSocketHandler();
// Register connection state callback BEFORE connecting
wsHandler.onConnectionStateChange = (isConnected) => {
this.updateConnectionUI(isConnected);
this.updateDialButton(isConnected);
};
// Register chat message callback BEFORE connecting
wsHandler.onChatMessage = (text, isUser) => {
this.addChatMessage(text, isUser);
};
// Register record button state callback BEFORE connecting
wsHandler.onRecordButtonStateChange = (isRecording) => {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
if (isRecording) {
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
} else {
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
}
}
};
const isConnected = await wsHandler.connect();
if (isConnected) {
// Check microphone availability (check again after connection)
const { checkMicrophoneAvailability } = await import('../core/audio/recorder.js?v=0127');
const micAvailable = await checkMicrophoneAvailability();
// 设置连接状态回调
wsHandler.onConnectionStateChange = (isConnected) => {
this.updateConnectionUI(isConnected);
this.updateDialButton(isConnected);
};
// 设置聊天消息回调
wsHandler.onChatMessage = (text, isUser) => {
this.addChatMessage(text, isUser);
};
// 设置录音按钮状态回调
wsHandler.onRecordButtonStateChange = (isRecording) => {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
if (isRecording) {
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
} else {
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
}
if (!micAvailable) {
const isHttp = window.isHttpNonLocalhost;
if (isHttp) {
this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
}
};
// Update global state
window.microphoneAvailable = false;
}
// 连接成功
this.addChatMessage('OTA连接成功，正在建立WebSocket连接...', false);
// 更新拨号按钮状态
// Update dial button state
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.disabled = false;
@@ -434,14 +485,14 @@ class UIController {
name: error.name
});
// 显示错误消息
// Show error message
const errorMessage = error.message.includes('Cannot set properties of null')
? '连接失败：请刷新页面重试'
? '连接失败：请检查设备连接'
: `连接失败: ${error.message}`;
this.addChatMessage(errorMessage, false);
// 恢复拨号按钮状态
// Restore dial button state
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.disabled = false;
@@ -452,7 +503,7 @@ class UIController {
}
}
// 添加MCP工具
// Add MCP tool
addMCPTool() {
const mcpToolsList = document.getElementById('mcpToolsList');
if (!mcpToolsList) return;
@@ -471,7 +522,7 @@ class UIController {
mcpToolsList.appendChild(toolDiv);
}
// 移除MCP工具
// Remove MCP tool
removeMCPTool(toolId) {
const toolElement = document.getElementById(toolId);
if (toolElement) {
@@ -479,24 +530,24 @@ class UIController {
}
}
// 更新音频统计信息
// Update audio statistics display
updateAudioStats() {
const audioPlayer = getAudioPlayer();
if (!audioPlayer) return;
const stats = audioPlayer.getAudioStats();
// 这里可以添加音频统计的UI更新逻辑
// Here can add audio statistics UI update logic
}
// 启动音频统计监控
// Start audio statistics monitor
startAudioStatsMonitor() {
// 每100ms更新一次音频统计
// Update audio statistics every 100ms
this.audioStatsTimer = setInterval(() => {
this.updateAudioStats();
}, 100);
}
// 停止音频统计监控
// Stop audio statistics monitor
stopAudioStatsMonitor() {
if (this.audioStatsTimer) {
clearInterval(this.audioStatsTimer);
@@ -504,7 +555,7 @@ class UIController {
}
}
// 绘制音频可视化效果
// Draw audio visualizer waveform
drawVisualizer(dataArray) {
if (!this.visualizerContext || !this.visualizerCanvas) return;
@@ -518,7 +569,7 @@ class UIController {
for (let i = 0; i < dataArray.length; i++) {
barHeight = dataArray[i] / 2;
// 创建渐变色：从紫色到蓝色到青色
// Create gradient color: from purple to blue to green
const gradient = this.visualizerContext.createLinearGradient(0, 0, 0, this.visualizerCanvas.height);
gradient.addColorStop(0, '#8e44ad');
gradient.addColorStop(0.5, '#3498db');
@@ -530,21 +581,21 @@ class UIController {
}
}
// 更新会话状态UI
// Update session status UI
updateSessionStatus(isSpeaking) {
// 这里可以添加会话状态的UI更新逻辑
// 例如：更新Live2D角色的表情或状态指示器
// Here can add session status UI update logic
// For example: update Live2D model's mouth movement status
}
// 更新会话表情
// Update session emotion
updateSessionEmotion(emoji) {
// 这里可以添加表情更新的逻辑
// 例如：在状态指示器中显示表情
// Here can add emotion update logic
// For example: display emoji in status indicator
}
}
// 创建全局实例
// Create singleton instance
export const uiController = new UIController();
// 导出类供其他模块使用
export { UIController };
// Export class for module usage
export { UIController };
main/xiaozhi-server/test/test_page.html
+10 -9
View File
@@ -5,7 +5,7 @@
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>小智服务器测试页面</title>
<link rel="stylesheet" href="css/test_page.css">
<link rel="stylesheet" href="css/test_page.css?v=0127">
<script>
// 检测是否使用file://协议打开
if (window.location.protocol === 'file:') {
@@ -143,7 +143,8 @@
</div>
<div class="config-item">
<label for="deviceName">设备名称:</label>
<input type="text" id="deviceName" value="Web测试设备" maxlength="50" placeholder="deviceName">
<input type="text" id="deviceName" value="Web测试设备" maxlength="50"
placeholder="deviceName">
</div>
</div>
</div>
@@ -237,23 +238,23 @@
</div>
<!-- 背景加载 -->
<script src="js/ui/background-load.js"></script>
<script src="js/ui/background-load.js?v=0127"></script>
<!-- PIXI.js 2D渲染引擎 -->
<script src="js/live2d/pixi.js"></script>
<script src="js/live2d/pixi.js?v=0127"></script>
<!-- Live2D Cubism 4.0 SDK -->
<script src="js/live2d/live2dcubismcore.min.js"></script>
<script src="js/live2d/cubism4.min.js"></script>
<script src="js/live2d/live2dcubismcore.min.js?v=0127"></script>
<script src="js/live2d/cubism4.min.js?v=0127"></script>
<!-- Live2D 管理器 -->
<script src="js/live2d/live2d.js"></script>
<script src="js/live2d/live2d.js?v=0127"></script>
<!-- Opus解码库 -->
<script src="js/utils/libopus.js"></script>
<script src="js/utils/libopus.js?v=0127"></script>
<!-- 主应用入口 -->
<script type="module" src="js/app.js"></script>
<script type="module" src="js/app.js?v=0127"></script>
<!-- 全局错误处理 -->
<script>