Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-28 10:03:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

update：增加live2d模型，修改测试页面样式

Browse Source
This commit is contained in:
rainv123
2026-01-12 11:00:50 +08:00
parent 1f6a89ada5
commit a9aa709502
37 changed files with 59977 additions and 1282 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/test/js/core/audio/recorder.js
+5
View File
@@ -326,6 +326,11 @@ export class AudioRecorder {
this.startVisualization(dataArray);
}
// 立即通知录音开始，更新按钮状态
if (this.onRecordingStart) {
this.onRecordingStart(0);
}
// 启动录音计时器
let recordingSeconds = 0;
this.recordingTimer = setInterval(() => {
main/xiaozhi-server/test/js/core/audio/stream-context.js
+11 -1
View File
@@ -23,6 +23,10 @@ export class StreamingContext {
this.totalSamples = 0; // 累积的总样本数
this.lastPlayTime = 0; // 上次播放的时间戳
this.scheduledEndTime = 0; // 已调度音频的结束时间
// 初始化分析器节点（供Live2D使用）
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 256;
}
// 缓存音频数组
@@ -108,6 +112,11 @@ export class StreamingContext {
log('音频缓冲已清空', 'success');
}
// 获取分析器节点（供Live2D使用）
getAnalyser() {
return this.analyser;
}
// 将Opus数据解码为PCM
async decodeOpusFrames() {
if (!this.opusDecoder) {
@@ -182,7 +191,8 @@ export class StreamingContext {
const currentTime = this.audioContext.currentTime;
const startTime = Math.max(this.scheduledEndTime, currentTime);
// 直接连接到输出
// 连接到分析器和输出
this.source.connect(this.analyser);
this.source.connect(this.audioContext.destination);
log(`调度播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'debug');
main/xiaozhi-server/test/js/core/mcp/tools.js
+8 -5
View File
@@ -48,7 +48,9 @@ function renderMcpTools() {
const container = document.getElementById('mcpToolsContainer');
const countSpan = document.getElementById('mcpToolsCount');
countSpan.textContent = `${mcpTools.length} 个工具`;
if (countSpan) {
countSpan.textContent = `${mcpTools.length} 个工具`;
}
if (mcpTools.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
@@ -65,12 +67,10 @@ function renderMcpTools() {
<div class="mcp-tool-header">
<div class="mcp-tool-name">${tool.name}</div>
<div class="mcp-tool-actions">
<button onclick="window.mcpModule.editMcpTool(${index})"
style="padding: 4px 10px; border: none; border-radius: 4px; background-color: #2196f3; color: white; cursor: pointer; font-size: 12px;">
<button class="mcp-edit-btn" onclick="window.mcpModule.editMcpTool(${index})">
✏️ 编辑
</button>
<button onclick="window.mcpModule.deleteMcpTool(${index})"
style="padding: 4px 10px; border: none; border-radius: 4px; background-color: #f44336; color: white; cursor: pointer; font-size: 12px;">
<button class="mcp-delete-btn" onclick="window.mcpModule.deleteMcpTool(${index})">
🗑️ 删除
</button>
</div>
@@ -218,6 +218,9 @@ function setupMcpEventListeners() {
panel.classList.toggle('expanded');
toggleBtn.textContent = isExpanded ? '展开' : '收起';
});
// 确保面板默认展开
panel.classList.add('expanded');
addBtn.addEventListener('click', () => openMcpModal());
closeBtn.addEventListener('click', closeMcpModal);
main/xiaozhi-server/test/js/core/network/websocket.js
+110 -11
View File
@@ -1,6 +1,5 @@
// WebSocket消息处理模块
import { log } from '../../utils/logger.js';
import { addMessage } from '../../ui/dom-helper.js';
import { webSocketConnect } from './ota-connector.js';
import { getConfig, saveConnectionUrls } from '../../config/manager.js';
import { getAudioPlayer } from '../audio/player.js';
@@ -15,6 +14,7 @@ export class WebSocketHandler {
this.onRecordButtonStateChange = null;
this.onSessionStateChange = null;
this.onSessionEmotionChange = null;
this.onChatMessage = null; // 新增：聊天消息回调
this.currentSessionId = null;
this.isRemoteSpeaking = false;
}
@@ -79,30 +79,45 @@ export class WebSocketHandler {
log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
} else if (message.type === 'stt') {
log(`识别结果: ${message.text}`, 'info');
addMessage(`${message.text}`, true);
// 使用新的聊天消息回调显示STT消息
if (this.onChatMessage && message.text) {
this.onChatMessage(message.text, true);
}
} else if (message.type === 'llm') {
log(`大模型回复: ${message.text}`, 'info');
// 使用新的聊天消息回调显示LLM回复
if (this.onChatMessage && message.text) {
this.onChatMessage(message.text, false);
}
// 如果包含表情，更新sessionStatus表情
// 如果包含表情，更新sessionStatus表情并触发Live2D动作
if (message.text && /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u.test(message.text)) {
// 提取表情符号
const emojiMatch = message.text.match(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u);
if (emojiMatch && this.onSessionEmotionChange) {
this.onSessionEmotionChange(emojiMatch[0]);
}
// 触发Live2D情绪动作
if (message.emotion) {
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
this.triggerLive2DEmotionAction(message.emotion);
}
}
// 只有当文本不仅仅是表情时，才添加到对话中
// 移除文本中的表情后检查是否还有内容
const textWithoutEmoji = message.text ? message.text.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim() : '';
if (textWithoutEmoji) {
addMessage(message.text);
if (textWithoutEmoji && this.onChatMessage) {
this.onChatMessage(message.text, false);
}
} else if (message.type === 'mcp') {
this.handleMCPMessage(message);
} else {
log(`未知消息类型: ${message.type}`, 'info');
addMessage(JSON.stringify(message, null, 2));
if (this.onChatMessage) {
this.onChatMessage(`未知消息类型: ${message.type}\n${JSON.stringify(message, null, 2)}`, false);
}
}
}
@@ -115,13 +130,26 @@ export class WebSocketHandler {
if (this.onSessionStateChange) {
this.onSessionStateChange(true);
}
// 启动Live2D说话动画
this.startLive2DTalking();
} else if (message.state === 'sentence_start') {
log(`服务器发送语音段: ${message.text}`, 'info');
if (message.text) {
addMessage(message.text);
this.ttsSentenceCount = (this.ttsSentenceCount || 0) + 1;
if (message.text && this.onChatMessage) {
this.onChatMessage(message.text, false);
}
// 确保动画在句子开始时运行
const live2dManager = window.chatApp?.live2dManager;
if (live2dManager && !live2dManager.isTalking) {
this.startLive2DTalking();
}
} else if (message.state === 'sentence_end') {
log(`语音段结束: ${message.text}`, 'info');
// 句子结束时不清除动画，等待下一个句子或最终停止
} else if (message.state === 'stop') {
log('服务器语音传输结束，清空所有音频缓冲', 'info');
@@ -136,6 +164,57 @@ export class WebSocketHandler {
if (this.onSessionStateChange) {
this.onSessionStateChange(false);
}
// 延迟停止Live2D说话动画，确保所有句子都播放完毕
setTimeout(() => {
this.stopLive2DTalking();
this.ttsSentenceCount = 0; // 重置计数器
}, 1000); // 1秒延迟，确保所有句子都完成
}
}
// 启动Live2D说话动画
startLive2DTalking() {
try {
// 获取Live2D管理器实例
const live2dManager = window.chatApp?.live2dManager;
if (live2dManager && live2dManager.live2dModel) {
// 使用音频播放器的分析器节点
live2dManager.startTalking();
log('Live2D说话动画已启动', 'info');
}
} catch (error) {
log(`启动Live2D说话动画失败: ${error.message}`, 'error');
}
}
// 停止Live2D说话动画
stopLive2DTalking() {
try {
const live2dManager = window.chatApp?.live2dManager;
if (live2dManager) {
live2dManager.stopTalking();
log('Live2D说话动画已停止', 'info');
}
} catch (error) {
log(`停止Live2D说话动画失败: ${error.message}`, 'error');
}
}
// 初始化Live2D音频分析器
initializeLive2DAudioAnalyzer() {
try {
const live2dManager = window.chatApp?.live2dManager;
if (live2dManager) {
// 初始化音频分析器（使用音频播放器的上下文）
if (live2dManager.initializeAudioAnalyzer()) {
log('Live2D音频分析器初始化完成，已连接到音频播放器', 'success');
} else {
log('Live2D音频分析器初始化失败，将使用模拟动画', 'warning');
}
}
} catch (error) {
log(`初始化Live2D音频分析器失败: ${error.message}`, 'error');
}
}
@@ -272,6 +351,9 @@ export class WebSocketHandler {
this.onSessionStateChange(false);
}
// 在WebSocket连接成功时初始化Live2D音频分析器
this.initializeLive2DAudioAnalyzer();
await this.sendHelloMessage();
};
@@ -304,9 +386,8 @@ export class WebSocketHandler {
}
} catch (error) {
log(`WebSocket消息处理错误: ${error.message}`, 'error');
if (typeof event.data === 'string') {
addMessage(event.data);
}
// 不再使用旧的addMessage函数，因为conversationDiv元素不存在
// 错误消息将通过其他方式显示
}
};
}
@@ -355,6 +436,24 @@ export class WebSocketHandler {
}
}
/**
* 触发Live2D情绪动作
* @param {string} emotion - 情绪名称
*/
triggerLive2DEmotionAction(emotion) {
try {
const live2dManager = window.chatApp?.live2dManager;
if (live2dManager && typeof live2dManager.triggerEmotionAction === 'function') {
live2dManager.triggerEmotionAction(emotion);
log(`触发Live2D情绪动作: ${emotion}`, 'info');
} else {
log(`无法触发Live2D情绪动作: Live2D管理器未找到或方法不可用`, 'warning');
}
} catch (error) {
log(`触发Live2D情绪动作失败: ${error.message}`, 'error');
}
}
// 获取WebSocket实例
getWebSocket() {
return this.websocket;