Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-21 22:53:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3044 from xinnan-tech/py-display

Browse Source 
添加“工具调用日志记录”与“设备端工具调用信息显示”
This commit is contained in:
wengzh
2026-03-31 15:12:11 +08:00
committed by GitHub
parent 8aa12d0961 64d25533d9
commit 24a8839753
10 changed files with 203 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/manager-api/src/main/resources/db/changelog/202603311200.sql
+2
View File
@@ -0,0 +1,2 @@
-- 修改聊天内容字段类型
ALTER TABLE ai_agent_chat_history MODIFY COLUMN content TEXT COMMENT '聊天内容';
main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
+8
View File
@@ -578,3 +578,11 @@ databaseChangeLog:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202603231037.sql
- changeSet:
id: 202603311200
author: cgd
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202603311200.sql
main/manager-web/src/components/ChatHistoryDialog.vue
+103 -3
View File
@@ -21,11 +21,35 @@
<div v-if="message.type === 'time'" class="time-divider">
{{ message.content }}
</div>
<div v-else class="message-item" :class="{ 'user-message': message.chatType === 1 }">
<div v-else class="message-item" :class="{ 'user-message': message.chatType === 1, 'tool-message': message.chatType === 3 }">
<img :src="message.chatType === 1 ? getUserAvatar(currentSessionId) : require('@/assets/xiaozhi-logo.png')"
class="avatar" />
<div class="message-content">
{{ extractContentFromString(message.content) }}
<template v-if="Array.isArray(extractContentFromString(message.content))">
<div class="content-wrapper">
<div v-for="(item, idx) in extractContentFromString(message.content)" :key="idx">
<div v-if="item.type === 'text'" class="text-content">{{ item.text }}</div>
<div v-else-if="item.type === 'tool'" class="tool-call-text">{{ item.text }}</div>
<div v-else-if="item.type === 'tool_result'" class="tool-call-text">
<div v-if="item.text && item.text.length > 80" class="tool-result-wrapper">
<div v-if="isToolResultCollapsed(index, idx)" class="tool-result-collapsed">
{{ getFirstLineText(item.text) }}
</div>
<div v-else class="tool-result-expanded">
{{ item.text }}
</div>
<span class="tool-toggle-btn" @click="toggleToolResult(index, idx)">
<i :class="isToolResultCollapsed(index, idx) ? 'el-icon-arrow-down' : 'el-icon-arrow-up'"></i>
</span>
</div>
<div v-else>{{ item.text }}</div>
</div>
</div>
</div>
</template>
<template v-else>
{{ extractContentFromString(message.content) }}
</template>
<i v-if="message.audioId" :class="getAudioIconClass(message)"
@click="playAudio(message)" class="audio-icon"></i>
</div>
@@ -82,7 +106,8 @@ export default {
scrollTimer: null,
isFirstLoad: true,
playingAudioId: null,
audioElement: null
audioElement: null,
expandedToolResults: {} // 跟踪工具结果的展开状态
};
},
watch: {
@@ -159,6 +184,13 @@ export default {
// 尝试解析为 JSON
try {
const jsonObj = JSON.parse(content);
// 如果是数组格式（包含 text 和 tool）
if (Array.isArray(jsonObj)) {
return jsonObj;
}
// 如果是对象且有 content 字段
if (jsonObj && typeof jsonObj === 'object' && jsonObj.content) {
return jsonObj.content;
}
@@ -169,6 +201,23 @@ export default {
// 如果不是 JSON 格式或没有 content 字段，直接返回原内容
return content;
},
// 切换工具结果的展开/折叠状态
toggleToolResult(messageIndex, itemIndex) {
const key = `${messageIndex}-${itemIndex}`;
this.$set(this.expandedToolResults, key, !this.expandedToolResults[key]);
},
// 判断工具结果是否处于折叠状态
isToolResultCollapsed(messageIndex, itemIndex) {
const key = `${messageIndex}-${itemIndex}`;
// 默认折叠（true表示折叠）
return !this.expandedToolResults[key];
},
// 获取截断的文本（只显示第一行）
getFirstLineText(text) {
if (!text) return '';
const firstLine = text.split('\n')[0];
return firstLine.length < text.length ? firstLine + '...' : text;
},
resetData() {
this.sessions = [];
this.messages = [];
@@ -178,6 +227,7 @@ export default {
this.loading = false;
this.hasMore = true;
this.isFirstLoad = true;
this.expandedToolResults = {};
},
handleClose() {
this.dialogVisible = false;
@@ -450,6 +500,56 @@ export default {
color: white;
}
.content-wrapper {
width: 100%;
}
.text-content {
display: block;
margin-bottom: 4px;
}
.tool-call-text {
color: #1890ff;
font-family: 'Courier New', monospace;
font-weight: 500;
font-size: 12px;
display: block;
margin-top: 4px;
}
.user-message .tool-call-text {
color: #e6f7ff;
}
.tool-message .message-content {
background-color: #f0f0f0;
}
.tool-result-wrapper {
position: relative;
padding-right: 20px;
}
.tool-result-collapsed {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.tool-toggle-btn {
position: absolute;
right: 0;
top: 0;
cursor: pointer;
color: #1890ff;
font-size: 12px;
}
.tool-toggle-btn:hover {
color: #40a9ff;
}
.loading,
.no-more {
text-align: center;
main/xiaozhi-server/core/connection.py
+15 -8
View File
@@ -24,7 +24,7 @@ from core.utils.modules_initialize import (
initialize_tts,
initialize_asr,
)
from core.handle.reportHandle import report
from core.handle.reportHandle import report, enqueue_tool_report
from core.providers.tts.default import DefaultTTS
from concurrent.futures import ThreadPoolExecutor
from core.utils.dialogue import Message, Dialogue
@@ -1063,6 +1063,10 @@ class ConnectionHandler:
)
if not bHasError and len(tool_calls_list) > 0:
self.logger.bind(tag=TAG).debug(
f"检测到 {len(tool_calls_list)} 个工具调用"
)
# 更新工具调用统计
if depth == 0:
current_turn = len(self.dialogue.dialogue) // 2
@@ -1079,10 +1083,6 @@ class ConnectionHandler:
self.dialogue.put(Message(role="assistant", content=text_buff))
response_message.clear()
self.logger.bind(tag=TAG).debug(
f"检测到 {len(tool_calls_list)} 个工具调用"
)
# 收集所有工具调用的 Future
futures_with_data = []
for tool_call_data in tool_calls_list:
@@ -1090,21 +1090,28 @@ class ConnectionHandler:
f"function_name={tool_call_data['name']}, function_id={tool_call_data['id']}, function_arguments={tool_call_data['arguments']}"
)
# 使用公共方法上报工具调用
tool_input = json.loads(tool_call_data.get("arguments") or "{}")
enqueue_tool_report(self, tool_call_data['name'], tool_input)
future = asyncio.run_coroutine_threadsafe(
self.func_handler.handle_llm_function_call(
self, tool_call_data
),
self.loop,
)
futures_with_data.append((future, tool_call_data))
futures_with_data.append((future, tool_call_data, tool_input))
# 等待协程结束（实际等待时长为最慢的那个）
tool_results = []
for future, tool_call_data in futures_with_data:
for future, tool_call_data, tool_input in futures_with_data:
result = future.result()
tool_results.append((result, tool_call_data))
# 统一处理所有工具调用结果
# 使用公共方法上报工具调用结果
enqueue_tool_report(self, tool_call_data['name'], tool_input, str(result.result) if result.result else None, report_tool_call=False)
# 统一处理工具调用结果
if tool_results:
self._handle_function_result(tool_results, depth=depth)
main/xiaozhi-server/core/handle/intentHandler.py
+15
View File
@@ -10,6 +10,7 @@ from core.providers.tts.dto.dto import ContentType
from core.handle.helloHandle import checkWakeupWords
from plugins_func.register import Action, ActionResponse
from core.handle.sendAudioHandle import send_stt_message
from core.handle.reportHandle import enqueue_tool_report
from core.utils.util import remove_punctuation_and_length
from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType
@@ -141,6 +142,17 @@ async def process_intent_result(
await send_stt_message(conn, original_text)
conn.client_abort = False
# 准备工具调用参数
tool_input = {}
if function_args:
if isinstance(function_args, str):
tool_input = json.loads(function_args) if function_args else {}
elif isinstance(function_args, dict):
tool_input = function_args
# 上报工具调用
enqueue_tool_report(conn, function_name, tool_input)
# 使用executor执行函数调用和结果处理
def process_function_call():
conn.dialogue.put(Message(role="user", content=original_text))
@@ -159,7 +171,10 @@ async def process_intent_result(
action=Action.ERROR, result=str(e), response=str(e)
)
# 上报工具调用结果
if result:
enqueue_tool_report(conn, function_name, tool_input, str(result.result) if result.result else None, report_tool_call=False)
if result.action == Action.RESPONSE: # 直接回复前端
text = result.response
if text is not None:
main/xiaozhi-server/core/handle/reportHandle.py
+41 -1
View File
@@ -10,6 +10,7 @@ TTS上报功能已集成到ConnectionHandler类中。
"""
import time
import json
import opuslib_next
from typing import TYPE_CHECKING
@@ -26,7 +27,7 @@ async def report(conn: "ConnectionHandler", type, text, opus_data, report_time):
Args:
conn: 连接对象
type: 上报类型，1为用户，2为智能体
type: 上报类型，1为用户，2为智能体，3为工具调用
text: 合成文本
opus_data: opus音频数据
report_time: 上报时间
@@ -132,6 +133,45 @@ def enqueue_tts_report(conn: "ConnectionHandler", text, opus_data):
conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
def enqueue_tool_report(conn: "ConnectionHandler", tool_name: str, tool_input: dict, tool_result: str = None, report_tool_call: bool = True):
"""将工具调用数据加入上报队列
Args:
conn: 连接对象
tool_name: 工具名称
tool_input: 工具输入参数
tool_result: 工具执行结果（可选）
report_tool_call: 是否上报工具调用本身，默认True；仅上报结果时设为False
"""
if not conn.read_config_from_api or conn.need_bind:
return
if conn.chat_history_conf == 0:
return
try:
timestamp = int(time.time())
# 构建工具调用内容
if report_tool_call:
tool_text = json.dumps(
[
{
"type": "tool",
"text": f"{tool_name}({json.dumps(tool_input, ensure_ascii=False)})",
}
]
)
conn.report_queue.put((3, tool_text, None, timestamp))
# 构建工具结果内容
if tool_result:
result_display = f'{{"result":"{str(tool_result)}"}}'
result_content = json.dumps([{"type": "tool_result", "text": result_display}], ensure_ascii=False)
conn.report_queue.put((3, result_content, None, timestamp + 1))
except Exception as e:
conn.logger.bind(tag=TAG).error(f"加入工具上报队列失败: {e}")
def enqueue_asr_report(conn: "ConnectionHandler", text, opus_data):
if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
return
main/xiaozhi-server/core/handle/sendAudioHandle.py
+10
View File
@@ -318,3 +318,13 @@ async def send_stt_message(conn: "ConnectionHandler", text):
await send_tts_message(conn, "start")
# 发送start消息后客户端状态会处于说话中状态，同步服务端状态
conn.client_is_speaking = True
async def send_display_message(conn: "ConnectionHandler", text):
"""发送纯显示消息"""
message = {
"type": "stt",
"text": text,
"session_id": conn.session_id
}
await conn.websocket.send(json.dumps(message))
main/xiaozhi-server/core/providers/asr/base.py
+2 -2
View File
@@ -168,10 +168,10 @@ class ASRProviderBase(ABC):
self.stop_ws_connection()
if text_len > 0:
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
audio_snapshot = asr_audio_task.copy()
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
except Exception as e:
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
import traceback
main/xiaozhi-server/core/providers/tools/unified_tool_handler.py
+7
View File
@@ -13,6 +13,7 @@ from .server_mcp import ServerMCPExecutor
from .device_iot import DeviceIoTExecutor
from .device_mcp import DeviceMCPExecutor
from .mcp_endpoint import MCPEndpointExecutor
from core.handle.sendAudioHandle import send_display_message
class UnifiedToolHandler:
@@ -167,6 +168,12 @@ class UnifiedToolHandler:
self.logger.debug(f"调用函数: {function_name}, 参数: {arguments}")
# 发送工具调用显示消息到设备
try:
await send_display_message(self.conn, f"% {function_name}")
except Exception as e:
self.logger.warning(f"发送工具调用显示消息失败: {e}")
# 执行工具调用
result = await self.tool_manager.execute_tool(function_name, arguments)
return result
main/xiaozhi-server/plugins_func/functions/play_music.py
-1
View File
@@ -217,7 +217,6 @@ async def play_local_music(conn: "ConnectionHandler", specific_file=None):
conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
return
text = _get_random_play_prompt(selected_music)
await send_stt_message(conn, text)
conn.dialogue.put(Message(role="assistant", content=text))
if conn.intent_type == "intent_llm":