mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-27 17:43:55 +08:00
perf:
1、新增工具调用Action枚举RECORD,通过完整的三段式 assistant(tool_calls) → tool → assistant(response)让模型准确学习到工具调用的正确行为,优化play_music的工具调用枚举NONE变更为RECORD,优化解决音乐播放工具调用缺陷问题。 2、处理悬空 tool_calls问题。 3、去除工具提示词强化注入逻辑。
This commit is contained in:
@@ -46,38 +46,6 @@ from core.utils import textUtils
|
|||||||
|
|
||||||
TAG = __name__
|
TAG = __name__
|
||||||
|
|
||||||
# 工具调用规则 - 用于动态注入提醒
|
|
||||||
TOOL_CALLING_RULES = """
|
|
||||||
<tool_calling>
|
|
||||||
【核心原则】你是拥有工具能力的智能助手。当用户请求需要实时信息或执行操作时,调用相应工具获取数据,禁止凭空编造答案。
|
|
||||||
|
|
||||||
- **何时必须调用工具:**
|
|
||||||
1. 实时信息查询(新闻、非本地天气、股价、汇率等)
|
|
||||||
2. 执行操作(播放音乐、控制设备、拍照、设置闹钟等)
|
|
||||||
3. 知识库检索(当工具列表包含 search_from_ragflow 时,结合用户意图判断是否需要调用)
|
|
||||||
4. 查询非今天的农历信息(明天农历、某日宜忌、节气等)
|
|
||||||
5. 用户说"拍照"时调用 self_camera_take_photo,默认 question 参数为"描述一下看到的物品"
|
|
||||||
|
|
||||||
- **何时无需调用工具:**
|
|
||||||
1. `<context>` 中已提供的信息(当前时间、今天日期、今天农历、本地天气等)
|
|
||||||
2. 普通对话、问候、闲聊、情感交流、讲故事
|
|
||||||
3. 通用知识问答(非实时信息)
|
|
||||||
|
|
||||||
- **调用规范:**
|
|
||||||
1. 每次请求独立判断,不复用历史工具结果,需重新获取最新数据
|
|
||||||
2. 多任务时依次调用所有需要的工具,并依次总结每个工具的结果,不得遗漏
|
|
||||||
3. 严格遵循工具的参数要求,提供所有必要参数
|
|
||||||
4. 不确定时引导用户澄清或告知能力限制,切勿猜测或编造
|
|
||||||
5. 不调用未提供的工具,对话中提及的旧工具若不可用则忽略或说明
|
|
||||||
|
|
||||||
- **反偷懒机制(最高优先级):**
|
|
||||||
1. **每次独立判断:** 无论对话历史中是否调用过工具,当前请求必须根据当前需求独立判断是否需要调用
|
|
||||||
2. **禁止模式模仿:** 即使之前的回复没有调用工具,也不代表本次可以不调用
|
|
||||||
3. **自我检查:** 回复前必须自问:"这个请求是否涉及实时信息或执行操作?如果是,我调用工具了吗?"
|
|
||||||
4. **历史不等于现在:** 对话历史中的行为模式不影响当前判断,每个用户请求都是全新的开始
|
|
||||||
</tool_calling>
|
|
||||||
"""
|
|
||||||
|
|
||||||
auto_import_modules("plugins_func.functions")
|
auto_import_modules("plugins_func.functions")
|
||||||
|
|
||||||
|
|
||||||
@@ -173,12 +141,6 @@ class ConnectionHandler:
|
|||||||
# llm相关变量
|
# llm相关变量
|
||||||
self.dialogue = Dialogue()
|
self.dialogue = Dialogue()
|
||||||
|
|
||||||
# 工具调用统计(用于监控和自动恢复)
|
|
||||||
self.tool_call_stats = {
|
|
||||||
'last_call_turn': -1, # 上次调用工具的轮数
|
|
||||||
'consecutive_no_call': 0, # 连续未调用次数
|
|
||||||
}
|
|
||||||
|
|
||||||
# tts相关变量
|
# tts相关变量
|
||||||
self.sentence_id = None
|
self.sentence_id = None
|
||||||
# 处理TTS响应没有文本返回
|
# 处理TTS响应没有文本返回
|
||||||
@@ -898,32 +860,6 @@ class ConnectionHandler:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 长对话工具调用提醒:当对话轮数较多时,提醒模型正确使用工具
|
|
||||||
force_reminder = False # 是否强制提醒
|
|
||||||
|
|
||||||
if depth == 0 and query is not None:
|
|
||||||
dialogue_length = len(self.dialogue.dialogue)
|
|
||||||
current_turn = dialogue_length // 2
|
|
||||||
|
|
||||||
# 检测距离上一次连续未调用工具的情况
|
|
||||||
if self.tool_call_stats['last_call_turn'] >= 0:
|
|
||||||
turns_since_last = current_turn - self.tool_call_stats['last_call_turn']
|
|
||||||
if turns_since_last > 3: # 超过3轮未调用
|
|
||||||
self.logger.bind(tag=TAG).warning(
|
|
||||||
f"检测到{turns_since_last}轮未调用工具,可能进入偷懒模式,将强制注入提醒"
|
|
||||||
)
|
|
||||||
force_reminder = True
|
|
||||||
|
|
||||||
# 对话历史截断:防止历史过长导致模型"偷懒模式"扩散
|
|
||||||
# 当对话历史超过阈值时,保留最近的 10 轮对话
|
|
||||||
# max_dialogue_turns = 10
|
|
||||||
# if dialogue_length > max_dialogue_turns * 2:
|
|
||||||
# removed = self.dialogue.trim_history(max_turns=max_dialogue_turns)
|
|
||||||
# if removed > 0:
|
|
||||||
# self.logger.bind(tag=TAG).info(
|
|
||||||
# f"对话历史过长({dialogue_length}条),已智能截断保留最近{max_dialogue_turns}轮,移除{removed}条消息"
|
|
||||||
# )
|
|
||||||
|
|
||||||
# Define intent functions
|
# Define intent functions
|
||||||
functions = None
|
functions = None
|
||||||
# 达到最大深度时,禁用工具调用,强制 LLM 直接回答
|
# 达到最大深度时,禁用工具调用,强制 LLM 直接回答
|
||||||
@@ -934,41 +870,8 @@ class ConnectionHandler:
|
|||||||
):
|
):
|
||||||
functions = self.func_handler.get_functions()
|
functions = self.func_handler.get_functions()
|
||||||
|
|
||||||
# 长对话工具调用规则强化:动态生成基于当前可用工具的提醒
|
|
||||||
tool_call_reminder = None
|
|
||||||
if depth == 0 and query is not None and functions is not None:
|
|
||||||
dialogue_length = len(self.dialogue.dialogue)
|
|
||||||
# 当对话历史超过4条消息时,注入规则强化
|
|
||||||
if dialogue_length > 4:
|
|
||||||
tool_summary = self._get_tool_summary(functions)
|
|
||||||
if tool_summary:
|
|
||||||
# 根据对话长度和偷懒检测,使用不同强度的提醒
|
|
||||||
if force_reminder:
|
|
||||||
# 强提醒 - 包含完整规则前缀
|
|
||||||
tool_call_reminder = (
|
|
||||||
TOOL_CALLING_RULES +
|
|
||||||
f"[重要提醒] 多轮未使用工具,检查回复是否遗漏了必要的工具调用!上一轮未使用工具,本轮必须重新判断是否需要工具。"
|
|
||||||
f"当前可用工具: {tool_summary}。"
|
|
||||||
)
|
|
||||||
reminder_level = "强"
|
|
||||||
else:
|
|
||||||
# 中等提醒 - 包含规则前缀
|
|
||||||
tool_call_reminder = (
|
|
||||||
TOOL_CALLING_RULES +
|
|
||||||
f"当前可用工具: {tool_summary}。"
|
|
||||||
f"仅当用户请求涉及实时信息查询或执行操作时调用,日常对话无需调用。"
|
|
||||||
)
|
|
||||||
reminder_level = "中"
|
|
||||||
self.logger.bind(tag=TAG).debug(
|
|
||||||
f"对话历史较长({dialogue_length}条),已注入{reminder_level}等级工具调用规则强化,当前可用工具:{tool_summary}"
|
|
||||||
)
|
|
||||||
|
|
||||||
response_message = []
|
response_message = []
|
||||||
|
|
||||||
# 如果有工具调用提醒,临时添加到对话中(标记为临时消息)
|
|
||||||
if tool_call_reminder:
|
|
||||||
self.dialogue.put(Message(role="user", content=tool_call_reminder, is_temporary=True))
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 使用带记忆的对话
|
# 使用带记忆的对话
|
||||||
memory_str = None
|
memory_str = None
|
||||||
@@ -1100,15 +1003,6 @@ class ConnectionHandler:
|
|||||||
f"检测到 {len(tool_calls_list)} 个工具调用"
|
f"检测到 {len(tool_calls_list)} 个工具调用"
|
||||||
)
|
)
|
||||||
|
|
||||||
# 更新工具调用统计
|
|
||||||
if depth == 0:
|
|
||||||
current_turn = len(self.dialogue.dialogue) // 2
|
|
||||||
self.tool_call_stats['last_call_turn'] = current_turn
|
|
||||||
self.tool_call_stats['consecutive_no_call'] = 0
|
|
||||||
self.logger.bind(tag=TAG).debug(
|
|
||||||
f"工具调用统计更新: 当前轮次={current_turn}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# LLM 流式阶段已播报过的文本
|
# LLM 流式阶段已播报过的文本
|
||||||
streamed_text = ""
|
streamed_text = ""
|
||||||
if len(response_message) > 0:
|
if len(response_message) > 0:
|
||||||
@@ -1170,10 +1064,6 @@ class ConnectionHandler:
|
|||||||
self.tts.store_tts_text(current_sentence_id, text_buff)
|
self.tts.store_tts_text(current_sentence_id, text_buff)
|
||||||
self.dialogue.put(Message(role="assistant", content=text_buff))
|
self.dialogue.put(Message(role="assistant", content=text_buff))
|
||||||
|
|
||||||
# 更新工具调用统计:如果没有调用工具,增加计数
|
|
||||||
if depth == 0 and not tool_call_flag:
|
|
||||||
self.tool_call_stats['consecutive_no_call'] += 1
|
|
||||||
|
|
||||||
if depth == 0:
|
if depth == 0:
|
||||||
self.tts.tts_text_queue.put(
|
self.tts.tts_text_queue.put(
|
||||||
TTSMessageDTO(
|
TTSMessageDTO(
|
||||||
@@ -1189,41 +1079,11 @@ class ConnectionHandler:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 清理临时插入的工具调用提醒消息(使用标记清理)
|
|
||||||
if tool_call_reminder and len(self.dialogue.dialogue) > 0:
|
|
||||||
original_length = len(self.dialogue.dialogue)
|
|
||||||
self.dialogue.dialogue = [
|
|
||||||
msg for msg in self.dialogue.dialogue
|
|
||||||
if not getattr(msg, 'is_temporary', False)
|
|
||||||
]
|
|
||||||
if len(self.dialogue.dialogue) < original_length:
|
|
||||||
self.logger.bind(tag=TAG).debug("已清理临时的工具调用提醒消息")
|
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _get_tool_summary(self, functions: list) -> str:
|
|
||||||
"""
|
|
||||||
从工具定义中提取摘要,用于规则强化注入
|
|
||||||
|
|
||||||
Args:
|
|
||||||
functions: 工具列表
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: 工具名称字符串
|
|
||||||
"""
|
|
||||||
if not functions:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
datas = []
|
|
||||||
for func in functions:
|
|
||||||
func_info = func.get("function", {})
|
|
||||||
name = func_info.get("name", "")
|
|
||||||
datas.append(name)
|
|
||||||
result = "、".join(datas)
|
|
||||||
return result
|
|
||||||
|
|
||||||
def _handle_function_result(self, tool_results, depth, streamed_text=""):
|
def _handle_function_result(self, tool_results, depth, streamed_text=""):
|
||||||
need_llm_tools = []
|
need_llm_tools = []
|
||||||
|
record_tools = []
|
||||||
|
|
||||||
for result, tool_call_data in tool_results:
|
for result, tool_call_data in tool_results:
|
||||||
if result.action in [
|
if result.action in [
|
||||||
@@ -1241,11 +1101,58 @@ class ConnectionHandler:
|
|||||||
self.tts.store_tts_text(self.sentence_id, text)
|
self.tts.store_tts_text(self.sentence_id, text)
|
||||||
self.dialogue.put(Message(role="assistant", content=text))
|
self.dialogue.put(Message(role="assistant", content=text))
|
||||||
elif result.action == Action.REQLLM:
|
elif result.action == Action.REQLLM:
|
||||||
# 收集需要 LLM 处理的工具
|
|
||||||
need_llm_tools.append((result, tool_call_data))
|
need_llm_tools.append((result, tool_call_data))
|
||||||
|
elif result.action == Action.RECORD:
|
||||||
|
record_tools.append((result, tool_call_data))
|
||||||
else:
|
else:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# Action.RECORD:写入完整工具调用链(assistant(tool_calls) → tool(result) → assistant(response))
|
||||||
|
# 模型从历史中学到工具调用模式,不额外调用LLM
|
||||||
|
if record_tools:
|
||||||
|
# 构造 assistant 消息(含 tool_calls),记录"模型调用了哪些工具"
|
||||||
|
all_tool_calls = [
|
||||||
|
{
|
||||||
|
"id": tool_call_data["id"],
|
||||||
|
"function": {
|
||||||
|
"arguments": (
|
||||||
|
"{}"
|
||||||
|
if tool_call_data["arguments"] == ""
|
||||||
|
else tool_call_data["arguments"]
|
||||||
|
),
|
||||||
|
"name": tool_call_data["name"],
|
||||||
|
},
|
||||||
|
"type": "function",
|
||||||
|
"index": idx,
|
||||||
|
}
|
||||||
|
for idx, (_, tool_call_data) in enumerate(record_tools)
|
||||||
|
]
|
||||||
|
self.dialogue.put(Message(role="assistant", tool_calls=all_tool_calls))
|
||||||
|
|
||||||
|
# 写入每条工具的执行结果,记录"工具返回了什么"
|
||||||
|
for result, tool_call_data in record_tools:
|
||||||
|
text = result.result or ""
|
||||||
|
self.dialogue.put(
|
||||||
|
Message(
|
||||||
|
role="tool",
|
||||||
|
tool_call_id=(
|
||||||
|
str(uuid.uuid4())
|
||||||
|
if tool_call_data["id"] is None
|
||||||
|
else tool_call_data["id"]
|
||||||
|
),
|
||||||
|
content=text,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 用固定文本作为最终回复,补全标准三段式,保证下一条消息是 user 而非接 tool
|
||||||
|
response_parts = []
|
||||||
|
for result, _ in record_tools:
|
||||||
|
resp = result.response or result.result
|
||||||
|
if resp:
|
||||||
|
response_parts.append(resp)
|
||||||
|
if response_parts:
|
||||||
|
self.dialogue.put(Message(role="assistant", content=",".join(response_parts)))
|
||||||
|
|
||||||
if need_llm_tools:
|
if need_llm_tools:
|
||||||
all_tool_calls = [
|
all_tool_calls = [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -123,6 +123,36 @@ class Dialogue:
|
|||||||
|
|
||||||
return removed_count
|
return removed_count
|
||||||
|
|
||||||
|
def _ensure_tool_calls_complete(self, messages: List[Message]) -> List[Message]:
|
||||||
|
"""
|
||||||
|
确保所有 tool_calls 都有对应的 tool 响应
|
||||||
|
修复被打断导致的悬空 tool_calls,防止大模型 API 报 400 错误
|
||||||
|
"""
|
||||||
|
pending_tool_calls = set()
|
||||||
|
result = []
|
||||||
|
|
||||||
|
for msg in messages:
|
||||||
|
result.append(msg)
|
||||||
|
|
||||||
|
if msg.role == "assistant" and msg.tool_calls:
|
||||||
|
for tc in msg.tool_calls:
|
||||||
|
tc_id = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
|
||||||
|
if tc_id:
|
||||||
|
pending_tool_calls.add(tc_id)
|
||||||
|
|
||||||
|
elif msg.role == "tool" and msg.tool_call_id:
|
||||||
|
pending_tool_calls.discard(msg.tool_call_id)
|
||||||
|
|
||||||
|
for missing_id in pending_tool_calls:
|
||||||
|
dummy_tool_msg = Message(
|
||||||
|
role="tool",
|
||||||
|
content='{"status": "interrupted", "message": "动作已取消/被打断"}',
|
||||||
|
tool_call_id=missing_id
|
||||||
|
)
|
||||||
|
result.append(dummy_tool_msg)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
def get_llm_dialogue_with_memory(
|
def get_llm_dialogue_with_memory(
|
||||||
self, memory_str: str = None, voiceprint_config: dict = None
|
self, memory_str: str = None, voiceprint_config: dict = None
|
||||||
) -> List[Dict[str, str]]:
|
) -> List[Dict[str, str]]:
|
||||||
@@ -175,8 +205,9 @@ class Dialogue:
|
|||||||
dialogue.append({"role": "system", "content": enhanced_system_prompt})
|
dialogue.append({"role": "system", "content": enhanced_system_prompt})
|
||||||
|
|
||||||
# 添加用户和助手的对话
|
# 添加用户和助手的对话
|
||||||
for m in self.dialogue:
|
non_system_messages = [m for m in self.dialogue if m.role != "system"]
|
||||||
if m.role != "system": # 跳过原始的系统消息
|
complete_messages = self._ensure_tool_calls_complete(non_system_messages)
|
||||||
self.getMessages(m, dialogue)
|
for m in complete_messages:
|
||||||
|
self.getMessages(m, dialogue)
|
||||||
|
|
||||||
return dialogue
|
return dialogue
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ def play_music(conn: "ConnectionHandler", song_name: str):
|
|||||||
task.add_done_callback(handle_done)
|
task.add_done_callback(handle_done)
|
||||||
|
|
||||||
return ActionResponse(
|
return ActionResponse(
|
||||||
action=Action.NONE, result="指令已接收", response="正在为您播放音乐"
|
action=Action.RECORD, result="指令已接收", response="正在为您播放音乐"
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
conn.logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
|
conn.logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
|
||||||
@@ -217,7 +217,7 @@ async def play_local_music(conn: "ConnectionHandler", specific_file=None):
|
|||||||
conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
|
conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
|
||||||
return
|
return
|
||||||
text = _get_random_play_prompt(selected_music)
|
text = _get_random_play_prompt(selected_music)
|
||||||
conn.dialogue.put(Message(role="assistant", content=text))
|
# conn.dialogue.put(Message(role="assistant", content=text))
|
||||||
|
|
||||||
if conn.intent_type == "intent_llm":
|
if conn.intent_type == "intent_llm":
|
||||||
conn.tts.tts_text_queue.put(
|
conn.tts.tts_text_queue.put(
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ class Action(Enum):
|
|||||||
NONE = (1, "啥也不干")
|
NONE = (1, "啥也不干")
|
||||||
RESPONSE = (2, "直接回复")
|
RESPONSE = (2, "直接回复")
|
||||||
REQLLM = (3, "调用函数后再请求llm生成回复")
|
REQLLM = (3, "调用函数后再请求llm生成回复")
|
||||||
|
RECORD = (4, "记录工具调用到对话历史,不调用LLM")
|
||||||
|
|
||||||
def __init__(self, code, message):
|
def __init__(self, code, message):
|
||||||
self.code = code
|
self.code = code
|
||||||
|
|||||||
Reference in New Issue
Block a user