Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

update:intent_llm加载动态插件

Browse Source
This commit is contained in:
hrz
2025-05-09 11:39:32 +08:00
parent 47f409246d
commit ff98f84f18
9 changed files with 113 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/config.yaml
+7
View File
@@ -175,6 +175,13 @@ Intent:
# 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型
# 如果你的不想使用selected_module.LLM意图识别，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM
llm: ChatGLMLLM
# plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用
# 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载
# 下面是加载查天气、角色切换、加载查新闻的插件示例
functions:
- get_weather
- get_news_from_newsnow
- play_music
function_call:
# 不需要动type
type: function_call
main/xiaozhi-server/core/connection.py
+17 -9
View File
@@ -58,9 +58,9 @@ class ConnectionHandler:
self.config = copy.deepcopy(config)
self.session_id = str(uuid.uuid4())
self.logger = setup_logging()
self.auth = AuthMiddleware(config)
self.server = server # 保存server实例的引用
self.auth = AuthMiddleware(config)
self.need_bind = False
self.bind_code = None
self.read_config_from_api = self.config.get("read_config_from_api", False)
@@ -128,8 +128,10 @@ class ConnectionHandler:
if len(cmd) > self.max_cmd_length:
self.max_cmd_length = len(cmd)
self.close_after_chat = False # 是否在聊天结束后关闭连接
self.use_function_call_mode = False
# 是否在聊天结束后关闭连接
self.close_after_chat = False
self.load_function_plugin = False
self.intent_type = "nointent"
self.timeout_task = None
self.timeout_seconds = (
@@ -365,11 +367,11 @@ class ConnectionHandler:
self.memory.init_memory(self.device_id, self.llm)
def _initialize_intent(self):
if (
self.config["Intent"][self.config["selected_module"]["Intent"]]["type"]
== "function_call"
):
self.use_function_call_mode = True
self.intent_type = self.config["Intent"][
self.config["selected_module"]["Intent"]
]["type"]
if self.intent_type == "function_call" or self.intent_type == "intent_llm":
self.load_function_plugin = True
"""初始化意图识别模块"""
# 获取意图识别配置
intent_config = self.config["Intent"]
@@ -757,7 +759,13 @@ class ConnectionHandler:
)
self.dialogue.put(
Message(role="tool", tool_call_id=function_id, content=text)
Message(
role="tool",
tool_call_id=(
str(uuid.uuid4()) if function_id is None else function_id
),
content=text,
)
)
self.chat_with_function_calling(text, tool_call=True)
elif result.action == Action.NOTFOUND or result.action == Action.ERROR:
main/xiaozhi-server/core/handle/intentHandler.py
+11 -6
View File
@@ -19,7 +19,7 @@ async def handle_user_intent(conn, text):
if await checkWakeupWords(conn, text):
return True
if conn.use_function_call_mode:
if conn.intent_type == "function_call":
# 使用支持function calling的聊天方法,不再进行意图分析
return False
# 使用LLM进行意图分析
@@ -103,7 +103,7 @@ async def process_intent_result(conn, intent_result, original_text):
conn, function_call_data
)
logger.bind(tag=TAG).debug(f"检测到Action : {result.action}")
if result:
if result.action == Action.RESPONSE: # 直接回复前端
text = result.response
@@ -118,14 +118,19 @@ async def process_intent_result(conn, intent_result, original_text):
conn.speak_and_play, text, text_index
)
conn.llm_finish_task = True
conn.tts_queue.put(future)
conn.tts_queue.put((future, text_index))
conn.dialogue.put(Message(role="assistant", content=text))
elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
text = result.result
if text is not None and len(text) > 0:
conn.dialogue.put(Message(role="tool", content=text))
conn.executor.submit(conn.chat_with_function_calling, text, True)
elif result.action == Action.NOTFOUND or result.action == Action.ERROR:
conn.executor.submit(
conn.chat_with_function_calling, text, True
)
elif (
result.action == Action.NOTFOUND
or result.action == Action.ERROR
):
text = result.result
if text is not None:
text_index = (
@@ -157,7 +162,7 @@ async def process_intent_result(conn, intent_result, original_text):
conn.speak_and_play, text, text_index
)
conn.llm_finish_task = True
conn.tts_queue.put(future)
conn.tts_queue.put((future, text_index))
conn.dialogue.put(Message(role="assistant", content=text))
# 将函数执行放在线程池中
main/xiaozhi-server/core/handle/iotHandle.py
+1 -1
View File
@@ -317,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors):
)
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
if conn.use_function_call_mode:
if conn.load_function_plugin:
# 注册或获取设备类型
type_id = register_device_type(descriptor)
device_functions = device_type_registry.get_device_functions(type_id)
main/xiaozhi-server/core/handle/receiveAudioHandle.py
+1 -1
View File
@@ -76,7 +76,7 @@ async def startToChat(conn, text):
# 意图未被处理，继续常规聊天流程
await send_stt_message(conn, text)
if conn.use_function_call_mode:
if conn.intent_type == "function_call":
# 使用支持function calling的聊天方法
conn.executor.submit(conn.chat_with_function_calling, text)
else:
main/xiaozhi-server/core/providers/intent/base.py
-12
View File
@@ -9,18 +9,6 @@ logger = setup_logging()
class IntentProviderBase(ABC):
def __init__(self, config):
self.config = config
self.intent_options = [
{
"name": "handle_exit_intent",
"desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
},
{
"name": "play_music",
"desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
},
{"name": "get_time", "desc": "获取今天日期或者当前时间信息"},
{"name": "continue_chat", "desc": "继续聊天"},
]
def set_llm(self, llm):
self.llm = llm
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+67 -39
View File
@@ -15,57 +15,71 @@ class IntentProvider(IntentProviderBase):
def __init__(self, config):
super().__init__(config)
self.llm = None
self.promot = self.get_intent_system_prompt()
self.promot = ""
# 添加缓存管理
self.intent_cache = {} # 缓存意图识别结果
self.cache_expiry = 600 # 缓存有效期10分钟
self.cache_max_size = 100 # 最多缓存100个意图
def get_intent_system_prompt(self) -> str:
def get_intent_system_prompt(self, functions_list: str) -> str:
"""
根据配置的意图选项动态生成系统提示词
根据配置的意图选项和可用函数动态生成系统提示词
Args:
functions: 可用的函数列表，JSON格式字符串
Returns:
格式化后的系统提示词
"""
# 构建函数说明部分
functions_desc = "可用的函数列表：\n"
for func in functions_list:
func_info = func.get("function", {})
name = func_info.get("name", "")
desc = func_info.get("description", "")
params = func_info.get("parameters", {})
functions_desc += f"\n函数名: {name}\n"
functions_desc += f"描述: {desc}\n"
if params:
functions_desc += "参数:\n"
for param_name, param_info in params.get("properties", {}).items():
param_desc = param_info.get("description", "")
param_type = param_info.get("type", "")
functions_desc += f"- {param_name} ({param_type}): {param_desc}\n"
functions_desc += "---\n"
prompt = (
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
"<start>"
f"{str(self.intent_options)}"
"<end>\n"
"处理步骤:"
"1. 思考意图类型，生成function_call格式"
"\n\n"
"返回格式示例：\n"
'1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n'
'2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
'3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n'
'4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n'
"\n"
"注意:\n"
'- 播放音乐：无歌名时，song_name设为"random"\n'
"- 如果没有明显的意图，应按照继续聊天意图处理\n"
"- 只返回纯JSON，不要任何其他内容\n"
"\n"
"示例分析:\n"
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
f"{functions_desc}\n"
"处理步骤:\n"
"1. 分析用户输入，确定用户意图\n"
"2. 从可用函数列表中选择最匹配的函数\n"
"3. 如果找到匹配的函数，生成对应的function_call 格式\n"
'4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
"返回格式要求：\n"
"1. 必须返回纯JSON格式\n"
"2. 必须包含function_call字段\n"
"3. function_call必须包含name字段\n"
"4. 如果函数需要参数，必须包含arguments字段\n\n"
"示例：\n"
"```\n"
"用户: 你也太搞笑了\n"
'返回: {"function_call": {"name": "continue_chat"}}\n'
"```\n"
"```\n"
"用户: 现在是几号了?现在几点了？\n"
"用户: 现在几点了？\n"
'返回: {"function_call": {"name": "get_time"}}\n'
"```\n"
"```\n"
"用户: 我们明天再聊吧\n"
'返回: {"function_call": {"name": "handle_exit_intent"}}\n'
"用户: 我想结束对话\n"
'返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
"```\n"
"```\n"
"用户: 播放中秋月\n"
'返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n'
"```\n"
"```\n"
"可用的音乐名称:\n"
"用户: 你好啊\n"
'返回: {"function_call": {"name": "continue_chat"}}\n'
"```\n\n"
"注意：\n"
"1. 只返回JSON格式，不要包含任何其他文字\n"
'2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
"3. 确保返回的JSON格式正确，包含所有必要的字段\n"
)
return prompt
@@ -118,6 +132,24 @@ class IntentProvider(IntentProviderBase):
# 清理缓存
self.clean_cache()
if self.promot == "":
if hasattr(conn, "func_handler"):
functions = conn.func_handler.get_functions()
self.promot = self.get_intent_system_prompt(functions)
music_config = initialize_music_handler(conn)
music_file_names = music_config["music_file_names"]
prompt_music = f"{self.promot}\n<musicNames>{music_file_names}\n</musicNames>"
devices = conn.config["plugins"]["home_assistant"].get("devices", [])
if len(devices) > 0:
hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
for device in devices:
hass_prompt += device + "\n"
prompt_music += hass_prompt
logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
# 构建用户最后一句话的提示
msgStr = ""
@@ -128,11 +160,7 @@ class IntentProvider(IntentProviderBase):
msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n"
msgStr += f"User: {text}\n"
user_prompt = f"当前的对话如下：\n{msgStr}"
music_config = initialize_music_handler(conn)
music_file_names = music_config["music_file_names"]
prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
user_prompt = f"current dialogue:\n{msgStr}"
# 记录预处理完成时间
preprocess_time = time.time() - total_start_time
main/xiaozhi-server/core/utils/dialogue.py
+7 -1
View File
@@ -33,7 +33,13 @@ class Dialogue:
dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
elif m.role == "tool":
dialogue.append(
{"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content}
{
"role": m.role,
"tool_call_id": (
str(uuid.uuid4()) if m.tool_call_id is None else m.tool_call_id
),
"content": m.content,
}
)
else:
dialogue.append({"role": m.role, "content": m.content})
main/xiaozhi-server/plugins_func/functions/hass_init.py
+2 -2
View File
@@ -8,7 +8,7 @@ HASS_CACHE = {}
def append_devices_to_prompt(conn):
if conn.use_function_call_mode:
if conn.intent_type == "function_call":
funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get(
"functions", []
)
@@ -27,7 +27,7 @@ def append_devices_to_prompt(conn):
def initialize_hass_handler(conn):
global HASS_CACHE
if HASS_CACHE == {}:
if conn.use_function_call_mode:
if conn.load_function_plugin:
funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get(
"functions", []
)