diff --git a/README.md b/README.md index b59d9d45..191790bc 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ - + 控制家电开关 @@ -95,6 +95,11 @@ + + + 播报新闻 + + @@ -185,7 +190,6 @@ server: | LLM | FastgptLLM | fastgpt 接口调用 | 免费/消耗 token | 本地化部署，注意配置提示词需在 Fastgpt 控制台设置 | | LLM | GeminiLLM | gemini 接口调用 | 免费 | [点击申请密钥](https://aistudio.google.com/apikey) | | LLM | CozeLLM | coze 接口调用 | 消耗 token | 需提供 bot_id、user_id 及个人令牌 | -| LLM | Home Assistant | homeassistant语音助手接口调用 | 免费 | 需提供home assistant令牌 | 实际上，任何支持 openai 接口调用的 LLM 均可接入使用。 @@ -370,7 +374,41 @@ VAD: ### 6、我想通过小智控制电灯、空调、远程开关机等操作 💡 -建议：在配置文件中将 `LLM` 设置为 `HomeAssistant`，通过 调用`HomeAssistant`接口实现相关控制。 +本项目，支持以工具调用的方式控制HomeAssistant设备 + +1、首先选择一款支持function call支持的LLM，例如`ChatGLMLLM`。 + +2、在配置文件中，将 `selected_module.Intent` 设置为 `function_call`。 + +3、登录`HomeAssistant`，点击`左下角个人`，切换`安全`导航栏，划到底部`长期访问令牌`生成api_key。 + +在配置文件中，配置好你的home assistant的`devices`（被控制的设备）和`api_key`和`base_url`等信息。例如： + +``` yaml +plugins + home_assistant: + devices: + - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 + - 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1 + base_url: http://你的homeassistant地址:8123 + api_key: 你的home assistant api访问令牌 +``` + +最后，允许function_call 插件在配置文件中启用`hass_get_state`(必须)、`hass_set_state`(必须)、`hass_play_music`(不想用ha听音乐可以不启动)，例如： + +``` yaml +Intent: + ... + function_call: + type: nointent + functions: + - change_role + - get_weather + - get_news + - hass_get_state + - hass_set_state + - hass_play_music +``` ### 7、更多问题，可联系我们反馈 💬 diff --git a/docs/images/demo0.png b/docs/images/demo0.png new file mode 100644 index 00000000..99592be6 Binary files /dev/null and b/docs/images/demo0.png differ diff --git a/docs/images/demo5.png b/docs/images/demo5.png index 8e03685b..c6da2997 100644 Binary files a/docs/images/demo5.png and b/docs/images/demo5.png differ diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ac62f988..9221b471 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -49,7 +49,7 @@ prompt: | 你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。 请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。 现在我正在和你进行语音聊天，我们开始吧。 - 如果用户希望结束对话，请在最后说“拜拜”或“再见”。 + # 使用完声音文件后删除文件(Delete the sound file when you are done using it) delete_audio: true @@ -99,6 +99,12 @@ Intent: - change_role - get_weather - get_news + # play_music是服务器自带的音乐播放，hass_play_music是通过home assistant控制的独立外部程序音乐播放 + # 如果用了hass_play_music，就不要开启play_music，两者只留一个 + - play_music + #- hass_get_state + #- hass_set_state + #- hass_play_music # 插件的基础配置 plugins: @@ -115,6 +121,13 @@ plugins: society: "https://www.chinanews.com.cn/rss/society.xml" world: "https://www.chinanews.com.cn/rss/world.xml" finance: "https://www.chinanews.com.cn/rss/finance.xml" + home_assistant: + devices: + - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 + - 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1 + base_url: http://homeassistant.local:8123 + api_key: 你的home assistant api访问令牌 + Memory: mem0ai: @@ -241,12 +254,6 @@ LLM: model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载 url: http://localhost:1234/v1 # LM Studio服务地址 api_key: lm-studio # LM Studio服务的固定API Key - HomeAssistant: - # 定义LLM API类型 - type: homeassistant - base_url: http://homeassistant.local:8123 - agent_id: conversation.chatgpt - api_key: 你的home assistant api访问令牌 FastgptLLM: # 定义LLM API类型 type: fastgpt @@ -500,20 +507,3 @@ module_test: - "你好，请介绍一下你自己" - "What's the weather like today?" - "请用100字概括量子计算的基本原理和应用前景" - -# 本地音乐播放配置 -music: - music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件 - music_ext: # 音乐文件类型，p3格式效率最高 - - ".mp3" - - ".wav" - - ".p3" - refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒 - -# 以下配置在小于等于0.0.9版本中的docker容器中可用 -# 0.0.9以后的新版本源码部署已经无法奏效 -manager: - enabled: false - ip: 0.0.0.0 - port: 8002 -use_private_config: false diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 718398c3..c0db5eb8 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -196,7 +196,7 @@ class ConnectionHandler: self.prompt = self.prompt + f"\n我在:{self.client_ip_info}" self.dialogue.put(Message(role="system", content=self.prompt)) - self.func_handler = FunctionHandler(self.config) + self.func_handler = FunctionHandler(self) def change_system_prompt(self, prompt): self.prompt = prompt diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index c7449973..e8056517 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -1,16 +1,16 @@ -import asyncio -from enum import Enum - from config.logger import setup_logging import json from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType +from plugins_func.functions.hass_init import append_devices_to_prompt + TAG = __name__ logger = setup_logging() class FunctionHandler: - def __init__(self, config): - self.config = config + def __init__(self, conn): + self.conn = conn + self.config = conn.config self.function_registry = FunctionRegistry() self.register_nessary_functions() self.register_config_functions() @@ -26,9 +26,10 @@ class FunctionHandler: func_names = ",".join(surport_plugins) for function_desc in self.functions_desc: if function_desc["function"]["name"] == "plugin_loader": - function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names) + function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", + func_names) break - + def upload_functions_desc(self): self.functions_desc = self.function_registry.get_all_function_desc() @@ -47,7 +48,6 @@ class FunctionHandler: def register_nessary_functions(self): """注册必要的函数""" self.function_registry.register_function("handle_exit_intent") - self.function_registry.register_function("play_music") self.function_registry.register_function("plugin_loader") self.function_registry.register_function("get_time") self.function_registry.register_function("raise_and_lower_the_volume") @@ -57,6 +57,9 @@ class FunctionHandler: for func in self.config["Intent"]["function_call"].get("functions", []): self.function_registry.register_function(func) + """home assistant需要初始化提示词""" + append_devices_to_prompt(self.conn) + def get_function(self, name): return self.function_registry.get_function(name) @@ -81,4 +84,4 @@ class FunctionHandler: except Exception as e: logger.bind(tag=TAG).error(f"处理function call错误: {e}") - return None \ No newline at end of file + return None diff --git a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py b/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py deleted file mode 100644 index 115e85e9..00000000 --- a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py +++ /dev/null @@ -1,62 +0,0 @@ -import requests -from requests.exceptions import RequestException -from config.logger import setup_logging -from core.providers.llm.base import LLMProviderBase - -TAG = __name__ -logger = setup_logging() - - -class LLMProvider(LLMProviderBase): - def __init__(self, config): - self.agent_id = config.get("agent_id") # 对应 agent_id - self.api_key = config.get("api_key") - self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url - self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL - - def response(self, session_id, dialogue): - print(dialogue) - try: - # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可 - - # 提取最后一个 role 为 'user' 的 content - input_text = None - if isinstance(dialogue, list): # 确保 dialogue 是一个列表 - # 逆序遍历，找到最后一个 role 为 'user' 的消息 - for message in reversed(dialogue): - if message.get("role") == "user": # 找到 role 为 'user' 的消息 - input_text = message.get("content", "") - break # 找到后立即退出循环 - - # 构造请求数据 - payload = { - "text": input_text, - "agent_id": self.agent_id, - "conversation_id": session_id # 使用 session_id 作为 conversation_id - } - # 设置请求头 - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - # 发起 POST 请求 - response = requests.post(self.api_url, json=payload, headers=headers) - - # 检查请求是否成功 - response.raise_for_status() - - # 解析返回数据 - data = response.json() - speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "") - - # 返回生成的内容 - if speech: - yield speech - else: - logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容") - - except RequestException as e: - logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}") - except Exception as e: - logger.bind(tag=TAG).error(f"生成响应时出错: {e}") diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index 8d2c161b..d4e66bb6 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -35,6 +35,15 @@ class Dialogue: self.getMessages(m, dialogue) return dialogue + def update_system_message(self, new_content: str): + """更新或添加系统消息""" + # 查找第一个系统消息 + system_msg = next((msg for msg in self.dialogue if msg.role == "system"), None) + if system_msg: + system_msg.content = new_content + else: + self.put(Message(role="system", content=new_content)) + def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]: if memory_str is None or len(memory_str) == 0: return self.get_llm_dialogue() diff --git a/main/xiaozhi-server/plugins_func/functions/hass_get_state.py b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py new file mode 100644 index 00000000..415f35c6 --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py @@ -0,0 +1,56 @@ +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from plugins_func.functions.hass_init import initialize_hass_handler +from config.logger import setup_logging +import asyncio +import requests + +TAG = __name__ +logger = setup_logging() + +hass_get_state_function_desc = { + "type": "function", + "function": { + "name": "hass_get_state", + "description": "获取homeassistant里设备的状态,包括灯光亮度,媒体播放器的音量,设备的暂停、继续操作", + "parameters": { + "type": "object", + "properties": { + "entity_id": { + "type": "string", + "description": "需要操作的设备id,homeassistant里的entity_id" + } + }, + "required": ["entity_id"] + } + } +} + + +@register_function("hass_get_state", hass_get_state_function_desc, ToolType.SYSTEM_CTL) +def hass_get_state(conn, entity_id=''): + try: + + future = asyncio.run_coroutine_threadsafe( + handle_hass_get_state(conn, entity_id), + conn.loop + ) + ha_response = future.result() + return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response) + except Exception as e: + logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}") + + +async def handle_hass_get_state(conn, entity_id): + HASS_CACHE = initialize_hass_handler(conn) + api_key = HASS_CACHE['api_key'] + base_url = HASS_CACHE['base_url'] + url = f"{base_url}/api/states/{entity_id}" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.json()['state'] + else: + return f"切换失败，错误码: {response.status_code}" diff --git a/main/xiaozhi-server/plugins_func/functions/hass_init.py b/main/xiaozhi-server/plugins_func/functions/hass_init.py new file mode 100644 index 00000000..e09eb67f --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_init.py @@ -0,0 +1,35 @@ +from config.logger import setup_logging +from core.utils.util import check_model_key + +TAG = __name__ +logger = setup_logging() + +HASS_CACHE = {} + + +def append_devices_to_prompt(conn): + if conn.use_function_call_mode: + funcs = conn.config["Intent"]["function_call"].get("functions", []) + if "hass_get_state" in funcs or "hass_get_state" in funcs: + prompt = "下面是我家智能设备，可以通过homeassistant控制\n" + devices = conn.config["plugins"]["home_assistant"].get("devices", []) + if len(devices) == 0: + return + for device in devices: + prompt += device + "\n" + conn.prompt += prompt + # 更新提示词 + conn.dialogue.update_system_message(conn.prompt) + + +def initialize_hass_handler(conn): + global HASS_CACHE + if HASS_CACHE == {}: + if conn.use_function_call_mode: + funcs = conn.config["Intent"]["function_call"].get("functions", []) + if "hass_get_state" in funcs or "hass_get_state" in funcs: + HASS_CACHE['base_url'] = conn.config["plugins"]["home_assistant"].get("base_url") + HASS_CACHE['api_key'] = conn.config["plugins"]["home_assistant"].get("api_key") + + check_model_key("home_assistant", HASS_CACHE['api_key']) + return HASS_CACHE diff --git a/main/xiaozhi-server/plugins_func/functions/hass_play_music.py b/main/xiaozhi-server/plugins_func/functions/hass_play_music.py new file mode 100644 index 00000000..502f784e --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_play_music.py @@ -0,0 +1,64 @@ +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from plugins_func.functions.hass_init import initialize_hass_handler +from config.logger import setup_logging +import asyncio +import requests + +TAG = __name__ +logger = setup_logging() + +hass_play_music_function_desc = { + "type": "function", + "function": { + "name": "hass_play_music", + "description": "用户想听音乐、有声书的时候使用，在房间的媒体播放器（media_player）里播放对应音频", + "parameters": { + "type": "object", + "properties": { + "media_content_id": { + "type": "string", + "description": "可以是音乐或有声书的专辑名称、歌曲名、演唱者,如果未指定就填random" + }, + "entity_id": { + "type": "string", + "description": "需要操作的音箱的设备id,homeassistant里的entity_id,media_player开头" + } + }, + "required": ["media_content_id", "entity_id"] + } + } +} + + +@register_function('hass_play_music', hass_play_music_function_desc, ToolType.SYSTEM_CTL) +def hass_play_music(conn, entity_id='', media_content_id='random'): + try: + # 执行音乐播放命令 + future = asyncio.run_coroutine_threadsafe( + handle_hass_play_music(conn, entity_id, media_content_id), + conn.loop + ) + ha_response = future.result() + return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=ha_response) + except Exception as e: + logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}") + + +async def handle_hass_play_music(conn, entity_id, media_content_id): + HASS_CACHE = initialize_hass_handler(conn) + api_key = HASS_CACHE['api_key'] + base_url = HASS_CACHE['base_url'] + url = f"{base_url}/api/services/music_assistant/play_media" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + data = { + "entity_id": entity_id, + "media_id": media_content_id + } + response = requests.post(url, headers=headers, json=data) + if response.status_code == 200: + return f"正在播放{media_content_id}的音乐" + else: + return f"音乐播放失败，错误码: {response.status_code}" diff --git a/main/xiaozhi-server/plugins_func/functions/hass_set_state.py b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py new file mode 100644 index 00000000..5e2b91a2 --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py @@ -0,0 +1,159 @@ +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from plugins_func.functions.hass_init import initialize_hass_handler +from config.logger import setup_logging +import asyncio +import requests + +TAG = __name__ +logger = setup_logging() + +hass_set_state_function_desc = { + "type": "function", + "function": { + "name": "hass_set_state", + "description": "设置homeassistant里设备的状态,包括开、关,调整灯光亮度,调整播放器的音量,设备的暂停、继续、静音操作", + "parameters": { + "type": "object", + "properties": { + "state": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "需要操作的动作,打开设备:turn_on,关闭设备:turn_off,增加亮度:brightness_up,降低亮度:brightness_down,设置亮度:brightness_value,增加>音量:,volume_up降低音量:volume_down,设置音量:volume_set,设备暂停:pause,设备继续:continue,静音/取消静音:volume_mute" + }, + "input": { + "type": "int", + "description": "只有在设置音量,设置亮度时候才需要,有效值为1-100,对应音量和亮度的1%-100%" + }, + "is_muted": { + "type": "string", + "description": "只有在设置静音操作时才需要,设置静音的时候该值为true,取消静音时该值为false" + } + }, + "required": ["type"] + }, + "entity_id": { + "type": "string", + "description": "需要操作的设备id,homeassistant里的entity_id" + } + }, + "required": ["state", "entity_id"] + } + } +} + + +@register_function('hass_set_state', hass_set_state_function_desc, ToolType.SYSTEM_CTL) +def hass_set_state(conn, entity_id='', state={}): + try: + future = asyncio.run_coroutine_threadsafe( + handle_hass_set_state(conn, entity_id, state), + conn.loop + ) + ha_response = future.result() + return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response) + except Exception as e: + logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}") + + +async def handle_hass_set_state(conn, entity_id, state): + HASS_CACHE = initialize_hass_handler(conn) + api_key = HASS_CACHE['api_key'] + base_url = HASS_CACHE['base_url'] + ''' + state = { "type":"brightness_up","input":"80","is_muted":"true"} + ''' + domains = entity_id.split(".") + if len(domains) > 1: + domain = domains[0] + else: + return "执行失败，错误的设备id" + action = '' + arg = '' + value = '' + if state['type'] == 'turn_on': + description = "设备已打开" + if domain == "cover": + action = "open_cover" + elif domain == "vacuum": + action = "start" + else: + action = "turn_on" + elif state['type'] == 'turn_off': + description = "设备已关闭" + if domain == 'cover': + action = "close_cover" + elif domain == 'vacuum': + action = "stop" + else: + action = "turn_off" + elif state['type'] == 'brightness_up': + description = "灯光已调亮" + action = 'turn_on' + arg = 'brightness_step_pct' + value = 10 + elif state['type'] == 'brightness_down': + description = "灯光已调暗" + action = 'turn_on' + arg = 'brightness_step_pct' + value = -10 + elif state['type'] == 'brightness_value': + description = f"亮度已调整到{state['input']}" + action = 'turn_on' + arg = 'brightness_pct' + value = state['input'] + elif state['type'] == 'volume_up': + description = "音量已调大" + action = state['type'] + elif state['type'] == 'volume_down': + description = "音量已调小" + action = state['type'] + elif state['type'] == 'volume_set': + description = f"音量已调整到{state['input']}" + action = state['type'] + arg = 'volume_level' + value = state['input'] + elif state['type'] == 'volume_mute': + description = f"设备已静音" + action = state['type'] + arg = 'is_volume_muted' + value = state['is_muted'] + elif state['type'] == 'pause': + description = f"设备已暂停" + action = state['type'] + if domain == 'media_player': + action = 'media_pause' + if domain == 'cover': + action = 'stop_cover' + if domain == 'vacuum': + action = 'pause' + elif state['type'] == 'continue': + description = f"设备已继续" + if domain == 'media_player': + action = 'media_play' + if domain == 'vacuum': + action = 'start' + else: + return f"{domain} {state.type}功能尚未支持" + + if arg == '': + data = { + "entity_id": entity_id, + } + else: + data = { + "entity_id": entity_id, + arg: value + } + url = f"{base_url}/api/services/{domain}/{action}" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + response = requests.post(url, headers=headers, json=data) + logger.bind(tag=TAG).info(f"设置状态:url:{url},return_code:{response.status_code}") + if response.status_code == 200: + return description + else: + return f"设置失败，错误码: {response.status_code}"