diff --git a/.gitignore b/.gitignore index a25b3401..675fb688 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,7 @@ tmp .history .DS_Store main/xiaozhi-server/data +main/xiaozhi-server/config/assets/wakeup_words.* main/manager-web/node_modules .config.yaml .secrets.yaml @@ -154,3 +155,4 @@ main/manager-web/node_modules # model files main/xiaozhi-server/models/SenseVoiceSmall/model.pt main/xiaozhi-server/models/sherpa-onnx* +my_wakeup_words.mp3 diff --git a/README.md b/README.md index b59d9d45..191790bc 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ - + 控制家电开关 @@ -95,6 +95,11 @@ + + + 播报新闻 + + @@ -185,7 +190,6 @@ server: | LLM | FastgptLLM | fastgpt 接口调用 | 免费/消耗 token | 本地化部署，注意配置提示词需在 Fastgpt 控制台设置 | | LLM | GeminiLLM | gemini 接口调用 | 免费 | [点击申请密钥](https://aistudio.google.com/apikey) | | LLM | CozeLLM | coze 接口调用 | 消耗 token | 需提供 bot_id、user_id 及个人令牌 | -| LLM | Home Assistant | homeassistant语音助手接口调用 | 免费 | 需提供home assistant令牌 | 实际上，任何支持 openai 接口调用的 LLM 均可接入使用。 @@ -370,7 +374,41 @@ VAD: ### 6、我想通过小智控制电灯、空调、远程开关机等操作 💡 -建议：在配置文件中将 `LLM` 设置为 `HomeAssistant`，通过 调用`HomeAssistant`接口实现相关控制。 +本项目，支持以工具调用的方式控制HomeAssistant设备 + +1、首先选择一款支持function call支持的LLM，例如`ChatGLMLLM`。 + +2、在配置文件中，将 `selected_module.Intent` 设置为 `function_call`。 + +3、登录`HomeAssistant`，点击`左下角个人`，切换`安全`导航栏，划到底部`长期访问令牌`生成api_key。 + +在配置文件中，配置好你的home assistant的`devices`（被控制的设备）和`api_key`和`base_url`等信息。例如： + +``` yaml +plugins + home_assistant: + devices: + - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 + - 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1 + base_url: http://你的homeassistant地址:8123 + api_key: 你的home assistant api访问令牌 +``` + +最后，允许function_call 插件在配置文件中启用`hass_get_state`(必须)、`hass_set_state`(必须)、`hass_play_music`(不想用ha听音乐可以不启动)，例如： + +``` yaml +Intent: + ... + function_call: + type: nointent + functions: + - change_role + - get_weather + - get_news + - hass_get_state + - hass_set_state + - hass_play_music +``` ### 7、更多问题，可联系我们反馈 💬 diff --git a/docs/images/demo0.png b/docs/images/demo0.png new file mode 100644 index 00000000..99592be6 Binary files /dev/null and b/docs/images/demo0.png differ diff --git a/docs/images/demo5.png b/docs/images/demo5.png index 8e03685b..c6da2997 100644 Binary files a/docs/images/demo5.png and b/docs/images/demo5.png differ diff --git a/main/manager-web/src/apis/api.js b/main/manager-web/src/apis/api.js index 23169518..e69beda2 100755 --- a/main/manager-web/src/apis/api.js +++ b/main/manager-web/src/apis/api.js @@ -3,6 +3,7 @@ import agent from './module/agent.js' import device from './module/device.js' import user from './module/user.js' import ota from './module/ota.js' +import admin from './module/admin.js' /** * 接口地址 @@ -29,5 +30,6 @@ export default { user, agent, device, - ota + ota, + admin } diff --git a/main/manager-web/src/apis/module/admin.js b/main/manager-web/src/apis/module/admin.js new file mode 100644 index 00000000..52468d6b --- /dev/null +++ b/main/manager-web/src/apis/module/admin.js @@ -0,0 +1,20 @@ +import RequestService from '../httpRequest' +import {getServiceUrl} from '../api' + + +export default { + // 用户列表 + getUserList(callback) { + RequestService.sendRequest().url(`${getServiceUrl()}/api/v1/admin/users`) + .method('GET') + .success((res) => { + RequestService.clearRequestTime() + callback(res) + }) + .fail(() => { + RequestService.reAjaxFun(() => { + this.getList() + }) + }).send() + }, +} diff --git a/main/manager-web/src/apis/module/user.js b/main/manager-web/src/apis/module/user.js index 28859db6..8cad2198 100755 --- a/main/manager-web/src/apis/module/user.js +++ b/main/manager-web/src/apis/module/user.js @@ -26,9 +26,9 @@ export default { .method('GET') .type('blob') .header({ - 'Content-Type': 'image/gif', - 'Pragma': 'No-cache', - 'Cache-Control': 'no-cache' + 'Content-Type': 'image/gif', + 'Pragma': 'No-cache', + 'Cache-Control': 'no-cache' }) .success((res) => { RequestService.clearRequestTime(); diff --git a/main/manager-web/src/components/AddModelDialog.vue b/main/manager-web/src/components/AddModelDialog.vue new file mode 100644 index 00000000..f8370a5f --- /dev/null +++ b/main/manager-web/src/components/AddModelDialog.vue @@ -0,0 +1,384 @@ + + + + + \ No newline at end of file diff --git a/main/manager-web/src/components/ChangePasswordDialog.vue b/main/manager-web/src/components/ChangePasswordDialog.vue new file mode 100644 index 00000000..19c20fd6 --- /dev/null +++ b/main/manager-web/src/components/ChangePasswordDialog.vue @@ -0,0 +1,141 @@ + + + + + \ No newline at end of file diff --git a/main/manager-web/src/components/EditVoiceDialog.vue b/main/manager-web/src/components/EditVoiceDialog.vue new file mode 100644 index 00000000..b7feb78d --- /dev/null +++ b/main/manager-web/src/components/EditVoiceDialog.vue @@ -0,0 +1,149 @@ + + + + + diff --git a/main/manager-web/src/components/HeaderBar.vue b/main/manager-web/src/components/HeaderBar.vue index 90c12db7..be3ebcad 100644 --- a/main/manager-web/src/components/HeaderBar.vue +++ b/main/manager-web/src/components/HeaderBar.vue @@ -13,6 +13,12 @@ 控制台 +
+ 用户管理 +
+
+ 模型配置 +
OTA管理 @@ -28,7 +34,7 @@ - {{ userInfo.username || '加载中...' }} + {{ userInfo.mobile || '加载中...' }} 个人中心 @@ -38,15 +44,21 @@
+ + + diff --git a/main/manager-web/src/components/ModelEditDialog.vue b/main/manager-web/src/components/ModelEditDialog.vue new file mode 100644 index 00000000..09e5570c --- /dev/null +++ b/main/manager-web/src/components/ModelEditDialog.vue @@ -0,0 +1,159 @@ + + + + + diff --git a/main/manager-web/src/components/TtsModel.vue b/main/manager-web/src/components/TtsModel.vue new file mode 100644 index 00000000..92b107ed --- /dev/null +++ b/main/manager-web/src/components/TtsModel.vue @@ -0,0 +1,194 @@ + + + + + diff --git a/main/manager-web/src/router/index.js b/main/manager-web/src/router/index.js index fb044879..4da67914 100644 --- a/main/manager-web/src/router/index.js +++ b/main/manager-web/src/router/index.js @@ -64,7 +64,29 @@ const routes = [ component: function () { return import('../views/ota.vue') } - } + }, + // 添加用户管理路由 + { + path: '/user-management', + name: 'UserManagement', + meta: { + menuCode: 'user', + }, + component: function () { + return import('../views/UserManagement.vue') + } + }, + { + path: '/model-config', + name: 'ModelConfig', + meta: { + menuCode: 'model', + }, + component: function () { + return import('../views/ModelConfig.vue') + } + }, + ] const router = new VueRouter({ diff --git a/main/manager-web/src/views/ModelConfig.vue b/main/manager-web/src/views/ModelConfig.vue new file mode 100644 index 00000000..088f810e --- /dev/null +++ b/main/manager-web/src/views/ModelConfig.vue @@ -0,0 +1,285 @@ + + + + + diff --git a/main/manager-web/src/views/UserManagement.vue b/main/manager-web/src/views/UserManagement.vue new file mode 100644 index 00000000..2122568e --- /dev/null +++ b/main/manager-web/src/views/UserManagement.vue @@ -0,0 +1,213 @@ + + + + + \ No newline at end of file diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 6bfe9322..ab505b5f 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -24,9 +24,9 @@ server: # - "24:0A:C4:1D:3B:F0" # MAC地址列表 log: # 设置控制台输出的日志格式，时间、日志级别、标签、消息 - log_format: "{time:YY-MM-DD HH:mm:ss}[{extra[tag]}] - {level} - {message}" + log_format: "{time:YYMMDD HH:mm:ss}[{version}_{selected_module}][{extra[tag]}]-{level}-{message}" # 设置日志文件输出的格式，时间、日志级别、标签、消息 - log_format_simple: "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}" + log_format_file: "{time:YYYY-MM-DD HH:mm:ss} - {version}_{selected_module} - {name} - {level} - {extra[tag]} - {message}" # 设置日志等级：INFO、DEBUG log_level: INFO # 设置日志路径 @@ -49,7 +49,7 @@ prompt: | 你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。 请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。 现在我正在和你进行语音聊天，我们开始吧。 - 如果用户希望结束对话，请在最后说“拜拜”或“再见”。 + # 使用完声音文件后删除文件(Delete the sound file when you are done using it) delete_audio: true @@ -57,6 +57,14 @@ delete_audio: true close_connection_no_voice_time: 120 # TTS请求超时时间(秒) tts_timeout: 10 +# 开启唤醒词加速 +enable_wakeup_words_response_cache: true +# 开场是否回复唤醒词 +enable_greeting: true +# 说完话是否开启提示音 +enable_stop_tts_notify: false +# 说完话是否开启提示音，音效地址 +stop_tts_notify_voice: "config/assets/tts_notify.mp3" CMD_exit: - "退出" @@ -99,6 +107,12 @@ Intent: - change_role - get_weather - get_news + # play_music是服务器自带的音乐播放，hass_play_music是通过home assistant控制的独立外部程序音乐播放 + # 如果用了hass_play_music，就不要开启play_music，两者只留一个 + - play_music + #- hass_get_state + #- hass_set_state + #- hass_play_music # 插件的基础配置 plugins: @@ -115,6 +129,20 @@ plugins: society: "https://www.chinanews.com.cn/rss/society.xml" world: "https://www.chinanews.com.cn/rss/world.xml" finance: "https://www.chinanews.com.cn/rss/finance.xml" + home_assistant: + devices: + - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 + - 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1 + base_url: http://homeassistant.local:8123 + api_key: 你的home assistant api访问令牌 + play_music: + music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件 + music_ext: # 音乐文件类型，p3格式效率最高 + - ".mp3" + - ".wav" + - ".p3" + refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒 + Memory: mem0ai: @@ -241,12 +269,6 @@ LLM: model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载 url: http://localhost:1234/v1 # LM Studio服务地址 api_key: lm-studio # LM Studio服务的固定API Key - HomeAssistant: - # 定义LLM API类型 - type: homeassistant - base_url: http://homeassistant.local:8123 - agent_id: conversation.chatgpt - api_key: 你的home assistant api访问令牌 FastgptLLM: # 定义LLM API类型 type: fastgpt @@ -501,24 +523,20 @@ module_test: - "What's the weather like today?" - "请用100字概括量子计算的基本原理和应用前景" -# 本地音乐播放配置 -music: - music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件 - music_ext: # 音乐文件类型，p3格式效率最高 - - ".mp3" - - ".wav" - - ".p3" - refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒 +# 唤醒词，用于识别唤醒词还是讲话内容 +wakeup_words: + - "你好小智" + - "你好小志" + - "小爱同学" + - "你好小鑫" + - "你好小新" + - "小美同学" + - "小龙小龙" + - "喵喵同学" + - "小滨小滨" + - "小冰小冰" # 远程配置 remote_config: enabled: true - url: http://192.168.5.11:8002/user/agent/loadAgentConfig/ - -# 以下配置在小于等于0.0.9版本中的docker容器中可用 -# 0.0.9以后的新版本源码部署已经无法奏效 -manager: - enabled: false - ip: 0.0.0.0 - port: 8002 -use_private_config: false + url: http://192.168.5.11:8002/user/agent/loadAgentConfig/ \ No newline at end of file diff --git a/main/xiaozhi-server/config/assets/tts_notify.mp3 b/main/xiaozhi-server/config/assets/tts_notify.mp3 new file mode 100644 index 00000000..ff071501 Binary files /dev/null and b/main/xiaozhi-server/config/assets/tts_notify.mp3 differ diff --git a/main/xiaozhi-server/config/assets/wakeup_words.wav b/main/xiaozhi-server/config/assets/wakeup_words.wav new file mode 100644 index 00000000..e5422555 Binary files /dev/null and b/main/xiaozhi-server/config/assets/wakeup_words.wav differ diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 99968ca1..2206e52c 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -3,12 +3,24 @@ import sys from loguru import logger from config.settings import load_config +SERVER_VERSION = "0.1.15" + def setup_logging(): """从配置文件中读取日志配置，并设置日志输出格式和级别""" config = load_config() log_config = config["log"] - log_format = log_config.get("log_format", "{time:YY-MM-DD HH:mm:ss}[{extra[tag]}] - {level} - {message}") - log_format_simple = log_config.get("log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}") + log_format = log_config.get("log_format", "{time:YYMMDD HH:mm:ss}[{version}_{selected_module}][{extra[tag]}]-{level}-{message}") + log_format_file = log_config.get("log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}") + + selected_module = config.get("selected_module") + selected_module_str = ''.join([key[0] + value[0] for key, value in selected_module.items()]) + + log_format = log_format.replace("{version}", SERVER_VERSION) + log_format = log_format.replace("{selected_module}", selected_module_str) + log_format_file = log_format_file.replace("{version}", SERVER_VERSION) + log_format_file = log_format_file.replace("{selected_module}", selected_module_str) + + log_level = log_config.get("log_level", "INFO") log_dir = log_config.get("log_dir", "tmp") log_file = log_config.get("log_file", "server.log") @@ -24,6 +36,6 @@ def setup_logging(): logger.add(sys.stdout, format=log_format, level=log_level) # 输出到文件 - logger.add(os.path.join(log_dir, log_file), format=log_format_simple, level=log_level) + logger.add(os.path.join(log_dir, log_file), format=log_format_file, level=log_level) return logger diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 718398c3..98cf5c5c 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -9,7 +9,7 @@ import traceback import threading import websockets from typing import Dict, Any -import plugins_func.loadplugins +from plugins_func.loadplugins import auto_import_modules from config.logger import setup_logging from core.utils.dialogue import Message, Dialogue from core.handle.textHandle import handleTextMessage @@ -25,6 +25,8 @@ from core.utils.auth_code_gen import AuthCodeGenerator TAG = __name__ +auto_import_modules('plugins_func.functions') + class TTSException(RuntimeError): pass @@ -109,11 +111,15 @@ class ConnectionHandler: # 进行认证 await self.auth.authenticate(self.headers) - device_id = self.headers.get("device-id", None) - self.memory.init_memory(device_id, self.llm) - self.intent.set_llm(self.llm) + # 认证通过,继续处理 + self.websocket = ws + self.session_id = str(uuid.uuid4()) + + self.welcome_msg = self.config["xiaozhi"] + self.welcome_msg["session_id"] = self.session_id + await self.websocket.send(json.dumps(self.welcome_msg)) # Load private configuration if device_id is provided bUsePrivateConfig = self.config.get("use_private_config", False) self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}") @@ -141,43 +147,32 @@ class ConnectionHandler: self.private_config = None raise - # 认证通过,继续处理 - self.websocket = ws - self.session_id = str(uuid.uuid4()) - - self.welcome_msg = self.config["xiaozhi"] - self.welcome_msg["session_id"] = self.session_id - await self.websocket.send(json.dumps(self.welcome_msg)) - # 异步初始化 - await self.loop.run_in_executor(None, self._initialize_components) - + self.executor.submit(self._initialize_components) # tts 消化线程 - tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True) - tts_priority.start() + self.tts_priority_thread = threading.Thread(target=self._tts_priority_thread, daemon=True) + self.tts_priority_thread.start() # 音频播放 消化线程 - audio_play_priority = threading.Thread(target=self._audio_play_priority_thread, daemon=True) - audio_play_priority.start() + self.audio_play_priority_thread = threading.Thread(target=self._audio_play_priority_thread, daemon=True) + self.audio_play_priority_thread.start() try: async for message in self.websocket: await self._route_message(message) except websockets.exceptions.ConnectionClosed: self.logger.bind(tag=TAG).info("客户端断开连接") - await self.close() except AuthenticationError as e: self.logger.bind(tag=TAG).error(f"Authentication failed: {str(e)}") - await ws.close() return except Exception as e: stack_trace = traceback.format_exc() self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}") - await ws.close() return finally: await self.memory.save_memory(self.dialogue.dialogue) + await self.close(ws) async def _route_message(self, message): """消息路由""" @@ -187,16 +182,26 @@ class ConnectionHandler: await handleAudioMessage(self, message) def _initialize_components(self): + """加载提示词""" self.prompt = self.config["prompt"] if self.private_config: self.prompt = self.private_config.private_config.get("prompt", self.prompt) - - self.client_ip_info = get_ip_info(self.client_ip) - self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}") - self.prompt = self.prompt + f"\n我在:{self.client_ip_info}" self.dialogue.put(Message(role="system", content=self.prompt)) - self.func_handler = FunctionHandler(self.config) + """加载插件""" + self.func_handler = FunctionHandler(self) + + """加载记忆""" + device_id = self.headers.get("device-id", None) + self.memory.init_memory(device_id, self.llm) + self.intent.set_llm(self.llm) + + """加载位置信息""" + self.client_ip_info = get_ip_info(self.client_ip) + if self.client_ip_info is not None and "city" in self.client_ip_info: + self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}") + self.prompt = self.prompt + f"\nuser location:{self.client_ip_info}" + self.dialogue.update_system_message(self.prompt) def change_system_prompt(self, prompt): self.prompt = prompt @@ -316,7 +321,9 @@ class ConnectionHandler: self.dialogue.put(Message(role="user", content=query)) # Define intent functions - functions = self.func_handler.get_functions() + functions = None + if hasattr(self, 'func_handler'): + functions = self.func_handler.get_functions() response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 @@ -492,7 +499,12 @@ class ConnectionHandler: while not self.stop_event.is_set(): text = None try: - future = self.tts_queue.get() + try: + future = self.tts_queue.get(timeout=1) + except queue.Empty: + if self.stop_event.is_set(): + break + continue if future is None: continue text = None @@ -533,7 +545,12 @@ class ConnectionHandler: while not self.stop_event.is_set(): text = None try: - opus_datas, text, text_index = self.audio_play_queue.get() + try: + opus_datas, text, text_index = self.audio_play_queue.get(timeout=1) + except queue.Empty: + if self.stop_event.is_set(): + break + continue future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index), self.loop) future.result() @@ -563,16 +580,41 @@ class ConnectionHandler: self.tts_first_text_index = text_index self.tts_last_text_index = text_index - async def close(self): + async def close(self, ws=None): """资源清理方法""" - # 清理其他资源 - self.stop_event.set() - self.executor.shutdown(wait=False) - if self.websocket: + # 触发停止事件并清理资源 + if self.stop_event: + self.stop_event.set() + + # 立即关闭线程池 + if self.executor: + self.executor.shutdown(wait=False, cancel_futures=True) + self.executor = None + + # 清空任务队列 + self._clear_queues() + + if ws: + await ws.close() + elif self.websocket: await self.websocket.close() self.logger.bind(tag=TAG).info("连接资源已释放") + def _clear_queues(self): + # 清空所有任务队列 + for q in [self.tts_queue, self.audio_play_queue]: + if not q: + continue + while not q.empty(): + try: + q.get_nowait() + except queue.Empty: + continue + q.queue.clear() + # 添加毒丸信号到队列，确保线程退出 + # q.queue.put(None) + def reset_vad_states(self): self.client_audio_buffer = bytes() self.client_have_voice = False diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index c7449973..9cad9499 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -1,16 +1,16 @@ -import asyncio -from enum import Enum - from config.logger import setup_logging import json from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType +from plugins_func.functions.hass_init import append_devices_to_prompt + TAG = __name__ logger = setup_logging() class FunctionHandler: - def __init__(self, config): - self.config = config + def __init__(self, conn): + self.conn = conn + self.config = conn.config self.function_registry = FunctionRegistry() self.register_nessary_functions() self.register_config_functions() @@ -26,9 +26,10 @@ class FunctionHandler: func_names = ",".join(surport_plugins) for function_desc in self.functions_desc: if function_desc["function"]["name"] == "plugin_loader": - function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names) + function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", + func_names) break - + def upload_functions_desc(self): self.functions_desc = self.function_registry.get_all_function_desc() @@ -47,16 +48,19 @@ class FunctionHandler: def register_nessary_functions(self): """注册必要的函数""" self.function_registry.register_function("handle_exit_intent") - self.function_registry.register_function("play_music") self.function_registry.register_function("plugin_loader") self.function_registry.register_function("get_time") - self.function_registry.register_function("raise_and_lower_the_volume") + self.function_registry.register_function("get_lunar") + self.function_registry.register_function("handle_device") def register_config_functions(self): """注册配置中的函数,可以不同客户端使用不同的配置""" for func in self.config["Intent"]["function_call"].get("functions", []): self.function_registry.register_function(func) + """home assistant需要初始化提示词""" + append_devices_to_prompt(self.conn) + def get_function(self, name): return self.function_registry.get_function(name) @@ -81,4 +85,4 @@ class FunctionHandler: except Exception as e: logger.bind(tag=TAG).error(f"处理function call错误: {e}") - return None \ No newline at end of file + return None diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index bafcf09d..4583a6e3 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,8 +1,74 @@ import json from config.logger import setup_logging +from core.handle.sendAudioHandle import send_stt_message +from core.utils.util import remove_punctuation_and_length +import shutil +import asyncio +import os +import random +import time logger = setup_logging() +WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10, + "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]} + async def handleHelloMessage(conn): await conn.websocket.send(json.dumps(conn.welcome_msg)) + + +async def checkWakeupWords(conn, text): + enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"] + """是否开启唤醒词加速""" + if not enable_wakeup_words_response_cache: + return False + """检查是否是唤醒词""" + _, text = remove_punctuation_and_length(text) + if text in conn.config.get("wakeup_words"): + await send_stt_message(conn, text) + conn.tts_first_text_index = 0 + conn.tts_last_text_index = 0 + conn.llm_finish_task = True + + file = getWakeupWordFile(WAKEUP_CONFIG["file_name"]) + if file is None: + asyncio.create_task(wakeupWordsResponse(conn)) + return False + opus_packets, duration = conn.tts.audio_to_opus_data(file) + conn.audio_play_queue.put((opus_packets, text, 0)) + if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]: + asyncio.create_task(wakeupWordsResponse(conn)) + return True + return False + + +def getWakeupWordFile(file_name): + for file in os.listdir(WAKEUP_CONFIG["dir"]): + if file.startswith("my_" + file_name): + """避免缓存文件是一个空文件""" + if os.stat(f"config/assets/{file}").st_size > (5 * 1024): + return f"config/assets/{file}" + + """查找config/assets/目录下名称为wakeup_words的文件""" + for file in os.listdir(WAKEUP_CONFIG["dir"]): + if file.startswith(file_name): + return f"config/assets/{file}" + return None + + +async def wakeupWordsResponse(conn): + """唤醒词响应""" + wakeup_word = random.choice(WAKEUP_CONFIG["words"]) + result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word) + tts_file = await asyncio.to_thread(conn.tts.to_tts, result) + if tts_file is not None and os.path.exists(tts_file): + file_type = os.path.splitext(tts_file)[1] + if file_type: + file_type = file_type.lstrip('.') + old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) + if old_file is not None: + os.remove(old_file) + """将文件挪到"wakeup_words.mp3""" + shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type) + WAKEUP_CONFIG["create_time"] = time.time() diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 1799516c..d8440fc6 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -2,6 +2,7 @@ from config.logger import setup_logging import json import uuid from core.handle.sendAudioHandle import send_stt_message +from core.handle.helloHandle import checkWakeupWords from core.utils.util import remove_punctuation_and_length TAG = __name__ @@ -12,6 +13,10 @@ async def handle_user_intent(conn, text): # 检查是否有明确的退出命令 if await check_direct_exit(conn, text): return True + # 检查是否是唤醒词 + if await checkWakeupWords(conn, text): + return True + if conn.use_function_call_mode: # 使用支持function calling的聊天方法,不再进行意图分析 return False @@ -35,6 +40,7 @@ async def check_direct_exit(conn, text): return False + async def analyze_intent_with_llm(conn, text): """使用LLM分析用户意图""" if not hasattr(conn, 'intent') or not conn.intent: diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 8999bf08..c0a664eb 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -7,40 +7,15 @@ from core.utils.util import remove_punctuation_and_length, get_string_no_punctua TAG = __name__ logger = setup_logging() + async def sendAudioMessage(conn, audios, text, text_index=0): # 发送句子开始消息 if text_index == conn.tts_first_text_index: logger.bind(tag=TAG).info(f"发送第一段语音: {text}") await send_tts_message(conn, "sentence_start", text) - # 流控参数优化 - original_frame_duration = 60 # 原始帧时长（毫秒） - adjusted_frame_duration = int(original_frame_duration * 0.8) # 缩短20% - total_frames = len(audios) # 获取总帧数 - compensation = total_frames * (original_frame_duration - adjusted_frame_duration) / 1000 # 补偿时间（秒） - - start_time = time.perf_counter() - play_position = 0 # 已播放时长（毫秒） - - for opus_packet in audios: - if conn.client_abort: - return - - # 计算带加速因子的预期时间 - expected_time = start_time + (play_position / 1000) - current_time = time.perf_counter() - - # 流控等待（使用加速后的帧时长） - delay = expected_time - current_time - if delay > 0: - await asyncio.sleep(delay) - - await conn.websocket.send(opus_packet) - play_position += adjusted_frame_duration # 使用调整后的帧时长 - - # 补偿因加速损失的时长 - if compensation > 0: - await asyncio.sleep(compensation) + # 播放音频 + await sendAudio(conn, audios) await send_tts_message(conn, "sentence_end", text) @@ -50,6 +25,38 @@ async def sendAudioMessage(conn, audios, text, text_index=0): if conn.close_after_chat: await conn.close() +# 播放音频 +async def sendAudio(conn, audios): + # 流控参数优化 + frame_duration = 60 # 帧时长（毫秒），匹配 Opus 编码 + start_time = time.perf_counter() + play_position = 0 + + # 预缓冲：发送前 3 帧 + pre_buffer = min(3, len(audios)) + for i in range(pre_buffer): + await conn.websocket.send(audios[i]) + conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms") + + # 正常播放剩余帧 + for opus_packet in audios[pre_buffer:]: + if conn.client_abort: + return + + # 计算预期发送时间 + expected_time = start_time + (play_position / 1000) + current_time = time.perf_counter() + delay = expected_time - current_time + if delay > 0: + await asyncio.sleep(delay) + + await conn.websocket.send(opus_packet) + conn.logger.bind(tag=TAG).debug(f"发送帧，位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms") + + play_position += frame_duration + + + async def send_tts_message(conn, state, text=None): """发送 TTS 状态消息""" message = { @@ -60,10 +67,20 @@ async def send_tts_message(conn, state, text=None): if text is not None: message["text"] = text - await conn.websocket.send(json.dumps(message)) + # TTS播放结束 if state == "stop": + # 播放提示音 + tts_notify = conn.config.get("enable_stop_tts_notify", False) + if tts_notify: + stop_tts_notify_voice = conn.config.get("stop_tts_notify_voice", "config/assets/tts_notify.mp3") + audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice) + await sendAudio(conn, audios) + # 清除服务端讲话状态 conn.clearSpeakStatus() + # 发送消息到客户端 + await conn.websocket.send(json.dumps(message)) + async def send_stt_message(conn, text): """发送 STT 状态消息""" diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index c63b9a33..3f60592a 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -2,7 +2,9 @@ from config.logger import setup_logging import json from core.handle.abortHandle import handleAbortMessage from core.handle.helloHandle import handleHelloMessage +from core.utils.util import remove_punctuation_and_length from core.handle.receiveAudioHandle import startToChat, handleAudioMessage +from core.handle.sendAudioHandle import send_stt_message, send_tts_message from core.handle.iotHandle import handleIotDescriptors, handleIotStatus TAG = __name__ @@ -38,7 +40,21 @@ async def handleTextMessage(conn, message): conn.client_have_voice = False conn.asr_audio.clear() if "text" in msg_json: - await startToChat(conn, msg_json["text"]) + text = msg_json["text"] + _, text = remove_punctuation_and_length(text) + + # 识别是否是唤醒词 + is_wakeup_words = text in conn.config.get("wakeup_words") + # 是否开启唤醒词回复 + enable_greeting = conn.config.get("enable_greeting", True) + + if is_wakeup_words and not enable_greeting: + # 如果是唤醒词，且关闭了唤醒词回复，就不用回答 + await send_stt_message(conn, text) + await send_tts_message(conn, "stop", None) + else: + # 否则需要LLM对文字内容进行答复 + await startToChat(conn, text) elif msg_json["type"] == "iot": if "descriptors" in msg_json: await handleIotDescriptors(conn, msg_json["descriptors"]) diff --git a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py b/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py deleted file mode 100644 index 115e85e9..00000000 --- a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py +++ /dev/null @@ -1,62 +0,0 @@ -import requests -from requests.exceptions import RequestException -from config.logger import setup_logging -from core.providers.llm.base import LLMProviderBase - -TAG = __name__ -logger = setup_logging() - - -class LLMProvider(LLMProviderBase): - def __init__(self, config): - self.agent_id = config.get("agent_id") # 对应 agent_id - self.api_key = config.get("api_key") - self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url - self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL - - def response(self, session_id, dialogue): - print(dialogue) - try: - # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可 - - # 提取最后一个 role 为 'user' 的 content - input_text = None - if isinstance(dialogue, list): # 确保 dialogue 是一个列表 - # 逆序遍历，找到最后一个 role 为 'user' 的消息 - for message in reversed(dialogue): - if message.get("role") == "user": # 找到 role 为 'user' 的消息 - input_text = message.get("content", "") - break # 找到后立即退出循环 - - # 构造请求数据 - payload = { - "text": input_text, - "agent_id": self.agent_id, - "conversation_id": session_id # 使用 session_id 作为 conversation_id - } - # 设置请求头 - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - # 发起 POST 请求 - response = requests.post(self.api_url, json=payload, headers=headers) - - # 检查请求是否成功 - response.raise_for_status() - - # 解析返回数据 - data = response.json() - speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "") - - # 返回生成的内容 - if speech: - yield speech - else: - logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容") - - except RequestException as e: - logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}") - except Exception as e: - logger.bind(tag=TAG).error(f"生成响应时出错: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 3146dc06..e39ca65c 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -36,7 +36,7 @@ class TTSProviderBase(ABC): return tmp_file except Exception as e: - logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}") + logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") return None @abstractmethod diff --git a/main/xiaozhi-server/core/providers/tts/edge.py b/main/xiaozhi-server/core/providers/tts/edge.py index 3c02597d..289961cb 100644 --- a/main/xiaozhi-server/core/providers/tts/edge.py +++ b/main/xiaozhi-server/core/providers/tts/edge.py @@ -14,5 +14,14 @@ class TTSProvider(TTSProviderBase): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") async def text_to_speak(self, text, output_file): - communicate = edge_tts.Communicate(text, voice=self.voice) # Use your preferred voice - await communicate.save(output_file) + communicate = edge_tts.Communicate(text, voice=self.voice) + # 确保目录存在并创建空文件 + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'wb') as f: + pass + + # 流式写入音频数据 + with open(output_file, 'ab') as f: # 改为追加模式避免覆盖 + async for chunk in communicate.stream(): + if chunk["type"] == "audio": # 只处理音频数据块 + f.write(chunk["data"]) diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index 8d2c161b..d4e66bb6 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -35,6 +35,15 @@ class Dialogue: self.getMessages(m, dialogue) return dialogue + def update_system_message(self, new_content: str): + """更新或添加系统消息""" + # 查找第一个系统消息 + system_msg = next((msg for msg in self.dialogue if msg.role == "system"), None) + if system_msg: + system_msg.content = new_content + else: + self.put(Message(role="system", content=new_content)) + def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]: if memory_str is None or len(memory_str) == 0: return self.get_llm_dialogue() diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 63cb5b9c..51841baf 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -73,9 +73,7 @@ def get_ip_info(ip_addr): resp = requests.get(url).json() ip_info = { - "city": resp.get("cityName"), - "region": resp.get("regionName"), - "country": resp.get("countryName") + "city": resp.get("cityName") } return ip_info except Exception as e: diff --git a/main/xiaozhi-server/core/websocket_server.py b/main/xiaozhi-server/core/websocket_server.py index 41f14515..6aa9a06d 100644 --- a/main/xiaozhi-server/core/websocket_server.py +++ b/main/xiaozhi-server/core/websocket_server.py @@ -63,8 +63,6 @@ class WebSocketServer: server_config = self.config["server"] host = server_config["ip"] port = server_config["port"] - selected_module = self.config.get("selected_module") - self.logger.bind(tag=TAG).info(f"selected_module values: {', '.join(selected_module.values())}") self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port) self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======") diff --git a/main/xiaozhi-server/plugins_func/functions/get_time.py b/main/xiaozhi-server/plugins_func/functions/get_time.py index 9a9aebfe..3eb7324c 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_time.py +++ b/main/xiaozhi-server/plugins_func/functions/get_time.py @@ -1,20 +1,19 @@ from datetime import datetime +import cnlunar from plugins_func.register import register_function, ToolType, ActionResponse, Action get_time_function_desc = { "type": "function", "function": { "name": "get_time", - "description": "获取当前时间、日期、星期几", + "description": "获取今天日期或者当前时间信息", 'parameters': {'type': 'object', 'properties': {}, 'required': []} } } - - @register_function('get_time', get_time_function_desc, ToolType.WAIT) def get_time(): """ - 获取当前时间、日期、星期几 + 获取当前的日期时间信息 """ now = datetime.now() current_time = now.strftime("%H:%M:%S") @@ -22,4 +21,65 @@ def get_time(): current_weekday = now.strftime("%A") response_text = f"当前日期: {current_date}，当前时间: {current_time}，星期: {current_weekday}" + return ActionResponse(Action.REQLLM, response_text, None) + + +get_lunar_function_desc = { + "type": "function", + "function": { + "name": "get_lunar", + "description": ( + "用于获取今天的阴历/农历和黄历信息。" + "用户可以指定查询内容，如：阴历日期、天干地支、节气、生肖、星座、八字、宜忌等。" + "如果没有指定查询内容，则默认查询干支年和农历日期。" + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "要查询的内容，例如阴历日期、天干地支、节日、节气、生肖、星座、八字、宜忌等" + } + }, + "required": [] + } + } +} +@register_function('get_lunar', get_lunar_function_desc, ToolType.WAIT) +def get_lunar(query): + """ + 用于获取当前的阴历/农历，和天干地支、节气、生肖、星座、八字、宜忌等黄历信息 + """ + now = datetime.now() + current_time = now.strftime("%H:%M:%S") + current_date = now.strftime("%Y-%m-%d") + current_weekday = now.strftime("%A") + response_text = f"根据以下信息回应用户的查询请求，并提供与{query}相关的信息：\n" + + lunar = cnlunar.Lunar(now, godType='8char') + response_text += ( + f"当前公历日期: {current_date}，当前时间: {current_time}，星期: {current_weekday}\n" + "农历信息：\n" + "%s年%s%s\n" % (lunar.lunarYearCn, lunar.lunarMonthCn[:-1], lunar.lunarDayCn) + + "干支: %s年 %s月 %s日\n" % (lunar.year8Char, lunar.month8Char, lunar.day8Char) + + "生肖: 属%s\n" % (lunar.chineseYearZodiac) + + "八字: %s\n" % (' '.join([lunar.year8Char, lunar.month8Char, lunar.day8Char, lunar.twohour8Char])) + + "今日节日: %s\n" % (",".join(filter(None, (lunar.get_legalHolidays(), lunar.get_otherHolidays(), lunar.get_otherLunarHolidays())))) + + "今日节气: %s\n" % (lunar.todaySolarTerms) + + "下一节气: %s %s年%s月%s日\n" % (lunar.nextSolarTerm, lunar.nextSolarTermYear, lunar.nextSolarTermDate[0], lunar.nextSolarTermDate[1]) + + "今年节气表: %s\n" % (', '.join([f"{term}({date[0]}月{date[1]}日)" for term, date in lunar.thisYearSolarTermsDic.items()])) + + "生肖冲煞: %s\n" % (lunar.chineseZodiacClash) + + "星座: %s\n" % (lunar.starZodiac) + + "纳音: %s\n" % lunar.get_nayin() + + "彭祖百忌: %s\n" % (lunar.get_pengTaboo(delimit=", ")) + + "值日: %s执位\n" % lunar.get_today12DayOfficer()[0] + + "值神: %s(%s)\n" % (lunar.get_today12DayOfficer()[1], lunar.get_today12DayOfficer()[2]) + + "廿八宿: %s\n" % lunar.get_the28Stars() + + "吉神方位: %s\n" % ' '.join(lunar.get_luckyGodsDirection()) + + "今日胎神: %s\n" % lunar.get_fetalGod() + + "宜: %s\n" % '、'.join(lunar.goodThing[:10]) + + "忌: %s\n" % '、'.join(lunar.badThing[:10]) + + "(默认返回干支年和农历日期；仅在要求查询宜忌信息时才返回本日宜忌)" + ) + return ActionResponse(Action.REQLLM, response_text, None) \ No newline at end of file diff --git a/main/xiaozhi-server/plugins_func/functions/get_weather.py b/main/xiaozhi-server/plugins_func/functions/get_weather.py index 75c0c1a3..8cdd6ea1 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_weather.py +++ b/main/xiaozhi-server/plugins_func/functions/get_weather.py @@ -39,6 +39,23 @@ HEADERS = { ) } +# 天气代码 https://dev.qweather.com/docs/resource/icons/#weather-icons +WEATHER_CODE_MAP = { + "100": "晴", "101": "多云", "102": "少云", "103": "晴间多云", "104": "阴", + "150": "晴", "151": "多云", "152": "少云", "153": "晴间多云", + "300": "阵雨", "301": "强阵雨", "302": "雷阵雨", "303": "强雷阵雨", "304": "雷阵雨伴有冰雹", + "305": "小雨", "306": "中雨", "307": "大雨", "308": "极端降雨", "309": "毛毛雨/细雨", + "310": "暴雨", "311": "大暴雨", "312": "特大暴雨", "313": "冻雨", "314": "小到中雨", + "315": "中到大雨", "316": "大到暴雨", "317": "暴雨到大暴雨", "318": "大暴雨到特大暴雨", + "350": "阵雨", "351": "强阵雨", "399": "雨", + "400": "小雪", "401": "中雪", "402": "大雪", "403": "暴雪", "404": "雨夹雪", + "405": "雨雪天气", "406": "阵雨夹雪", "407": "阵雪", "408": "小到中雪", "409": "中到大雪", "410": "大到暴雪", + "456": "阵雨夹雪", "457": "阵雪", "499": "雪", + "500": "薄雾", "501": "雾", "502": "霾", "503": "扬沙", "504": "浮尘", + "507": "沙尘暴", "508": "强沙尘暴", + "509": "浓雾", "510": "强浓雾", "511": "中度霾", "512": "重度霾", "513": "严重霾", "514": "大雾", "515": "特强浓雾", + "900": "热", "901": "冷", "999": "未知" +} def fetch_city_info(location, api_key): url = f"https://geoapi.qweather.com/v2/city/lookup?key={api_key}&location={location}&lang=zh" @@ -67,9 +84,11 @@ def parse_weather_info(soup): temps_list = [] for row in soup.select(".city-forecast-tabs__row")[:7]: # 取前7天的数据 date = row.select_one(".date-bg .date").get_text(strip=True) + weather_code = row.select_one(".date-bg .icon")["src"].split("/")[-1].split(".")[0] + weather = WEATHER_CODE_MAP.get(weather_code, "未知") temps = [span.get_text(strip=True) for span in row.select(".tmp-cont .temp")] high_temp, low_temp = (temps[0], temps[-1]) if len(temps) >= 2 else (None, None) - temps_list.append((date, high_temp, low_temp)) + temps_list.append((date, weather, high_temp, low_temp)) return city_name, current_abstract, current_basic, temps_list @@ -91,13 +110,13 @@ def get_weather(conn, location: str = None, lang: str = "zh_CN"): city_name, current_abstract, current_basic, temps_list = parse_weather_info(soup) weather_report = f"根据下列数据，用{lang}回应用户的查询天气请求：\n{city_name}未来7天天气:\n" - for i, (date, high, low) in enumerate(temps_list): + for i, (date, weather, high, low) in enumerate(temps_list): if high and low: - weather_report += f"{date}: {low}到{high}\n" + weather_report += f"{date}: {low}到{high}, {weather}\n" weather_report += ( f"当前天气: {current_abstract}\n" f"当前天气参数: {current_basic}\n" - f"(确保只报告指定单日的气温范围，除非用户明确要求想要了解多日天气，如果未指定，默认报告今天的温度范围。" + f"(确保只报告指定单日的天气情况，除非未来会出现异常天气；或者用户明确要求想要了解多日天气，如果未指定，默认报告今天的天气。" "参数为0的值不需要报告给用户，每次都报告体感温度，根据语境选择合适的参数内容告知用户，并对参数给出相应评价)" ) diff --git a/main/xiaozhi-server/plugins_func/functions/handle_device.py b/main/xiaozhi-server/plugins_func/functions/handle_device.py new file mode 100644 index 00000000..d564562b --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/handle_device.py @@ -0,0 +1,105 @@ +from config.logger import setup_logging +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from core.handle.iotHandle import get_iot_status, send_iot_conn +import asyncio + +TAG = __name__ +logger = setup_logging() + +async def _get_device_status(conn, device_name, device_type, property_name): + """获取设备状态""" + status = await get_iot_status(conn, device_type, property_name) + if status is None: + raise Exception(f"你的设备不支持{device_name}控制") + return status + +async def _set_device_property(conn, device_name, device_type, method_name, property_name, new_value=None, action=None, step=10): + """设置设备属性""" + current_value = await _get_device_status(conn, device_name, device_type, property_name) + + if action == 'raise': + current_value += step + elif action == 'lower': + current_value -= step + elif action == 'set': + if new_value is None: + raise Exception(f"缺少{property_name}参数") + current_value = new_value + + # 限制属性范围在0到100之间 + current_value = max(0, min(100, current_value)) + + await send_iot_conn(conn, device_type, method_name, {property_name: current_value}) + return current_value + +def _handle_device_action(conn, func, success_message, error_message, *args, **kwargs): + """处理设备操作的通用函数""" + future = asyncio.run_coroutine_threadsafe(func(conn, *args, **kwargs), conn.loop) + try: + result = future.result() + logger.bind(tag=TAG).info(f"{success_message}: {result}") + response = f"{success_message}{result}" + return ActionResponse(action=Action.RESPONSE, result=result, response=response) + except Exception as e: + logger.bind(tag=TAG).error(f"{error_message}: {e}") + response = f"{error_message}: {e}" + return ActionResponse(action=Action.RESPONSE, result=None, response=response) + +# 设备控制 +handle_device_function_desc = { + "type": "function", + "function": { + "name": "handle_device", + "description": ( + "用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。" + "比如用户说现在亮度多少，参数为：device_type:Backlight,action:get。" + "比如用户说设置音量为50，参数为：device_type:Speaker,action:set,value:50。" + "比如用户说亮度太高了，参数为：device_type:Backlight,action:lower。" + "比如用户说调大音量，参数为：device_type:Speaker,action:raise。" + ), + "parameters": { + "type": "object", + "properties": { + "device_type": { + "type": "string", + "description": "设备类型，可选值：Speaker(音量),Backlight(亮度)" + }, + "action": { + "type": "string", + "description": "动作名称，可选值：get(获取),set(设置),raise(提高),lower(降低)" + }, + "value": { + "type": "int", + "description": "值大小，可选值：0-100之间的整数" + } + }, + "required": ["device_type", "action"] + } + } +} + +@register_function('handle_device', handle_device_function_desc, ToolType.IOT_CTL) +def handle_device(conn, device_type: str, action: str, value: int = None): + if device_type == "Speaker": + method_name, property_name, device_name = "SetVolume", "volume", "音量" + elif device_type == "Backlight": + method_name, property_name, device_name = "SetBrightness", "brightness", "亮度" + else: + raise Exception(f"未识别的设备类型: {device_type}") + + if action not in ["get", "set", "raise", "lower"]: + raise Exception(f"未识别的动作名称: {action}") + + if action == "get": + # get + return _handle_device_action( + conn, _get_device_status, f"当前{device_name}", f"获取{device_name}失败", + device_name=device_name, device_type=device_type, property_name=property_name, + ) + else: + # set, raise, lower + return _handle_device_action( + conn, _set_device_property, f"{device_name}已调整到", f"{device_name}调整失败", + device_name=device_name, device_type=device_type, method_name=method_name, + property_name=property_name, new_value=value, action=action + ) diff --git a/main/xiaozhi-server/plugins_func/functions/hass_get_state.py b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py new file mode 100644 index 00000000..415f35c6 --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py @@ -0,0 +1,56 @@ +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from plugins_func.functions.hass_init import initialize_hass_handler +from config.logger import setup_logging +import asyncio +import requests + +TAG = __name__ +logger = setup_logging() + +hass_get_state_function_desc = { + "type": "function", + "function": { + "name": "hass_get_state", + "description": "获取homeassistant里设备的状态,包括灯光亮度,媒体播放器的音量,设备的暂停、继续操作", + "parameters": { + "type": "object", + "properties": { + "entity_id": { + "type": "string", + "description": "需要操作的设备id,homeassistant里的entity_id" + } + }, + "required": ["entity_id"] + } + } +} + + +@register_function("hass_get_state", hass_get_state_function_desc, ToolType.SYSTEM_CTL) +def hass_get_state(conn, entity_id=''): + try: + + future = asyncio.run_coroutine_threadsafe( + handle_hass_get_state(conn, entity_id), + conn.loop + ) + ha_response = future.result() + return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response) + except Exception as e: + logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}") + + +async def handle_hass_get_state(conn, entity_id): + HASS_CACHE = initialize_hass_handler(conn) + api_key = HASS_CACHE['api_key'] + base_url = HASS_CACHE['base_url'] + url = f"{base_url}/api/states/{entity_id}" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.json()['state'] + else: + return f"切换失败，错误码: {response.status_code}" diff --git a/main/xiaozhi-server/plugins_func/functions/hass_init.py b/main/xiaozhi-server/plugins_func/functions/hass_init.py new file mode 100644 index 00000000..e09eb67f --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_init.py @@ -0,0 +1,35 @@ +from config.logger import setup_logging +from core.utils.util import check_model_key + +TAG = __name__ +logger = setup_logging() + +HASS_CACHE = {} + + +def append_devices_to_prompt(conn): + if conn.use_function_call_mode: + funcs = conn.config["Intent"]["function_call"].get("functions", []) + if "hass_get_state" in funcs or "hass_get_state" in funcs: + prompt = "下面是我家智能设备，可以通过homeassistant控制\n" + devices = conn.config["plugins"]["home_assistant"].get("devices", []) + if len(devices) == 0: + return + for device in devices: + prompt += device + "\n" + conn.prompt += prompt + # 更新提示词 + conn.dialogue.update_system_message(conn.prompt) + + +def initialize_hass_handler(conn): + global HASS_CACHE + if HASS_CACHE == {}: + if conn.use_function_call_mode: + funcs = conn.config["Intent"]["function_call"].get("functions", []) + if "hass_get_state" in funcs or "hass_get_state" in funcs: + HASS_CACHE['base_url'] = conn.config["plugins"]["home_assistant"].get("base_url") + HASS_CACHE['api_key'] = conn.config["plugins"]["home_assistant"].get("api_key") + + check_model_key("home_assistant", HASS_CACHE['api_key']) + return HASS_CACHE diff --git a/main/xiaozhi-server/plugins_func/functions/hass_play_music.py b/main/xiaozhi-server/plugins_func/functions/hass_play_music.py new file mode 100644 index 00000000..502f784e --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_play_music.py @@ -0,0 +1,64 @@ +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from plugins_func.functions.hass_init import initialize_hass_handler +from config.logger import setup_logging +import asyncio +import requests + +TAG = __name__ +logger = setup_logging() + +hass_play_music_function_desc = { + "type": "function", + "function": { + "name": "hass_play_music", + "description": "用户想听音乐、有声书的时候使用，在房间的媒体播放器（media_player）里播放对应音频", + "parameters": { + "type": "object", + "properties": { + "media_content_id": { + "type": "string", + "description": "可以是音乐或有声书的专辑名称、歌曲名、演唱者,如果未指定就填random" + }, + "entity_id": { + "type": "string", + "description": "需要操作的音箱的设备id,homeassistant里的entity_id,media_player开头" + } + }, + "required": ["media_content_id", "entity_id"] + } + } +} + + +@register_function('hass_play_music', hass_play_music_function_desc, ToolType.SYSTEM_CTL) +def hass_play_music(conn, entity_id='', media_content_id='random'): + try: + # 执行音乐播放命令 + future = asyncio.run_coroutine_threadsafe( + handle_hass_play_music(conn, entity_id, media_content_id), + conn.loop + ) + ha_response = future.result() + return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=ha_response) + except Exception as e: + logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}") + + +async def handle_hass_play_music(conn, entity_id, media_content_id): + HASS_CACHE = initialize_hass_handler(conn) + api_key = HASS_CACHE['api_key'] + base_url = HASS_CACHE['base_url'] + url = f"{base_url}/api/services/music_assistant/play_media" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + data = { + "entity_id": entity_id, + "media_id": media_content_id + } + response = requests.post(url, headers=headers, json=data) + if response.status_code == 200: + return f"正在播放{media_content_id}的音乐" + else: + return f"音乐播放失败，错误码: {response.status_code}" diff --git a/main/xiaozhi-server/plugins_func/functions/hass_set_state.py b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py new file mode 100644 index 00000000..5e2b91a2 --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py @@ -0,0 +1,159 @@ +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from plugins_func.functions.hass_init import initialize_hass_handler +from config.logger import setup_logging +import asyncio +import requests + +TAG = __name__ +logger = setup_logging() + +hass_set_state_function_desc = { + "type": "function", + "function": { + "name": "hass_set_state", + "description": "设置homeassistant里设备的状态,包括开、关,调整灯光亮度,调整播放器的音量,设备的暂停、继续、静音操作", + "parameters": { + "type": "object", + "properties": { + "state": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "需要操作的动作,打开设备:turn_on,关闭设备:turn_off,增加亮度:brightness_up,降低亮度:brightness_down,设置亮度:brightness_value,增加>音量:,volume_up降低音量:volume_down,设置音量:volume_set,设备暂停:pause,设备继续:continue,静音/取消静音:volume_mute" + }, + "input": { + "type": "int", + "description": "只有在设置音量,设置亮度时候才需要,有效值为1-100,对应音量和亮度的1%-100%" + }, + "is_muted": { + "type": "string", + "description": "只有在设置静音操作时才需要,设置静音的时候该值为true,取消静音时该值为false" + } + }, + "required": ["type"] + }, + "entity_id": { + "type": "string", + "description": "需要操作的设备id,homeassistant里的entity_id" + } + }, + "required": ["state", "entity_id"] + } + } +} + + +@register_function('hass_set_state', hass_set_state_function_desc, ToolType.SYSTEM_CTL) +def hass_set_state(conn, entity_id='', state={}): + try: + future = asyncio.run_coroutine_threadsafe( + handle_hass_set_state(conn, entity_id, state), + conn.loop + ) + ha_response = future.result() + return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response) + except Exception as e: + logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}") + + +async def handle_hass_set_state(conn, entity_id, state): + HASS_CACHE = initialize_hass_handler(conn) + api_key = HASS_CACHE['api_key'] + base_url = HASS_CACHE['base_url'] + ''' + state = { "type":"brightness_up","input":"80","is_muted":"true"} + ''' + domains = entity_id.split(".") + if len(domains) > 1: + domain = domains[0] + else: + return "执行失败，错误的设备id" + action = '' + arg = '' + value = '' + if state['type'] == 'turn_on': + description = "设备已打开" + if domain == "cover": + action = "open_cover" + elif domain == "vacuum": + action = "start" + else: + action = "turn_on" + elif state['type'] == 'turn_off': + description = "设备已关闭" + if domain == 'cover': + action = "close_cover" + elif domain == 'vacuum': + action = "stop" + else: + action = "turn_off" + elif state['type'] == 'brightness_up': + description = "灯光已调亮" + action = 'turn_on' + arg = 'brightness_step_pct' + value = 10 + elif state['type'] == 'brightness_down': + description = "灯光已调暗" + action = 'turn_on' + arg = 'brightness_step_pct' + value = -10 + elif state['type'] == 'brightness_value': + description = f"亮度已调整到{state['input']}" + action = 'turn_on' + arg = 'brightness_pct' + value = state['input'] + elif state['type'] == 'volume_up': + description = "音量已调大" + action = state['type'] + elif state['type'] == 'volume_down': + description = "音量已调小" + action = state['type'] + elif state['type'] == 'volume_set': + description = f"音量已调整到{state['input']}" + action = state['type'] + arg = 'volume_level' + value = state['input'] + elif state['type'] == 'volume_mute': + description = f"设备已静音" + action = state['type'] + arg = 'is_volume_muted' + value = state['is_muted'] + elif state['type'] == 'pause': + description = f"设备已暂停" + action = state['type'] + if domain == 'media_player': + action = 'media_pause' + if domain == 'cover': + action = 'stop_cover' + if domain == 'vacuum': + action = 'pause' + elif state['type'] == 'continue': + description = f"设备已继续" + if domain == 'media_player': + action = 'media_play' + if domain == 'vacuum': + action = 'start' + else: + return f"{domain} {state.type}功能尚未支持" + + if arg == '': + data = { + "entity_id": entity_id, + } + else: + data = { + "entity_id": entity_id, + arg: value + } + url = f"{base_url}/api/services/{domain}/{action}" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + response = requests.post(url, headers=headers, json=data) + logger.bind(tag=TAG).info(f"设置状态:url:{url},return_code:{response.status_code}") + if response.status_code == 200: + return description + else: + return f"设置失败，错误码: {response.status_code}" diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 75f3834d..20148b5f 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -21,13 +21,13 @@ play_music_function_desc = { "type": "function", "function": { "name": "play_music", - "description": "唱歌、听歌、播放音乐方法。比如用户说播放音乐，参数为：random，比如用户说播放两只老虎，参数为：两只老虎", + "description": "唱歌、听歌、播放音乐的方法。", "parameters": { "type": "object", "properties": { "song_name": { "type": "string", - "description": "歌曲名称，如果没有指定具体歌名则为'random'" + "description": "歌曲名称，如果用户没有指定具体歌名则为'random', 明确指定的时返回音乐的名字 示例: ```用户:播放两只老虎\n参数：两只老虎``` ```用户:播放音乐 \n参数：random ```" } }, "required": ["song_name"] @@ -112,8 +112,8 @@ def get_music_files(music_dir, music_ext): def initialize_music_handler(conn): global MUSIC_CACHE if MUSIC_CACHE == {}: - if "music" in conn.config: - MUSIC_CACHE["music_config"] = conn.config["music"] + if "play_music" in conn.config["plugins"]: + MUSIC_CACHE["music_config"] = conn.config["plugins"]["play_music"] MUSIC_CACHE["music_dir"] = os.path.abspath( MUSIC_CACHE["music_config"].get("music_dir", "./music") # 默认路径修改 ) diff --git a/main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py b/main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py deleted file mode 100644 index c77cb78b..00000000 --- a/main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py +++ /dev/null @@ -1,64 +0,0 @@ -from config.logger import setup_logging -from plugins_func.register import register_function, ToolType, ActionResponse, Action -from core.handle.iotHandle import get_iot_status, send_iot_conn -import asyncio - -TAG = __name__ -logger = setup_logging() - -raise_and_lower_the_volume_function_desc = { - "type": "function", - "function": { - "name": "raise_and_lower_the_volume", - "description": "用户觉得声音过高或过低，或者用户想提高或降低音量。比如用户说太大声了，参数为：lower，比如用户说提高音量，参数为：raise", - "parameters": { - "type": "object", - "properties": { - "action": { - "type": "string", - "description": "动作名称，要么是raise，要么是lower" - } - }, - "required": ["action"] - } - } -} - - -@register_function('raise_and_lower_the_volume', raise_and_lower_the_volume_function_desc, ToolType.IOT_CTL) -def raise_and_lower_the_volume(conn, action: str): - """ - 获取当前设备音量 - """ - - future = asyncio.run_coroutine_threadsafe( - _raise_and_lower_the_volume(conn, action), - conn.loop - ) - - try: - new_volume = future.result() # 同步等待异步操作完成 - logger.bind(tag=TAG).info(f"音量操作完成: {new_volume}") - response = f"音量已调整到{new_volume}" - except Exception as e: - logger.bind(tag=TAG).error(f"音量操作失败: {e}") - response = f"音量调整失败: {e}" - - return ActionResponse(action=Action.RESPONSE, result="指令已接收", response=response) - - -async def _raise_and_lower_the_volume(conn, action): - volume = await get_iot_status(conn, "Speaker", "volume") - if volume is None: - raise Exception("你的设备不支持音量控制") - if action == 'raise': - volume += 10 - elif action == 'lower': - volume -= 10 - # 限制音量范围在0到100之间 - if volume < 0: - volume = 0 - elif volume > 100: - volume = 100 - await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": volume}) - return volume diff --git a/main/xiaozhi-server/plugins_func/loadplugins.py b/main/xiaozhi-server/plugins_func/loadplugins.py index d826fac8..9b20355e 100644 --- a/main/xiaozhi-server/plugins_func/loadplugins.py +++ b/main/xiaozhi-server/plugins_func/loadplugins.py @@ -22,6 +22,4 @@ def auto_import_modules(package_name): # 导入模块 full_module_name = f"{package_name}.{module_name}" importlib.import_module(full_module_name) - #logger.bind(tag=TAG).info(f"模块 '{full_module_name}' 已加载") - -auto_import_modules('plugins_func.functions') \ No newline at end of file + #logger.bind(tag=TAG).info(f"模块 '{full_module_name}' 已加载") \ No newline at end of file diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index 6afec6b5..51433cb9 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -22,3 +22,4 @@ mem0ai==0.1.62 bs4==0.0.2 modelscope==1.23.2 sherpa_onnx==1.11.0 +cnlunar==0.2.0 \ No newline at end of file