@@ -64,6 +62,12 @@ export default {
// 跳转到首页
this.$router.push('/')
},
+ goUserManagement() {
+ this.$router.push('/user-management')
+ },
+ goModelConfig() {
+ this.$router.push('/model-config')
+ },
// 获取用户信息
fetchUserInfo() {
userApi.getUserInfo(({data}) => {
@@ -117,18 +121,24 @@ export default {
}
.equipment-management2 {
- width: 87px;
- height: 22px;
- border-radius: 11px;
+ width: 82px;
+ height: 24px;
+ border-radius: 12px;
background: #fff;
display: flex;
justify-content: center;
- font-size: 9px;
- font-weight: 400;
+ font-size: 10px;
+ font-weight: 500;
gap: 7px;
color: #3d4566;
- margin-left: 2px;
+ margin-left: 1px;
align-items: center;
+ transition: all 0.3s ease;
+}
+
+.equipment-management2.active-tab {
+ background: #5778ff !important;
+ color: #fff !important;
}
.header {
@@ -168,28 +178,4 @@ export default {
color: #3d4566;
}
-.console {
- width: 90px;
- height: 22px;
- border-radius: 11px;
- background: radial-gradient(50% 50% at 50% 50%, #fff 0%, #e8f0ff 100%);
- display: flex;
- align-items: center;
- justify-content: center;
- font-size: 9px;
- color: #979db1;
- font-weight: 400;
- gap: 7px;
- margin-left: 15px;
-}
-
-.el-dropdown-link {
- cursor: pointer;
- color: #5778ff;
-}
-
-.el-icon-arrow-down {
- font-size: 12px;
-}
-
diff --git a/main/manager-web/src/components/ModelEditDialog.vue b/main/manager-web/src/components/ModelEditDialog.vue
new file mode 100644
index 00000000..09e5570c
--- /dev/null
+++ b/main/manager-web/src/components/ModelEditDialog.vue
@@ -0,0 +1,159 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 设成默认
+
+ 是否启用
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/main/manager-web/src/components/TtsModel.vue b/main/manager-web/src/components/TtsModel.vue
new file mode 100644
index 00000000..92b107ed
--- /dev/null
+++ b/main/manager-web/src/components/TtsModel.vue
@@ -0,0 +1,194 @@
+
+
+
+
+
+
+
+
+
+
+ 查询
+ 添加音色
+ 批量删除
+
+
+
+
+
+
+
+
+
+ 修改
+ 删除
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/main/manager-web/src/router/index.js b/main/manager-web/src/router/index.js
index d3add00a..6bc58823 100644
--- a/main/manager-web/src/router/index.js
+++ b/main/manager-web/src/router/index.js
@@ -39,15 +39,30 @@ const routes = [
return import('../views/register.vue')
}
},
- // 新增设备管理页面路由
+ // 设备管理页面路由
{
path: '/device-management',
name: 'DeviceManagement',
component: function () {
return import('../views/DeviceManagement.vue')
}
+ },
+ // 添加用户管理路由
+ {
+ path: '/user-management',
+ name: 'UserManagement',
+ component: function () {
+ return import('../views/UserManagement.vue')
+ }
+ },
+ {
+ path: '/model-config',
+ name: 'ModelConfig',
+ component: function () {
+ return import('../views/ModelConfig.vue')
+ }
+ },
- }
]
const router = new VueRouter({
diff --git a/main/manager-web/src/views/DeviceManagement.vue b/main/manager-web/src/views/DeviceManagement.vue
index 06c4317c..03939a28 100644
--- a/main/manager-web/src/views/DeviceManagement.vue
+++ b/main/manager-web/src/views/DeviceManagement.vue
@@ -9,11 +9,11 @@
-
-
-
-
-
+
+
+
+
+
@@ -85,7 +85,7 @@ export default {
remark: '生产环境设备',
isEdit: false,
otaSwitch: false
- }
+ },
]
};
},
diff --git a/main/manager-web/src/views/ModelConfig.vue b/main/manager-web/src/views/ModelConfig.vue
new file mode 100644
index 00000000..720b7c36
--- /dev/null
+++ b/main/manager-web/src/views/ModelConfig.vue
@@ -0,0 +1,199 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 模型设置
+ 导入导出配置
+
+
+
+
+
+
+
+
+ 查询
+ 增加模型
+ 批量删除
+
+
+
+
+
+
+
+
+ {{ scope.row.isApplied ? '是' : '否' }}
+
+
+
+
+
+
+ 修改
+ 删除
+
+
+
+
+
+
+
+
+ ©2025 xiaozhi-esp32-server
+
+
+
+
+
+
+
+
+
+
diff --git a/main/manager-web/src/views/UserManagement.vue b/main/manager-web/src/views/UserManagement.vue
new file mode 100644
index 00000000..9ccc5989
--- /dev/null
+++ b/main/manager-web/src/views/UserManagement.vue
@@ -0,0 +1,145 @@
+
+
+
+
+
+
+
+ 查询
+ 批量删除
+ 批量禁用
+
+
+
+
+
+
+
+
+
+ 重置密码
+ 禁用
+ 恢复
+ 删除用户
+
+
+
+
+
+
+
+
+ ©2025 xiaozhi-esp32-server
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/main/manager-web/src/views/roleConfig.vue b/main/manager-web/src/views/roleConfig.vue
index 77ecc85f..d0f0000e 100644
--- a/main/manager-web/src/views/roleConfig.vue
+++ b/main/manager-web/src/views/roleConfig.vue
@@ -125,11 +125,11 @@ export default {
],
models: [
{ label: '大语言模型(LLM)', key: 'llm' },
- { label: '语音转文本模型(ASR)', key: 'asr' },
+ { label: '语音识别(ASR)', key: 'asr' },
{ label: '语音活动检测模型(VAD)', key: 'vad' },
- { label: '语音生成模型(TTS)', key: 'tts' },
- { label: '意图分类模型(Intent)', key: 'intent' },
- { label: '记忆增强模型(Memory)', key: 'memory' }
+ { label: '语音合成模型(TTS)', key: 'tts' },
+ { label: '意图识别模型(Intent)', key: 'intent' },
+ { label: '记忆模型(Memory)', key: 'memory' }
],
templates: ['台湾女友', '土豆子', '英语老师', '好奇小男孩', '汪汪队队长']
diff --git a/main/manager-web/vue.config.js b/main/manager-web/vue.config.js
index 9e72e068..c10f5e55 100644
--- a/main/manager-web/vue.config.js
+++ b/main/manager-web/vue.config.js
@@ -7,6 +7,7 @@ dotenv.config();
module.exports = defineConfig({
devServer: {
// Bug 修复:将代理配置为环境变量中定义的 API 基础 URL
+ port: 8001, // 指定端口为 8001
proxy: {
'/xiaozhi-esp32-api': {
target: process.env.VUE_APP_API_BASE_URL || 'http://localhost:8002', // 后端 API 的基础 URL
diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml
index 587543dc..2fe438cf 100644
--- a/main/xiaozhi-server/config.yaml
+++ b/main/xiaozhi-server/config.yaml
@@ -49,7 +49,7 @@ prompt: |
你是一个叫小智/小志的台湾女孩,说话机车,声音好听,习惯简短表达,爱用网络梗。
请注意,要像一个人一样说话,请不要回复表情符号、代码、和xml标签。
现在我正在和你进行语音聊天,我们开始吧。
- 如果用户希望结束对话,请在最后说“拜拜”或“再见”。
+
# 使用完声音文件后删除文件(Delete the sound file when you are done using it)
delete_audio: true
@@ -57,6 +57,14 @@ delete_audio: true
close_connection_no_voice_time: 120
# TTS请求超时时间(秒)
tts_timeout: 10
+# 开启唤醒词加速
+enable_wakeup_words_response_cache: true
+# 开场是否回复唤醒词
+enable_greeting: true
+# 说完话是否开启提示音
+enable_stop_tts_notify: false
+# 说完话是否开启提示音,音效地址
+stop_tts_notify_voice: "config/assets/tts_notify.mp3"
CMD_exit:
- "退出"
@@ -99,6 +107,12 @@ Intent:
- change_role
- get_weather
- get_news
+ # play_music是服务器自带的音乐播放,hass_play_music是通过home assistant控制的独立外部程序音乐播放
+ # 如果用了hass_play_music,就不要开启play_music,两者只留一个
+ - play_music
+ #- hass_get_state
+ #- hass_set_state
+ #- hass_play_music
# 插件的基础配置
plugins:
@@ -115,6 +129,20 @@ plugins:
society: "https://www.chinanews.com.cn/rss/society.xml"
world: "https://www.chinanews.com.cn/rss/world.xml"
finance: "https://www.chinanews.com.cn/rss/finance.xml"
+ home_assistant:
+ devices:
+ - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1
+ - 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1
+ base_url: http://homeassistant.local:8123
+ api_key: 你的home assistant api访问令牌
+ play_music:
+ music_dir: "./music" # 音乐文件存放路径,将从该目录及子目录下搜索音乐文件
+ music_ext: # 音乐文件类型,p3格式效率最高
+ - ".mp3"
+ - ".wav"
+ - ".p3"
+ refresh_time: 300 # 刷新音乐列表的时间间隔,单位为秒
+
Memory:
mem0ai:
@@ -165,6 +193,18 @@ LLM:
top_p: 1
top_k: 50
frequency_penalty: 0 # 频率惩罚
+ AliAppLLM:
+ # 定义LLM API类型
+ type: AliBL
+ base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
+ app_id: 你的app_id
+ # 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key
+ api_key: 你的api_key
+ # 是否不使用本地prompt:true|false (默不用请在百练应用中设置prompt)
+ is_no_prompt: true
+ # Ali_memory_id:false(不使用)|你的memory_id(请在百练应用中设置中获取)
+ # Tips!:Ali_memory未实现多用户存储记忆(记忆按id调用)
+ ali_memory_id: false
DoubaoLLM:
# 定义LLM API类型
type: openai
@@ -229,12 +269,6 @@ LLM:
model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称,需要预先在社区下载
url: http://localhost:1234/v1 # LM Studio服务地址
api_key: lm-studio # LM Studio服务的固定API Key
- HomeAssistant:
- # 定义LLM API类型
- type: homeassistant
- base_url: http://homeassistant.local:8123
- agent_id: conversation.chatgpt
- api_key: 你的home assistant api访问令牌
FastgptLLM:
# 定义LLM API类型
type: fastgpt
@@ -489,19 +523,15 @@ module_test:
- "What's the weather like today?"
- "请用100字概括量子计算的基本原理和应用前景"
-# 本地音乐播放配置
-music:
- music_dir: "./music" # 音乐文件存放路径,将从该目录及子目录下搜索音乐文件
- music_ext: # 音乐文件类型,p3格式效率最高
- - ".mp3"
- - ".wav"
- - ".p3"
- refresh_time: 300 # 刷新音乐列表的时间间隔,单位为秒
-
-# 以下配置在小于等于0.0.9版本中的docker容器中可用
-# 0.0.9以后的新版本源码部署已经无法奏效
-manager:
- enabled: false
- ip: 0.0.0.0
- port: 8002
-use_private_config: false
+# 唤醒词,用于识别唤醒词还是讲话内容
+wakeup_words:
+ - "你好小智"
+ - "你好小志"
+ - "小爱同学"
+ - "你好小鑫"
+ - "你好小新"
+ - "小美同学"
+ - "小龙小龙"
+ - "喵喵同学"
+ - "小滨小滨"
+ - "小冰小冰"
\ No newline at end of file
diff --git a/main/xiaozhi-server/config/assets/tts_notify.mp3 b/main/xiaozhi-server/config/assets/tts_notify.mp3
new file mode 100644
index 00000000..ff071501
Binary files /dev/null and b/main/xiaozhi-server/config/assets/tts_notify.mp3 differ
diff --git a/main/xiaozhi-server/config/assets/wakeup_words.wav b/main/xiaozhi-server/config/assets/wakeup_words.wav
new file mode 100644
index 00000000..e5422555
Binary files /dev/null and b/main/xiaozhi-server/config/assets/wakeup_words.wav differ
diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py
index 718398c3..c9a52b12 100644
--- a/main/xiaozhi-server/core/connection.py
+++ b/main/xiaozhi-server/core/connection.py
@@ -9,7 +9,7 @@ import traceback
import threading
import websockets
from typing import Dict, Any
-import plugins_func.loadplugins
+from plugins_func.loadplugins import auto_import_modules
from config.logger import setup_logging
from core.utils.dialogue import Message, Dialogue
from core.handle.textHandle import handleTextMessage
@@ -25,6 +25,8 @@ from core.utils.auth_code_gen import AuthCodeGenerator
TAG = __name__
+auto_import_modules('plugins_func.functions')
+
class TTSException(RuntimeError):
pass
@@ -109,11 +111,15 @@ class ConnectionHandler:
# 进行认证
await self.auth.authenticate(self.headers)
-
device_id = self.headers.get("device-id", None)
- self.memory.init_memory(device_id, self.llm)
- self.intent.set_llm(self.llm)
+ # 认证通过,继续处理
+ self.websocket = ws
+ self.session_id = str(uuid.uuid4())
+
+ self.welcome_msg = self.config["xiaozhi"]
+ self.welcome_msg["session_id"] = self.session_id
+ await self.websocket.send(json.dumps(self.welcome_msg))
# Load private configuration if device_id is provided
bUsePrivateConfig = self.config.get("use_private_config", False)
self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
@@ -141,17 +147,8 @@ class ConnectionHandler:
self.private_config = None
raise
- # 认证通过,继续处理
- self.websocket = ws
- self.session_id = str(uuid.uuid4())
-
- self.welcome_msg = self.config["xiaozhi"]
- self.welcome_msg["session_id"] = self.session_id
- await self.websocket.send(json.dumps(self.welcome_msg))
-
# 异步初始化
- await self.loop.run_in_executor(None, self._initialize_components)
-
+ self.executor.submit(self._initialize_components)
# tts 消化线程
tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
tts_priority.start()
@@ -187,16 +184,26 @@ class ConnectionHandler:
await handleAudioMessage(self, message)
def _initialize_components(self):
+ """加载插件"""
+ self.func_handler = FunctionHandler(self)
+
+ """加载提示词"""
self.prompt = self.config["prompt"]
if self.private_config:
self.prompt = self.private_config.private_config.get("prompt", self.prompt)
-
- self.client_ip_info = get_ip_info(self.client_ip)
- self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
- self.prompt = self.prompt + f"\n我在:{self.client_ip_info}"
self.dialogue.put(Message(role="system", content=self.prompt))
- self.func_handler = FunctionHandler(self.config)
+ """加载记忆"""
+ device_id = self.headers.get("device-id", None)
+ self.memory.init_memory(device_id, self.llm)
+ self.intent.set_llm(self.llm)
+
+ """加载位置信息"""
+ self.client_ip_info = get_ip_info(self.client_ip)
+ if self.client_ip_info is not None and "city" in self.client_ip_info:
+ self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
+ self.prompt = self.prompt + f"\nuser location:{self.client_ip_info}"
+ self.dialogue.update_system_message(self.prompt)
def change_system_prompt(self, prompt):
self.prompt = prompt
diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py
index c7449973..e8056517 100644
--- a/main/xiaozhi-server/core/handle/functionHandler.py
+++ b/main/xiaozhi-server/core/handle/functionHandler.py
@@ -1,16 +1,16 @@
-import asyncio
-from enum import Enum
-
from config.logger import setup_logging
import json
from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+from plugins_func.functions.hass_init import append_devices_to_prompt
+
TAG = __name__
logger = setup_logging()
class FunctionHandler:
- def __init__(self, config):
- self.config = config
+ def __init__(self, conn):
+ self.conn = conn
+ self.config = conn.config
self.function_registry = FunctionRegistry()
self.register_nessary_functions()
self.register_config_functions()
@@ -26,9 +26,10 @@ class FunctionHandler:
func_names = ",".join(surport_plugins)
for function_desc in self.functions_desc:
if function_desc["function"]["name"] == "plugin_loader":
- function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names)
+ function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]",
+ func_names)
break
-
+
def upload_functions_desc(self):
self.functions_desc = self.function_registry.get_all_function_desc()
@@ -47,7 +48,6 @@ class FunctionHandler:
def register_nessary_functions(self):
"""注册必要的函数"""
self.function_registry.register_function("handle_exit_intent")
- self.function_registry.register_function("play_music")
self.function_registry.register_function("plugin_loader")
self.function_registry.register_function("get_time")
self.function_registry.register_function("raise_and_lower_the_volume")
@@ -57,6 +57,9 @@ class FunctionHandler:
for func in self.config["Intent"]["function_call"].get("functions", []):
self.function_registry.register_function(func)
+ """home assistant需要初始化提示词"""
+ append_devices_to_prompt(self.conn)
+
def get_function(self, name):
return self.function_registry.get_function(name)
@@ -81,4 +84,4 @@ class FunctionHandler:
except Exception as e:
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
- return None
\ No newline at end of file
+ return None
diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py
index bafcf09d..cc968862 100644
--- a/main/xiaozhi-server/core/handle/helloHandle.py
+++ b/main/xiaozhi-server/core/handle/helloHandle.py
@@ -1,8 +1,72 @@
import json
from config.logger import setup_logging
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils.util import remove_punctuation_and_length
+import shutil
+import asyncio
+import os
+import random
+import time
logger = setup_logging()
+WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10,
+ "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]}
+
async def handleHelloMessage(conn):
await conn.websocket.send(json.dumps(conn.welcome_msg))
+
+
+async def checkWakeupWords(conn, text):
+ enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"]
+ """是否开启唤醒词加速"""
+ if not enable_wakeup_words_response_cache:
+ return False
+ """检查是否是唤醒词"""
+ _, text = remove_punctuation_and_length(text)
+ if text in conn.config.get("wakeup_words"):
+ await send_stt_message(conn, text)
+ conn.tts_first_text_index = 0
+ conn.tts_last_text_index = 0
+ conn.llm_finish_task = True
+
+ file = getWakeupWordFile(WAKEUP_CONFIG["file_name"])
+ if file is None:
+ asyncio.create_task(wakeupWordsResponse(conn))
+ return False
+ opus_packets, duration = conn.tts.audio_to_opus_data(file)
+ conn.audio_play_queue.put((opus_packets, text, 0))
+ if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
+ asyncio.create_task(wakeupWordsResponse(conn))
+ return True
+ return False
+
+
+def getWakeupWordFile(file_name):
+ for file in os.listdir(WAKEUP_CONFIG["dir"]):
+ if "my_"+file_name in file:
+ return f"config/assets/{file}"
+
+ """查找config/assets/目录下名称为wakeup_words的文件"""
+ for file in os.listdir(WAKEUP_CONFIG["dir"]):
+ if file_name in file:
+ return f"config/assets/{file}"
+ return None
+
+
+async def wakeupWordsResponse(conn):
+ """唤醒词响应"""
+ wakeup_word = random.choice(WAKEUP_CONFIG["words"])
+ result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
+ tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
+ if tts_file is not None and os.path.exists(tts_file):
+ file_type = os.path.splitext(tts_file)[1]
+ if file_type:
+ file_type = file_type.lstrip('.')
+ old_file = getWakeupWordFile("my_" +WAKEUP_CONFIG["file_name"])
+ if old_file is not None:
+ os.remove(old_file)
+ """将文件挪到"wakeup_words.mp3"""
+ shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" +WAKEUP_CONFIG["file_name"] + "." + file_type)
+ WAKEUP_CONFIG["create_time"] = time.time()
diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py
index 1799516c..d8440fc6 100644
--- a/main/xiaozhi-server/core/handle/intentHandler.py
+++ b/main/xiaozhi-server/core/handle/intentHandler.py
@@ -2,6 +2,7 @@ from config.logger import setup_logging
import json
import uuid
from core.handle.sendAudioHandle import send_stt_message
+from core.handle.helloHandle import checkWakeupWords
from core.utils.util import remove_punctuation_and_length
TAG = __name__
@@ -12,6 +13,10 @@ async def handle_user_intent(conn, text):
# 检查是否有明确的退出命令
if await check_direct_exit(conn, text):
return True
+ # 检查是否是唤醒词
+ if await checkWakeupWords(conn, text):
+ return True
+
if conn.use_function_call_mode:
# 使用支持function calling的聊天方法,不再进行意图分析
return False
@@ -35,6 +40,7 @@ async def check_direct_exit(conn, text):
return False
+
async def analyze_intent_with_llm(conn, text):
"""使用LLM分析用户意图"""
if not hasattr(conn, 'intent') or not conn.intent:
diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py
index a6fed97b..ffb36398 100644
--- a/main/xiaozhi-server/core/handle/sendAudioHandle.py
+++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py
@@ -7,12 +7,26 @@ from core.utils.util import remove_punctuation_and_length, get_string_no_punctua
TAG = __name__
logger = setup_logging()
+
async def sendAudioMessage(conn, audios, text, text_index=0):
# 发送句子开始消息
if text_index == conn.tts_first_text_index:
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
await send_tts_message(conn, "sentence_start", text)
+ # 播放音频
+ await sendAudio(conn, audios)
+
+ await send_tts_message(conn, "sentence_end", text)
+
+ # 发送结束消息(如果是最后一个文本)
+ if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+ await send_tts_message(conn, 'stop', None)
+ if conn.close_after_chat:
+ await conn.close()
+
+# 播放音频
+async def sendAudio(conn, audios):
# 流控参数优化
frame_duration = 62 # 帧时长(毫秒),增加余量
start_time = time.perf_counter()
@@ -46,13 +60,6 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
# 动态调整下次延迟,补偿发送耗时
play_position += frame_duration # 更新播放位置
- await send_tts_message(conn, "sentence_end", text)
-
- # 发送结束消息(如果是最后一个文本)
- if conn.llm_finish_task and text_index == conn.tts_last_text_index:
- await send_tts_message(conn, 'stop', None)
- if conn.close_after_chat:
- await conn.close()
async def send_tts_message(conn, state, text=None):
"""发送 TTS 状态消息"""
@@ -64,10 +71,20 @@ async def send_tts_message(conn, state, text=None):
if text is not None:
message["text"] = text
- await conn.websocket.send(json.dumps(message))
+ # TTS播放结束
if state == "stop":
+ # 播放提示音
+ tts_notify = conn.config.get("enable_stop_tts_notify", False)
+ if tts_notify:
+ stop_tts_notify_voice = conn.config.get("stop_tts_notify_voice", "config/assets/tts_notify.mp3")
+ audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
+ await sendAudio(conn, audios)
+ # 清除服务端讲话状态
conn.clearSpeakStatus()
+ # 发送消息到客户端
+ await conn.websocket.send(json.dumps(message))
+
async def send_stt_message(conn, text):
"""发送 STT 状态消息"""
diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py
index c63b9a33..3f60592a 100644
--- a/main/xiaozhi-server/core/handle/textHandle.py
+++ b/main/xiaozhi-server/core/handle/textHandle.py
@@ -2,7 +2,9 @@ from config.logger import setup_logging
import json
from core.handle.abortHandle import handleAbortMessage
from core.handle.helloHandle import handleHelloMessage
+from core.utils.util import remove_punctuation_and_length
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
+from core.handle.sendAudioHandle import send_stt_message, send_tts_message
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
TAG = __name__
@@ -38,7 +40,21 @@ async def handleTextMessage(conn, message):
conn.client_have_voice = False
conn.asr_audio.clear()
if "text" in msg_json:
- await startToChat(conn, msg_json["text"])
+ text = msg_json["text"]
+ _, text = remove_punctuation_and_length(text)
+
+ # 识别是否是唤醒词
+ is_wakeup_words = text in conn.config.get("wakeup_words")
+ # 是否开启唤醒词回复
+ enable_greeting = conn.config.get("enable_greeting", True)
+
+ if is_wakeup_words and not enable_greeting:
+ # 如果是唤醒词,且关闭了唤醒词回复,就不用回答
+ await send_stt_message(conn, text)
+ await send_tts_message(conn, "stop", None)
+ else:
+ # 否则需要LLM对文字内容进行答复
+ await startToChat(conn, text)
elif msg_json["type"] == "iot":
if "descriptors" in msg_json:
await handleIotDescriptors(conn, msg_json["descriptors"])
diff --git a/main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py b/main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py
new file mode 100644
index 00000000..51efdc06
--- /dev/null
+++ b/main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py
@@ -0,0 +1,52 @@
+from config.logger import setup_logging
+from http import HTTPStatus
+from dashscope import Application
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProvider(LLMProviderBase):
+ def __init__(self, config):
+ self.api_key = config["api_key"]
+ self.app_id = config["app_id"]
+ self.base_url = config.get("base_url")
+ self.is_No_prompt = config.get("is_no_prompt")
+ self.memory_id = config.get("ali_memory_id")
+
+ def response(self, session_id, dialogue):
+ try:
+ # 处理dialogue
+ if self.is_No_prompt:
+ dialogue.pop(0)
+ logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
+
+ # 构造调用参数
+ call_params = {
+ "api_key": self.api_key,
+ "app_id": self.app_id,
+ "session_id": session_id,
+ "messages": dialogue
+ }
+ if self.memory_id != False:
+ # 百练memory需要prompt参数
+ prompt = dialogue[-1].get("content")
+ call_params["memory_id"] = self.memory_id
+ call_params["prompt"] = prompt
+ logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
+
+ responses = Application.call(**call_params)
+ if responses.status_code != HTTPStatus.OK:
+ logger.bind(tag=TAG).error(
+ f"code={responses.status_code}, "
+ f"message={responses.message}, "
+ f"请参考文档:https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
+ )
+ yield "【阿里百练API服务响应异常】"
+ else:
+ logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
+ yield responses.output.text
+
+ except Exception as e:
+ logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
+ yield "【LLM服务响应异常】"
diff --git a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py b/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py
deleted file mode 100644
index 115e85e9..00000000
--- a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import requests
-from requests.exceptions import RequestException
-from config.logger import setup_logging
-from core.providers.llm.base import LLMProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-
-class LLMProvider(LLMProviderBase):
- def __init__(self, config):
- self.agent_id = config.get("agent_id") # 对应 agent_id
- self.api_key = config.get("api_key")
- self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url
- self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
-
- def response(self, session_id, dialogue):
- print(dialogue)
- try:
- # home assistant语音助手自带意图,无需使用xiaozhi ai自带的,只需要把用户说的话传递给home assistant即可
-
- # 提取最后一个 role 为 'user' 的 content
- input_text = None
- if isinstance(dialogue, list): # 确保 dialogue 是一个列表
- # 逆序遍历,找到最后一个 role 为 'user' 的消息
- for message in reversed(dialogue):
- if message.get("role") == "user": # 找到 role 为 'user' 的消息
- input_text = message.get("content", "")
- break # 找到后立即退出循环
-
- # 构造请求数据
- payload = {
- "text": input_text,
- "agent_id": self.agent_id,
- "conversation_id": session_id # 使用 session_id 作为 conversation_id
- }
- # 设置请求头
- headers = {
- "Authorization": f"Bearer {self.api_key}",
- "Content-Type": "application/json"
- }
-
- # 发起 POST 请求
- response = requests.post(self.api_url, json=payload, headers=headers)
-
- # 检查请求是否成功
- response.raise_for_status()
-
- # 解析返回数据
- data = response.json()
- speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
-
- # 返回生成的内容
- if speech:
- yield speech
- else:
- logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
-
- except RequestException as e:
- logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
- except Exception as e:
- logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py
index 3146dc06..e39ca65c 100644
--- a/main/xiaozhi-server/core/providers/tts/base.py
+++ b/main/xiaozhi-server/core/providers/tts/base.py
@@ -36,7 +36,7 @@ class TTSProviderBase(ABC):
return tmp_file
except Exception as e:
- logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
+ logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
return None
@abstractmethod
diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py
index 8d2c161b..d4e66bb6 100644
--- a/main/xiaozhi-server/core/utils/dialogue.py
+++ b/main/xiaozhi-server/core/utils/dialogue.py
@@ -35,6 +35,15 @@ class Dialogue:
self.getMessages(m, dialogue)
return dialogue
+ def update_system_message(self, new_content: str):
+ """更新或添加系统消息"""
+ # 查找第一个系统消息
+ system_msg = next((msg for msg in self.dialogue if msg.role == "system"), None)
+ if system_msg:
+ system_msg.content = new_content
+ else:
+ self.put(Message(role="system", content=new_content))
+
def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
if memory_str is None or len(memory_str) == 0:
return self.get_llm_dialogue()
diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py
index 63cb5b9c..51841baf 100644
--- a/main/xiaozhi-server/core/utils/util.py
+++ b/main/xiaozhi-server/core/utils/util.py
@@ -73,9 +73,7 @@ def get_ip_info(ip_addr):
resp = requests.get(url).json()
ip_info = {
- "city": resp.get("cityName"),
- "region": resp.get("regionName"),
- "country": resp.get("countryName")
+ "city": resp.get("cityName")
}
return ip_info
except Exception as e:
diff --git a/main/xiaozhi-server/plugins_func/functions/hass_get_state.py b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py
new file mode 100644
index 00000000..415f35c6
--- /dev/null
+++ b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py
@@ -0,0 +1,56 @@
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+from plugins_func.functions.hass_init import initialize_hass_handler
+from config.logger import setup_logging
+import asyncio
+import requests
+
+TAG = __name__
+logger = setup_logging()
+
+hass_get_state_function_desc = {
+ "type": "function",
+ "function": {
+ "name": "hass_get_state",
+ "description": "获取homeassistant里设备的状态,包括灯光亮度,媒体播放器的音量,设备的暂停、继续操作",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "entity_id": {
+ "type": "string",
+ "description": "需要操作的设备id,homeassistant里的entity_id"
+ }
+ },
+ "required": ["entity_id"]
+ }
+ }
+}
+
+
+@register_function("hass_get_state", hass_get_state_function_desc, ToolType.SYSTEM_CTL)
+def hass_get_state(conn, entity_id=''):
+ try:
+
+ future = asyncio.run_coroutine_threadsafe(
+ handle_hass_get_state(conn, entity_id),
+ conn.loop
+ )
+ ha_response = future.result()
+ return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response)
+ except Exception as e:
+ logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}")
+
+
+async def handle_hass_get_state(conn, entity_id):
+ HASS_CACHE = initialize_hass_handler(conn)
+ api_key = HASS_CACHE['api_key']
+ base_url = HASS_CACHE['base_url']
+ url = f"{base_url}/api/states/{entity_id}"
+ headers = {
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json"
+ }
+ response = requests.get(url, headers=headers)
+ if response.status_code == 200:
+ return response.json()['state']
+ else:
+ return f"切换失败,错误码: {response.status_code}"
diff --git a/main/xiaozhi-server/plugins_func/functions/hass_init.py b/main/xiaozhi-server/plugins_func/functions/hass_init.py
new file mode 100644
index 00000000..e09eb67f
--- /dev/null
+++ b/main/xiaozhi-server/plugins_func/functions/hass_init.py
@@ -0,0 +1,35 @@
+from config.logger import setup_logging
+from core.utils.util import check_model_key
+
+TAG = __name__
+logger = setup_logging()
+
+HASS_CACHE = {}
+
+
+def append_devices_to_prompt(conn):
+ if conn.use_function_call_mode:
+ funcs = conn.config["Intent"]["function_call"].get("functions", [])
+ if "hass_get_state" in funcs or "hass_get_state" in funcs:
+ prompt = "下面是我家智能设备,可以通过homeassistant控制\n"
+ devices = conn.config["plugins"]["home_assistant"].get("devices", [])
+ if len(devices) == 0:
+ return
+ for device in devices:
+ prompt += device + "\n"
+ conn.prompt += prompt
+ # 更新提示词
+ conn.dialogue.update_system_message(conn.prompt)
+
+
+def initialize_hass_handler(conn):
+ global HASS_CACHE
+ if HASS_CACHE == {}:
+ if conn.use_function_call_mode:
+ funcs = conn.config["Intent"]["function_call"].get("functions", [])
+ if "hass_get_state" in funcs or "hass_get_state" in funcs:
+ HASS_CACHE['base_url'] = conn.config["plugins"]["home_assistant"].get("base_url")
+ HASS_CACHE['api_key'] = conn.config["plugins"]["home_assistant"].get("api_key")
+
+ check_model_key("home_assistant", HASS_CACHE['api_key'])
+ return HASS_CACHE
diff --git a/main/xiaozhi-server/plugins_func/functions/hass_play_music.py b/main/xiaozhi-server/plugins_func/functions/hass_play_music.py
new file mode 100644
index 00000000..502f784e
--- /dev/null
+++ b/main/xiaozhi-server/plugins_func/functions/hass_play_music.py
@@ -0,0 +1,64 @@
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+from plugins_func.functions.hass_init import initialize_hass_handler
+from config.logger import setup_logging
+import asyncio
+import requests
+
+TAG = __name__
+logger = setup_logging()
+
+hass_play_music_function_desc = {
+ "type": "function",
+ "function": {
+ "name": "hass_play_music",
+ "description": "用户想听音乐、有声书的时候使用,在房间的媒体播放器(media_player)里播放对应音频",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "media_content_id": {
+ "type": "string",
+ "description": "可以是音乐或有声书的专辑名称、歌曲名、演唱者,如果未指定就填random"
+ },
+ "entity_id": {
+ "type": "string",
+ "description": "需要操作的音箱的设备id,homeassistant里的entity_id,media_player开头"
+ }
+ },
+ "required": ["media_content_id", "entity_id"]
+ }
+ }
+}
+
+
+@register_function('hass_play_music', hass_play_music_function_desc, ToolType.SYSTEM_CTL)
+def hass_play_music(conn, entity_id='', media_content_id='random'):
+ try:
+ # 执行音乐播放命令
+ future = asyncio.run_coroutine_threadsafe(
+ handle_hass_play_music(conn, entity_id, media_content_id),
+ conn.loop
+ )
+ ha_response = future.result()
+ return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=ha_response)
+ except Exception as e:
+ logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+
+
+async def handle_hass_play_music(conn, entity_id, media_content_id):
+ HASS_CACHE = initialize_hass_handler(conn)
+ api_key = HASS_CACHE['api_key']
+ base_url = HASS_CACHE['base_url']
+ url = f"{base_url}/api/services/music_assistant/play_media"
+ headers = {
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json"
+ }
+ data = {
+ "entity_id": entity_id,
+ "media_id": media_content_id
+ }
+ response = requests.post(url, headers=headers, json=data)
+ if response.status_code == 200:
+ return f"正在播放{media_content_id}的音乐"
+ else:
+ return f"音乐播放失败,错误码: {response.status_code}"
diff --git a/main/xiaozhi-server/plugins_func/functions/hass_set_state.py b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py
new file mode 100644
index 00000000..5e2b91a2
--- /dev/null
+++ b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py
@@ -0,0 +1,159 @@
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+from plugins_func.functions.hass_init import initialize_hass_handler
+from config.logger import setup_logging
+import asyncio
+import requests
+
+TAG = __name__
+logger = setup_logging()
+
+hass_set_state_function_desc = {
+ "type": "function",
+ "function": {
+ "name": "hass_set_state",
+ "description": "设置homeassistant里设备的状态,包括开、关,调整灯光亮度,调整播放器的音量,设备的暂停、继续、静音操作",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "state": {
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "需要操作的动作,打开设备:turn_on,关闭设备:turn_off,增加亮度:brightness_up,降低亮度:brightness_down,设置亮度:brightness_value,增加>音量:,volume_up降低音量:volume_down,设置音量:volume_set,设备暂停:pause,设备继续:continue,静音/取消静音:volume_mute"
+ },
+ "input": {
+ "type": "int",
+ "description": "只有在设置音量,设置亮度时候才需要,有效值为1-100,对应音量和亮度的1%-100%"
+ },
+ "is_muted": {
+ "type": "string",
+ "description": "只有在设置静音操作时才需要,设置静音的时候该值为true,取消静音时该值为false"
+ }
+ },
+ "required": ["type"]
+ },
+ "entity_id": {
+ "type": "string",
+ "description": "需要操作的设备id,homeassistant里的entity_id"
+ }
+ },
+ "required": ["state", "entity_id"]
+ }
+ }
+}
+
+
+@register_function('hass_set_state', hass_set_state_function_desc, ToolType.SYSTEM_CTL)
+def hass_set_state(conn, entity_id='', state={}):
+ try:
+ future = asyncio.run_coroutine_threadsafe(
+ handle_hass_set_state(conn, entity_id, state),
+ conn.loop
+ )
+ ha_response = future.result()
+ return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response)
+ except Exception as e:
+ logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}")
+
+
+async def handle_hass_set_state(conn, entity_id, state):
+ HASS_CACHE = initialize_hass_handler(conn)
+ api_key = HASS_CACHE['api_key']
+ base_url = HASS_CACHE['base_url']
+ '''
+ state = { "type":"brightness_up","input":"80","is_muted":"true"}
+ '''
+ domains = entity_id.split(".")
+ if len(domains) > 1:
+ domain = domains[0]
+ else:
+ return "执行失败,错误的设备id"
+ action = ''
+ arg = ''
+ value = ''
+ if state['type'] == 'turn_on':
+ description = "设备已打开"
+ if domain == "cover":
+ action = "open_cover"
+ elif domain == "vacuum":
+ action = "start"
+ else:
+ action = "turn_on"
+ elif state['type'] == 'turn_off':
+ description = "设备已关闭"
+ if domain == 'cover':
+ action = "close_cover"
+ elif domain == 'vacuum':
+ action = "stop"
+ else:
+ action = "turn_off"
+ elif state['type'] == 'brightness_up':
+ description = "灯光已调亮"
+ action = 'turn_on'
+ arg = 'brightness_step_pct'
+ value = 10
+ elif state['type'] == 'brightness_down':
+ description = "灯光已调暗"
+ action = 'turn_on'
+ arg = 'brightness_step_pct'
+ value = -10
+ elif state['type'] == 'brightness_value':
+ description = f"亮度已调整到{state['input']}"
+ action = 'turn_on'
+ arg = 'brightness_pct'
+ value = state['input']
+ elif state['type'] == 'volume_up':
+ description = "音量已调大"
+ action = state['type']
+ elif state['type'] == 'volume_down':
+ description = "音量已调小"
+ action = state['type']
+ elif state['type'] == 'volume_set':
+ description = f"音量已调整到{state['input']}"
+ action = state['type']
+ arg = 'volume_level'
+ value = state['input']
+ elif state['type'] == 'volume_mute':
+ description = f"设备已静音"
+ action = state['type']
+ arg = 'is_volume_muted'
+ value = state['is_muted']
+ elif state['type'] == 'pause':
+ description = f"设备已暂停"
+ action = state['type']
+ if domain == 'media_player':
+ action = 'media_pause'
+ if domain == 'cover':
+ action = 'stop_cover'
+ if domain == 'vacuum':
+ action = 'pause'
+ elif state['type'] == 'continue':
+ description = f"设备已继续"
+ if domain == 'media_player':
+ action = 'media_play'
+ if domain == 'vacuum':
+ action = 'start'
+ else:
+ return f"{domain} {state.type}功能尚未支持"
+
+ if arg == '':
+ data = {
+ "entity_id": entity_id,
+ }
+ else:
+ data = {
+ "entity_id": entity_id,
+ arg: value
+ }
+ url = f"{base_url}/api/services/{domain}/{action}"
+ headers = {
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json"
+ }
+ response = requests.post(url, headers=headers, json=data)
+ logger.bind(tag=TAG).info(f"设置状态:url:{url},return_code:{response.status_code}")
+ if response.status_code == 200:
+ return description
+ else:
+ return f"设置失败,错误码: {response.status_code}"
diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py
index 75f3834d..20148b5f 100644
--- a/main/xiaozhi-server/plugins_func/functions/play_music.py
+++ b/main/xiaozhi-server/plugins_func/functions/play_music.py
@@ -21,13 +21,13 @@ play_music_function_desc = {
"type": "function",
"function": {
"name": "play_music",
- "description": "唱歌、听歌、播放音乐方法。比如用户说播放音乐,参数为:random,比如用户说播放两只老虎,参数为:两只老虎",
+ "description": "唱歌、听歌、播放音乐的方法。",
"parameters": {
"type": "object",
"properties": {
"song_name": {
"type": "string",
- "description": "歌曲名称,如果没有指定具体歌名则为'random'"
+ "description": "歌曲名称,如果用户没有指定具体歌名则为'random', 明确指定的时返回音乐的名字 示例: ```用户:播放两只老虎\n参数:两只老虎``` ```用户:播放音乐 \n参数:random ```"
}
},
"required": ["song_name"]
@@ -112,8 +112,8 @@ def get_music_files(music_dir, music_ext):
def initialize_music_handler(conn):
global MUSIC_CACHE
if MUSIC_CACHE == {}:
- if "music" in conn.config:
- MUSIC_CACHE["music_config"] = conn.config["music"]
+ if "play_music" in conn.config["plugins"]:
+ MUSIC_CACHE["music_config"] = conn.config["plugins"]["play_music"]
MUSIC_CACHE["music_dir"] = os.path.abspath(
MUSIC_CACHE["music_config"].get("music_dir", "./music") # 默认路径修改
)
diff --git a/main/xiaozhi-server/plugins_func/loadplugins.py b/main/xiaozhi-server/plugins_func/loadplugins.py
index d826fac8..9b20355e 100644
--- a/main/xiaozhi-server/plugins_func/loadplugins.py
+++ b/main/xiaozhi-server/plugins_func/loadplugins.py
@@ -22,6 +22,4 @@ def auto_import_modules(package_name):
# 导入模块
full_module_name = f"{package_name}.{module_name}"
importlib.import_module(full_module_name)
- #logger.bind(tag=TAG).info(f"模块 '{full_module_name}' 已加载")
-
-auto_import_modules('plugins_func.functions')
\ No newline at end of file
+ #logger.bind(tag=TAG).info(f"模块 '{full_module_name}' 已加载")
\ No newline at end of file