-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
![验证码]()
+
+
+
+
+
+
+
+
+ {{ countdown > 0 ? `${countdown}秒后重试` : '发送验证码' }}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
![验证码]()
+
+
+
+
+
@@ -81,7 +121,7 @@
+@import './auth.scss';
+
+.send-captcha-btn {
+ margin-right: -5px;
+ min-width: 100px;
+ height: 40px;
+ line-height: 40px;
+ border-radius: 4px;
+ font-size: 14px;
+ background: rgb(87, 120, 255);
+ border: none;
+ padding: 0px;
+
+ &:disabled {
+ background: #c0c4cc;
+ cursor: not-allowed;
+ }
+}
+
diff --git a/main/manager-web/src/views/retrievePassword.vue b/main/manager-web/src/views/retrievePassword.vue
new file mode 100644
index 00000000..9072baae
--- /dev/null
+++ b/main/manager-web/src/views/retrievePassword.vue
@@ -0,0 +1,275 @@
+
+
+
+
+
+
+

+

+
+
+
+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/main/xiaozhi-server/app.py b/main/xiaozhi-server/app.py
index 619fb5c7..63849c58 100644
--- a/main/xiaozhi-server/app.py
+++ b/main/xiaozhi-server/app.py
@@ -102,7 +102,7 @@ async def main():
await asyncio.wait(
[stdin_task, ws_task, ota_task] if ota_task else [stdin_task, ws_task],
timeout=3.0,
- return_when=asyncio.ALL_COMPLETED
+ return_when=asyncio.ALL_COMPLETED,
)
print("服务器已关闭,程序退出。")
diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml
index c1ca2dbb..05756cd7 100644
--- a/main/xiaozhi-server/config.yaml
+++ b/main/xiaozhi-server/config.yaml
@@ -411,7 +411,7 @@ LLM:
base_url: https://host/api/v1
# 你可以在这里找到你的api_key
# https://cloud.tryfastgpt.ai/account/apikey
- api_key: fastgpt-xxx
+ api_key: 你的fastgpt密钥
variables:
k: "v"
k2: "v2"
diff --git a/main/xiaozhi-server/config/config_loader.py b/main/xiaozhi-server/config/config_loader.py
index 35802a67..f14e4aa3 100644
--- a/main/xiaozhi-server/config/config_loader.py
+++ b/main/xiaozhi-server/config/config_loader.py
@@ -82,6 +82,8 @@ def ensure_directories(config):
# ASR/TTS模块输出目录
for module in ["ASR", "TTS"]:
+ if config.get(module) is None:
+ continue
for provider in config.get(module, {}).values():
output_dir = provider.get("output_dir", "")
if output_dir:
@@ -93,6 +95,10 @@ def ensure_directories(config):
selected_provider = selected_modules.get(module_type)
if not selected_provider:
continue
+ if config.get(module) is None:
+ continue
+ if config.get(selected_provider) is None:
+ continue
provider_config = config.get(module_type, {}).get(selected_provider, {})
output_dir = provider_config.get("output_dir")
if output_dir:
diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py
index 8866b1a8..5588e00f 100644
--- a/main/xiaozhi-server/config/logger.py
+++ b/main/xiaozhi-server/config/logger.py
@@ -4,7 +4,7 @@ from loguru import logger
from config.config_loader import load_config
from config.settings import check_config_file
-SERVER_VERSION = "0.4.3"
+SERVER_VERSION = "0.4.4"
def get_module_abbreviation(module_name, module_dict):
diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py
index 0abb3cec..74b612fe 100644
--- a/main/xiaozhi-server/core/connection.py
+++ b/main/xiaozhi-server/core/connection.py
@@ -22,6 +22,7 @@ from core.utils.util import (
initialize_modules,
check_vad_update,
check_asr_update,
+ filter_sensitive_info,
)
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from core.handle.sendAudioHandle import sendAudioMessage
@@ -273,9 +274,10 @@ class ConnectionHandler:
await self.websocket.send(
json.dumps(
{
- "type": "server_response",
+ "type": "server",
"status": "success",
"message": "服务器重启中...",
+ "content": {"action": "restart"},
}
)
)
@@ -302,9 +304,10 @@ class ConnectionHandler:
await self.websocket.send(
json.dumps(
{
- "type": "server_response",
+ "type": "server",
"status": "error",
"message": f"Restart failed: {str(e)}",
+ "content": {"action": "restart"},
}
)
)
@@ -1087,37 +1090,3 @@ class ConnectionHandler:
break
except Exception as e:
self.logger.bind(tag=TAG).error(f"超时检查任务出错: {e}")
-
-
-def filter_sensitive_info(config: dict) -> dict:
- """
- 过滤配置中的敏感信息
- Args:
- config: 原始配置字典
- Returns:
- 过滤后的配置字典
- """
- sensitive_keys = [
- "api_key",
- "personal_access_token",
- "access_token",
- "token",
- "secret",
- "access_key_secret",
- "secret_key",
- ]
-
- def _filter_dict(d: dict) -> dict:
- filtered = {}
- for k, v in d.items():
- if any(sensitive in k.lower() for sensitive in sensitive_keys):
- filtered[k] = "***"
- elif isinstance(v, dict):
- filtered[k] = _filter_dict(v)
- elif isinstance(v, list):
- filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
- else:
- filtered[k] = v
- return filtered
-
- return _filter_dict(copy.deepcopy(config))
diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py
index b09a369c..5f620880 100644
--- a/main/xiaozhi-server/core/handle/functionHandler.py
+++ b/main/xiaozhi-server/core/handle/functionHandler.py
@@ -1,6 +1,12 @@
from config.logger import setup_logging
import json
-from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+from plugins_func.register import (
+ FunctionRegistry,
+ ActionResponse,
+ Action,
+ ToolType,
+ DeviceTypeRegistry,
+)
from plugins_func.functions.hass_init import append_devices_to_prompt
TAG = __name__
@@ -10,6 +16,7 @@ class FunctionHandler:
def __init__(self, conn):
self.conn = conn
self.config = conn.config
+ self.device_type_registry = DeviceTypeRegistry()
self.function_registry = FunctionRegistry()
self.register_nessary_functions()
self.register_config_functions()
@@ -54,7 +61,7 @@ class FunctionHandler:
self.function_registry.register_function("plugin_loader")
self.function_registry.register_function("get_time")
self.function_registry.register_function("get_lunar")
- self.function_registry.register_function("handle_device")
+ self.function_registry.register_function("handle_speaker_volume_or_screen_brightness")
def register_config_functions(self):
"""注册配置中的函数,可以不同客户端使用不同的配置"""
diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py
index f902a64f..e29ef450 100644
--- a/main/xiaozhi-server/core/handle/helloHandle.py
+++ b/main/xiaozhi-server/core/handle/helloHandle.py
@@ -40,8 +40,8 @@ async def checkWakeupWords(conn, text):
if not enable_wakeup_words_response_cache:
return False
"""检查是否是唤醒词"""
- _, text = remove_punctuation_and_length(text)
- if text in conn.config.get("wakeup_words"):
+ _, filtered_text = remove_punctuation_and_length(text)
+ if filtered_text in conn.config.get("wakeup_words"):
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py
index 1cae8bab..495da96f 100644
--- a/main/xiaozhi-server/core/handle/intentHandler.py
+++ b/main/xiaozhi-server/core/handle/intentHandler.py
@@ -13,10 +13,11 @@ TAG = __name__
async def handle_user_intent(conn, text):
# 检查是否有明确的退出命令
- if await check_direct_exit(conn, text):
+ filtered_text = remove_punctuation_and_length(text)[1]
+ if await check_direct_exit(conn, filtered_text):
return True
# 检查是否是唤醒词
- if await checkWakeupWords(conn, text):
+ if await checkWakeupWords(conn, filtered_text):
return True
if conn.intent_type == "function_call":
diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py
index 5b1bb9bb..5cf2d204 100644
--- a/main/xiaozhi-server/core/handle/iotHandle.py
+++ b/main/xiaozhi-server/core/handle/iotHandle.py
@@ -1,9 +1,8 @@
import json
import asyncio
-from config.logger import setup_logging
from plugins_func.register import (
- device_type_registry,
- register_function,
+ FunctionItem,
+ register_device_function,
ActionResponse,
Action,
ToolType,
@@ -177,7 +176,7 @@ class IotDescriptor:
self.methods.append(method)
-def register_device_type(descriptor):
+def register_device_type(descriptor, device_type_registry):
"""注册设备类型及其功能"""
device_name = descriptor["name"]
type_id = device_type_registry.generate_device_type_id(descriptor)
@@ -213,10 +212,12 @@ def register_device_type(descriptor):
},
}
query_func = create_iot_query_function(device_name, prop_name, prop_info)
- decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
- query_func
+ decorated_func = register_device_function(
+ func_name, func_desc, ToolType.IOT_CTL
+ )(query_func)
+ functions[func_name] = FunctionItem(
+ func_name, func_desc, decorated_func, ToolType.IOT_CTL
)
- functions[func_name] = decorated_func
# 为每个方法创建控制函数
for method_name, method_info in descriptor["methods"].items():
@@ -267,10 +268,12 @@ def register_device_type(descriptor):
},
}
control_func = create_iot_function(device_name, method_name, method_info)
- decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
- control_func
+ decorated_func = register_device_function(
+ func_name, func_desc, ToolType.IOT_CTL
+ )(control_func)
+ functions[func_name] = FunctionItem(
+ func_name, func_desc, decorated_func, ToolType.IOT_CTL
)
- functions[func_name] = decorated_func
device_type_registry.register_device_type(type_id, functions)
return type_id
@@ -289,7 +292,6 @@ async def handleIotDescriptors(conn, descriptors):
functions_changed = False
for descriptor in descriptors:
-
# 如果descriptor没有properties和methods,则直接跳过
if "properties" not in descriptor and "methods" not in descriptor:
continue
@@ -319,13 +321,16 @@ async def handleIotDescriptors(conn, descriptors):
if conn.load_function_plugin:
# 注册或获取设备类型
- type_id = register_device_type(descriptor)
+ device_type_registry = conn.func_handler.device_type_registry
+ type_id = register_device_type(descriptor, device_type_registry)
device_functions = device_type_registry.get_device_functions(type_id)
# 在连接级注册设备函数
if hasattr(conn, "func_handler"):
- for func_name in device_functions:
- conn.func_handler.function_registry.register_function(func_name)
+ for func_name, func_item in device_functions.items():
+ conn.func_handler.function_registry.register_function(
+ func_name, func_item
+ )
conn.logger.bind(tag=TAG).info(
f"注册IOT函数到function handler: {func_name}"
)
diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py
index ee6a1d5c..9c693751 100644
--- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py
+++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py
@@ -39,14 +39,14 @@ async def handleAudioMessage(conn, audio):
if len(conn.asr_audio) < 15:
conn.asr_server_receive = True
else:
- text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
- conn.logger.bind(tag=TAG).info(f"识别文本: {text}")
- text_len, _ = remove_punctuation_and_length(text)
+ raw_text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) # 确保ASR模块返回原始文本
+ conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+ text_len, _ = remove_punctuation_and_length(raw_text)
if text_len > 0:
# 使用自定义模块进行上报
- enqueue_asr_report(conn, text, copy.deepcopy(conn.asr_audio))
+ enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio))
- await startToChat(conn, text)
+ await startToChat(conn, raw_text)
else:
conn.asr_server_receive = True
conn.asr_audio.clear()
diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py
index 79874f1c..da6ede47 100644
--- a/main/xiaozhi-server/core/handle/textHandle.py
+++ b/main/xiaozhi-server/core/handle/textHandle.py
@@ -1,7 +1,7 @@
import json
from core.handle.abortHandle import handleAbortMessage
from core.handle.helloHandle import handleHelloMessage
-from core.utils.util import remove_punctuation_and_length
+from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
@@ -13,17 +13,20 @@ TAG = __name__
async def handleTextMessage(conn, message):
"""处理文本消息"""
- conn.logger.bind(tag=TAG).info(f"收到文本消息:{message}")
try:
msg_json = json.loads(message)
if isinstance(msg_json, int):
+ conn.logger.bind(tag=TAG).info(f"收到文本消息:{message}")
await conn.websocket.send(message)
return
if msg_json["type"] == "hello":
+ conn.logger.bind(tag=TAG).info(f"收到hello消息:{message}")
await handleHelloMessage(conn, msg_json)
elif msg_json["type"] == "abort":
+ conn.logger.bind(tag=TAG).info(f"收到abort消息:{message}")
await handleAbortMessage(conn)
elif msg_json["type"] == "listen":
+ conn.logger.bind(tag=TAG).info(f"收到listen消息:{message}")
if "mode" in msg_json:
conn.client_listen_mode = msg_json["mode"]
conn.logger.bind(tag=TAG).debug(
@@ -42,17 +45,17 @@ async def handleTextMessage(conn, message):
conn.client_have_voice = False
conn.asr_audio.clear()
if "text" in msg_json:
- text = msg_json["text"]
- _, text = remove_punctuation_and_length(text)
+ original_text = msg_json["text"] # 保留原始文本
+ filtered_len, filtered_text = remove_punctuation_and_length(original_text)
# 识别是否是唤醒词
- is_wakeup_words = text in conn.config.get("wakeup_words")
+ is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
# 是否开启唤醒词回复
enable_greeting = conn.config.get("enable_greeting", True)
if is_wakeup_words and not enable_greeting:
# 如果是唤醒词,且关闭了唤醒词回复,就不用回答
- await send_stt_message(conn, text)
+ await send_stt_message(conn, original_text)
await send_tts_message(conn, "stop", None)
elif is_wakeup_words:
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
@@ -60,15 +63,20 @@ async def handleTextMessage(conn, message):
await startToChat(conn, "嘿,你好呀")
else:
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
- enqueue_asr_report(conn, text, [])
+ enqueue_asr_report(conn, original_text, [])
# 否则需要LLM对文字内容进行答复
- await startToChat(conn, text)
+ await startToChat(conn, original_text)
elif msg_json["type"] == "iot":
+ conn.logger.bind(tag=TAG).info(f"收到iot消息:{message}")
if "descriptors" in msg_json:
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
if "states" in msg_json:
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
elif msg_json["type"] == "server":
+ # 记录日志时过滤敏感信息
+ conn.logger.bind(tag=TAG).info(
+ f"收到服务器消息:{filter_sensitive_info(msg_json)}"
+ )
# 如果配置是从API读取的,则需要验证secret
if not conn.read_config_from_api:
return
@@ -95,9 +103,10 @@ async def handleTextMessage(conn, message):
await conn.websocket.send(
json.dumps(
{
- "type": "config_update_response",
+ "type": "server",
"status": "error",
"message": "无法获取服务器实例",
+ "content": {"action": "update_config"},
}
)
)
@@ -107,9 +116,10 @@ async def handleTextMessage(conn, message):
await conn.websocket.send(
json.dumps(
{
- "type": "config_update_response",
+ "type": "server",
"status": "error",
"message": "更新服务器配置失败",
+ "content": {"action": "update_config"},
}
)
)
@@ -119,9 +129,10 @@ async def handleTextMessage(conn, message):
await conn.websocket.send(
json.dumps(
{
- "type": "config_update_response",
+ "type": "server",
"status": "success",
"message": "配置更新成功",
+ "content": {"action": "update_config"},
}
)
)
@@ -130,9 +141,10 @@ async def handleTextMessage(conn, message):
await conn.websocket.send(
json.dumps(
{
- "type": "config_update_response",
+ "type": "server",
"status": "error",
"message": f"更新配置失败: {str(e)}",
+ "content": {"action": "update_config"},
}
)
)
diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
index 195d058a..ee657420 100644
--- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
@@ -53,6 +53,8 @@ class IntentProvider(IntentProviderBase):
prompt = (
"你是一个意图识别助手。请分析用户的最后一句话,判断用户意图并调用相应的函数。\n\n"
+ "- 如果用户使用疑问词(如'怎么'、'为什么'、'如何')询问退出相关的问题(例如'怎么退出了?'),注意这不是让你退出,请返回 {'function_call': {'name': 'continue_chat'}\n"
+ "- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时,才触发 handle_exit_intent\n\n"
f"{functions_desc}\n"
"处理步骤:\n"
"1. 分析用户输入,确定用户意图\n"
@@ -218,6 +220,15 @@ class IntentProvider(IntentProviderBase):
f"llm 识别到意图: {function_name}, 参数: {function_args}"
)
+ # 如果是继续聊天,清理工具调用相关的历史消息
+ if function_name == "continue_chat":
+ # 保留非工具相关的消息
+ clean_history = [
+ msg for msg in conn.dialogue.dialogue
+ if msg.role not in ["tool", "function"]
+ ]
+ conn.dialogue.dialogue = clean_history
+
# 添加到缓存
self.intent_cache[cache_key] = {
"intent": intent,
diff --git a/main/xiaozhi-server/core/providers/tts/fishspeech.py b/main/xiaozhi-server/core/providers/tts/fishspeech.py
index bf195101..71f1ac88 100644
--- a/main/xiaozhi-server/core/providers/tts/fishspeech.py
+++ b/main/xiaozhi-server/core/providers/tts/fishspeech.py
@@ -130,7 +130,7 @@ class TTSProvider(TTSProviderBase):
"yes",
)
self.use_memory_cache = config.get("use_memory_cache", "on")
- self.seed = config.get("seed") or None
+ self.seed = int(config.get("seed")) if config.get("seed") else None
self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
def generate_filename(self, extension=".wav"):
diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py
index 9ab77a5a..0b7e46a1 100644
--- a/main/xiaozhi-server/core/utils/util.py
+++ b/main/xiaozhi-server/core/utils/util.py
@@ -9,6 +9,7 @@ import opuslib_next
from pydub import AudioSegment
from typing import Dict, Any
from core.utils import tts, llm, intent, memory, vad, asr
+import copy
TAG = __name__
emoji_map = {
@@ -319,7 +320,7 @@ def initialize_modules(
modules["memory"] = memory.create_instance(
memory_type,
config["Memory"][select_memory_module],
- config.get('summaryMemory', None),
+ config.get("summaryMemory", None),
)
logger.bind(tag=TAG).info(f"初始化组件: memory成功 {select_memory_module}")
@@ -956,3 +957,37 @@ def check_asr_update(before_config, new_config):
)
update_asr = current_asr_type != new_asr_type
return update_asr
+
+
+def filter_sensitive_info(config: dict) -> dict:
+ """
+ 过滤配置中的敏感信息
+ Args:
+ config: 原始配置字典
+ Returns:
+ 过滤后的配置字典
+ """
+ sensitive_keys = [
+ "api_key",
+ "personal_access_token",
+ "access_token",
+ "token",
+ "secret",
+ "access_key_secret",
+ "secret_key",
+ ]
+
+ def _filter_dict(d: dict) -> dict:
+ filtered = {}
+ for k, v in d.items():
+ if any(sensitive in k.lower() for sensitive in sensitive_keys):
+ filtered[k] = "***"
+ elif isinstance(v, dict):
+ filtered[k] = _filter_dict(v)
+ elif isinstance(v, list):
+ filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
+ else:
+ filtered[k] = v
+ return filtered
+
+ return _filter_dict(copy.deepcopy(config))
diff --git a/main/xiaozhi-server/core/websocket_server.py b/main/xiaozhi-server/core/websocket_server.py
index 441fd4e1..3f9e2fd7 100644
--- a/main/xiaozhi-server/core/websocket_server.py
+++ b/main/xiaozhi-server/core/websocket_server.py
@@ -82,11 +82,13 @@ class WebSocketServer:
if new_config is None:
self.logger.bind(tag=TAG).error("获取新配置失败")
return False
-
+ self.logger.bind(tag=TAG).info(f"获取新配置成功")
# 检查 VAD 和 ASR 类型是否需要更新
update_vad = check_vad_update(self.config, new_config)
update_asr = check_asr_update(self.config, new_config)
-
+ self.logger.bind(tag=TAG).info(
+ f"检查VAD和ASR类型是否需要更新: {update_vad} {update_asr}"
+ )
# 更新配置
self.config = new_config
# 重新初始化组件
@@ -114,7 +116,7 @@ class WebSocketServer:
self._intent = modules["intent"]
if "memory" in modules:
self._memory = modules["memory"]
-
+ self.logger.bind(tag=TAG).info(f"更新配置任务执行完毕")
return True
except Exception as e:
self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")
diff --git a/main/xiaozhi-server/performance_tester.py b/main/xiaozhi-server/performance_tester.py
index 38daea00..80503166 100644
--- a/main/xiaozhi-server/performance_tester.py
+++ b/main/xiaozhi-server/performance_tester.py
@@ -1,15 +1,17 @@
-import time
-import aiohttp
import asyncio
+import logging
+import os
+import statistics
+import time
+from typing import Dict
+
+import aiohttp
from tabulate import tabulate
-from typing import Dict, List
+
+from config.settings import load_config
+from core.utils.asr import create_instance as create_stt_instance
from core.utils.llm import create_instance as create_llm_instance
from core.utils.tts import create_instance as create_tts_instance
-import statistics
-from config.settings import load_config
-import inspect
-import os
-import logging
# 设置全局日志级别为WARNING,抑制INFO级别日志
logging.basicConfig(level=logging.WARNING)
@@ -26,7 +28,17 @@ class AsyncPerformanceTester:
"请用100字概括量子计算的基本原理和应用前景",
],
)
- self.results = {"llm": {}, "tts": {}, "combinations": []}
+
+ self.test_wav_list = []
+ self.wav_root = r"config/assets"
+ for file_name in os.listdir(self.wav_root):
+ file_path = os.path.join(self.wav_root, file_name)
+ # 检查文件大小是否大于300KB
+ if os.path.getsize(file_path) > 300 * 1024: # 300KB = 300 * 1024 bytes
+ with open(file_path, "rb") as f:
+ self.test_wav_list.append(f.read())
+
+ self.results = {"llm": {}, "tts": {}, "stt": {}, "combinations": []}
async def _check_ollama_service(self, base_url: str, model_name: str) -> bool:
"""异步检查Ollama服务状态"""
@@ -109,6 +121,57 @@ class AsyncPerformanceTester:
print(f"⚠️ {tts_name} 测试失败: {str(e)}")
return {"name": tts_name, "type": "tts", "errors": 1}
+ async def _test_stt(self, stt_name: str, config: Dict) -> Dict:
+ """异步测试单个STT性能"""
+ try:
+ logging.getLogger("core.providers.asr.base").setLevel(logging.WARNING)
+ token_fields = ["access_token", "api_key", "token"]
+ if any(
+ field in config
+ and any(x in config[field] for x in ["你的", "placeholder"])
+ for field in token_fields
+ ):
+ print(f"⏭️ STT {stt_name} 未配置access_token/api_key,已跳过")
+ return {"name": stt_name, "type": "stt", "errors": 1}
+
+ module_type = config.get("type", stt_name)
+ stt = create_stt_instance(module_type, config, delete_audio_file=True)
+ stt.audio_format = "pcm"
+
+ print(f"🎵 测试 STT: {stt_name}")
+
+ text, _ = await stt.speech_to_text([self.test_wav_list[0]], "1")
+
+ if text is None:
+ print(f"❌ {stt_name} 连接失败")
+ return {"name": stt_name, "type": "stt", "errors": 1}
+
+ total_time = 0
+ test_count = len(self.test_wav_list)
+
+ for i, sentence in enumerate(self.test_wav_list, 1):
+ start = time.time()
+ text, _ = await stt.speech_to_text([sentence], "1")
+ duration = time.time() - start
+ total_time += duration
+
+ if text:
+ print(f"✓ {stt_name} [{i}/{test_count}]")
+ else:
+ print(f"✗ {stt_name} [{i}/{test_count}]")
+ return {"name": stt_name, "type": "stt", "errors": 1}
+
+ return {
+ "name": stt_name,
+ "type": "stt",
+ "avg_time": total_time / test_count,
+ "errors": 0,
+ }
+
+ except Exception as e:
+ print(f"⚠️ {stt_name} 测试失败: {str(e)}")
+ return {"name": stt_name, "type": "stt", "errors": 1}
+
async def _test_llm(self, llm_name: str, config: Dict) -> Dict:
"""异步测试单个LLM性能"""
try:
@@ -234,6 +297,7 @@ class AsyncPerformanceTester:
if v["errors"] == 0 and v["avg_first_token"] >= 0.05
]
valid_tts = [k for k, v in self.results["tts"].items() if v["errors"] == 0]
+ valid_stt = [k for k, v in self.results["stt"].items() if v["errors"] == 0]
# 找出基准值
min_first_token = (
@@ -246,42 +310,53 @@ class AsyncPerformanceTester:
if valid_tts
else 1
)
+ min_stt_time = (
+ min([self.results["stt"][stt]["avg_time"] for stt in valid_stt])
+ if valid_stt
+ else 1
+ )
for llm in valid_llms:
for tts in valid_tts:
- # 计算相对性能分数(越小越好)
- llm_score = (
- self.results["llm"][llm]["avg_first_token"] / min_first_token
- )
- tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
+ for stt in valid_stt:
+ # 计算相对性能分数(越小越好)
+ llm_score = (
+ self.results["llm"][llm]["avg_first_token"] / min_first_token
+ )
+ tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
+ stt_score = self.results["stt"][stt]["avg_time"] / min_stt_time
- # 计算稳定性分数(标准差/平均值,越小越稳定)
- llm_stability = (
- self.results["llm"][llm]["std_first_token"]
- / self.results["llm"][llm]["avg_first_token"]
- )
+ # 计算稳定性分数(标准差/平均值,越小越稳定)
+ llm_stability = (
+ self.results["llm"][llm]["std_first_token"]
+ / self.results["llm"][llm]["avg_first_token"]
+ )
- # 综合得分(考虑性能和稳定性)
- # 性能权重0.7,稳定性权重0.3
- llm_final_score = llm_score * 0.7 + llm_stability * 0.3
+ # 综合得分(考虑性能和稳定性)
+ # LLM得分: 性能权重(70%) + 稳定性权重(30%)
+ llm_final_score = llm_score * 0.7 + llm_stability * 0.3
- # 总分 = LLM得分(70%) + TTS得分(30%)
- total_score = llm_final_score * 0.7 + tts_score * 0.3
+ # 总分 = LLM得分(70%) + TTS得分(30%) + STT得分(30%)
+ total_score = (
+ llm_final_score * 0.7 + tts_score * 0.3 + stt_score * 0.3
+ )
- self.results["combinations"].append(
- {
- "llm": llm,
- "tts": tts,
- "score": total_score,
- "details": {
- "llm_first_token": self.results["llm"][llm][
- "avg_first_token"
- ],
- "llm_stability": llm_stability,
- "tts_time": self.results["tts"][tts]["avg_time"],
- },
- }
- )
+ self.results["combinations"].append(
+ {
+ "llm": llm,
+ "tts": tts,
+ "stt": stt,
+ "score": total_score,
+ "details": {
+ "llm_first_token": self.results["llm"][llm][
+ "avg_first_token"
+ ],
+ "llm_stability": llm_stability,
+ "tts_time": self.results["tts"][tts]["avg_time"],
+ "stt_time": self.results["stt"][stt]["avg_time"],
+ },
+ }
+ )
# 分数越小越好
self.results["combinations"].sort(key=lambda x: x["score"])
@@ -302,7 +377,7 @@ class AsyncPerformanceTester:
)
if llm_table:
- print("\nLLM 性能排行:")
+ print("\nLLM 性能排行:\n")
print(
tabulate(
llm_table,
@@ -321,7 +396,7 @@ class AsyncPerformanceTester:
tts_table.append([name, f"{data['avg_time']:.3f}秒"]) # 不需要固定宽度
if tts_table:
- print("\nTTS 性能排行:")
+ print("\nTTS 性能排行:\n")
print(
tabulate(
tts_table,
@@ -334,17 +409,37 @@ class AsyncPerformanceTester:
else:
print("\n⚠️ 没有可用的TTS模块进行测试。")
+ stt_table = []
+ for name, data in self.results["stt"].items():
+ if data["errors"] == 0:
+ stt_table.append([name, f"{data['avg_time']:.3f}秒"]) # 不需要固定宽度
+
+ if stt_table:
+ print("\nSTT 性能排行:\n")
+ print(
+ tabulate(
+ stt_table,
+ headers=["模型名称", "合成耗时"],
+ tablefmt="github",
+ colalign=("left", "right"),
+ disable_numparse=True,
+ )
+ )
+ else:
+ print("\n⚠️ 没有可用的STT模块进行测试。")
+
if self.results["combinations"]:
- print("\n推荐配置组合 (得分越小越好):")
+ print("\n推荐配置组合 (得分越小越好):\n")
combo_table = []
- for combo in self.results["combinations"][:5]:
+ for combo in self.results["combinations"][:]:
combo_table.append(
[
- f"{combo['llm']} + {combo['tts']}", # 不需要固定宽度
+ f"{combo['llm']} + {combo['tts']} + {combo['stt']}", # 不需要固定宽度
f"{combo['score']:.3f}",
f"{combo['details']['llm_first_token']:.3f}秒",
f"{combo['details']['llm_stability']:.3f}",
f"{combo['details']['tts_time']:.3f}秒",
+ f"{combo['details']['stt_time']:.3f}秒",
]
)
@@ -357,9 +452,10 @@ class AsyncPerformanceTester:
"LLM首字耗时",
"稳定性",
"TTS合成耗时",
+ "STT合成耗时",
],
tablefmt="github",
- colalign=("left", "right", "right", "right", "right"),
+ colalign=("left", "right", "right", "right", "right", "right"),
disable_numparse=True,
)
)
@@ -372,8 +468,12 @@ class AsyncPerformanceTester:
if result["errors"] == 0:
if result["type"] == "llm":
self.results["llm"][result["name"]] = result
- else:
+ elif result["type"] == "tts":
self.results["tts"][result["name"]] = result
+ elif result["type"] == "stt":
+ self.results["stt"][result["name"]] = result
+ else:
+ pass
async def run(self):
"""执行全量异步测试"""
@@ -383,52 +483,73 @@ class AsyncPerformanceTester:
all_tasks = []
# LLM测试任务
- for llm_name, config in self.config.get("LLM", {}).items():
- # 检查配置有效性
- if llm_name == "CozeLLM":
- if any(x in config.get("bot_id", "") for x in ["你的"]) or any(
- x in config.get("user_id", "") for x in ["你的"]
+ if self.config.get("LLM") is not None:
+ for llm_name, config in self.config.get("LLM", {}).items():
+ # 检查配置有效性
+ if llm_name == "CozeLLM":
+ if any(x in config.get("bot_id", "") for x in ["你的"]) or any(
+ x in config.get("user_id", "") for x in ["你的"]
+ ):
+ print(f"⏭️ LLM {llm_name} 未配置bot_id/user_id,已跳过")
+ continue
+ elif "api_key" in config and any(
+ x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]
):
- print(f"⏭️ LLM {llm_name} 未配置bot_id/user_id,已跳过")
- continue
- elif "api_key" in config and any(
- x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]
- ):
- print(f"⏭️ LLM {llm_name} 未配置api_key,已跳过")
- continue
-
- # 对于Ollama,先检查服务状态
- if llm_name == "Ollama":
- base_url = config.get("base_url", "http://localhost:11434")
- model_name = config.get("model_name")
- if not model_name:
- print(f"🚫 Ollama未配置model_name")
+ print(f"⏭️ LLM {llm_name} 未配置api_key,已跳过")
continue
- if not await self._check_ollama_service(base_url, model_name):
- continue
+ # 对于Ollama,先检查服务状态
+ if llm_name == "Ollama":
+ base_url = config.get("base_url", "http://localhost:11434")
+ model_name = config.get("model_name")
+ if not model_name:
+ print(f"🚫 Ollama未配置model_name")
+ continue
- print(f"📋 添加LLM测试任务: {llm_name}")
- module_type = config.get("type", llm_name)
- llm = create_llm_instance(module_type, config)
+ if not await self._check_ollama_service(base_url, model_name):
+ continue
- # 为每个句子创建独立任务
- for sentence in self.test_sentences:
- sentence = sentence.encode("utf-8").decode("utf-8")
- all_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
+ print(f"📋 添加LLM测试任务: {llm_name}")
+ module_type = config.get("type", llm_name)
+ llm = create_llm_instance(module_type, config)
+
+ # 为每个句子创建独立任务
+ for sentence in self.test_sentences:
+ sentence = sentence.encode("utf-8").decode("utf-8")
+ all_tasks.append(
+ self._test_single_sentence(llm_name, llm, sentence)
+ )
# TTS测试任务
- for tts_name, config in self.config.get("TTS", {}).items():
- token_fields = ["access_token", "api_key", "token"]
- if any(
- field in config
- and any(x in config[field] for x in ["你的", "placeholder"])
- for field in token_fields
- ):
- print(f"⏭️ TTS {tts_name} 未配置access_token/api_key,已跳过")
- continue
- print(f"🎵 添加TTS测试任务: {tts_name}")
- all_tasks.append(self._test_tts(tts_name, config))
+ if self.config.get("TTS") is not None:
+ for tts_name, config in self.config.get("TTS", {}).items():
+ token_fields = ["access_token", "api_key", "token"]
+ if any(
+ field in config
+ and any(x in config[field] for x in ["你的", "placeholder"])
+ for field in token_fields
+ ):
+ print(f"⏭️ TTS {tts_name} 未配置access_token/api_key,已跳过")
+ continue
+ print(f"🎵 添加TTS测试任务: {tts_name}")
+ all_tasks.append(self._test_tts(tts_name, config))
+
+ # STT测试任务
+ if len(self.test_wav_list) >= 1:
+ if self.config.get("ASR") is not None:
+ for stt_name, config in self.config.get("ASR", {}).items():
+ token_fields = ["access_token", "api_key", "token"]
+ if any(
+ field in config
+ and any(x in config[field] for x in ["你的", "placeholder"])
+ for field in token_fields
+ ):
+ print(f"⏭️ ASR {stt_name} 未配置access_token/api_key,已跳过")
+ continue
+ print(f"🎵 添加ASR测试任务: {stt_name}")
+ all_tasks.append(self._test_stt(stt_name, config))
+ else:
+ print(f"\n⚠️ {self.wav_root} 路径下没有音频文件,已跳过STT测试任务")
print(
f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块"
@@ -436,6 +557,9 @@ class AsyncPerformanceTester:
print(
f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块"
)
+ print(
+ f"✅ 找到 {len([t for t in all_tasks if '_test_stt' in str(t)])} 个可用STT模块"
+ )
print("\n⏳ 开始并发测试所有模块...\n")
# 并发执行所有测试任务
@@ -494,6 +618,15 @@ class AsyncPerformanceTester:
if result["errors"] == 0:
self.results["tts"][result["name"]] = result
+ # 处理STT结果
+ for result in [
+ r
+ for r in all_results
+ if r and isinstance(r, dict) and r.get("type") == "stt"
+ ]:
+ if result["errors"] == 0:
+ self.results["stt"][result["name"]] = result
+
# 生成组合建议并打印结果
print("\n📊 生成测试报告...")
self._generate_combinations()
diff --git a/main/xiaozhi-server/plugins_func/functions/handle_device.py b/main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py
similarity index 70%
rename from main/xiaozhi-server/plugins_func/functions/handle_device.py
rename to main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py
index 70902c55..fa357ff5 100644
--- a/main/xiaozhi-server/plugins_func/functions/handle_device.py
+++ b/main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py
@@ -54,20 +54,29 @@ def _handle_device_action(conn, func, success_message, error_message, *args, **k
handle_device_function_desc = {
"type": "function",
"function": {
- "name": "handle_device",
+ "name": "handle_speaker_volume_or_screen_brightness",
"description": (
- "用户想要获取或者设置设备的音量/亮度大小,或者用户觉得声音/亮度过高或过低,或者用户想提高或降低音量/亮度。"
- "比如用户说现在亮度多少,参数为:device_type:Screen,action:get。"
- "比如用户说设置音量为50,参数为:device_type:Speaker,action:set,value:50。"
- "比如用户说亮度太高了,参数为:device_type:Screen,action:lower。"
- "比如用户说调大音量,参数为:device_type:Speaker,action:raise。"
+ "用户想要获取或者设置设备的音量/亮度大小,或者用户觉得声音/亮度过高或过低,或者用户想提高或降低音量/亮度。\n"
+ "**严格限制**:仅当用户明确操作 **Speaker(音量)或Screen(亮度)** 时才能调用此函数!\n"
+ "对于其他设备(如AC、Battery、Switch等),请不要调用此函数,而是继续正常的对话。\n\n"
+ "示例:\n"
+ "- 用户说『现在亮度多少』 → 调用函数:device_type: Screen, action: get\n"
+ "- 用户说『设置音量为50』 → 调用函数:device_type: Speaker, action: set, value: 50\n"
+ "- 用户说『亮度太高了』 → 调用函数:device_type: Screen, action: lower\n"
+ "- 用户说『调大音量』 → 调用函数:device_type: Speaker, action: raise\n\n"
+ "**拒绝调用示例**(应继续对话而非调用本函数):\n"
+ "- 用户说『空调调低一度』 → 不调用(设备类型为AC)\n"
+ "- 用户说『开关灯』 → 不调用(设备类型为Switch)\n"
+ "- 用户说『电量多少』 → 不调用(设备类型为Battery)\n"
),
"parameters": {
"type": "object",
"properties": {
"device_type": {
"type": "string",
- "description": "设备类型,可选值:Speaker(音量),Screen(亮度)"
+ "description": "设备类型,**严格限定为Speaker(音量)或Screen(亮度)**,其他设备类型禁止调用此函数",
+ "enum": ["Speaker", "Screen"]
+
},
"action": {
"type": "string",
@@ -84,8 +93,8 @@ handle_device_function_desc = {
}
-@register_function('handle_device', handle_device_function_desc, ToolType.IOT_CTL)
-def handle_device(conn, device_type: str, action: str, value: int = None):
+@register_function('handle_speaker_volume_or_screen_brightness', handle_device_function_desc, ToolType.IOT_CTL)
+def handle_speaker_volume_or_screen_brightness(conn, device_type: str, action: str, value: int = None):
if device_type == "Speaker":
method_name, property_name, device_name = "SetVolume", "volume", "音量"
elif device_type == "Screen":
diff --git a/main/xiaozhi-server/plugins_func/register.py b/main/xiaozhi-server/plugins_func/register.py
index b96f2f29..873c61e1 100644
--- a/main/xiaozhi-server/plugins_func/register.py
+++ b/main/xiaozhi-server/plugins_func/register.py
@@ -77,7 +77,6 @@ class DeviceTypeRegistry:
# 初始化函数注册字典
all_function_registry = {}
-device_type_registry = DeviceTypeRegistry()
def register_function(name, desc, type=None):
@@ -91,13 +90,29 @@ def register_function(name, desc, type=None):
return decorator
+def register_device_function(name, desc, type=None):
+ """注册设备级别的函数到函数注册字典的装饰器"""
+
+ def decorator(func):
+ logger.bind(tag=TAG).debug(f"设备函数 '{name}' 已加载")
+ return func
+
+ return decorator
+
+
class FunctionRegistry:
def __init__(self):
self.function_registry = {}
self.logger = setup_logging()
- def register_function(self, name):
- # 查找all_function_registry中是否有对应的函数
+ def register_function(self, name, func_item=None):
+ # 如果提供了func_item,直接注册
+ if func_item:
+ self.function_registry[name] = func_item
+ self.logger.bind(tag=TAG).debug(f"函数 '{name}' 直接注册成功")
+ return func_item
+
+ # 否则从all_function_registry中查找
func = all_function_registry.get(name)
if not func:
self.logger.bind(tag=TAG).error(f"函数 '{name}' 未找到")