Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-28 20:33:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into tts-response

Browse Source 
# Conflicts:
#	main/xiaozhi-server/core/handle/intentHandler.py
This commit is contained in:
hrz
2025-05-23 09:15:15 +08:00
parent fdc6dcb26e 0a9e5d2cea
commit 8119897818
23 changed files with 224 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/app.py
+1 -1
View File
@@ -102,7 +102,7 @@ async def main():
await asyncio.wait(
[stdin_task, ws_task, ota_task] if ota_task else [stdin_task, ws_task],
timeout=3.0,
return_when=asyncio.ALL_COMPLETED
return_when=asyncio.ALL_COMPLETED,
)
print("服务器已关闭，程序退出。")
main/xiaozhi-server/config.yaml
+16 -9
View File
@@ -686,17 +686,24 @@ TTS:
speed: 1
output_dir: tmp/
CustomTTS:
# 自定义的TTS接口服务，请求参数可自定义
# 要求接口使用GET方式请求，并返回音频文件
# 自定义的TTS接口服务，请求参数可自定义，可接入众多TTS服务
# 以本地部署的KokoroTTS为例
# 如果只有cpu运行：docker run -p 8880:8880 ghcr.io/remsky/kokoro-fastapi-cpu:latest
# 如果只有gpu运行：docker run --gpus all -p 8880:8880 ghcr.io/remsky/kokoro-fastapi-gpu:latest
# 要求接口使用POST方式请求，并返回音频文件
type: custom
url: "http://127.0.0.1:9880/tts"
method: POST
url: "http://127.0.0.1:8880/v1/audio/speech"
params: # 自定义请求参数
# text: "{prompt_text}" # {prompt_text}会被替换为实际的提示词内容
# speaker: jok老师
# speed: 1
# foo: bar
# testabc: 123456
input: "{prompt_text}"
response_format: "mp3"
download_format: "mp3"
voice: "zf_xiaoxiao"
lang_code: "z"
return_download_link: true
speed: 1
stream: false
headers: # 自定义请求头
# Authorization: Bearer xxxx
format: wav # 接口返回的音频格式
format: mp3 # 接口返回的音频格式
output_dir: tmp/
main/xiaozhi-server/core/handle/functionHandler.py
+9 -2
View File
@@ -1,6 +1,12 @@
from config.logger import setup_logging
import json
from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
from plugins_func.register import (
FunctionRegistry,
ActionResponse,
Action,
ToolType,
DeviceTypeRegistry,
)
from plugins_func.functions.hass_init import append_devices_to_prompt
TAG = __name__
@@ -10,6 +16,7 @@ class FunctionHandler:
def __init__(self, conn):
self.conn = conn
self.config = conn.config
self.device_type_registry = DeviceTypeRegistry()
self.function_registry = FunctionRegistry()
self.register_nessary_functions()
self.register_config_functions()
@@ -54,7 +61,7 @@ class FunctionHandler:
self.function_registry.register_function("plugin_loader")
self.function_registry.register_function("get_time")
self.function_registry.register_function("get_lunar")
self.function_registry.register_function("handle_device")
self.function_registry.register_function("handle_speaker_volume_or_screen_brightness")
def register_config_functions(self):
"""注册配置中的函数,可以不同客户端使用不同的配置"""
main/xiaozhi-server/core/handle/helloHandle.py
+2 -2
View File
@@ -40,8 +40,8 @@ async def checkWakeupWords(conn, text):
if not enable_wakeup_words_response_cache:
return False
"""检查是否是唤醒词"""
_, text = remove_punctuation_and_length(text)
if text in conn.config.get("wakeup_words"):
_, filtered_text = remove_punctuation_and_length(text)
if filtered_text in conn.config.get("wakeup_words"):
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
main/xiaozhi-server/core/handle/intentHandler.py
+4 -3
View File
@@ -12,11 +12,12 @@ TAG = __name__
async def handle_user_intent(conn, text):
# 检查是否有明确的退出命令
if await check_direct_exit(conn, text):
filtered_text = remove_punctuation_and_length(text)[1]
if await check_direct_exit(conn, filtered_text):
return True
# 4月4日因流式改造暂时关闭唤醒词加速功能
# # 检查是否是唤醒词
# if await checkWakeupWords(conn, text):
# 检查是否是唤醒词
# if await checkWakeupWords(conn, filtered_text):
# return True
if conn.intent_type == "function_call":
main/xiaozhi-server/core/handle/iotHandle.py
+19 -14
View File
@@ -1,9 +1,8 @@
import json
import asyncio
from config.logger import setup_logging
from plugins_func.register import (
device_type_registry,
register_function,
FunctionItem,
register_device_function,
ActionResponse,
Action,
ToolType,
@@ -177,7 +176,7 @@ class IotDescriptor:
self.methods.append(method)
def register_device_type(descriptor):
def register_device_type(descriptor, device_type_registry):
"""注册设备类型及其功能"""
device_name = descriptor["name"]
type_id = device_type_registry.generate_device_type_id(descriptor)
@@ -213,10 +212,12 @@ def register_device_type(descriptor):
},
}
query_func = create_iot_query_function(device_name, prop_name, prop_info)
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
query_func
decorated_func = register_device_function(
func_name, func_desc, ToolType.IOT_CTL
)(query_func)
functions[func_name] = FunctionItem(
func_name, func_desc, decorated_func, ToolType.IOT_CTL
)
functions[func_name] = decorated_func
# 为每个方法创建控制函数
for method_name, method_info in descriptor["methods"].items():
@@ -267,10 +268,12 @@ def register_device_type(descriptor):
},
}
control_func = create_iot_function(device_name, method_name, method_info)
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
control_func
decorated_func = register_device_function(
func_name, func_desc, ToolType.IOT_CTL
)(control_func)
functions[func_name] = FunctionItem(
func_name, func_desc, decorated_func, ToolType.IOT_CTL
)
functions[func_name] = decorated_func
device_type_registry.register_device_type(type_id, functions)
return type_id
@@ -289,7 +292,6 @@ async def handleIotDescriptors(conn, descriptors):
functions_changed = False
for descriptor in descriptors:
# 如果descriptor没有properties和methods，则直接跳过
if "properties" not in descriptor and "methods" not in descriptor:
continue
@@ -319,13 +321,16 @@ async def handleIotDescriptors(conn, descriptors):
if conn.load_function_plugin:
# 注册或获取设备类型
type_id = register_device_type(descriptor)
device_type_registry = conn.func_handler.device_type_registry
type_id = register_device_type(descriptor, device_type_registry)
device_functions = device_type_registry.get_device_functions(type_id)
# 在连接级注册设备函数
if hasattr(conn, "func_handler"):
for func_name in device_functions:
conn.func_handler.function_registry.register_function(func_name)
for func_name, func_item in device_functions.items():
conn.func_handler.function_registry.register_function(
func_name, func_item
)
conn.logger.bind(tag=TAG).info(
f"注册IOT函数到function handler: {func_name}"
)
main/xiaozhi-server/core/handle/receiveAudioHandle.py
+5 -5
View File
@@ -39,14 +39,14 @@ async def handleAudioMessage(conn, audio):
if len(conn.asr_audio) < 15:
conn.asr_server_receive = True
else:
text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
conn.logger.bind(tag=TAG).info(f"识别文本: {text}")
text_len, _ = remove_punctuation_and_length(text)
raw_text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) # 确保ASR模块返回原始文本
conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
text_len, _ = remove_punctuation_and_length(raw_text)
if text_len > 0:
# 使用自定义模块进行上报
enqueue_asr_report(conn, text, copy.deepcopy(conn.asr_audio))
enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio))
await startToChat(conn, text)
await startToChat(conn, raw_text)
else:
conn.asr_server_receive = True
conn.asr_audio.clear()
main/xiaozhi-server/core/handle/textHandle.py
+6 -6
View File
@@ -45,17 +45,17 @@ async def handleTextMessage(conn, message):
conn.client_have_voice = False
conn.asr_audio.clear()
if "text" in msg_json:
text = msg_json["text"]
_, text = remove_punctuation_and_length(text)
original_text = msg_json["text"] # 保留原始文本
filtered_len, filtered_text = remove_punctuation_and_length(original_text)
# 识别是否是唤醒词
is_wakeup_words = text in conn.config.get("wakeup_words")
is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
# 是否开启唤醒词回复
enable_greeting = conn.config.get("enable_greeting", True)
if is_wakeup_words and not enable_greeting:
# 如果是唤醒词，且关闭了唤醒词回复，就不用回答
await send_stt_message(conn, text)
await send_stt_message(conn, original_text)
await send_tts_message(conn, "stop", None)
elif is_wakeup_words:
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
@@ -63,9 +63,9 @@ async def handleTextMessage(conn, message):
await startToChat(conn, "嘿，你好呀")
else:
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
enqueue_asr_report(conn, text, [])
enqueue_asr_report(conn, original_text, [])
# 否则需要LLM对文字内容进行答复
await startToChat(conn, text)
await startToChat(conn, original_text)
elif msg_json["type"] == "iot":
conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}")
if "descriptors" in msg_json:
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+2
View File
@@ -53,6 +53,8 @@ class IntentProvider(IntentProviderBase):
prompt = (
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
"- 如果用户使用疑问词（如'怎么'、'为什么'、'如何'）询问退出相关的问题（例如'怎么退出了？'），注意这不是让你退出，请返回 {'function_call': {'name': 'continue_chat'}\n"
"- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时，才触发 handle_exit_intent\n\n"
f"{functions_desc}\n"
"处理步骤:\n"
"1. 分析用户输入，确定用户意图\n"
main/xiaozhi-server/core/providers/tts/custom.py
+16 -2
View File
@@ -1,4 +1,5 @@
import os
import json
import uuid
import requests
from pydub import AudioSegment
@@ -16,11 +17,21 @@ class TTSProvider(TTSProviderBase):
def __init__(self, config, delete_audio_file):
super().__init__(config, delete_audio_file)
self.url = config.get("url")
self.method = config.get("method", "GET")
self.headers = config.get("headers", {})
self.params = config.get("params")
self.format = config.get("format", "wav")
self.output_file = config.get("output_dir", "tmp/")
self.params = config.get("params")
if isinstance(self.params, str):
try:
self.params = json.loads(self.params)
except json.JSONDecodeError:
raise ValueError("Custom TTS配置参数出错,无法将字符串解析为对象")
elif not isinstance(self.params, dict):
raise TypeError("Custom TTS配置参数出错, 请参考配置说明")
def generate_filename(self):
return os.path.join(
self.output_file,
@@ -35,7 +46,10 @@ class TTSProvider(TTSProviderBase):
v = v.replace("{prompt_text}", text)
request_params[k] = v
resp = requests.get(self.url, params=request_params, headers=self.headers)
if self.method.upper() == "POST":
resp = requests.post(self.url, json=request_params, headers=self.headers)
else:
resp = requests.get(self.url, params=request_params, headers=self.headers)
if resp.status_code == 200:
with open(tmp_file, "wb") as file:
file.write(resp.content)
main/xiaozhi-server/plugins_func/functions/handle_device.py → main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py
+18 -9
View File
@@ -54,20 +54,29 @@ def _handle_device_action(conn, func, success_message, error_message, *args, **k
handle_device_function_desc = {
"type": "function",
"function": {
"name": "handle_device",
"name": "handle_speaker_volume_or_screen_brightness",
"description": (
"用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。"
"比如用户说现在亮度多少，参数为：device_type:Screen,action:get。"
"比如用户说设置音量为50，参数为：device_type:Speaker,action:set,value:50。"
"比如用户说亮度太高了，参数为：device_type:Screen,action:lower。"
"比如用户说调大音量，参数为：device_type:Speaker,action:raise。"
"用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。\n"
"**严格限制**：仅当用户明确操作 **Speaker（音量）或Screen（亮度）** 时才能调用此函数！\n"
"对于其他设备（如AC、Battery、Switch等），请不要调用此函数，而是继续正常的对话。\n\n"
"示例：\n"
"- 用户说『现在亮度多少』 → 调用函数：device_type: Screen, action: get\n"
"- 用户说『设置音量为50』 → 调用函数：device_type: Speaker, action: set, value: 50\n"
"- 用户说『亮度太高了』 → 调用函数：device_type: Screen, action: lower\n"
"- 用户说『调大音量』 → 调用函数：device_type: Speaker, action: raise\n\n"
"**拒绝调用示例**（应继续对话而非调用本函数）：\n"
"- 用户说『空调调低一度』 → 不调用（设备类型为AC）\n"
"- 用户说『开关灯』 → 不调用（设备类型为Switch）\n"
"- 用户说『电量多少』 → 不调用（设备类型为Battery）\n"
),
"parameters": {
"type": "object",
"properties": {
"device_type": {
"type": "string",
"description": "设备类型，可选值：Speaker(音量),Screen(亮度)"
"description": "设备类型，**严格限定为Speaker（音量）或Screen（亮度）**，其他设备类型禁止调用此函数",
"enum": ["Speaker", "Screen"]
},
"action": {
"type": "string",
@@ -84,8 +93,8 @@ handle_device_function_desc = {
}
@register_function('handle_device', handle_device_function_desc, ToolType.IOT_CTL)
def handle_device(conn, device_type: str, action: str, value: int = None):
@register_function('handle_speaker_volume_or_screen_brightness', handle_device_function_desc, ToolType.IOT_CTL)
def handle_speaker_volume_or_screen_brightness(conn, device_type: str, action: str, value: int = None):
if device_type == "Speaker":
method_name, property_name, device_name = "SetVolume", "volume", "音量"
elif device_type == "Screen":
main/xiaozhi-server/plugins_func/register.py
+18 -3
View File
@@ -77,7 +77,6 @@ class DeviceTypeRegistry:
# 初始化函数注册字典
all_function_registry = {}
device_type_registry = DeviceTypeRegistry()
def register_function(name, desc, type=None):
@@ -91,13 +90,29 @@ def register_function(name, desc, type=None):
return decorator
def register_device_function(name, desc, type=None):
"""注册设备级别的函数到函数注册字典的装饰器"""
def decorator(func):
logger.bind(tag=TAG).debug(f"设备函数 '{name}' 已加载")
return func
return decorator
class FunctionRegistry:
def __init__(self):
self.function_registry = {}
self.logger = setup_logging()
def register_function(self, name):
# 查找all_function_registry中是否有对应的函数
def register_function(self, name, func_item=None):
# 如果提供了func_item，直接注册
if func_item:
self.function_registry[name] = func_item
self.logger.bind(tag=TAG).debug(f"函数 '{name}' 直接注册成功")
return func_item
# 否则从all_function_registry中查找
func = all_function_registry.get(name)
if not func:
self.logger.bind(tag=TAG).error(f"函数 '{name}' 未找到")