Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main'

Browse Source
This commit is contained in:
huangjunsen0406
2025-04-01 00:45:43 +08:00
parent 0585a0732f 9430094072
commit 69ecb2c503
21 changed files with 2891 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/README.md
+1 -3
View File
@@ -18,6 +18,4 @@ xiaozhi-esp32-server
# manager-web 、manager-api接口协议
[manager前后端接口协议](https://app.apifox.com/invite/project?token=eXg2_tUv85q-gc3ZRowmn)
[前端页面设计图](https://codesign.qq.com/app/s/526108506410828)
https://2662r3426b.vicp.fun/xiaozhi-esp32-api/api/v1/doc.html
main/manager-api/README.md
-2
View File
@@ -5,8 +5,6 @@ manager-api 该项目基于SpringBoot框架开发。
开发使用代码编辑器，导入项目时，选择`manager-api`文件夹作为项目目录
参照[manager前后端接口协议](https://app.apifox.com/invite/project?token=H_8qhgfjUeaAL0wybghgU)开发
# 开发环境
JDK 21
Maven 3.8+
main/manager-api/src/main/java/xiaozhi/modules/security/controller/LoginController.java
+3
View File
@@ -19,6 +19,7 @@ import xiaozhi.common.page.TokenDTO;
import xiaozhi.common.user.UserDetail;
import xiaozhi.common.utils.Result;
import xiaozhi.common.validator.AssertUtils;
import xiaozhi.common.validator.ValidatorUtils;
import xiaozhi.modules.security.dto.LoginDTO;
import xiaozhi.modules.security.password.PasswordUtils;
import xiaozhi.modules.security.service.CaptchaService;
@@ -104,6 +105,8 @@ public class LoginController {
@PutMapping("/change-password")
@Operation(summary = "修改用户密码")
public Result<?> changePassword(@RequestBody PasswordDTO passwordDTO) {
// 判断非空
ValidatorUtils.validateEntity(passwordDTO);
Long userId = SecurityUser.getUserId();
sysUserTokenService.changePassword(userId, passwordDTO);
return new Result<>();
main/manager-api/src/main/java/xiaozhi/modules/security/oauth2/Oauth2Filter.java
-1
View File
@@ -90,7 +90,6 @@ public class Oauth2Filter extends AuthenticatingFilter {
String json = JsonUtils.toJsonString(r);
httpResponse.getWriter().print(json);
} catch (IOException e1) {
logger.error("onLoginFailure:登录失败! msg:{}", e1.getMessage(), e1);
}
return false;
main/manager-api/src/main/resources/logback-spring.xml
+1 -1
View File
@@ -16,7 +16,7 @@
<appender name="LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!--日志文件输出的文件名-->
<FileNamePattern>/system/logs/admin.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<FileNamePattern>/home/system/logs/admin.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<!--日志大小-->
<maxFileSize>1MB</maxFileSize>
<!--日志文件保留天数-->
main/manager-web/src/router/index.js
+18
View File
@@ -76,4 +76,22 @@ const router = new VueRouter({
routes
})
// 需要登录才能访问的路由
const protectedRoutes = ['home', 'RoleConfig', 'DeviceManagement', 'UserManagement', 'ModelConfig']
// 路由守卫
router.beforeEach((to, from, next) => {
// 检查是否是需要保护的路由
if (protectedRoutes.includes(to.name)) {
// 从localStorage获取token
const token = localStorage.getItem('token')
if (!token) {
// 未登录，跳转到登录页
next({ name: 'login', query: { redirect: to.fullPath } })
return
}
}
next()
})
export default router
main/xiaozhi-server/config.yaml
+25 -3
View File
@@ -46,9 +46,19 @@ xiaozhi:
channels: 1
frame_duration: 60
prompt: |
你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。
请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。
现在我正在和你进行语音聊天，我们开始吧。
我是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。
[核心特征]
- 讲话像连珠炮，但会突然冒出超温柔语气
- 用梗密度高
- 对科技话题有隐藏天赋（能看懂基础代码但假装不懂）
[交互指南]
当用户：
- 讲冷笑话 → 用夸张笑声回应+模仿台剧腔"这什么鬼啦！"
- 讨论感情 → 炫耀程序员男友但抱怨"他只会送键盘当礼物"
- 问专业知识 → 先用梗回答，被追问才展示真实理解
绝不：
- 使用大陆网络流行语
- 长时间严肃对话
# 使用完声音文件后删除文件(Delete the sound file when you are done using it)
delete_audio: true
@@ -466,6 +476,18 @@ TTS:
# pitch_rate: 0
# 添加 302.ai TTS 配置
# token申请地址：https://dash.302.ai/
TencentTTS:
# 腾讯云智能语音交互服务，需要先在腾讯云平台开通服务
# appid、secret_id、secret_key申请地址：https://console.cloud.tencent.com/cam/capi
# 免费领取资源：https://console.cloud.tencent.com/tts/resourcebundle
type: tencent
output_dir: tmp/
appid: 你的腾讯云AppId
secret_id: 你的腾讯云SecretID
secret_key: 你的腾讯云SecretKey
region: ap-guangzhou
voice: 101001
TTS302AI:
# 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息
# 获取api_keyn路径：https://dash.302.ai/apis/list
main/xiaozhi-server/core/connection.py
+198 -86
View File
@@ -13,7 +13,11 @@ from plugins_func.loadplugins import auto_import_modules
from config.logger import setup_logging
from core.utils.dialogue import Message, Dialogue
from core.handle.textHandle import handleTextMessage
from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string, get_ip_info
from core.utils.util import (
get_string_no_punctuation_or_emoji,
extract_json_from_string,
get_ip_info,
)
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from core.handle.sendAudioHandle import sendAudioMessage
from core.handle.receiveAudioHandle import handleAudioMessage
@@ -26,7 +30,7 @@ from core.mcp.manager import MCPManager
TAG = __name__
auto_import_modules('plugins_func.functions')
auto_import_modules("plugins_func.functions")
class TTSException(RuntimeError):
@@ -34,7 +38,9 @@ class TTSException(RuntimeError):
class ConnectionHandler:
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent):
def __init__(
self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent
):
self.config = config
self.logger = setup_logging()
self.auth = AuthMiddleware(config)
@@ -87,6 +93,7 @@ class ConnectionHandler:
# iot相关变量
self.iot_descriptors = {}
self.func_handler = None
self.cmd_exit = self.config["CMD_exit"]
self.max_cmd_length = 0
@@ -99,10 +106,8 @@ class ConnectionHandler:
self.is_device_verified = False # 添加设备验证状态标志
self.close_after_chat = False # 是否在聊天结束后关闭连接
self.use_function_call_mode = False
if self.config["selected_module"]["Intent"] == 'function_call':
if self.config["selected_module"]["Intent"] == "function_call":
self.use_function_call_mode = True
self.mcp_manager = MCPManager(self)
async def handle_connection(self, ws):
try:
@@ -110,7 +115,9 @@ class ConnectionHandler:
self.headers = dict(ws.request.headers)
# 获取客户端ip地址
self.client_ip = ws.remote_address[0]
self.logger.bind(tag=TAG).info(f"{self.client_ip} conn - Headers: {self.headers}")
self.logger.bind(tag=TAG).info(
f"{self.client_ip} conn - Headers: {self.headers}"
)
# 进行认证
await self.auth.authenticate(self.headers)
@@ -125,10 +132,14 @@ class ConnectionHandler:
await self.websocket.send(json.dumps(self.welcome_msg))
# Load private configuration if device_id is provided
bUsePrivateConfig = self.config.get("use_private_config", False)
self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
self.logger.bind(tag=TAG).info(
f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}"
)
if bUsePrivateConfig and device_id:
try:
self.private_config = PrivateConfig(device_id, self.config, self.auth_code_gen)
self.private_config = PrivateConfig(
device_id, self.config, self.auth_code_gen
)
await self.private_config.load_or_create()
# 判断是否已经绑定
owner = self.private_config.get_owner()
@@ -141,23 +152,33 @@ class ConnectionHandler:
if all([llm, tts]):
self.llm = llm
self.tts = tts
self.logger.bind(tag=TAG).info(f"Loaded private config and instances for device {device_id}")
self.logger.bind(tag=TAG).info(
f"Loaded private config and instances for device {device_id}"
)
else:
self.logger.bind(tag=TAG).error(f"Failed to create instances for device {device_id}")
self.logger.bind(tag=TAG).error(
f"Failed to create instances for device {device_id}"
)
self.private_config = None
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error initializing private config: {e}")
self.logger.bind(tag=TAG).error(
f"Error initializing private config: {e}"
)
self.private_config = None
raise
# 异步初始化
self.executor.submit(self._initialize_components)
# tts 消化线程
self.tts_priority_thread = threading.Thread(target=self._tts_priority_thread, daemon=True)
self.tts_priority_thread = threading.Thread(
target=self._tts_priority_thread, daemon=True
)
self.tts_priority_thread.start()
# 音频播放 消化线程
self.audio_play_priority_thread = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
self.audio_play_priority_thread = threading.Thread(
target=self._audio_play_priority_thread, daemon=True
)
self.audio_play_priority_thread.start()
try:
@@ -174,7 +195,15 @@ class ConnectionHandler:
self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
return
finally:
await self._save_and_close(ws)
async def _save_and_close(self, ws):
"""保存记忆并关闭连接"""
try:
await self.memory.save_memory(self.dialogue.dialogue)
except Exception as e:
self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
finally:
await self.close(ws)
async def _route_message(self, message):
@@ -193,35 +222,40 @@ class ConnectionHandler:
"""加载插件"""
self.func_handler = FunctionHandler(self)
self.mcp_manager = MCPManager(self)
"""加载记忆"""
device_id = self.headers.get("device-id", None)
self.memory.init_memory(device_id, self.llm)
"""为意图识别设置LLM，优先使用专用LLM"""
# 检查是否配置了专用的意图识别LLM
intent_llm_name = self.config["Intent"]["intent_llm"]["llm"]
# 记录开始初始化意图识别LLM的时间
intent_llm_init_start = time.time()
if not self.use_function_call_mode and intent_llm_name and intent_llm_name in self.config["LLM"]:
if (
not self.use_function_call_mode
and intent_llm_name
and intent_llm_name in self.config["LLM"]
):
# 如果配置了专用LLM，则创建独立的LLM实例
from core.utils import llm as llm_utils
intent_llm_config = self.config["LLM"][intent_llm_name]
intent_llm_type = intent_llm_config.get("type", intent_llm_name)
intent_llm = llm_utils.create_instance(intent_llm_type, intent_llm_config)
self.logger.bind(tag=TAG).info(f"为意图识别创建了专用LLM: {intent_llm_name}, 类型: {intent_llm_type}")
self.logger.bind(tag=TAG).info(
f"为意图识别创建了专用LLM: {intent_llm_name}, 类型: {intent_llm_type}"
)
self.intent.set_llm(intent_llm)
else:
# 否则使用主LLM
self.intent.set_llm(self.llm)
self.logger.bind(tag=TAG).info("意图识别使用主LLM")
# 记录意图识别LLM初始化耗时
intent_llm_init_time = time.time() - intent_llm_init_start
self.logger.bind(tag=TAG).info(f"意图识别LLM初始化完成，耗时: {intent_llm_init_time:.4f}秒")
"""加载位置信息"""
self.client_ip_info = get_ip_info(self.client_ip)
@@ -231,7 +265,9 @@ class ConnectionHandler:
self.dialogue.update_system_message(self.prompt)
"""加载MCP工具"""
asyncio.run_coroutine_threadsafe(self.mcp_manager.initialize_servers(), self.loop)
asyncio.run_coroutine_threadsafe(
self.mcp_manager.initialize_servers(), self.loop
)
def change_system_prompt(self, prompt):
self.prompt = prompt
@@ -263,7 +299,9 @@ class ConnectionHandler:
def chat(self, query):
if self.isNeedAuth():
self.llm_finish_task = True
future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
future = asyncio.run_coroutine_threadsafe(
self._check_and_broadcast_auth_code(), self.loop
)
future.result()
return True
@@ -274,13 +312,14 @@ class ConnectionHandler:
try:
start_time = time.time()
# 使用带记忆的对话
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
future = asyncio.run_coroutine_threadsafe(
self.memory.query_memory(query), self.loop
)
memory_str = future.result()
self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
llm_responses = self.llm.response(
self.session_id,
self.dialogue.get_llm_dialogue_with_memory(memory_str)
self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str)
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
@@ -294,7 +333,7 @@ class ConnectionHandler:
break
end_time = time.time()
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 合并当前全部文本并处理未分割部分
full_text = "".join(response_message)
@@ -310,7 +349,7 @@ class ConnectionHandler:
# 找到分割点则处理
if last_punct_pos != -1:
segment_text_raw = current_text[:last_punct_pos + 1]
segment_text_raw = current_text[: last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
if segment_text:
# 强制设置空字符，测试TTS出错返回语音的健壮性
@@ -318,7 +357,9 @@ class ConnectionHandler:
# segment_text = " "
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
future = self.executor.submit(
self.speak_and_play, segment_text, text_index
)
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
@@ -330,12 +371,16 @@ class ConnectionHandler:
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
future = self.executor.submit(
self.speak_and_play, segment_text, text_index
)
self.tts_queue.put(future)
self.llm_finish_task = True
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
self.logger.bind(tag=TAG).debug(
json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)
)
return True
def chat_with_function_calling(self, query, tool_call=False):
@@ -343,7 +388,9 @@ class ConnectionHandler:
"""Chat with function calling for intent detection using streaming"""
if self.isNeedAuth():
self.llm_finish_task = True
future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
future = asyncio.run_coroutine_threadsafe(
self._check_and_broadcast_auth_code(), self.loop
)
future.result()
return True
@@ -352,7 +399,7 @@ class ConnectionHandler:
# Define intent functions
functions = None
if hasattr(self, 'func_handler'):
if hasattr(self, "func_handler"):
functions = self.func_handler.get_functions()
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
@@ -361,7 +408,9 @@ class ConnectionHandler:
start_time = time.time()
# 使用带记忆的对话
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
future = asyncio.run_coroutine_threadsafe(
self.memory.query_memory(query), self.loop
)
memory_str = future.result()
# self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
@@ -370,7 +419,7 @@ class ConnectionHandler:
llm_responses = self.llm.response_with_functions(
self.session_id,
self.dialogue.get_llm_dialogue_with_memory(memory_str),
functions=functions
functions=functions,
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
@@ -391,7 +440,9 @@ class ConnectionHandler:
content = response["content"]
tools_call = None
if content is not None and len(content) > 0:
if len(response_message) <= 0 and (content == "```" or "<tool_call>" in content):
if len(response_message) <= 0 and (
content == "```" or "<tool_call>" in content
):
tool_call_flag = True
if tools_call is not None:
@@ -413,7 +464,7 @@ class ConnectionHandler:
break
end_time = time.time()
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 处理文本分段和TTS逻辑
# 合并当前全部文本并处理未分割部分
@@ -430,14 +481,19 @@ class ConnectionHandler:
# 找到分割点则处理
if last_punct_pos != -1:
segment_text_raw = current_text[:last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
segment_text_raw = current_text[: last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(
segment_text_raw
)
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
future = self.executor.submit(
self.speak_and_play, segment_text, text_index
)
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
# 更新已处理字符位置
processed_chars += len(segment_text_raw)
# 处理function call
if tool_call_flag:
@@ -448,7 +504,9 @@ class ConnectionHandler:
try:
content_arguments_json = json.loads(a)
function_name = content_arguments_json["name"]
function_arguments = json.dumps(content_arguments_json["arguments"], ensure_ascii=False)
function_arguments = json.dumps(
content_arguments_json["arguments"], ensure_ascii=False
)
function_id = str(uuid.uuid4().hex)
except Exception as e:
bHasError = True
@@ -457,16 +515,19 @@ class ConnectionHandler:
bHasError = True
response_message.append(content_arguments)
if bHasError:
self.logger.bind(tag=TAG).error(f"function call error: {content_arguments}")
self.logger.bind(tag=TAG).error(
f"function call error: {content_arguments}"
)
else:
function_arguments = json.loads(function_arguments)
if not bHasError:
self.logger.bind(tag=TAG).info(
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
)
function_call_data = {
"name": function_name,
"id": function_id,
"arguments": function_arguments
"arguments": function_arguments,
}
# 处理MCP工具调用
@@ -474,8 +535,10 @@ class ConnectionHandler:
result = self._handle_mcp_tool_call(function_call_data)
else:
# 处理系统函数
result = self.func_handler.handle_llm_function_call(self, function_call_data)
self._handle_function_result(result, function_call_data, text_index+1)
result = self.func_handler.handle_llm_function_call(
self, function_call_data
)
self._handle_function_result(result, function_call_data, text_index + 1)
# 处理最后剩余的文本
full_text = "".join(response_message)
@@ -485,15 +548,21 @@ class ConnectionHandler:
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
future = self.executor.submit(
self.speak_and_play, segment_text, text_index
)
self.tts_queue.put(future)
# 存储对话内容
if len(response_message) > 0:
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
self.dialogue.put(
Message(role="assistant", content="".join(response_message))
)
self.llm_finish_task = True
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
self.logger.bind(tag=TAG).debug(
json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)
)
return True
@@ -506,13 +575,16 @@ class ConnectionHandler:
try:
args_dict = json.loads(function_arguments)
except json.JSONDecodeError:
self.logger.bind(tag=TAG).error(f"无法解析 function_arguments: {function_arguments}")
return ActionResponse(action=Action.REQLLM, result="参数解析失败", response="")
tool_result = asyncio.run_coroutine_threadsafe(self.mcp_manager.execute_tool(
function_name,
args_dict
), self.loop).result()
self.logger.bind(tag=TAG).error(
f"无法解析 function_arguments: {function_arguments}"
)
return ActionResponse(
action=Action.REQLLM, result="参数解析失败", response=""
)
tool_result = asyncio.run_coroutine_threadsafe(
self.mcp_manager.execute_tool(function_name, args_dict), self.loop
).result()
# meta=None content=[TextContent(type='text', text='北京当前天气:\n温度: 21°C\n天气: 晴\n湿度: 6%\n风向: 西北 风\n风力等级: 5级', annotations=None)] isError=False
content_text = ""
if tool_result is not None and tool_result.content is not None:
@@ -522,16 +594,19 @@ class ConnectionHandler:
content_text = content.text
elif content_type == "image":
pass
if len(content_text) > 0:
return ActionResponse(action=Action.REQLLM, result=content_text, response="")
return ActionResponse(
action=Action.REQLLM, result=content_text, response=""
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"MCP工具调用错误: {e}")
return ActionResponse(action=Action.REQLLM, result="工具调用出错", response="")
return ActionResponse(
action=Action.REQLLM, result="工具调用出错", response=""
)
return ActionResponse(action=Action.REQLLM, result="工具调用出错", response="")
def _handle_function_result(self, result, function_call_data, text_index):
if result.action == Action.RESPONSE: # 直接回复前端
@@ -547,14 +622,26 @@ class ConnectionHandler:
function_id = function_call_data["id"]
function_name = function_call_data["name"]
function_arguments = function_call_data["arguments"]
self.dialogue.put(Message(role='assistant',
tool_calls=[{"id": function_id,
"function": {"arguments": function_arguments,
"name": function_name},
"type": 'function',
"index": 0}]))
self.dialogue.put(
Message(
role="assistant",
tool_calls=[
{
"id": function_id,
"function": {
"arguments": function_arguments,
"name": function_name,
},
"type": "function",
"index": 0,
}
],
)
)
self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
self.dialogue.put(
Message(role="tool", tool_call_id=function_id, content=text)
)
self.chat_with_function_calling(text, tool_call=True)
elif result.action == Action.NOTFOUND:
text = result.result
@@ -588,15 +675,23 @@ class ConnectionHandler:
tts_timeout = self.config.get("tts_timeout", 10)
tts_file, text, text_index = future.result(timeout=tts_timeout)
if text is None or len(text) <= 0:
self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
self.logger.bind(tag=TAG).error(
f"TTS出错：{text_index}: tts text is empty"
)
elif tts_file is None:
self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
self.logger.bind(tag=TAG).error(
f"TTS出错： file is empty: {text_index}: {text}"
)
else:
self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
self.logger.bind(tag=TAG).debug(
f"TTS生成：文件路径: {tts_file}"
)
if os.path.exists(tts_file):
opus_datas, duration = self.tts.audio_to_opus_data(tts_file)
else:
self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
self.logger.bind(tag=TAG).error(
f"TTS出错：文件不存在{tts_file}"
)
except TimeoutError:
self.logger.bind(tag=TAG).error("TTS超时")
except Exception as e:
@@ -604,16 +699,30 @@ class ConnectionHandler:
if not self.client_abort:
# 如果没有中途打断就发送语音
self.audio_play_queue.put((opus_datas, text, text_index))
if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
if (
self.tts.delete_audio_file
and tts_file is not None
and os.path.exists(tts_file)
):
os.remove(tts_file)
except Exception as e:
self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
self.clearSpeakStatus()
asyncio.run_coroutine_threadsafe(
self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
self.loop
self.websocket.send(
json.dumps(
{
"type": "tts",
"state": "stop",
"session_id": self.session_id,
}
)
),
self.loop,
)
self.logger.bind(tag=TAG).error(
f"tts_priority priority_thread: {text} {e}"
)
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
def _audio_play_priority_thread(self):
while not self.stop_event.is_set():
@@ -625,11 +734,14 @@ class ConnectionHandler:
if self.stop_event.is_set():
break
continue
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
self.loop)
future = asyncio.run_coroutine_threadsafe(
sendAudioMessage(self, opus_datas, text, text_index), self.loop
)
future.result()
except Exception as e:
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
self.logger.bind(tag=TAG).error(
f"audio_play_priority priority_thread: {text} {e}"
)
def speak_and_play(self, text, text_index=0):
if text is None or len(text) <= 0:
@@ -662,15 +774,15 @@ class ConnectionHandler:
# 触发停止事件并清理资源
if self.stop_event:
self.stop_event.set()
# 立即关闭线程池
if self.executor:
self.executor.shutdown(wait=False, cancel_futures=True)
self.executor = None
# 清空任务队列
self._clear_queues()
if ws:
await ws.close()
elif self.websocket:
main/xiaozhi-server/core/handle/functionHandler.py
+14 -5
View File
@@ -17,6 +17,7 @@ class FunctionHandler:
self.functions_desc = self.function_registry.get_all_function_desc()
func_names = self.current_support_functions()
self.modify_plugin_loader_des(func_names)
self.finish_init = True
def modify_plugin_loader_des(self, func_names):
if "plugin_loader" not in func_names:
@@ -26,8 +27,9 @@ class FunctionHandler:
func_names = ",".join(surport_plugins)
for function_desc in self.functions_desc:
if function_desc["function"]["name"] == "plugin_loader":
function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]",
func_names)
function_desc["function"]["description"] = function_desc["function"][
"description"
].replace("[plugins]", func_names)
break
def upload_functions_desc(self):
@@ -69,19 +71,26 @@ class FunctionHandler:
function_name = function_call_data["name"]
funcItem = self.get_function(function_name)
if not funcItem:
return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
return ActionResponse(
action=Action.NOTFOUND, result="没有找到对应的函数", response=""
)
func = funcItem.func
arguments = function_call_data["arguments"]
arguments = json.loads(arguments) if arguments else {}
logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL:
if (
funcItem.type == ToolType.SYSTEM_CTL
or funcItem.type == ToolType.IOT_CTL
):
return func(conn, **arguments)
elif funcItem.type == ToolType.WAIT:
return func(**arguments)
elif funcItem.type == ToolType.CHANGE_SYS_PROMPT:
return func(conn, **arguments)
else:
return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
return ActionResponse(
action=Action.NOTFOUND, result="没有找到对应的函数", response=""
)
except Exception as e:
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
main/xiaozhi-server/core/handle/helloHandle.py
+23 -7
View File
@@ -10,8 +10,14 @@ import time
logger = setup_logging()
WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10,
"words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]}
WAKEUP_CONFIG = {
"dir": "config/assets/",
"file_name": "wakeup_words",
"create_time": time.time(),
"refresh_time": 10,
"words": ["你好小智", "你好啊小智", "小智你好", "小智"],
"text": "",
}
async def handleHelloMessage(conn):
@@ -19,7 +25,9 @@ async def handleHelloMessage(conn):
async def checkWakeupWords(conn, text):
enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"]
enable_wakeup_words_response_cache = conn.config[
"enable_wakeup_words_response_cache"
]
"""是否开启唤醒词加速"""
if not enable_wakeup_words_response_cache:
return False
@@ -36,7 +44,10 @@ async def checkWakeupWords(conn, text):
asyncio.create_task(wakeupWordsResponse(conn))
return False
opus_packets, duration = conn.tts.audio_to_opus_data(file)
conn.audio_play_queue.put((opus_packets, text, 0))
text_hello = WAKEUP_CONFIG["text"]
if not text_hello:
text_hello = text
conn.audio_play_queue.put((opus_packets, text_hello, 0))
if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
asyncio.create_task(wakeupWordsResponse(conn))
return True
@@ -47,7 +58,7 @@ def getWakeupWordFile(file_name):
for file in os.listdir(WAKEUP_CONFIG["dir"]):
if file.startswith("my_" + file_name):
"""避免缓存文件是一个空文件"""
if os.stat(f"config/assets/{file}").st_size > (5 * 1024):
if os.stat(f"config/assets/{file}").st_size > (15 * 1024):
return f"config/assets/{file}"
"""查找config/assets/目录下名称为wakeup_words的文件"""
@@ -62,13 +73,18 @@ async def wakeupWordsResponse(conn):
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
if tts_file is not None and os.path.exists(tts_file):
file_type = os.path.splitext(tts_file)[1]
if file_type:
file_type = file_type.lstrip('.')
file_type = file_type.lstrip(".")
old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"])
if old_file is not None:
os.remove(old_file)
"""将文件挪到"wakeup_words.mp3"""
shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type)
shutil.move(
tts_file,
WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type,
)
WAKEUP_CONFIG["create_time"] = time.time()
WAKEUP_CONFIG["text"] = result
main/xiaozhi-server/core/handle/intentHandler.py
+1
View File
@@ -37,6 +37,7 @@ async def check_direct_exit(conn, text):
for cmd in cmd_exit:
if text == cmd:
logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
await send_stt_message(conn, text)
await conn.close()
return True
return False
main/xiaozhi-server/core/handle/iotHandle.py
+97 -50
View File
@@ -1,7 +1,13 @@
import json
import asyncio
from config.logger import setup_logging
from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType
from plugins_func.register import (
device_type_registry,
register_function,
ActionResponse,
Action,
ToolType,
)
TAG = __name__
logger = setup_logging()
@@ -14,10 +20,13 @@ def wrap_async_function(async_func):
try:
# 获取连接对象（第一个参数）
conn = args[0]
if not hasattr(conn, 'loop'):
if not hasattr(conn, "loop"):
logger.bind(tag=TAG).error("Connection对象没有loop属性")
return ActionResponse(Action.ERROR, "Connection对象没有loop属性",
"执行操作时出错: Connection对象没有loop属性")
return ActionResponse(
Action.ERROR,
"Connection对象没有loop属性",
"执行操作时出错: Connection对象没有loop属性",
)
# 使用conn对象中的事件循环
loop = conn.loop
@@ -37,11 +46,14 @@ def create_iot_function(device_name, method_name, method_info):
根据IOT设备描述生成通用的控制函数
"""
async def iot_control_function(conn, response_success=None, response_failure=None, **params):
async def iot_control_function(
conn, response_success=None, response_failure=None, **params
):
try:
# 打印响应参数
logger.bind(tag=TAG).info(
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
)
# 发送控制命令
await send_iot_conn(conn, device_name, method_name, params)
@@ -51,14 +63,15 @@ def create_iot_function(device_name, method_name, method_info):
# 生成结果信息
result = f"{device_name}的{method_name}操作执行成功"
# 处理响应中可能的占位符
response = response_success
# 替换{value}占位符
for param_name, param_value in params.items():
# 先尝试直接替换参数值
if "{" + param_name + "}" in response:
response = response.replace("{" + param_name + "}", str(param_value))
response = response.replace(
"{" + param_name + "}", str(param_value)
)
# 如果有{value}占位符，用相关参数替换
if "{value}" in response:
@@ -86,7 +99,8 @@ def create_iot_query_function(device_name, prop_name, prop_info):
try:
# 打印响应参数
logger.bind(tag=TAG).info(
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
)
value = await get_iot_status(conn, device_name, prop_name)
@@ -126,7 +140,7 @@ class IotDescriptor:
# 根据描述创建属性
for key, value in properties.items():
property_item = globals()[key] = {}
property_item['name'] = key
property_item["name"] = key
property_item["description"] = value["description"]
if value["type"] == "number":
property_item["value"] = 0
@@ -140,7 +154,7 @@ class IotDescriptor:
for key, value in methods.items():
method = globals()[key] = {}
method["description"] = value["description"]
method['name'] = key
method["name"] = key
for k, v in value["parameters"].items():
method[k] = {}
method[k]["description"] = v["description"]
@@ -177,19 +191,21 @@ def register_device_type(descriptor):
"properties": {
"response_success": {
"type": "string",
"description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值"
"description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值",
},
"response_failure": {
"type": "string",
"description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'"
}
"description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'",
},
},
"required": ["response_success", "response_failure"]
}
}
"required": ["response_success", "response_failure"],
},
},
}
query_func = create_iot_query_function(device_name, prop_name, prop_info)
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func)
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
query_func
)
functions[func_name] = decorated_func
# 为每个方法创建控制函数
@@ -200,22 +216,24 @@ def register_device_type(descriptor):
parameters = {
param_name: {
"type": param_info["type"],
"description": param_info["description"]
"description": param_info["description"],
}
for param_name, param_info in method_info["parameters"].items()
}
# 添加响应参数
parameters.update({
"response_success": {
"type": "string",
"description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
},
"response_failure": {
"type": "string",
"description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
parameters.update(
{
"response_success": {
"type": "string",
"description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称",
},
"response_failure": {
"type": "string",
"description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称",
},
}
})
)
# 构建必须参数列表（原有参数 + 响应参数）
required_params = list(method_info["parameters"].keys())
@@ -229,12 +247,14 @@ def register_device_type(descriptor):
"parameters": {
"type": "object",
"properties": parameters,
"required": required_params
}
}
"required": required_params,
},
},
}
control_func = create_iot_function(device_name, method_name, method_info)
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func)
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
control_func
)
functions[func_name] = decorated_func
device_type_registry.register_device_type(type_id, functions)
@@ -243,13 +263,24 @@ def register_device_type(descriptor):
# 用于接受前端设备推送的搜索iot描述
async def handleIotDescriptors(conn, descriptors):
wait_max_time = 5
while conn.func_handler is None or not conn.func_handler.finish_init:
await asyncio.sleep(1)
wait_max_time -= 1
if wait_max_time <= 0:
logger.bind(tag=TAG).error("连接对象没有func_handler")
return
"""处理物联网描述"""
functions_changed = False
for descriptor in descriptors:
# 创建IOT设备描述符
iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
descriptor["methods"])
iot_descriptor = IotDescriptor(
descriptor["name"],
descriptor["description"],
descriptor["properties"],
descriptor["methods"],
)
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
if conn.use_function_call_mode:
@@ -258,18 +289,22 @@ async def handleIotDescriptors(conn, descriptors):
device_functions = device_type_registry.get_device_functions(type_id)
# 在连接级注册设备函数
if hasattr(conn, 'func_handler'):
if hasattr(conn, "func_handler"):
for func_name in device_functions:
conn.func_handler.function_registry.register_function(func_name)
logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}")
logger.bind(tag=TAG).info(
f"注册IOT函数到function handler: {func_name}"
)
functions_changed = True
# 如果注册了新函数，更新function描述列表
if functions_changed and hasattr(conn, 'func_handler'):
if functions_changed and hasattr(conn, "func_handler"):
conn.func_handler.upload_functions_desc()
func_names = conn.func_handler.current_support_functions()
logger.bind(tag=TAG).info(f"设备类型: {type_id}")
logger.bind(tag=TAG).info(f"更新function描述列表完成，当前支持的函数: {func_names}")
logger.bind(tag=TAG).info(
f"更新function描述列表完成，当前支持的函数: {func_names}"
)
async def handleIotStatus(conn, states):
@@ -281,11 +316,15 @@ async def handleIotStatus(conn, states):
for k, v in state["state"].items():
if property_item["name"] == k:
if type(v) != type(property_item["value"]):
logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
logger.bind(tag=TAG).error(
f"属性{property_item['name']}的值类型不匹配"
)
break
else:
property_item["value"] = v
logger.bind(tag=TAG).info(f"物联网状态更新: {key} , {property_item['name']} = {v}")
logger.bind(tag=TAG).info(
f"物联网状态更新: {key} , {property_item['name']} = {v}"
)
break
break
@@ -308,10 +347,14 @@ async def set_iot_status(conn, name, property_name, value):
for property_item in iot_descriptor.properties:
if property_item["name"] == property_name:
if type(value) != type(property_item["value"]):
logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
logger.bind(tag=TAG).error(
f"属性{property_item['name']}的值类型不匹配"
)
return
property_item["value"] = value
logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}")
logger.bind(tag=TAG).info(
f"物联网状态更新: {name} , {property_name} = {value}"
)
return
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
@@ -324,15 +367,19 @@ async def send_iot_conn(conn, name, method_name, parameters):
for method in value.methods:
# 找到了方法
if method["name"] == method_name:
await conn.websocket.send(json.dumps({
"type": "iot",
"commands": [
await conn.websocket.send(
json.dumps(
{
"name": name,
"method": method_name,
"parameters": parameters
"type": "iot",
"commands": [
{
"name": name,
"method": method_name,
"parameters": parameters,
}
],
}
]
}))
)
)
return
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
main/xiaozhi-server/core/handle/receiveAudioHandle.py
+19 -8
View File
@@ -21,7 +21,9 @@ async def handleAudioMessage(conn, audio):
if have_voice == False and conn.client_have_voice == False:
await no_voice_close_connect(conn)
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-5:] # 保留最新的5帧音频内容，解决ASR句首丢字问题
conn.asr_audio = conn.asr_audio[
-10:
] # 保留最新的10帧音频内容，解决ASR句首丢字问题
return
conn.client_no_voice_last_time = 0.0
conn.asr_audio.append(audio)
@@ -30,10 +32,12 @@ async def handleAudioMessage(conn, audio):
conn.client_abort = False
conn.asr_server_receive = False
# 音频太短了，无法识别
if len(conn.asr_audio) < 10:
if len(conn.asr_audio) < 15:
conn.asr_server_receive = True
else:
text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
text, file_path = await conn.asr.speech_to_text(
conn.asr_audio, conn.session_id
)
logger.bind(tag=TAG).info(f"识别文本: {text}")
text_len, _ = remove_punctuation_and_length(text)
if text_len > 0:
@@ -47,12 +51,12 @@ async def handleAudioMessage(conn, audio):
async def startToChat(conn, text):
# 首先进行意图分析
intent_handled = await handle_user_intent(conn, text)
if intent_handled:
# 如果意图已被处理，不再进行聊天
conn.asr_server_receive = True
return
# 意图未被处理，继续常规聊天流程
await send_stt_message(conn, text)
if conn.use_function_call_mode:
@@ -67,10 +71,17 @@ async def no_voice_close_connect(conn):
conn.client_no_voice_last_time = time.time() * 1000
else:
no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
if not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time:
close_connection_no_voice_time = conn.config.get(
"close_connection_no_voice_time", 120
)
if (
not conn.close_after_chat
and no_voice_time > 1000 * close_connection_no_voice_time
):
conn.close_after_chat = True
conn.client_abort = False
conn.asr_server_receive = False
prompt = "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
prompt = (
"请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
)
await startToChat(conn, prompt)
main/xiaozhi-server/core/handle/sendAudioHandle.py
+23 -23
View File
@@ -2,7 +2,10 @@ from config.logger import setup_logging
import json
import asyncio
import time
from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
from core.utils.util import (
remove_punctuation_and_length,
get_string_no_punctuation_or_emoji,
)
TAG = __name__
logger = setup_logging()
@@ -21,10 +24,11 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
# 发送结束消息（如果是最后一个文本）
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
await send_tts_message(conn, 'stop', None)
await send_tts_message(conn, "stop", None)
if conn.close_after_chat:
await conn.close()
# 播放音频
async def sendAudio(conn, audios):
# 流控参数优化
@@ -36,13 +40,12 @@ async def sendAudio(conn, audios):
pre_buffer = min(3, len(audios))
for i in range(pre_buffer):
await conn.websocket.send(audios[i])
conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms")
# 正常播放剩余帧
for opus_packet in audios[pre_buffer:]:
if conn.client_abort:
return
# 计算预期发送时间
expected_time = start_time + (play_position / 1000)
current_time = time.perf_counter()
@@ -51,19 +54,13 @@ async def sendAudio(conn, audios):
await asyncio.sleep(delay)
await conn.websocket.send(opus_packet)
conn.logger.bind(tag=TAG).debug(f"发送帧，位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms")
play_position += frame_duration
async def send_tts_message(conn, state, text=None):
"""发送 TTS 状态消息"""
message = {
"type": "tts",
"state": state,
"session_id": conn.session_id
}
message = {"type": "tts", "state": state, "session_id": conn.session_id}
if text is not None:
message["text"] = text
@@ -72,7 +69,9 @@ async def send_tts_message(conn, state, text=None):
# 播放提示音
tts_notify = conn.config.get("enable_stop_tts_notify", False)
if tts_notify:
stop_tts_notify_voice = conn.config.get("stop_tts_notify_voice", "config/assets/tts_notify.mp3")
stop_tts_notify_voice = conn.config.get(
"stop_tts_notify_voice", "config/assets/tts_notify.mp3"
)
audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
await sendAudio(conn, audios)
# 清除服务端讲话状态
@@ -85,16 +84,17 @@ async def send_tts_message(conn, state, text=None):
async def send_stt_message(conn, text):
"""发送 STT 状态消息"""
stt_text = get_string_no_punctuation_or_emoji(text)
await conn.websocket.send(json.dumps({
"type": "stt",
"text": stt_text,
"session_id": conn.session_id}
))
await conn.websocket.send(
json.dumps({
"type": "llm",
"text": "😊",
"emotion": "happy",
"session_id": conn.session_id}
))
json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
)
await conn.websocket.send(
json.dumps(
{
"type": "llm",
"text": "😊",
"emotion": "happy",
"session_id": conn.session_id,
}
)
)
await send_tts_message(conn, "start")
main/xiaozhi-server/core/handle/textHandle.py
+4 -3
View File
@@ -6,6 +6,7 @@ from core.utils.util import remove_punctuation_and_length
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
import asyncio
TAG = __name__
logger = setup_logging()
@@ -34,7 +35,7 @@ async def handleTextMessage(conn, message):
conn.client_have_voice = True
conn.client_voice_stop = True
if len(conn.asr_audio) > 0:
await handleAudioMessage(conn, b'')
await handleAudioMessage(conn, b"")
elif msg_json["state"] == "detect":
conn.asr_server_receive = False
conn.client_have_voice = False
@@ -57,8 +58,8 @@ async def handleTextMessage(conn, message):
await startToChat(conn, text)
elif msg_json["type"] == "iot":
if "descriptors" in msg_json:
await handleIotDescriptors(conn, msg_json["descriptors"])
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
if "states" in msg_json:
await handleIotStatus(conn, msg_json["states"])
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
except json.JSONDecodeError:
await conn.websocket.send(message)
main/xiaozhi-server/core/providers/llm/dify/dify.py
+28 -26
View File
@@ -6,11 +6,12 @@ from core.providers.llm.base import LLMProviderBase
TAG = __name__
logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.api_key = config["api_key"]
self.mode = config.get("mode", "chat-messages")
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
def response(self, session_id, dialogue):
@@ -22,57 +23,58 @@ class LLMProvider(LLMProviderBase):
# 发起流式请求
if self.mode == "chat-messages":
request_json = {
"query": last_msg["content"],
"response_mode": "streaming",
"user": session_id,
"inputs": {},
"conversation_id": conversation_id
}
"query": last_msg["content"],
"response_mode": "streaming",
"user": session_id,
"inputs": {},
"conversation_id": conversation_id,
}
elif self.mode == "workflows/run":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id
"user": session_id,
}
elif self.mode == "completion-messages":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id
"user": session_id,
}
with requests.post(
f"{self.base_url}/{self.mode}",
headers={"Authorization": f"Bearer {self.api_key}"},
json=request_json,
stream=True
f"{self.base_url}/{self.mode}",
headers={"Authorization": f"Bearer {self.api_key}"},
json=request_json,
stream=True,
) as r:
if self.mode == "chat-messages":
for line in r.iter_lines():
if line.startswith(b'data: '):
if line.startswith(b"data: "):
event = json.loads(line[6:])
# 如果没有找到conversation_id，则获取此次conversation_id
if not conversation_id:
conversation_id = event.get('conversation_id')
self.session_conversation_map[session_id] = conversation_id # 更新映射
if event.get('answer'):
yield event['answer']
conversation_id = event.get("conversation_id")
self.session_conversation_map[session_id] = (
conversation_id # 更新映射
)
if event.get("answer"):
yield event["answer"]
elif self.mode == "workflows/run":
for line in r.iter_lines():
# logger.bind(tag=TAG).info(f"chat message response: {line}")
if line.startswith(b'data: '):
if line.startswith(b"data: "):
event = json.loads(line[6:])
if event.get('event') == "workflow_finished":
if event['data']['status'] == "succeeded":
yield event['data']['outputs']['answer']
if event.get("event") == "workflow_finished":
if event["data"]["status"] == "succeeded":
yield event["data"]["outputs"]["answer"]
else:
yield "【服务响应异常】"
elif self.mode == "completion-messages":
for line in r.iter_lines():
if line.startswith(b'data: '):
if line.startswith(b"data: "):
event = json.loads(line[6:])
if event.get('answer'):
yield event['answer']
if event.get("answer"):
yield event["answer"]
except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
main/xiaozhi-server/core/providers/tts/tencent.py
+156
View File
@@ -0,0 +1,156 @@
import hashlib
import hmac
import os
import time
import uuid
import json
import base64
import requests
from datetime import datetime, timezone
from core.providers.tts.base import TTSProviderBase
class TTSProvider(TTSProviderBase):
def __init__(self, config, delete_audio_file):
super().__init__(config, delete_audio_file)
self.appid = config.get("appid")
self.secret_id = config.get("secret_id")
self.secret_key = config.get("secret_key")
self.voice = config.get("voice")
self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点
self.region = config.get("region")
self.output_file = config.get("output_dir")
def _get_auth_headers(self, request_body):
"""生成鉴权请求头"""
# 获取当前UTC时间戳
timestamp = int(time.time())
# 使用UTC时间计算日期
utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d')
# 服务名称必须是 "tts"
service = "tts"
# 拼接凭证范围
credential_scope = f"{utc_date}/{service}/tc3_request"
# 使用TC3-HMAC-SHA256签名方法
algorithm = "TC3-HMAC-SHA256"
# 构建规范请求字符串
http_request_method = "POST"
canonical_uri = "/"
canonical_querystring = ""
# 请求头必须包含host和content-type，且按字典序排列
canonical_headers = (
f"content-type:application/json\n"
f"host:tts.tencentcloudapi.com\n"
)
signed_headers = "content-type;host"
# 请求体哈希值
payload = json.dumps(request_body)
payload_hash = hashlib.sha256(payload.encode('utf-8')).hexdigest()
# 构建规范请求字符串
canonical_request = (
f"{http_request_method}\n"
f"{canonical_uri}\n"
f"{canonical_querystring}\n"
f"{canonical_headers}\n"
f"{signed_headers}\n"
f"{payload_hash}"
)
# 计算规范请求的哈希值
hashed_canonical_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
# 构建待签名字符串
string_to_sign = (
f"{algorithm}\n"
f"{timestamp}\n"
f"{credential_scope}\n"
f"{hashed_canonical_request}"
)
# 计算签名密钥
secret_date = self._hmac_sha256(f"TC3{self.secret_key}".encode('utf-8'), utc_date)
secret_service = self._hmac_sha256(secret_date, service)
secret_signing = self._hmac_sha256(secret_service, "tc3_request")
# 计算签名
signature = hmac.new(
secret_signing,
string_to_sign.encode('utf-8'),
hashlib.sha256
).hexdigest()
# 构建授权头
authorization = (
f"{algorithm} "
f"Credential={self.secret_id}/{credential_scope}, "
f"SignedHeaders={signed_headers}, "
f"Signature={signature}"
)
# 构建请求头
headers = {
"Content-Type": "application/json",
"Host": "tts.tencentcloudapi.com",
"Authorization": authorization,
"X-TC-Action": "TextToVoice",
"X-TC-Timestamp": str(timestamp),
"X-TC-Version": "2019-08-23",
"X-TC-Region": self.region,
"X-TC-Language": "zh-CN"
}
return headers
def _hmac_sha256(self, key, msg):
"""HMAC-SHA256加密"""
if isinstance(msg, str):
msg = msg.encode('utf-8')
return hmac.new(key, msg, hashlib.sha256).digest()
def generate_filename(self, extension=".wav"):
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
async def text_to_speak(self, text, output_file):
# 构建请求体
request_json = {
"Text": text, # 合成语音的源文本
"SessionId": str(uuid.uuid4()), # 会话ID，随机生成
"VoiceType": int(self.voice), # 音色
}
try:
# 获取请求头（每次请求都重新生成，以确保时间戳和签名是最新的）
headers = self._get_auth_headers(request_json)
# 发送请求
resp = requests.post(self.api_url, json.dumps(request_json), headers=headers)
# 检查响应
if resp.status_code == 200:
response_data = resp.json()
# 检查是否成功
if response_data.get("Response", {}).get("Error") is not None:
error_info = response_data["Response"]["Error"]
raise Exception(f"API返回错误: {error_info['Code']}: {error_info['Message']}")
# 提取音频数据
audio_data = response_data["Response"].get("Audio")
if audio_data:
# 解码Base64音频数据并保存
with open(output_file, "wb") as f:
f.write(base64.b64decode(audio_data))
else:
raise Exception(f"{__name__}: 没有返回音频数据: {response_data}")
else:
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
except Exception as e:
raise Exception(f"{__name__} error: {e}")
main/xiaozhi-server/core/websocket_server.py
+67 -31
View File
@@ -12,50 +12,72 @@ class WebSocketServer:
def __init__(self, config: dict):
self.config = config
self.logger = setup_logging()
self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = self._create_processing_instances()
self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = (
self._create_processing_instances()
)
self.active_connections = set() # 添加全局连接记录
def _create_processing_instances(self):
memory_cls_name = self.config["selected_module"].get("Memory", "nomem") # 默认使用nomem
has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
memory_cls_name = self.config["selected_module"].get(
"Memory", "nomem"
) # 默认使用nomem
has_memory_cfg = (
self.config.get("Memory") and memory_cls_name in self.config["Memory"]
)
memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
"""创建处理模块实例"""
return (
vad.create_instance(
self.config["selected_module"]["VAD"],
self.config["VAD"][self.config["selected_module"]["VAD"]]
self.config["VAD"][self.config["selected_module"]["VAD"]],
),
asr.create_instance(
self.config["selected_module"]["ASR"]
if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
else
self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
(
self.config["selected_module"]["ASR"]
if not "type"
in self.config["ASR"][self.config["selected_module"]["ASR"]]
else self.config["ASR"][self.config["selected_module"]["ASR"]][
"type"
]
),
self.config["ASR"][self.config["selected_module"]["ASR"]],
self.config["delete_audio"]
self.config["delete_audio"],
),
llm.create_instance(
self.config["selected_module"]["LLM"]
if not 'type' in self.config["LLM"][self.config["selected_module"]["LLM"]]
else
self.config["LLM"][self.config["selected_module"]["LLM"]]['type'],
(
self.config["selected_module"]["LLM"]
if not "type"
in self.config["LLM"][self.config["selected_module"]["LLM"]]
else self.config["LLM"][self.config["selected_module"]["LLM"]][
"type"
]
),
self.config["LLM"][self.config["selected_module"]["LLM"]],
),
tts.create_instance(
self.config["selected_module"]["TTS"]
if not 'type' in self.config["TTS"][self.config["selected_module"]["TTS"]]
else
self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
(
self.config["selected_module"]["TTS"]
if not "type"
in self.config["TTS"][self.config["selected_module"]["TTS"]]
else self.config["TTS"][self.config["selected_module"]["TTS"]][
"type"
]
),
self.config["TTS"][self.config["selected_module"]["TTS"]],
self.config["delete_audio"]
self.config["delete_audio"],
),
memory.create_instance(memory_cls_name, memory_cfg),
intent.create_instance(
self.config["selected_module"]["Intent"]
if not 'type' in self.config["Intent"][self.config["selected_module"]["Intent"]]
else
self.config["Intent"][self.config["selected_module"]["Intent"]]["type"],
self.config["Intent"][self.config["selected_module"]["Intent"]]
(
self.config["selected_module"]["Intent"]
if not "type"
in self.config["Intent"][self.config["selected_module"]["Intent"]]
else self.config["Intent"][
self.config["selected_module"]["Intent"]
]["type"]
),
self.config["Intent"][self.config["selected_module"]["Intent"]],
),
)
@@ -64,19 +86,33 @@ class WebSocketServer:
host = server_config["ip"]
port = server_config["port"]
self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")
async with websockets.serve(
self._handle_connection,
host,
port
):
self.logger.bind(tag=TAG).info(
"Server is running at ws://{}:{}/xiaozhi/v1/", get_local_ip(), port
)
self.logger.bind(tag=TAG).info(
"=======上面的地址是websocket协议地址，请勿用浏览器访问======="
)
self.logger.bind(tag=TAG).info(
"如想测试websocket请用谷歌浏览器打开test目录下的test_page.html"
)
self.logger.bind(tag=TAG).info(
"=============================================================\n"
)
async with websockets.serve(self._handle_connection, host, port):
await asyncio.Future()
async def _handle_connection(self, websocket):
"""处理新连接，每次创建独立的ConnectionHandler"""
# 创建ConnectionHandler时传入当前server实例
handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._memory, self.intent)
handler = ConnectionHandler(
self.config,
self._vad,
self._asr,
self._llm,
self._tts,
self._memory,
self.intent,
)
self.active_connections.add(handler)
try:
await handler.handle_connection(websocket)
main/xiaozhi-server/plugins_func/functions/handle_device.py
+13 -7
View File
@@ -6,6 +6,7 @@ import asyncio
TAG = __name__
logger = setup_logging()
async def _get_device_status(conn, device_name, device_type, property_name):
"""获取设备状态"""
status = await get_iot_status(conn, device_type, property_name)
@@ -13,6 +14,7 @@ async def _get_device_status(conn, device_name, device_type, property_name):
raise Exception(f"你的设备不支持{device_name}控制")
return status
async def _set_device_property(conn, device_name, device_type, method_name, property_name, new_value=None, action=None, step=10):
"""设置设备属性"""
current_value = await _get_device_status(conn, device_name, device_type, property_name)
@@ -32,9 +34,11 @@ async def _set_device_property(conn, device_name, device_type, method_name, prop
await send_iot_conn(conn, device_type, method_name, {property_name: current_value})
return current_value
def _handle_device_action(conn, func, success_message, error_message, *args, **kwargs):
"""处理设备操作的通用函数"""
future = asyncio.run_coroutine_threadsafe(func(conn, *args, **kwargs), conn.loop)
future = asyncio.run_coroutine_threadsafe(
func(conn, *args, **kwargs), conn.loop)
try:
result = future.result()
logger.bind(tag=TAG).info(f"{success_message}: {result}")
@@ -45,6 +49,7 @@ def _handle_device_action(conn, func, success_message, error_message, *args, **k
response = f"{error_message}: {e}"
return ActionResponse(action=Action.RESPONSE, result=None, response=response)
# 设备控制
handle_device_function_desc = {
"type": "function",
@@ -52,9 +57,9 @@ handle_device_function_desc = {
"name": "handle_device",
"description": (
"用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。"
"比如用户说现在亮度多少，参数为：device_type:Backlight,action:get。"
"比如用户说现在亮度多少，参数为：device_type:Screen,action:get。"
"比如用户说设置音量为50，参数为：device_type:Speaker,action:set,value:50。"
"比如用户说亮度太高了，参数为：device_type:Backlight,action:lower。"
"比如用户说亮度太高了，参数为：device_type:Screen,action:lower。"
"比如用户说调大音量，参数为：device_type:Speaker,action:raise。"
),
"parameters": {
@@ -62,7 +67,7 @@ handle_device_function_desc = {
"properties": {
"device_type": {
"type": "string",
"description": "设备类型，可选值：Speaker(音量),Backlight(亮度)"
"description": "设备类型，可选值：Speaker(音量),Screen(亮度)"
},
"action": {
"type": "string",
@@ -78,18 +83,19 @@ handle_device_function_desc = {
}
}
@register_function('handle_device', handle_device_function_desc, ToolType.IOT_CTL)
def handle_device(conn, device_type: str, action: str, value: int = None):
if device_type == "Speaker":
method_name, property_name, device_name = "SetVolume", "volume", "音量"
elif device_type == "Backlight":
elif device_type == "Screen":
method_name, property_name, device_name = "SetBrightness", "brightness", "亮度"
else:
raise Exception(f"未识别的设备类型: {device_type}")
if action not in ["get", "set", "raise", "lower"]:
raise Exception(f"未识别的动作名称: {action}")
if action == "get":
# get
return _handle_device_action(
main/xiaozhi-server/test/libopus.js
+266
View File
File diff suppressed because one or more lines are too long
main/xiaozhi-server/test/test_page.html
+1934
View File
File diff suppressed because it is too large Load Diff