Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py_test_tts' of https://github.com/xinnan-tech/xiaozhi-esp32-server into py_test_tts

Browse Source
This commit is contained in:
Sakura-RanChen
2025-07-23 14:40:14 +08:00
parent e93053d412 2370936dfd
commit 83ded8458a
12 changed files with 164 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -217,7 +217,7 @@ Websocket接口地址: wss://2662r3426b.vicp.fun/xiaozhi/v1/
| ASR(语音识别) | FunASR(本地) | 👍FunASRServer 或 👍DoubaoStreamASR |
| LLM(大模型) | ChatGLMLLM(智谱glm-4-flash) | 👍DoubaoLLM(火山doubao-1-5-pro-32k-250115) |
| VLLM(视觉大模型) | ChatGLMVLLM(智谱glm-4v-flash) | 👍QwenVLVLLM(千问qwen2.5-vl-3b-instructh) |
| TTS(语音合成) | ✅LinkeraiTTS(灵犀流式) | 👍HuoshanDoubleStreamTTS(火山双流式语音合成) |
| TTS(语音合成) | ✅LinkeraiTTS(灵犀流式) | 👍HuoshanDoubleStreamTTS(火山双流式语音合成) 或 👍AliyunStreamTTS(阿里云流式语音合成) |
| Intent(意图识别) | function_call(函数调用) | function_call(函数调用) |
| Memory(记忆功能) | mem_local_short(本地短期记忆） | mem_local_short（本地短期记忆） |
docs/voiceprint-integration.md
+47 -2
View File
@@ -1,8 +1,9 @@
# 声纹识别启用指南
本教程包含2个部分
本教程包含3个部分
- 1、如何部署声纹识别这个服务
- 2、全模块部署时，怎么配置声纹识别接口
- 3、最简化部署时，怎么配置声纹识别
# 1、如何部署声纹识别这个服务
@@ -185,4 +186,48 @@ http://192.168.1.25:8005/voiceprint/health?key=abcd
## 第三步 和你的智能体聊天
将你的设备通电，问它，你知道我是谁吗？如果他能回答得出，说明声纹识别功能正常。
将你的设备通电，问它，你知道我是谁吗？如果他能回答得出，说明声纹识别功能正常。
# 3、最简化部署时，怎么配置声纹识别
## 第一步 配置接口
打开 `xiaozhi-server/data/.config.yaml` 文件（如果没有需要创建），然后添加/修改以下内容：
```
# 声纹识别配置
voiceprint:
# 声纹接口地址
url: 你的声纹接口地址
# 说话人配置：speaker_id,名称,描述
speakers:
- "test1,张三,张三是一个程序员"
- "test2,李四,李四是一个产品经理"
- "test3,王五,王五是一个设计师"
```
把上一步得来的 `声纹接口地址` 粘贴到 `url` 里。然后保存。
`speakers` 参数依据需求添加。这里需要注意这个 `speaker_id` 参数，后面注册声纹会用到。
## 第二步 注册声纹
如果你已经启动了声纹服务，本地浏览器里访问 `http://localhost:8005/voiceprint/docs` 即可查看 API 文档，这里只说明注册声纹的 API 如何使用。
注册声纹的 API 地址为 `http://localhost:8005/voiceprint/register`，请求方式为 POST。
请求头需要包含 Bearer Token 认证，token 为 `声纹接口地址` 中 `?key=` 后的部分，比如如果我的声纹注册地址为 `http://127.0.0.1:8005/voiceprint/health?key=abcd`，那么我的 token 就是`abcd`。
请求体包含说话人 ID（speaker_id），和 WAV 音频文件（file），请求示例如下：
```
curl -X POST \
-H "Authorization: Bearer YOUR_ACCESS_TOKEN" \
-F "speaker_id=your_speaker_id_here" \
-F "file=@/path/to/your/file" \
http://localhost:8005/voiceprint/register
```
这里的 `file` 是要注册的说话人说话的音频文件， `speaker_id` 需要和第一步配置接口的 `speaker_id` 保持一致。比如说我需要注册张三的声纹，在 `.config.yaml` 中填的张三的 `speaker_id` 为 `test1`，那么我注册张三声纹的时候，请求体里填的 `speaker_id` 就是 `test1`， `file` 填的就是张三说一段话的音频文件。
## 第三步 启动服务
启动小智服务器和声纹服务，即可正常使用。
main/manager-api/src/main/java/xiaozhi/common/utils/JsonRpcTwo.java
+21
View File
@@ -0,0 +1,21 @@
package xiaozhi.common.utils;
import lombok.Data;
/**
* JSON-RPC2.0 格式规范对象
*/
@Data
public class JsonRpcTwo {
private String jsonrpc = "2.0";
private String method;
private Object params;
private Integer id;
public JsonRpcTwo(String method, Object params, Integer id) {
this.method = method;
this.params = params;
this.id = id;
}
}
main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/XiaoZhiMcpJsonRpcJson.java
+44
View File
@@ -0,0 +1,44 @@
package xiaozhi.modules.agent.Enums;
import xiaozhi.common.utils.JsonUtils;
import xiaozhi.common.utils.JsonRpcTwo;
import java.util.Map;
/**
* 小智MCP JSON-RPC 请求json
*/
public class XiaoZhiMcpJsonRpcJson {
//小智初始化mcp请求json
private static final String INITIALIZE_JSON;
//小智mcp初始化成功，返回通知请求json
private static final String NOTIFICATIONS_INITIALIZED_JSON;
//小智mcp获取mcp工具集合请求json
private static final String TOOLS_LIST_REQUEST;
// 延迟加载
static {
INITIALIZE_JSON = JsonUtils.toJsonString(new JsonRpcTwo("initialize",
Map.of(
"protocolVersion", "2024-11-05",
"capabilities", Map.of(
"roots", Map.of("listChanged", false),
"sampling", Map.of()),
"clientInfo", Map.of(
"name", "xz-mcp-broker",
"version", "0.0.1")),
1));
NOTIFICATIONS_INITIALIZED_JSON = "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}";
TOOLS_LIST_REQUEST = JsonUtils.toJsonString(new JsonRpcTwo("tools/list", null, 2));
}
public static String getInitializeJson(){
return INITIALIZE_JSON;
}
public static String getNotificationsInitializedJson(){
return NOTIFICATIONS_INITIALIZED_JSON;
}
public static String getToolsListJson(){
return TOOLS_LIST_REQUEST;
}
}
main/manager-api/src/main/java/xiaozhi/modules/agent/dto/McpJsonRpcRequest.java
-32
View File
@@ -1,32 +0,0 @@
package xiaozhi.modules.agent.dto;
import lombok.Data;
/**
* MCP JSON-RPC 请求 DTO
*/
@Data
public class McpJsonRpcRequest {
private String jsonrpc = "2.0";
private String method;
private Object params;
private Integer id;
public McpJsonRpcRequest() {
}
public McpJsonRpcRequest(String method) {
this.method = method;
}
public McpJsonRpcRequest(String method, Object params, Integer id) {
this.method = method;
this.params = params;
this.id = id;
}
public McpJsonRpcRequest(String method, Object params) {
this.method = method;
this.params = params;
}
}
main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentMcpAccessPointServiceImpl.java
+5 -17
View File
@@ -18,7 +18,7 @@ import xiaozhi.common.constant.Constant;
import xiaozhi.common.utils.AESUtils;
import xiaozhi.common.utils.HashEncryptionUtil;
import xiaozhi.common.utils.JsonUtils;
import xiaozhi.modules.agent.dto.McpJsonRpcRequest;
import xiaozhi.modules.agent.Enums.XiaoZhiMcpJsonRpcJson;
import xiaozhi.modules.agent.service.AgentMcpAccessPointService;
import xiaozhi.modules.sys.service.SysParamsService;
import xiaozhi.modules.sys.utils.WebSocketClientManager;
@@ -65,22 +65,13 @@ public class AgentMcpAccessPointServiceImpl implements AgentMcpAccessPointServic
try (WebSocketClientManager client = WebSocketClientManager.build(
new WebSocketClientManager.Builder()
.uri(wsUrl)
.bufferSize(1024 * 1024)
.connectTimeout(8, TimeUnit.SECONDS)
.maxSessionDuration(10, TimeUnit.SECONDS))) {
// 步骤1: 发送初始化消息并等待响应
log.info("发送MCP初始化消息，智能体ID: {}", id);
McpJsonRpcRequest initializeRequest = new McpJsonRpcRequest("initialize",
Map.of(
"protocolVersion", "2024-11-05",
"capabilities", Map.of(
"roots", Map.of("listChanged", false),
"sampling", Map.of()),
"clientInfo", Map.of(
"name", "xz-mcp-broker",
"version", "0.0.1")),
1);
client.sendJson(initializeRequest);
client.sendText(XiaoZhiMcpJsonRpcJson.getInitializeJson());
// 等待初始化响应 (id=1) - 移除固定延迟，改为响应驱动
List<String> initResponses = client.listenerWithoutClose(response -> {
@@ -124,13 +115,10 @@ public class AgentMcpAccessPointServiceImpl implements AgentMcpAccessPointServic
// 步骤2: 发送初始化完成通知 - 只有在收到initialize响应后才发送
log.info("发送MCP初始化完成通知，智能体ID: {}", id);
String notificationJson = "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}";
client.sendText(notificationJson);
client.sendText(XiaoZhiMcpJsonRpcJson.getNotificationsInitializedJson());
// 步骤3: 发送工具列表请求 - 立即发送，无需额外延迟
log.info("发送MCP工具列表请求，智能体ID: {}", id);
McpJsonRpcRequest toolsRequest = new McpJsonRpcRequest("tools/list", null, 2);
client.sendJson(toolsRequest);
client.sendText(XiaoZhiMcpJsonRpcJson.getToolsListJson());
// 等待工具列表响应 (id=2)
List<String> toolsResponses = client.listener(response -> {
main/manager-api/src/main/java/xiaozhi/modules/sys/utils/WebSocketClientManager.java
+12 -2
View File
@@ -86,10 +86,14 @@ public class WebSocketClientManager implements Closeable {
if (sess == null || !sess.isOpen()) {
throw new IOException("握手失败或会话未打开");
}
// 设置缓冲区
sess.setTextMessageSizeLimit(b.bufferSize);
sess.setBinaryMessageSizeLimit(b.bufferSize);
ws.session = sess;
return ws;
}
/**
* 发送 Text
*/
@@ -308,10 +312,11 @@ public class WebSocketClientManager implements Closeable {
if (stopWatch.isRunning()) {
stopWatch.stop();
}
log.info("ws连接关闭, 目标URI: {}, 关闭时间: {}, 连接总时长: {}s",
log.info("ws连接关闭, 目标URI: {}, 关闭时间: {}, 连接总时长: {}s,断开原因：{}",
targetUri, DateUtils.getDateTimeNow(DateUtils.DATE_TIME_MILLIS_PATTERN),
DateUtils.millsToSecond(stopWatch.getTotalTimeMillis()));
DateUtils.millsToSecond(stopWatch.getTotalTimeMillis()),status);
}
}
public static class Builder {
@@ -321,6 +326,7 @@ public class WebSocketClientManager implements Closeable {
private long maxSessionDuration = 5; // 最大连线时间，默认5秒
private TimeUnit maxSessionDurationUnit = TimeUnit.SECONDS; // 最大连线时间单位
private int queueCapacity = 100; // 消息队列容量
private int bufferSize = 8 * 1024; //默认 8kb
private WebSocketHttpHeaders headers; // 请求头
/**
@@ -352,6 +358,10 @@ public class WebSocketClientManager implements Closeable {
this.queueCapacity = c;
return this;
}
public Builder bufferSize(int c) {
this.bufferSize = c;
return this;
}
public WebSocketClientManager build()
throws InterruptedException, ExecutionException, TimeoutException, IOException {
main/xiaozhi-server/core/connection.py
+5
View File
@@ -499,6 +499,11 @@ class ConnectionHandler:
self.config["selected_module"]["LLM"] = private_config["selected_module"][
"LLM"
]
if private_config.get("VLLM", None) is not None:
self.config["VLLM"] = private_config["VLLM"]
self.config["selected_module"]["VLLM"] = private_config["selected_module"][
"VLLM"
]
if private_config.get("Memory", None) is not None:
init_memory = True
self.config["Memory"] = private_config["Memory"]
main/xiaozhi-server/core/handle/intentHandler.py
+10
View File
@@ -13,6 +13,16 @@ TAG = __name__
async def handle_user_intent(conn, text):
# 预处理输入文本，处理可能的JSON格式
try:
if text.strip().startswith('{') and text.strip().endswith('}'):
parsed_data = json.loads(text)
if isinstance(parsed_data, dict) and "content" in parsed_data:
text = parsed_data["content"] # 提取content用于意图分析
conn.current_speaker = parsed_data.get("speaker") # 保留说话人信息
except (json.JSONDecodeError, TypeError):
pass
# 检查是否有明确的退出命令
filtered_text = remove_punctuation_and_length(text)[1]
if await check_direct_exit(conn, filtered_text):
main/xiaozhi-server/core/handle/sendAudioHandle.py
+3
View File
@@ -109,6 +109,9 @@ async def send_stt_message(conn, text):
if isinstance(parsed_data, dict) and "content" in parsed_data:
# 如果是包含说话人信息的JSON格式，只显示content部分
display_text = parsed_data["content"]
# 保存说话人信息到conn对象
if "speaker" in parsed_data:
conn.current_speaker = parsed_data["speaker"]
except (json.JSONDecodeError, TypeError):
# 如果不是JSON格式，直接使用原始文本
display_text = text
main/xiaozhi-server/plugins_func/functions/hass_get_state.py
+8 -2
View File
@@ -33,10 +33,16 @@ def hass_get_state(conn, entity_id=""):
future = asyncio.run_coroutine_threadsafe(
handle_hass_get_state(conn, entity_id), conn.loop
)
ha_response = future.result()
# 添加10秒超时
ha_response = future.result(timeout=10)
return ActionResponse(Action.REQLLM, ha_response, None)
except asyncio.TimeoutError:
logger.bind(tag=TAG).error("获取Home Assistant状态超时")
return ActionResponse(Action.ERROR, "请求超时", None)
except Exception as e:
logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}")
error_msg = f"执行Home Assistant操作失败"
logger.bind(tag=TAG).error(error_msg)
return ActionResponse(Action.ERROR, error_msg, None)
async def handle_hass_get_state(conn, entity_id):
main/xiaozhi-server/plugins_func/functions/hass_set_state.py
+8 -2
View File
@@ -55,10 +55,16 @@ def hass_set_state(conn, entity_id="", state={}):
future = asyncio.run_coroutine_threadsafe(
handle_hass_set_state(conn, entity_id, state), conn.loop
)
ha_response = future.result()
# 添加10秒超时
ha_response = future.result(timeout=10)
return ActionResponse(Action.REQLLM, ha_response, None)
except asyncio.TimeoutError:
logger.bind(tag=TAG).error("设置Home Assistant状态超时")
return ActionResponse(Action.ERROR, "请求超时", None)
except Exception as e:
logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}")
error_msg = f"执行Home Assistant操作失败"
logger.bind(tag=TAG).error(error_msg)
return ActionResponse(Action.ERROR, error_msg, None)
async def handle_hass_set_state(conn, entity_id, state):