diff --git a/README.md b/README.md index d4b8422b..febc1f6e 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Websocket接口地址: wss://2662r3426b.vicp.fun/xiaozhi/v1/ | ASR(语音识别) | FunASR(本地) | 👍FunASRServer 或 👍DoubaoStreamASR | | LLM(大模型) | ChatGLMLLM(智谱glm-4-flash) | 👍DoubaoLLM(火山doubao-1-5-pro-32k-250115) | | VLLM(视觉大模型) | ChatGLMVLLM(智谱glm-4v-flash) | 👍QwenVLVLLM(千问qwen2.5-vl-3b-instructh) | -| TTS(语音合成) | ✅LinkeraiTTS(灵犀流式) | 👍HuoshanDoubleStreamTTS(火山双流式语音合成) | +| TTS(语音合成) | ✅LinkeraiTTS(灵犀流式) | 👍HuoshanDoubleStreamTTS(火山双流式语音合成) 或 👍AliyunStreamTTS(阿里云流式语音合成) | | Intent(意图识别) | function_call(函数调用) | function_call(函数调用) | | Memory(记忆功能) | mem_local_short(本地短期记忆） | mem_local_short（本地短期记忆） | diff --git a/docs/voiceprint-integration.md b/docs/voiceprint-integration.md index 4364faa9..9db1af0c 100644 --- a/docs/voiceprint-integration.md +++ b/docs/voiceprint-integration.md @@ -1,8 +1,9 @@ # 声纹识别启用指南 -本教程包含2个部分 +本教程包含3个部分 - 1、如何部署声纹识别这个服务 - 2、全模块部署时，怎么配置声纹识别接口 +- 3、最简化部署时，怎么配置声纹识别 # 1、如何部署声纹识别这个服务 @@ -185,4 +186,48 @@ http://192.168.1.25:8005/voiceprint/health?key=abcd ## 第三步 和你的智能体聊天 -将你的设备通电，问它，你知道我是谁吗？如果他能回答得出，说明声纹识别功能正常。 \ No newline at end of file +将你的设备通电，问它，你知道我是谁吗？如果他能回答得出，说明声纹识别功能正常。 + +# 3、最简化部署时，怎么配置声纹识别 + +## 第一步 配置接口 +打开 `xiaozhi-server/data/.config.yaml` 文件（如果没有需要创建），然后添加/修改以下内容： + +``` +# 声纹识别配置 +voiceprint: + # 声纹接口地址 + url: 你的声纹接口地址 + # 说话人配置：speaker_id,名称,描述 + speakers: + - "test1,张三,张三是一个程序员" + - "test2,李四,李四是一个产品经理" + - "test3,王五,王五是一个设计师" +``` + +把上一步得来的 `声纹接口地址` 粘贴到 `url` 里。然后保存。 + +`speakers` 参数依据需求添加。这里需要注意这个 `speaker_id` 参数，后面注册声纹会用到。 + +## 第二步 注册声纹 +如果你已经启动了声纹服务，本地浏览器里访问 `http://localhost:8005/voiceprint/docs` 即可查看 API 文档，这里只说明注册声纹的 API 如何使用。 + +注册声纹的 API 地址为 `http://localhost:8005/voiceprint/register`，请求方式为 POST。 + +请求头需要包含 Bearer Token 认证，token 为 `声纹接口地址` 中 `?key=` 后的部分，比如如果我的声纹注册地址为 `http://127.0.0.1:8005/voiceprint/health?key=abcd`，那么我的 token 就是`abcd`。 + +请求体包含说话人 ID（speaker_id），和 WAV 音频文件（file），请求示例如下： + +``` +curl -X POST \ + -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \ + -F "speaker_id=your_speaker_id_here" \ + -F "file=@/path/to/your/file" \ + http://localhost:8005/voiceprint/register +``` + + 这里的 `file` 是要注册的说话人说话的音频文件， `speaker_id` 需要和第一步配置接口的 `speaker_id` 保持一致。比如说我需要注册张三的声纹，在 `.config.yaml` 中填的张三的 `speaker_id` 为 `test1`，那么我注册张三声纹的时候，请求体里填的 `speaker_id` 就是 `test1`， `file` 填的就是张三说一段话的音频文件。 + + ## 第三步 启动服务 + +启动小智服务器和声纹服务，即可正常使用。 diff --git a/main/manager-api/src/main/java/xiaozhi/common/utils/JsonRpcTwo.java b/main/manager-api/src/main/java/xiaozhi/common/utils/JsonRpcTwo.java new file mode 100644 index 00000000..c522a61d --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/common/utils/JsonRpcTwo.java @@ -0,0 +1,21 @@ +package xiaozhi.common.utils; + +import lombok.Data; +/** + * JSON-RPC2.0 格式规范对象 + */ +@Data +public class JsonRpcTwo { + private String jsonrpc = "2.0"; + private String method; + private Object params; + private Integer id; + + public JsonRpcTwo(String method, Object params, Integer id) { + this.method = method; + this.params = params; + this.id = id; + } + + +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/XiaoZhiMcpJsonRpcJson.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/XiaoZhiMcpJsonRpcJson.java new file mode 100644 index 00000000..114b1ab9 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/XiaoZhiMcpJsonRpcJson.java @@ -0,0 +1,44 @@ +package xiaozhi.modules.agent.Enums; + +import xiaozhi.common.utils.JsonUtils; +import xiaozhi.common.utils.JsonRpcTwo; + +import java.util.Map; + + +/** + * 小智MCP JSON-RPC 请求json + */ +public class XiaoZhiMcpJsonRpcJson { + //小智初始化mcp请求json + private static final String INITIALIZE_JSON; + //小智mcp初始化成功，返回通知请求json + private static final String NOTIFICATIONS_INITIALIZED_JSON; + //小智mcp获取mcp工具集合请求json + private static final String TOOLS_LIST_REQUEST; + // 延迟加载 + static { + INITIALIZE_JSON = JsonUtils.toJsonString(new JsonRpcTwo("initialize", + Map.of( + "protocolVersion", "2024-11-05", + "capabilities", Map.of( + "roots", Map.of("listChanged", false), + "sampling", Map.of()), + "clientInfo", Map.of( + "name", "xz-mcp-broker", + "version", "0.0.1")), + 1)); + NOTIFICATIONS_INITIALIZED_JSON = "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}"; + TOOLS_LIST_REQUEST = JsonUtils.toJsonString(new JsonRpcTwo("tools/list", null, 2)); + } + public static String getInitializeJson(){ + return INITIALIZE_JSON; + } + public static String getNotificationsInitializedJson(){ + return NOTIFICATIONS_INITIALIZED_JSON; + } + public static String getToolsListJson(){ + return TOOLS_LIST_REQUEST; + } + +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/McpJsonRpcRequest.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/McpJsonRpcRequest.java deleted file mode 100644 index e4fac002..00000000 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/McpJsonRpcRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -package xiaozhi.modules.agent.dto; - -import lombok.Data; - -/** - * MCP JSON-RPC 请求 DTO - */ -@Data -public class McpJsonRpcRequest { - private String jsonrpc = "2.0"; - private String method; - private Object params; - private Integer id; - - public McpJsonRpcRequest() { - } - - public McpJsonRpcRequest(String method) { - this.method = method; - } - - public McpJsonRpcRequest(String method, Object params, Integer id) { - this.method = method; - this.params = params; - this.id = id; - } - - public McpJsonRpcRequest(String method, Object params) { - this.method = method; - this.params = params; - } -} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentMcpAccessPointServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentMcpAccessPointServiceImpl.java index bc982f51..7d96f748 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentMcpAccessPointServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentMcpAccessPointServiceImpl.java @@ -18,7 +18,7 @@ import xiaozhi.common.constant.Constant; import xiaozhi.common.utils.AESUtils; import xiaozhi.common.utils.HashEncryptionUtil; import xiaozhi.common.utils.JsonUtils; -import xiaozhi.modules.agent.dto.McpJsonRpcRequest; +import xiaozhi.modules.agent.Enums.XiaoZhiMcpJsonRpcJson; import xiaozhi.modules.agent.service.AgentMcpAccessPointService; import xiaozhi.modules.sys.service.SysParamsService; import xiaozhi.modules.sys.utils.WebSocketClientManager; @@ -65,22 +65,13 @@ public class AgentMcpAccessPointServiceImpl implements AgentMcpAccessPointServic try (WebSocketClientManager client = WebSocketClientManager.build( new WebSocketClientManager.Builder() .uri(wsUrl) + .bufferSize(1024 * 1024) .connectTimeout(8, TimeUnit.SECONDS) .maxSessionDuration(10, TimeUnit.SECONDS))) { // 步骤1: 发送初始化消息并等待响应 log.info("发送MCP初始化消息，智能体ID: {}", id); - McpJsonRpcRequest initializeRequest = new McpJsonRpcRequest("initialize", - Map.of( - "protocolVersion", "2024-11-05", - "capabilities", Map.of( - "roots", Map.of("listChanged", false), - "sampling", Map.of()), - "clientInfo", Map.of( - "name", "xz-mcp-broker", - "version", "0.0.1")), - 1); - client.sendJson(initializeRequest); + client.sendText(XiaoZhiMcpJsonRpcJson.getInitializeJson()); // 等待初始化响应 (id=1) - 移除固定延迟，改为响应驱动 List initResponses = client.listenerWithoutClose(response -> { @@ -124,13 +115,10 @@ public class AgentMcpAccessPointServiceImpl implements AgentMcpAccessPointServic // 步骤2: 发送初始化完成通知 - 只有在收到initialize响应后才发送 log.info("发送MCP初始化完成通知，智能体ID: {}", id); - String notificationJson = "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}"; - client.sendText(notificationJson); - + client.sendText(XiaoZhiMcpJsonRpcJson.getNotificationsInitializedJson()); // 步骤3: 发送工具列表请求 - 立即发送，无需额外延迟 log.info("发送MCP工具列表请求，智能体ID: {}", id); - McpJsonRpcRequest toolsRequest = new McpJsonRpcRequest("tools/list", null, 2); - client.sendJson(toolsRequest); + client.sendText(XiaoZhiMcpJsonRpcJson.getToolsListJson()); // 等待工具列表响应 (id=2) List toolsResponses = client.listener(response -> { diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/utils/WebSocketClientManager.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/utils/WebSocketClientManager.java index d7171ae7..d4203c75 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/utils/WebSocketClientManager.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/utils/WebSocketClientManager.java @@ -86,10 +86,14 @@ public class WebSocketClientManager implements Closeable { if (sess == null || !sess.isOpen()) { throw new IOException("握手失败或会话未打开"); } + // 设置缓冲区 + sess.setTextMessageSizeLimit(b.bufferSize); + sess.setBinaryMessageSizeLimit(b.bufferSize); ws.session = sess; return ws; } + /** * 发送 Text */ @@ -308,10 +312,11 @@ public class WebSocketClientManager implements Closeable { if (stopWatch.isRunning()) { stopWatch.stop(); } - log.info("ws连接关闭, 目标URI: {}, 关闭时间: {}, 连接总时长: {}s", + log.info("ws连接关闭, 目标URI: {}, 关闭时间: {}, 连接总时长: {}s,断开原因：{}", targetUri, DateUtils.getDateTimeNow(DateUtils.DATE_TIME_MILLIS_PATTERN), - DateUtils.millsToSecond(stopWatch.getTotalTimeMillis())); + DateUtils.millsToSecond(stopWatch.getTotalTimeMillis()),status); } + } public static class Builder { @@ -321,6 +326,7 @@ public class WebSocketClientManager implements Closeable { private long maxSessionDuration = 5; // 最大连线时间，默认5秒 private TimeUnit maxSessionDurationUnit = TimeUnit.SECONDS; // 最大连线时间单位 private int queueCapacity = 100; // 消息队列容量 + private int bufferSize = 8 * 1024; //默认 8kb private WebSocketHttpHeaders headers; // 请求头 /** @@ -352,6 +358,10 @@ public class WebSocketClientManager implements Closeable { this.queueCapacity = c; return this; } + public Builder bufferSize(int c) { + this.bufferSize = c; + return this; + } public WebSocketClientManager build() throws InterruptedException, ExecutionException, TimeoutException, IOException { diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 6d8fc11e..992e01fa 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -499,6 +499,11 @@ class ConnectionHandler: self.config["selected_module"]["LLM"] = private_config["selected_module"][ "LLM" ] + if private_config.get("VLLM", None) is not None: + self.config["VLLM"] = private_config["VLLM"] + self.config["selected_module"]["VLLM"] = private_config["selected_module"][ + "VLLM" + ] if private_config.get("Memory", None) is not None: init_memory = True self.config["Memory"] = private_config["Memory"] diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index ccff094c..81911f55 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -13,6 +13,16 @@ TAG = __name__ async def handle_user_intent(conn, text): + # 预处理输入文本，处理可能的JSON格式 + try: + if text.strip().startswith('{') and text.strip().endswith('}'): + parsed_data = json.loads(text) + if isinstance(parsed_data, dict) and "content" in parsed_data: + text = parsed_data["content"] # 提取content用于意图分析 + conn.current_speaker = parsed_data.get("speaker") # 保留说话人信息 + except (json.JSONDecodeError, TypeError): + pass + # 检查是否有明确的退出命令 filtered_text = remove_punctuation_and_length(text)[1] if await check_direct_exit(conn, filtered_text): diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 82c168d4..aa59a4d1 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -109,6 +109,9 @@ async def send_stt_message(conn, text): if isinstance(parsed_data, dict) and "content" in parsed_data: # 如果是包含说话人信息的JSON格式，只显示content部分 display_text = parsed_data["content"] + # 保存说话人信息到conn对象 + if "speaker" in parsed_data: + conn.current_speaker = parsed_data["speaker"] except (json.JSONDecodeError, TypeError): # 如果不是JSON格式，直接使用原始文本 display_text = text diff --git a/main/xiaozhi-server/plugins_func/functions/hass_get_state.py b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py index e8167db4..94478e35 100644 --- a/main/xiaozhi-server/plugins_func/functions/hass_get_state.py +++ b/main/xiaozhi-server/plugins_func/functions/hass_get_state.py @@ -33,10 +33,16 @@ def hass_get_state(conn, entity_id=""): future = asyncio.run_coroutine_threadsafe( handle_hass_get_state(conn, entity_id), conn.loop ) - ha_response = future.result() + # 添加10秒超时 + ha_response = future.result(timeout=10) return ActionResponse(Action.REQLLM, ha_response, None) + except asyncio.TimeoutError: + logger.bind(tag=TAG).error("获取Home Assistant状态超时") + return ActionResponse(Action.ERROR, "请求超时", None) except Exception as e: - logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}") + error_msg = f"执行Home Assistant操作失败" + logger.bind(tag=TAG).error(error_msg) + return ActionResponse(Action.ERROR, error_msg, None) async def handle_hass_get_state(conn, entity_id): diff --git a/main/xiaozhi-server/plugins_func/functions/hass_set_state.py b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py index 704148ee..430679bb 100644 --- a/main/xiaozhi-server/plugins_func/functions/hass_set_state.py +++ b/main/xiaozhi-server/plugins_func/functions/hass_set_state.py @@ -55,10 +55,16 @@ def hass_set_state(conn, entity_id="", state={}): future = asyncio.run_coroutine_threadsafe( handle_hass_set_state(conn, entity_id, state), conn.loop ) - ha_response = future.result() + # 添加10秒超时 + ha_response = future.result(timeout=10) return ActionResponse(Action.REQLLM, ha_response, None) + except asyncio.TimeoutError: + logger.bind(tag=TAG).error("设置Home Assistant状态超时") + return ActionResponse(Action.ERROR, "请求超时", None) except Exception as e: - logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}") + error_msg = f"执行Home Assistant操作失败" + logger.bind(tag=TAG).error(error_msg) + return ActionResponse(Action.ERROR, error_msg, None) async def handle_hass_set_state(conn, entity_id, state):