From f92c0313a6f8d07cdcb03cf19498148e8800f8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 30 Apr 2025 09:41:38 +0800 Subject: [PATCH 01/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=B5=8B=E8=AF=95=E6=96=B9=E6=B3=95=E5=92=8C?= =?UTF-8?q?=E6=A8=A1=E6=8B=9F=E8=AE=BE=E5=A4=87=E8=BF=9E=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E8=BF=87=E6=9D=A5=E7=9A=84=E6=96=B9=E6=B3=95=EF=BC=8C=E6=96=B9?= =?UTF-8?q?=E4=BE=BF=E6=96=B0=E5=BC=80=E5=8F=91=E8=80=85=E8=B0=83=E8=AF=95?= =?UTF-8?q?=20--DeviceTest.java?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi/modules/device/DeviceTest.java | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java b/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java index 8647692c..4c13bcdb 100644 --- a/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java +++ b/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java @@ -10,6 +10,10 @@ import org.springframework.test.context.ActiveProfiles; import lombok.extern.slf4j.Slf4j; import xiaozhi.common.redis.RedisKeys; import xiaozhi.common.redis.RedisUtils; +import xiaozhi.modules.sys.dto.SysUserDTO; +import xiaozhi.modules.sys.service.SysUserService; + +import java.util.HashMap; @Slf4j @SpringBootTest @@ -19,18 +23,38 @@ public class DeviceTest { @Autowired private RedisUtils redisUtils; + @Autowired + private SysUserService sysUserService; + + @Test + public void testSaveUser() { + SysUserDTO userDTO = new SysUserDTO(); + userDTO.setUsername("13536468486"); + userDTO.setPassword("0218jianyuQ!"); + sysUserService.save(userDTO); + } @Test @DisplayName("测试写入设备信息") public void testWriteDeviceInfo() { log.info("开始测试写入设备信息..."); - // 模拟设备MAC地址 - String macAddress = "00:11:22:33:44:55"; + String macAddress = "00:11:22:33:44:66"; // 模拟设备验证码 String deviceCode = "123456"; - String redisKey = RedisKeys.getDeviceCaptchaKey(deviceCode); + HashMap map = new HashMap<>(); + map.put("mac_address",macAddress); + map.put("activation_code",deviceCode); + map.put("board","硬件型号"); + map.put("app_version","0.3.13"); + + String safeDeviceId = macAddress.replace(":", "_").toLowerCase(); + String cacheDeviceKey = String.format("ota:activation:data:%s", safeDeviceId); + redisUtils.set(cacheDeviceKey, map, 300); + + + String redisKey = "ota:activation:code:"+deviceCode; log.info("Redis Key: {}", redisKey); // 将设备信息写入Redis From 9b0088f4f0a13276c94b78b956dc4d39022614d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 30 Apr 2025 10:44:54 +0800 Subject: [PATCH 02/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E6=AD=A4=E6=99=BA=E8=83=BD=E4=BD=93=E5=85=A8?= =?UTF-8?q?=E9=83=A8=E8=AE=BE=E5=A4=87=E7=9A=84=E6=9C=80=E5=90=8E=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=97=B6=E9=97=B4=EF=BC=8C=E6=96=B9=E6=B3=95=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E5=92=8Csql=EF=BC=8C=E4=B8=8D=E4=BD=BF=E7=94=A8mysql-?= =?UTF-8?q?plus=E7=9A=84=E6=96=B9=E6=B3=95=EF=BC=8C=E6=98=AF=E4=B8=BA?= =?UTF-8?q?=E4=BA=86=E5=87=8F=E5=B0=91=E6=95=B0=E6=8D=AE=E5=BA=93=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=BC=A0=E8=BE=93=EF=BC=8C=E5=9B=A0=E4=B8=BA=E5=8F=AA?= =?UTF-8?q?=E9=9C=80=E8=A6=81=E6=9C=80=E5=90=8E=E8=BF=9E=E6=8E=A5=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E5=AD=97=E6=AE=B5=20--DeviceDao.java=20=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=AE=9A=E4=B9=89=20--DeviceDao.xml=20sql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/xiaozhi/modules/device/dao/DeviceDao.java | 10 ++++++++++ .../src/main/resources/mapper/device/DeviceDao.xml | 8 ++++++++ 2 files changed, 18 insertions(+) create mode 100644 main/manager-api/src/main/resources/mapper/device/DeviceDao.xml diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java b/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java index 5ede3d6e..abff8ab3 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java @@ -6,6 +6,16 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; import xiaozhi.modules.device.entity.DeviceEntity; +import java.util.Date; +import java.util.List; + @Mapper public interface DeviceDao extends BaseMapper { + /** + * 获取此智能体全部设备的最后连接时间 + * @param agentId 智能体id + * @return + */ + List getAllLastConnectedAtByAgentId(String agentId); + } \ No newline at end of file diff --git a/main/manager-api/src/main/resources/mapper/device/DeviceDao.xml b/main/manager-api/src/main/resources/mapper/device/DeviceDao.xml new file mode 100644 index 00000000..09f16760 --- /dev/null +++ b/main/manager-api/src/main/resources/mapper/device/DeviceDao.xml @@ -0,0 +1,8 @@ + + + + + + \ No newline at end of file From 1d627d570d37446b73a8fbefeaeb88d6120bcb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 30 Apr 2025 10:45:49 +0800 Subject: [PATCH 03/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E8=BF=99=E4=B8=AA=E6=99=BA=E8=83=BD=E4=BD=93=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E7=90=86=E7=9A=84=E6=9C=80=E8=BF=91=E7=9A=84=E6=9C=80=E5=90=8E?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=97=B6=E9=97=B4=E5=AE=9A=E4=B9=89=E5=92=8C?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20--DeviceService.java=20=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=20--DeviceServiceImpl.java=20=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/device/service/DeviceService.java | 10 ++++++++++ .../device/service/impl/DeviceServiceImpl.java | 13 +++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/DeviceService.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/DeviceService.java index aff31bff..fb91ff89 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/DeviceService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/DeviceService.java @@ -1,5 +1,6 @@ package xiaozhi.modules.device.service; +import java.util.Date; import java.util.List; import xiaozhi.common.page.PageData; @@ -78,4 +79,13 @@ public interface DeviceService extends BaseService { * @return 激活码 */ String geCodeByDeviceId(String deviceId); + + /** + * 获取这个智能体设备理的最近的最后连接时间 + * @param agentId 智能体id + * @return 返回设备最近的最后连接时间 + */ + Date getLatestLastConnectionTime(String agentId); + + } \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index 8eb23a93..99b40a38 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -1,12 +1,7 @@ package xiaozhi.modules.device.service.impl; import java.time.Instant; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.UUID; +import java.util.*; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -258,6 +253,12 @@ public class DeviceServiceImpl extends BaseServiceImpl return null; } + @Override + public Date getLatestLastConnectionTime(String agentId) { + List list = deviceDao.getAllLastConnectedAtByAgentId(agentId); + return Collections.max(list, Comparator.comparing(Date::getTime)); + } + private String getDeviceCacheKey(String deviceId) { String safeDeviceId = deviceId.replace(":", "_").toLowerCase(); String dataKey = String.format("ota:activation:data:%s", safeDeviceId); From 7d74e853ae0ec674e3b6bc9c20b4a195331e242e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 30 Apr 2025 10:46:34 +0800 Subject: [PATCH 04/24] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E6=9C=80=E8=BF=91=E7=9A=84=E6=9C=80=E5=90=8E=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=97=B6=E9=97=B4=E4=B8=BA=E7=A9=BA=E7=9A=84bug=20--A?= =?UTF-8?q?gentServiceImpl.java=20=E4=BF=AE=E5=A4=8Dbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/agent/service/impl/AgentServiceImpl.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java index ff284ef0..ff883a57 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java @@ -21,6 +21,7 @@ import xiaozhi.modules.agent.dao.AgentDao; import xiaozhi.modules.agent.dto.AgentDTO; import xiaozhi.modules.agent.entity.AgentEntity; import xiaozhi.modules.agent.service.AgentService; +import xiaozhi.modules.device.service.DeviceService; import xiaozhi.modules.model.service.ModelConfigService; import xiaozhi.modules.timbre.service.TimbreService; @@ -31,6 +32,9 @@ public class AgentServiceImpl extends BaseServiceImpl imp @Autowired private TimbreService timbreModelService; + @Autowired + private DeviceService deviceService; + @Autowired private ModelConfigService modelConfigService; @@ -104,6 +108,11 @@ public class AgentServiceImpl extends BaseServiceImpl imp // 获取设备数量 dto.setDeviceCount(getDeviceCountByAgentId(agent.getId())); + // 获取智能体最近的最后连接时长 + dto.setLastConnectedAt(deviceService.getLatestLastConnectionTime(agent.getId())); + + + return dto; }).collect(Collectors.toList()); } From 6f7ff9f858bee0c154aa00ed552bed5636c96a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 30 Apr 2025 11:05:20 +0800 Subject: [PATCH 05/24] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E8=BF=99=E4=B8=AA=E6=99=BA=E8=83=BD=E4=BD=93=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E7=90=86=E7=9A=84=E6=9C=80=E8=BF=91=E7=9A=84=E6=9C=80=E5=90=8E?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=97=B6=E9=97=B4=E6=96=B9=E6=B3=95=EF=BC=8C?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=97=B6=E9=97=B4=E5=92=8C=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E6=95=B0=E9=87=8F=20--DeviceServiceImpl.java=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../device/service/impl/DeviceServiceImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index 99b40a38..0475f2e9 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -255,8 +255,19 @@ public class DeviceServiceImpl extends BaseServiceImpl @Override public Date getLatestLastConnectionTime(String agentId) { + // 查询是否有缓存时间，有则返回 + Date cachedDate = (Date) redisUtils.get(RedisKeys.getAgentDeviceLastConnectedAtById(agentId)); + if (cachedDate != null) { + return cachedDate; + } List list = deviceDao.getAllLastConnectedAtByAgentId(agentId); - return Collections.max(list, Comparator.comparing(Date::getTime)); + Date maxDate = Collections.max(list, Comparator.comparing(Date::getTime)); + // 将结果存入Redis, 存储设备最近最后连接时间，和设备数量 + redisUtils.set(RedisKeys.getAgentDeviceCountById(agentId), list.size(), 60); + if (maxDate != null) { + redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), maxDate, 60); + } + return maxDate; } private String getDeviceCacheKey(String deviceId) { From 09c17b2dc5325754bf579af8d8f30ab83dffa7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 30 Apr 2025 11:07:53 +0800 Subject: [PATCH 06/24] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=96=B9=E6=B3=95?= =?UTF-8?q?=EF=BC=8C=E6=8A=8A=E8=8E=B7=E5=8F=96=E6=9C=80=E8=BF=91=E6=9C=80?= =?UTF-8?q?=E5=90=8E=E6=97=B6=E9=97=B4=E8=8E=B7=E5=8F=96=EF=BC=8C=E6=94=BE?= =?UTF-8?q?=E5=9C=A8=E8=8E=B7=E5=8F=96=E8=AE=BE=E5=A4=87=E6=95=B0=E9=87=8F?= =?UTF-8?q?=E5=89=8D=E9=9D=A2=E5=89=8D=EF=BC=8C=E5=8F=AF=E4=BB=A5=E5=87=8F?= =?UTF-8?q?=E5=B0=91=E4=B8=80=E6=AC=A1sql=E6=9F=A5=E8=AF=A2=EF=BC=8C?= =?UTF-8?q?=E5=9B=A0=E4=B8=BA=E5=9C=A8=E8=8E=B7=E5=8F=96=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E5=80=99=EF=BC=8C=E5=B7=B2=E7=BB=8F=E9=A1=BA?= =?UTF-8?q?=E4=BE=BF=E7=BC=93=E5=AD=98=E7=9A=84=E8=AE=BE=E5=A4=87=E6=95=B0?= =?UTF-8?q?=E9=87=8F=E4=BA=86=20--AgentServiceImpl.java=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=EF=BC=9A=E5=87=8F=E5=B0=91sql=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/agent/service/impl/AgentServiceImpl.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java index ff883a57..c731c2fa 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java @@ -105,14 +105,11 @@ public class AgentServiceImpl extends BaseServiceImpl imp // 获取 TTS 音色名称 dto.setTtsVoiceName(timbreModelService.getTimbreNameById(agent.getTtsVoiceId())); - // 获取设备数量 - dto.setDeviceCount(getDeviceCountByAgentId(agent.getId())); - // 获取智能体最近的最后连接时长 dto.setLastConnectedAt(deviceService.getLatestLastConnectionTime(agent.getId())); - - + // 获取设备数量 + dto.setDeviceCount(getDeviceCountByAgentId(agent.getId())); return dto; }).collect(Collectors.toList()); } From b4944f23ef873b27eb808b23fb97e0e4766d36b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 30 Apr 2025 11:08:59 +0800 Subject: [PATCH 07/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E6=96=B0=E7=9A=84redis:key=20--RedisKeys.java=20?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E8=AE=BE=E5=A4=87=E6=9C=80=E8=BF=91=E6=9C=80?= =?UTF-8?q?=E4=B9=85=E6=97=B6=E9=97=B4=E7=9A=84key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/xiaozhi/common/redis/RedisKeys.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java b/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java index 5d3b4f32..5e5ec309 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java +++ b/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java @@ -62,6 +62,13 @@ public class RedisKeys { return "agent:device:count:" + id; } + /** + * 获取智能体最后连接时间缓存key + */ + public static String getAgentDeviceLastConnectedAtById(String id) { + return "agent:device:lastConnected:" + id; + } + /** * 获取系统配置缓存key */ From bde260b33077cd68df35113d6830883805a26609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E5=87=A4=E7=A7=91=E6=8A=80?= Date: Tue, 6 May 2025 14:57:29 +0800 Subject: [PATCH 08/24] =?UTF-8?q?PCM=E9=9F=B3=E9=A2=91=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 17 ++++++++----- .../xiaozhi-server/core/handle/helloHandle.py | 10 +++++++- main/xiaozhi-server/core/handle/textHandle.py | 2 +- .../core/providers/asr/aliyun.py | 15 +---------- .../core/providers/asr/baidu.py | 16 ------------ .../xiaozhi-server/core/providers/asr/base.py | 22 +++++++++++++++- .../core/providers/asr/doubao.py | 15 ----------- .../core/providers/asr/fun_local.py | 23 ++++------------- .../core/providers/asr/fun_server.py | 17 ------------- .../core/providers/asr/sherpa_onnx_local.py | 15 ----------- .../core/providers/asr/tencent.py | 16 ------------ .../xiaozhi-server/core/providers/tts/base.py | 25 +++++++++++++------ 12 files changed, 66 insertions(+), 127 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index f08a61ff..0fa8b715 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -131,6 +131,8 @@ class ConnectionHandler: int(self.config.get("close_connection_no_voice_time", 120)) + 60 ) # 在原来第一道关闭的基础上加60秒，进行二道关闭 + self.audio_format = "opus" + async def handle_connection(self, ws): try: # 获取并验证headers @@ -867,7 +869,7 @@ class ConnectionHandler: if future is None: continue text = None - opus_datas, text_index, tts_file = [], 0, None + audio_datas, text_index, tts_file = [], 0, None try: self.logger.bind(tag=TAG).debug("正在处理TTS任务...") tts_timeout = int(self.config.get("tts_timeout", 10)) @@ -885,9 +887,12 @@ class ConnectionHandler: f"TTS生成：文件路径: {tts_file}" ) if os.path.exists(tts_file): - opus_datas, _ = self.tts.audio_to_opus_data(tts_file) + if self.audio_format == "pcm": + audio_datas, _ = self.tts.audio_to_pcm_data(tts_file) + else: + audio_datas, _ = self.tts.audio_to_opus_data(tts_file) # 在这里上报TTS数据（使用文件路径） - enqueue_tts_report(self, 2, text, opus_datas) + enqueue_tts_report(self, 2, text, audio_datas) else: self.logger.bind(tag=TAG).error( f"TTS出错：文件不存在{tts_file}" @@ -898,7 +903,7 @@ class ConnectionHandler: self.logger.bind(tag=TAG).error(f"TTS出错: {e}") if not self.client_abort: # 如果没有中途打断就发送语音 - self.audio_play_queue.put((opus_datas, text, text_index)) + self.audio_play_queue.put((audio_datas, text, text_index)) if ( self.tts.delete_audio_file and tts_file is not None @@ -929,13 +934,13 @@ class ConnectionHandler: text = None try: try: - opus_datas, text, text_index = self.audio_play_queue.get(timeout=1) + audio_datas, text, text_index = self.audio_play_queue.get(timeout=1) except queue.Empty: if self.stop_event.is_set(): break continue future = asyncio.run_coroutine_threadsafe( - sendAudioMessage(self, opus_datas, text, text_index), self.loop + sendAudioMessage(self, audio_datas, text, text_index), self.loop ) future.result() except Exception as e: diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 51782bb9..5da1fa53 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -21,7 +21,15 @@ WAKEUP_CONFIG = { } -async def handleHelloMessage(conn): +async def handleHelloMessage(conn, msg_json): + """处理hello消息""" + audio_params = msg_json.get("audio_params") + if audio_params: + format = audio_params.get("format") + logger.bind(tag=TAG).info(f"客户端音频格式: {format}") + conn.audio_format = format + conn.welcome_msg['audio_params'] = audio_params + await conn.websocket.send(json.dumps(conn.welcome_msg)) diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index d350f78c..33f94cf6 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -22,7 +22,7 @@ async def handleTextMessage(conn, message): await conn.websocket.send(message) return if msg_json["type"] == "hello": - await handleHelloMessage(conn) + await handleHelloMessage(conn, msg_json) elif msg_json["type"] == "abort": await handleAbortMessage(conn) elif msg_json["type"] == "listen": diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index 250d3a66..6ef9cc3d 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -158,20 +158,7 @@ class ASRProvider(ASRProviderBase): request += "&enable_inverse_text_normalization=true" request += "&enable_voice_detection=false" return request - - def decode_opus(self, opus_data: List[bytes], session_id: str) -> List[bytes]: - """将Opus数据解码为PCM""" - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" diff --git a/main/xiaozhi-server/core/providers/asr/baidu.py b/main/xiaozhi-server/core/providers/asr/baidu.py index 94996a59..628c38be 100644 --- a/main/xiaozhi-server/core/providers/asr/baidu.py +++ b/main/xiaozhi-server/core/providers/asr/baidu.py @@ -49,22 +49,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - async def speech_to_text( self, opus_data: List[bytes], session_id: str ) -> Tuple[Optional[str], Optional[str]]: diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 6d9c2c90..61c1b754 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from typing import Optional, Tuple, List - +import opuslib_next from config.logger import setup_logging TAG = __name__ @@ -17,3 +17,23 @@ class ASRProviderBase(ABC): async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" pass + + def set_audio_format(self, format: str) -> None: + """设置音频格式""" + self.audio_format = format + + @staticmethod + def decode_opus(opus_data: List[bytes]) -> bytes: + """将Opus音频数据解码为PCM数据""" + + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return pcm_data \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index ce7e05ee..f9df3b3a 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -226,21 +226,6 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True) return None - @staticmethod - def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - @staticmethod def slice_data(data: bytes, chunk_size: int) -> (list, bool): """ diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 3724ec2e..373d9c6a 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -6,9 +6,7 @@ import io from config.logger import setup_logging from typing import Optional, Tuple, List import uuid -import opuslib_next from core.providers.asr.base import ASRProviderBase - from funasr import AutoModel from funasr.utils.postprocess_utils import rich_transcription_postprocess @@ -64,27 +62,16 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: # 合并所有opus数据包 - pcm_data = self.decode_opus(opus_data, session_id) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data, session_id) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index 501d43bd..2c11b8b3 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -44,23 +44,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - - - async def _receive_responses(self, ws) -> None: ''' Asynchronous generator to receive messages from the WebSocket. diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 3e720210..7e3fb6dd 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -97,21 +97,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]: """ Args: diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index dbdcbe26..bc7c5928 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -45,22 +45,6 @@ class ASRProvider(ASRProviderBase): return file_path - @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index e668459c..11960221 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -53,8 +53,15 @@ class TTSProviderBase(ABC): async def text_to_speak(self, text, output_file): pass + def audio_to_pcm_data(self, audio_file_path): + """音频文件转换为PCM编码""" + return self.audio_to_data(audio_file_path, is_opus=False) + def audio_to_opus_data(self, audio_file_path): """音频文件转换为Opus编码""" + return self.audio_to_data(audio_file_path, is_opus=True) + + def audio_to_data(self, audio_file_path, is_opus=True): # 获取文件后缀名 file_type = os.path.splitext(audio_file_path)[1] if file_type: @@ -80,7 +87,7 @@ class TTSProviderBase(ABC): frame_duration = 60 # 60ms per frame frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame - opus_datas = [] + datas = [] # 按帧处理所有音频数据（包括最后一帧可能补零） for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample # 获取当前帧的二进制数据 @@ -89,12 +96,16 @@ class TTSProviderBase(ABC): # 如果最后一帧不足，补零 if len(chunk) < frame_size * 2: chunk += b"\x00" * (frame_size * 2 - len(chunk)) + + if is_opus: + # 转换为numpy数组处理 + np_frame = np.frombuffer(chunk, dtype=np.int16) + # 编码Opus数据 + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + else: + frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) - # 转换为numpy数组处理 - np_frame = np.frombuffer(chunk, dtype=np.int16) - # 编码Opus数据 - opus_data = encoder.encode(np_frame.tobytes(), frame_size) - opus_datas.append(opus_data) + datas.append(frame_data) - return opus_datas, duration + return datas, duration From 770a198772c573a62e0f4c211a7d258d58306d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=89=91=E9=9B=A8?= <2375294554@qq.com> Date: Wed, 7 May 2025 11:04:14 +0800 Subject: [PATCH 09/24] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E6=9C=80=E5=A4=A7=E7=9A=84=E6=9C=80=E8=BF=91=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E3=80=82=E6=94=B9=E4=B8=BA=E5=9C=A8=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E9=87=8C=E6=8E=92=E5=BA=8F=E5=A5=BD=E5=90=8E?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=E7=BB=99=E7=B3=BB=E7=BB=9F=EF=BC=8C=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9=E4=B8=BA2?= =?UTF-8?q?=E5=88=86=E9=92=9F=20--DeviceDao.java=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E8=BF=94=E5=9B=9E=E5=80=BC=20--DeviceDao.xml?= =?UTF-8?q?=20=E4=BF=AE=E6=94=B9sql=EF=BC=8C=E5=9C=A8=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E9=87=8C=E6=8E=92=E5=BA=8F=E8=BF=94=E5=9B=9E=20--Devi?= =?UTF-8?q?ceServiceImpl.java=20=E4=BF=AE=E6=94=B9=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=B8=BA2=E5=88=86=E9=92=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/xiaozhi/modules/device/dao/DeviceDao.java | 2 +- .../modules/device/service/impl/DeviceServiceImpl.java | 7 ++----- .../src/main/resources/mapper/device/DeviceDao.xml | 6 +++++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java b/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java index abff8ab3..6570b89b 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java @@ -16,6 +16,6 @@ public interface DeviceDao extends BaseMapper { * @param agentId 智能体id * @return */ - List getAllLastConnectedAtByAgentId(String agentId); + Date getAllLastConnectedAtByAgentId(String agentId); } \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index 0475f2e9..59e329fa 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -260,12 +260,9 @@ public class DeviceServiceImpl extends BaseServiceImpl if (cachedDate != null) { return cachedDate; } - List list = deviceDao.getAllLastConnectedAtByAgentId(agentId); - Date maxDate = Collections.max(list, Comparator.comparing(Date::getTime)); - // 将结果存入Redis, 存储设备最近最后连接时间，和设备数量 - redisUtils.set(RedisKeys.getAgentDeviceCountById(agentId), list.size(), 60); + Date maxDate = deviceDao.getAllLastConnectedAtByAgentId(agentId); if (maxDate != null) { - redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), maxDate, 60); + redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), maxDate, 60 * 2); } return maxDate; } diff --git a/main/manager-api/src/main/resources/mapper/device/DeviceDao.xml b/main/manager-api/src/main/resources/mapper/device/DeviceDao.xml index 09f16760..54de47c9 100644 --- a/main/manager-api/src/main/resources/mapper/device/DeviceDao.xml +++ b/main/manager-api/src/main/resources/mapper/device/DeviceDao.xml @@ -3,6 +3,10 @@ \ No newline at end of file From ba86b34a8c5fb2e6897b7f79f0ea671f3d05aa80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=84=E5=87=A4=E7=A7=91=E6=8A=80?= Date: Wed, 7 May 2025 11:34:29 +0800 Subject: [PATCH 10/24] =?UTF-8?q?pcm=E6=A8=A1=E5=BC=8F=EF=BC=8C=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E5=85=B6=E4=BB=96asr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/asr/aliyun.py | 5 ++++- main/xiaozhi-server/core/providers/asr/baidu.py | 5 ++++- main/xiaozhi-server/core/providers/asr/doubao.py | 5 ++++- main/xiaozhi-server/core/providers/asr/fun_server.py | 5 ++++- main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py | 5 ++++- main/xiaozhi-server/core/providers/asr/tencent.py | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index 6ef9cc3d..be5541ee 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -232,7 +232,10 @@ class ASRProvider(ASRProviderBase): file_path = None try: # 解码Opus为PCM - pcm_data = self.decode_opus(opus_data, session_id) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data, session_id) combined_pcm_data = b''.join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/baidu.py b/main/xiaozhi-server/core/providers/asr/baidu.py index 628c38be..58b26375 100644 --- a/main/xiaozhi-server/core/providers/asr/baidu.py +++ b/main/xiaozhi-server/core/providers/asr/baidu.py @@ -65,7 +65,10 @@ class ASRProvider(ASRProviderBase): return None, file_path # 将Opus音频数据解码为PCM - pcm_data = self.decode_opus(opus_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index f9df3b3a..0ca089c9 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -250,7 +250,10 @@ class ASRProvider(ASRProviderBase): file_path = None try: # 合并所有opus数据包 - pcm_data = self.decode_opus(opus_data, session_id) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data, session_id) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index 2c11b8b3..f20d5be7 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -106,7 +106,10 @@ class ASRProvider(ASRProviderBase): :return: Tuple containing recognized text and optional timestamp. ''' file_path = None - pcm_data = self.decode_opus(opus_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 7e3fb6dd..45e22336 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -129,7 +129,10 @@ class ASRProvider(ASRProviderBase): try: # 保存音频文件 start_time = time.time() - pcm_data = self.decode_opus(opus_data, session_id) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data, session_id) file_path = self.save_audio_to_file(pcm_data, session_id) logger.bind(tag=TAG).debug( f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index bc7c5928..2aa7bec5 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -59,7 +59,10 @@ class ASRProvider(ASRProviderBase): return None, file_path # 将Opus音频数据解码为PCM - pcm_data = self.decode_opus(opus_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 From bbc31e01c40c00b6d590cb7320db00e7dbb646d8 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 7 May 2025 14:31:59 +0800 Subject: [PATCH 11/24] =?UTF-8?q?fix:tts=E9=98=9F=E5=88=97=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E5=85=83=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 060de4c9..6836c08e 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -510,7 +510,7 @@ class ConnectionHandler: future = self.executor.submit( self.speak_and_play, segment_text, text_index ) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) self.llm_finish_task = True self.dialogue.put(Message(role="assistant", content="".join(response_message))) @@ -685,7 +685,7 @@ class ConnectionHandler: future = self.executor.submit( self.speak_and_play, segment_text, text_index ) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) # 存储对话内容 if len(response_message) > 0: @@ -747,7 +747,7 @@ class ConnectionHandler: text = result.response self.recode_first_last_text(text, text_index) future = self.executor.submit(self.speak_and_play, text, text_index) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) self.dialogue.put(Message(role="assistant", content=text)) elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result @@ -780,7 +780,7 @@ class ConnectionHandler: text = result.result self.recode_first_last_text(text, text_index) future = self.executor.submit(self.speak_and_play, text, text_index) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) self.dialogue.put(Message(role="assistant", content=text)) else: pass @@ -908,6 +908,8 @@ class ConnectionHandler: if text is None or len(text) <= 0: self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") return None, text, text_index + if text_index>2: + time.sleep(20) tts_file = self.tts.to_tts(text) if tts_file is None: self.logger.bind(tag=TAG).error(f"tts转换失败，{text}") From 4a3ac2cfcd6c09392190f84d417ccde64a7564b2 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 7 May 2025 14:35:41 +0800 Subject: [PATCH 12/24] no message --- main/xiaozhi-server/core/connection.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 6836c08e..56c745fe 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -908,8 +908,6 @@ class ConnectionHandler: if text is None or len(text) <= 0: self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") return None, text, text_index - if text_index>2: - time.sleep(20) tts_file = self.tts.to_tts(text) if tts_file is None: self.logger.bind(tag=TAG).error(f"tts转换失败，{text}") From 44e1f00ffc8e7b89f9a741f21fb1ec54d93e3e83 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 7 May 2025 15:17:34 +0800 Subject: [PATCH 13/24] =?UTF-8?q?fix:=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=EF=BC=8C=E6=96=B9=E4=BE=BF=E6=84=8F=E5=9B=BE=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/plugins_func/functions/play_music.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 0b283c8e..4ea9bb57 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -10,6 +10,7 @@ from pathlib import Path from core.utils import p3 from core.handle.sendAudioHandle import send_stt_message from plugins_func.register import register_function, ToolType, ActionResponse, Action +from core.utils.dialogue import Message TAG = __name__ @@ -214,6 +215,7 @@ async def play_local_music(conn, specific_file=None): return text = _get_random_play_prompt(selected_music) await send_stt_message(conn, text) + conn.dialogue.put(Message(role="assistant", content=text)) conn.tts_first_text_index = 0 conn.tts_last_text_index = 0 From ea5f54e4210ffccdc7fdf8e763912beaa2c6ff3e Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 7 May 2025 18:06:13 +0800 Subject: [PATCH 14/24] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E5=AF=B9=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 124 ++++++++---------- .../xiaozhi-server/core/handle/abortHandle.py | 9 +- .../core/handle/functionHandler.py | 11 +- .../xiaozhi-server/core/handle/helloHandle.py | 3 +- .../core/handle/intentHandler.py | 14 +- main/xiaozhi-server/core/handle/iotHandle.py | 41 +++--- .../core/handle/receiveAudioHandle.py | 12 +- .../core/handle/sendAudioHandle.py | 6 +- main/xiaozhi-server/core/handle/textHandle.py | 8 +- .../core/handle/ttsReportHandle.py | 17 +-- main/xiaozhi-server/core/mcp/manager.py | 23 ++-- main/xiaozhi-server/core/utils/dialogue.py | 27 ++-- main/xiaozhi-server/core/utils/util.py | 48 ++++++- main/xiaozhi-server/core/websocket_server.py | 54 +++----- .../plugins_func/functions/play_music.py | 26 ++-- .../plugins_func/functions/plugin_loader.py | 69 +++++----- 16 files changed, 249 insertions(+), 243 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 060de4c9..b49917ba 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -18,6 +18,8 @@ from core.utils.util import ( get_string_no_punctuation_or_emoji, extract_json_from_string, initialize_modules, + check_vad_update, + check_asr_update, ) from concurrent.futures import ThreadPoolExecutor, TimeoutError from core.handle.sendAudioHandle import sendAudioMessage @@ -52,7 +54,9 @@ class ConnectionHandler: _intent, server=None, ): - self.config = config + self.common_config = config + self.config = copy.deepcopy(config) + self.session_id = str(uuid.uuid4()) self.logger = setup_logging() self.auth = AuthMiddleware(config) self.server = server # 保存server实例的引用 @@ -66,7 +70,6 @@ class ConnectionHandler: self.device_id = None self.client_ip = None self.client_ip_info = {} - self.session_id = None self.prompt = None self.welcome_msg = None self.max_output_size = 0 @@ -87,8 +90,10 @@ class ConnectionHandler: self.tts_report_thread = None # 依赖的组件 - self.vad = _vad - self.asr = _asr + self.vad = None + self.asr = None + self._asr = _asr + self._vad = _vad self.llm = _llm self.tts = _tts self.memory = _memory @@ -166,7 +171,6 @@ class ConnectionHandler: # 认证通过,继续处理 self.websocket = ws self.device_id = self.headers.get("device-id", None) - self.session_id = str(uuid.uuid4()) # 启动超时检查任务 self.timeout_task = asyncio.create_task(self._check_timeout()) @@ -176,9 +180,9 @@ class ConnectionHandler: await self.websocket.send(json.dumps(self.welcome_msg)) # 获取差异化配置 - private_config = self._initialize_private_config() + self._initialize_private_config() # 异步初始化 - self.executor.submit(self._initialize_components, private_config) + self.executor.submit(self._initialize_components) # tts 消化线程 self.tts_priority_thread = threading.Thread( target=self._tts_priority_thread, daemon=True @@ -233,13 +237,17 @@ class ConnectionHandler: elif isinstance(message, bytes): await handleAudioMessage(self, message) - def _initialize_components(self, private_config): + def _initialize_components(self): """初始化组件""" - if private_config is not None: - self._initialize_models(private_config) - else: - self.prompt = self.config["prompt"] - self.change_system_prompt(self.prompt) + self.prompt = self.config["prompt"] + self.change_system_prompt(self.prompt) + self.logger.bind(tag=TAG).info(f"初始化组件: prompt成功 {self.prompt[:50]}...") + + """初始化本地组件""" + if self.vad is None: + self.vad = self._vad + if self.asr is None: + self.asr = self._asr """加载记忆""" self._initialize_memory() """加载意图识别""" @@ -286,54 +294,21 @@ class ConnectionHandler: self.logger.bind(tag=TAG).error(f"获取差异化配置失败: {e}") private_config = {} - init_tts = False - if private_config.get("TTS", None) is not None: - init_tts = True - self.config["TTS"] = private_config["TTS"] - self.config["selected_module"]["TTS"] = private_config["selected_module"][ - "TTS" - ] - - try: - modules = initialize_modules( - self.logger, - private_config, - False, - False, - False, - init_tts, - False, - False, - ) - except Exception as e: - self.logger.bind(tag=TAG).error(f"初始化组件失败: {e}") - modules = {} - if modules.get("tts", None) is not None: - self.tts = modules["tts"] - if modules.get("prompt", None) is not None: - self.change_system_prompt(modules["prompt"]) - private_config["prompt"] = None - return private_config - - def _initialize_models(self, private_config): - init_vad, init_asr, init_llm, init_memory, init_intent = ( - False, + init_llm, init_tts, init_memory, init_intent = ( False, False, False, False, ) - if private_config.get("VAD", None) is not None: - init_vad = True - self.config["VAD"] = private_config["VAD"] - self.config["selected_module"]["VAD"] = private_config["selected_module"][ - "VAD" - ] - if private_config.get("ASR", None) is not None: - init_asr = True - self.config["ASR"] = private_config["ASR"] - self.config["selected_module"]["ASR"] = private_config["selected_module"][ - "ASR" + + init_vad = check_vad_update(self.common_config, private_config) + init_asr = check_asr_update(self.common_config, private_config) + + if private_config.get("TTS", None) is not None: + init_tts = True + self.config["TTS"] = private_config["TTS"] + self.config["selected_module"]["TTS"] = private_config["selected_module"][ + "TTS" ] if private_config.get("LLM", None) is not None: init_llm = True @@ -353,8 +328,11 @@ class ConnectionHandler: self.config["selected_module"]["Intent"] = private_config[ "selected_module" ]["Intent"] + if private_config.get("prompt", None) is not None: + self.config["prompt"] = private_config["prompt"] if private_config.get("device_max_output_size", None) is not None: self.max_output_size = int(private_config["device_max_output_size"]) + try: modules = initialize_modules( self.logger, @@ -362,13 +340,15 @@ class ConnectionHandler: init_vad, init_asr, init_llm, - False, + init_tts, init_memory, init_intent, ) except Exception as e: self.logger.bind(tag=TAG).error(f"初始化组件失败: {e}") modules = {} + if modules.get("tts", None) is not None: + self.tts = modules["tts"] if modules.get("vad", None) is not None: self.vad = modules["vad"] if modules.get("asr", None) is not None: @@ -446,10 +426,12 @@ class ConnectionHandler: processed_chars = 0 # 跟踪已处理的字符位置 try: # 使用带记忆的对话 - future = asyncio.run_coroutine_threadsafe( - self.memory.query_memory(query), self.loop - ) - memory_str = future.result() + memory_str = None + if self.memory is not None: + future = asyncio.run_coroutine_threadsafe( + self.memory.query_memory(query), self.loop + ) + memory_str = future.result() self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}") llm_responses = self.llm.response( @@ -510,7 +492,7 @@ class ConnectionHandler: future = self.executor.submit( self.speak_and_play, segment_text, text_index ) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) self.llm_finish_task = True self.dialogue.put(Message(role="assistant", content="".join(response_message))) @@ -537,10 +519,12 @@ class ConnectionHandler: start_time = time.time() # 使用带记忆的对话 - future = asyncio.run_coroutine_threadsafe( - self.memory.query_memory(query), self.loop - ) - memory_str = future.result() + memory_str = None + if self.memory is not None: + future = asyncio.run_coroutine_threadsafe( + self.memory.query_memory(query), self.loop + ) + memory_str = future.result() # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") @@ -685,7 +669,7 @@ class ConnectionHandler: future = self.executor.submit( self.speak_and_play, segment_text, text_index ) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) # 存储对话内容 if len(response_message) > 0: @@ -747,7 +731,7 @@ class ConnectionHandler: text = result.response self.recode_first_last_text(text, text_index) future = self.executor.submit(self.speak_and_play, text, text_index) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) self.dialogue.put(Message(role="assistant", content=text)) elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result @@ -780,7 +764,7 @@ class ConnectionHandler: text = result.result self.recode_first_last_text(text, text_index) future = self.executor.submit(self.speak_and_play, text, text_index) - self.tts_queue.put(future) + self.tts_queue.put((future, text_index)) self.dialogue.put(Message(role="assistant", content=text)) else: pass @@ -801,7 +785,7 @@ class ConnectionHandler: if future is None: continue text = None - opus_datas, tts_file = [], None + opus_datas, tts_file = [], None try: self.logger.bind(tag=TAG).debug("正在处理TTS任务...") tts_timeout = int(self.config.get("tts_timeout", 10)) diff --git a/main/xiaozhi-server/core/handle/abortHandle.py b/main/xiaozhi-server/core/handle/abortHandle.py index 35385b89..fe271511 100644 --- a/main/xiaozhi-server/core/handle/abortHandle.py +++ b/main/xiaozhi-server/core/handle/abortHandle.py @@ -3,15 +3,16 @@ import queue from config.logger import setup_logging TAG = __name__ -logger = setup_logging() async def handleAbortMessage(conn): - logger.bind(tag=TAG).info("Abort message received") + conn.logger.bind(tag=TAG).info("Abort message received") # 设置成打断状态，会自动打断llm、tts任务 conn.client_abort = True conn.clear_queues() # 打断客户端说话状态 - await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})) + await conn.websocket.send( + json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}) + ) conn.clearSpeakStatus() - logger.bind(tag=TAG).info("Abort message received-end") + conn.logger.bind(tag=TAG).info("Abort message received-end") diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index 77d22bb9..b09a369c 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -4,7 +4,6 @@ from plugins_func.register import FunctionRegistry, ActionResponse, Action, Tool from plugins_func.functions.hass_init import append_devices_to_prompt TAG = __name__ -logger = setup_logging() class FunctionHandler: @@ -40,7 +39,9 @@ class FunctionHandler: for func in self.functions_desc: func_names.append(func["function"]["name"]) # 打印当前支持的函数列表 - logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}") + self.conn.logger.bind(tag=TAG, session_id=self.conn.session_id).info( + f"当前支持的函数列表: {func_names}" + ) return func_names def get_functions(self): @@ -79,7 +80,9 @@ class FunctionHandler: func = funcItem.func arguments = function_call_data["arguments"] arguments = json.loads(arguments) if arguments else {} - logger.bind(tag=TAG).debug(f"调用函数: {function_name}, 参数: {arguments}") + self.conn.logger.bind(tag=TAG).debug( + f"调用函数: {function_name}, 参数: {arguments}" + ) if ( funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL @@ -94,6 +97,6 @@ class FunctionHandler: action=Action.NOTFOUND, result="没有找到对应的函数", response="" ) except Exception as e: - logger.bind(tag=TAG).error(f"处理function call错误: {e}") + self.conn.logger.bind(tag=TAG).error(f"处理function call错误: {e}") return None diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index e84a798f..b23c2f0a 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -9,7 +9,6 @@ import random import time TAG = __name__ -logger = setup_logging() WAKEUP_CONFIG = { "dir": "config/assets/", @@ -75,7 +74,7 @@ async def wakeupWordsResponse(conn): await asyncio.sleep(1) wait_max_time -= 1 if wait_max_time <= 0: - logger.bind(tag=TAG).error("连接对象没有llm") + conn.logger.bind(tag=TAG).error("连接对象没有llm") return """唤醒词响应""" diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 90d8e1d5..0174bc43 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -5,10 +5,8 @@ from core.handle.sendAudioHandle import send_stt_message from core.handle.helloHandle import checkWakeupWords from core.utils.util import remove_punctuation_and_length from core.utils.dialogue import Message -from loguru import logger TAG = __name__ -logger = setup_logging() async def handle_user_intent(conn, text): @@ -36,7 +34,7 @@ async def check_direct_exit(conn, text): cmd_exit = conn.cmd_exit for cmd in cmd_exit: if text == cmd: - logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}") + conn.logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}") await send_stt_message(conn, text) await conn.close() return True @@ -46,7 +44,7 @@ async def check_direct_exit(conn, text): async def analyze_intent_with_llm(conn, text): """使用LLM分析用户意图""" if not hasattr(conn, "intent") or not conn.intent: - logger.bind(tag=TAG).warning("意图识别服务未初始化") + conn.logger.bind(tag=TAG).warning("意图识别服务未初始化") return None # 对话历史记录 @@ -55,7 +53,7 @@ async def analyze_intent_with_llm(conn, text): intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text) return intent_result except Exception as e: - logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}") + conn.logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}") return None @@ -69,7 +67,7 @@ async def process_intent_result(conn, intent_result, original_text): # 检查是否有function_call if "function_call" in intent_data: # 直接从意图识别获取了function_call - logger.bind(tag=TAG).debug( + conn.logger.bind(tag=TAG).debug( f"检测到function_call格式的意图结果: {intent_data['function_call']['name']}" ) function_name = intent_data["function_call"]["name"] @@ -118,7 +116,7 @@ async def process_intent_result(conn, intent_result, original_text): conn.speak_and_play, text, text_index ) conn.llm_finish_task = True - conn.tts_queue.put(future) + conn.tts_queue.put((future, text_index)) conn.dialogue.put(Message(role="assistant", content=text)) # 将函数执行放在线程池中 @@ -126,7 +124,7 @@ async def process_intent_result(conn, intent_result, original_text): return True return False except json.JSONDecodeError as e: - logger.bind(tag=TAG).error(f"处理意图结果时出错: {e}") + conn.logger.bind(tag=TAG).error(f"处理意图结果时出错: {e}") return False diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index c6d9f4d1..12432102 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -10,7 +10,6 @@ from plugins_func.register import ( ) TAG = __name__ -logger = setup_logging() def wrap_async_function(async_func): @@ -21,7 +20,7 @@ def wrap_async_function(async_func): # 获取连接对象（第一个参数） conn = args[0] if not hasattr(conn, "loop"): - logger.bind(tag=TAG).error("Connection对象没有loop属性") + conn.logger.bind(tag=TAG).error("Connection对象没有loop属性") return ActionResponse( Action.ERROR, "Connection对象没有loop属性", @@ -35,7 +34,7 @@ def wrap_async_function(async_func): # 等待结果返回 return future.result() except Exception as e: - logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}") + conn.logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}") return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}") return wrapper @@ -57,7 +56,7 @@ def create_iot_function(device_name, method_name, method_info): response_failure = "操作失败" # 打印响应参数 - logger.bind(tag=TAG).debug( + conn.logger.bind(tag=TAG).debug( f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'" ) @@ -86,7 +85,9 @@ def create_iot_function(device_name, method_name, method_info): return ActionResponse(Action.RESPONSE, result, response) except Exception as e: - logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}") + conn.logger.bind(tag=TAG).error( + f"执行{device_name}的{method_name}操作失败: {e}" + ) # 操作失败时使用大模型提供的失败响应 response = response_failure @@ -104,7 +105,7 @@ def create_iot_query_function(device_name, prop_name, prop_info): async def iot_query_function(conn, response_success=None, response_failure=None): try: # 打印响应参数 - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'" ) @@ -122,7 +123,9 @@ def create_iot_query_function(device_name, prop_name, prop_info): return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response) except Exception as e: - logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}") + conn.logger.bind(tag=TAG).error( + f"查询{device_name}的{prop_name}时出错: {e}" + ) # 查询出错时使用大模型提供的失败响应 response = response_failure @@ -280,7 +283,7 @@ async def handleIotDescriptors(conn, descriptors): await asyncio.sleep(1) wait_max_time -= 1 if wait_max_time <= 0: - logger.bind(tag=TAG).debug("连接对象没有func_handler") + conn.logger.bind(tag=TAG).debug("连接对象没有func_handler") return """处理物联网描述""" functions_changed = False @@ -323,7 +326,7 @@ async def handleIotDescriptors(conn, descriptors): if hasattr(conn, "func_handler"): for func_name in device_functions: conn.func_handler.function_registry.register_function(func_name) - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"注册IOT函数到function handler: {func_name}" ) functions_changed = True @@ -332,8 +335,8 @@ async def handleIotDescriptors(conn, descriptors): if functions_changed and hasattr(conn, "func_handler"): conn.func_handler.upload_functions_desc() func_names = conn.func_handler.current_support_functions() - logger.bind(tag=TAG).info(f"设备类型: {type_id}") - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info(f"设备类型: {type_id}") + conn.logger.bind(tag=TAG).info( f"更新function描述列表完成，当前支持的函数: {func_names}" ) @@ -347,13 +350,13 @@ async def handleIotStatus(conn, states): for k, v in state["state"].items(): if property_item["name"] == k: if type(v) != type(property_item["value"]): - logger.bind(tag=TAG).error( + conn.logger.bind(tag=TAG).error( f"属性{property_item['name']}的值类型不匹配" ) break else: property_item["value"] = v - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"物联网状态更新: {key} , {property_item['name']} = {v}" ) break @@ -367,7 +370,7 @@ async def get_iot_status(conn, name, property_name): for property_item in value.properties: if property_item["name"] == property_name: return property_item["value"] - logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") + conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") return None @@ -378,16 +381,16 @@ async def set_iot_status(conn, name, property_name, value): for property_item in iot_descriptor.properties: if property_item["name"] == property_name: if type(value) != type(property_item["value"]): - logger.bind(tag=TAG).error( + conn.logger.bind(tag=TAG).error( f"属性{property_item['name']}的值类型不匹配" ) return property_item["value"] = value - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"物联网状态更新: {name} , {property_name} = {value}" ) return - logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") + conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") async def send_iot_conn(conn, name, method_name, parameters): @@ -409,6 +412,6 @@ async def send_iot_conn(conn, name, method_name, parameters): command["parameters"] = parameters send_message = json.dumps({"type": "iot", "commands": [command]}) await conn.websocket.send(send_message) - logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}") + conn.logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}") return - logger.bind(tag=TAG).error(f"未找到方法{method_name}") + conn.logger.bind(tag=TAG).error(f"未找到方法{method_name}") diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index ff6aaad5..383616c3 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -1,4 +1,3 @@ -from config.logger import setup_logging import time import copy from core.utils.util import remove_punctuation_and_length @@ -9,12 +8,13 @@ from core.handle.ttsReportHandle import enqueue_tts_report from core.providers.tts.base import audio_to_opus_data TAG = __name__ -logger = setup_logging() async def handleAudioMessage(conn, audio): + if conn.vad is None: + return if not conn.asr_server_receive: - logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收") + conn.logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收") return if conn.client_listen_mode == "auto": have_voice = conn.vad.is_vad(conn, audio) @@ -40,7 +40,7 @@ async def handleAudioMessage(conn, audio): conn.asr_server_receive = True else: text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) - logger.bind(tag=TAG).info(f"识别文本: {text}") + conn.logger.bind(tag=TAG).info(f"识别文本: {text}") text_len, _ = remove_punctuation_and_length(text) if text_len > 0: # 使用自定义模块进行上报 @@ -120,7 +120,7 @@ async def check_bind_device(conn): if conn.bind_code: # 确保bind_code是6位数字 if len(conn.bind_code) != 6: - logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}") + conn.logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}") text = "绑定码格式错误，请检查配置。" await send_stt_message(conn, text) return @@ -144,7 +144,7 @@ async def check_bind_device(conn): num_packets, _ = audio_to_opus_data(num_path) conn.audio_play_queue.put((num_packets, None, i + 1)) except Exception as e: - logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") + conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") continue else: text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。" diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index b9148238..78fe743c 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -1,11 +1,9 @@ -from config.logger import setup_logging import json import asyncio import time from core.utils.util import get_string_no_punctuation_or_emoji, analyze_emotion TAG = __name__ -logger = setup_logging() emoji_map = { "neutral": "😶", @@ -49,10 +47,10 @@ async def sendAudioMessage(conn, audios, text, text_index=0): ) if text_index == conn.tts_first_text_index: - logger.bind(tag=TAG).info(f"发送第一段语音: {text}") + conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}") await send_tts_message(conn, "sentence_start", text) - is_first_audio = (text_index == conn.tts_first_text_index) + is_first_audio = text_index == conn.tts_first_text_index await sendAudio(conn, audios, pre_buffer=is_first_audio) await send_tts_message(conn, "sentence_end", text) diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index 0bded15b..f446edf8 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -1,4 +1,3 @@ -from config.logger import setup_logging import json from core.handle.abortHandle import handleAbortMessage from core.handle.helloHandle import handleHelloMessage @@ -10,12 +9,11 @@ from core.handle.ttsReportHandle import enqueue_tts_report import asyncio TAG = __name__ -logger = setup_logging() async def handleTextMessage(conn, message): """处理文本消息""" - logger.bind(tag=TAG).info(f"收到文本消息：{message}") + conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}") try: msg_json = json.loads(message) if isinstance(msg_json, int): @@ -28,7 +26,9 @@ async def handleTextMessage(conn, message): elif msg_json["type"] == "listen": if "mode" in msg_json: conn.client_listen_mode = msg_json["mode"] - logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}") + conn.logger.bind(tag=TAG).debug( + f"客户端拾音模式：{conn.client_listen_mode}" + ) if msg_json["state"] == "start": conn.client_have_voice = True conn.client_voice_stop = False diff --git a/main/xiaozhi-server/core/handle/ttsReportHandle.py b/main/xiaozhi-server/core/handle/ttsReportHandle.py index 0935b9de..ae8928ce 100644 --- a/main/xiaozhi-server/core/handle/ttsReportHandle.py +++ b/main/xiaozhi-server/core/handle/ttsReportHandle.py @@ -9,16 +9,11 @@ TTS上报功能已集成到ConnectionHandler类中。 具体实现请参考core/connection.py中的相关代码。 """ -import os -import uuid -import wave import opuslib_next -from config.logger import setup_logging from config.manage_api_client import report TAG = __name__ -logger = setup_logging() def report_tts(conn, type, text, opus_data): @@ -32,7 +27,7 @@ def report_tts(conn, type, text, opus_data): """ try: if opus_data: - audio_data = opus_to_wav(opus_data) + audio_data = opus_to_wav(conn, opus_data) else: audio_data = None # 执行上报 @@ -44,10 +39,10 @@ def report_tts(conn, type, text, opus_data): audio=audio_data, ) except Exception as e: - logger.bind(tag=TAG).error(f"TTS上报失败: {e}") + conn.logger.bind(tag=TAG).error(f"TTS上报失败: {e}") -def opus_to_wav(opus_data): +def opus_to_wav(conn, opus_data): """将Opus数据转换为WAV格式的字节流 Args: @@ -65,7 +60,7 @@ def opus_to_wav(opus_data): pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms pcm_data.append(pcm_frame) except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) if not pcm_data: raise ValueError("没有有效的PCM数据") @@ -108,8 +103,8 @@ def enqueue_tts_report(conn, type, text, opus_data): # 使用连接对象的队列，传入文本和二进制数据而非文件路径 conn.tts_report_queue.put((type, text, opus_data)) - logger.bind(tag=TAG).debug( + conn.logger.bind(tag=TAG).debug( f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} " ) except Exception as e: - logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}") + conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}") diff --git a/main/xiaozhi-server/core/mcp/manager.py b/main/xiaozhi-server/core/mcp/manager.py index cc72228a..dc2f9de9 100644 --- a/main/xiaozhi-server/core/mcp/manager.py +++ b/main/xiaozhi-server/core/mcp/manager.py @@ -1,9 +1,9 @@ """MCP服务管理器""" + import asyncio import os, json from typing import Dict, Any, List from .MCPClient import MCPClient -from config.logger import setup_logging from plugins_func.register import register_function, ToolType from config.config_loader import get_project_dir @@ -18,11 +18,10 @@ class MCPManager: 初始化MCP管理器 """ self.conn = conn - self.logger = setup_logging() self.config_path = get_project_dir() + "data/.mcp_server_settings.json" if os.path.exists(self.config_path) == False: self.config_path = "" - self.logger.bind(tag=TAG).warning( + self.conn.logger.bind(tag=TAG).warning( f"请检查mcp服务配置文件：data/.mcp_server_settings.json" ) self.client: Dict[str, MCPClient] = {} @@ -41,7 +40,7 @@ class MCPManager: config = json.load(f) return config.get("mcpServers", {}) except Exception as e: - self.logger.bind(tag=TAG).error( + self.conn.logger.bind(tag=TAG).error( f"Error loading MCP config from {self.config_path}: {e}" ) return {} @@ -51,7 +50,7 @@ class MCPManager: config = self.load_config() for name, srv_config in config.items(): if not srv_config.get("command") and not srv_config.get("url"): - self.logger.bind(tag=TAG).warning( + self.conn.logger.bind(tag=TAG).warning( f"Skipping server {name}: neither command nor url specified" ) continue @@ -60,7 +59,7 @@ class MCPManager: client = MCPClient(srv_config) await client.initialize() self.client[name] = client - self.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}") + self.conn.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}") client_tools = client.get_available_tools() self.tools.extend(client_tools) for tool in client_tools: @@ -73,7 +72,7 @@ class MCPManager: ) except Exception as e: - self.logger.bind(tag=TAG).error( + self.conn.logger.bind(tag=TAG).error( f"Failed to initialize MCP server {name}: {e}" ) self.conn.func_handler.upload_functions_desc() @@ -94,8 +93,8 @@ class MCPManager: """ for tool in self.tools: if ( - tool.get("function") != None - and tool["function"].get("name") == tool_name + tool.get("function") != None + and tool["function"].get("name") == tool_name ): return True return False @@ -110,7 +109,7 @@ class MCPManager: Raises: ValueError: 工具未找到时抛出 """ - self.logger.bind(tag=TAG).info( + self.conn.logger.bind(tag=TAG).info( f"Executing tool {tool_name} with arguments: {arguments}" ) for client in self.client.values(): @@ -124,9 +123,9 @@ class MCPManager: for name, client in list(self.client.items()): try: await asyncio.wait_for(client.cleanup(), timeout=20) - self.logger.bind(tag=TAG).info(f"MCP client closed: {name}") + self.conn.logger.bind(tag=TAG).info(f"MCP client closed: {name}") except (asyncio.TimeoutError, Exception) as e: - self.logger.bind(tag=TAG).error( + self.conn.logger.bind(tag=TAG).error( f"Error closing MCP client {name}: {e}" ) self.client.clear() diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index d4e66bb6..8b79a35a 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -4,7 +4,14 @@ from datetime import datetime class Message: - def __init__(self, role: str, content: str = None, uniq_id: str = None, tool_calls = None, tool_call_id=None): + def __init__( + self, + role: str, + content: str = None, + uniq_id: str = None, + tool_calls=None, + tool_call_id=None, + ): self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4()) self.role = role self.content = content @@ -16,7 +23,7 @@ class Dialogue: def __init__(self): self.dialogue: List[Message] = [] # 获取当前时间 - self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + self.current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") def put(self, message: Message): self.dialogue.append(message) @@ -25,7 +32,9 @@ class Dialogue: if m.tool_calls is not None: dialogue.append({"role": m.role, "tool_calls": m.tool_calls}) elif m.role == "tool": - dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content}) + dialogue.append( + {"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content} + ) else: dialogue.append({"role": m.role, "content": m.content}) @@ -44,23 +53,23 @@ class Dialogue: else: self.put(Message(role="system", content=new_content)) - def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]: + def get_llm_dialogue_with_memory( + self, memory_str: str = None + ) -> List[Dict[str, str]]: if memory_str is None or len(memory_str) == 0: return self.get_llm_dialogue() - + # 构建带记忆的对话 dialogue = [] - + # 添加系统提示和记忆 system_message = next( (msg for msg in self.dialogue if msg.role == "system"), None ) - if system_message: enhanced_system_prompt = ( - f"{system_message.content}\n\n" - f"相关记忆：\n{memory_str}" + f"{system_message.content}\n\n" f"相关记忆：\n{memory_str}" ) dialogue.append({"role": "system", "content": enhanced_system_prompt}) diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 6e5b1028..b75a24da 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -350,12 +350,6 @@ def initialize_modules( str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"), ) logger.bind(tag=TAG).info(f"初始化组件: asr成功 {select_asr_module}") - - # 初始化自定义prompt - if config.get("prompt", None) is not None: - modules["prompt"] = config["prompt"] - logger.bind(tag=TAG).info(f"初始化组件: prompt成功 {modules['prompt'][:50]}...") - return modules @@ -913,3 +907,45 @@ def audio_to_opus_data(audio_file_path): opus_datas.append(opus_data) return opus_datas, duration + + +def check_vad_update(before_config, new_config): + if new_config["selected_module"].get("VAD") is None: + return False + update_vad = False + current_vad_module = before_config["selected_module"]["VAD"] + new_vad_module = new_config["selected_module"]["VAD"] + current_vad_type = ( + current_vad_module + if "type" not in before_config["VAD"][current_vad_module] + else before_config["VAD"][current_vad_module]["type"] + ) + new_vad_type = ( + new_vad_module + if "type" not in new_config["VAD"][new_vad_module] + else new_config["VAD"][new_vad_module]["type"] + ) + print(f"前vad:{current_vad_type}，后vad:{new_vad_type}") + update_vad = current_vad_type != new_vad_type + return update_vad + + +def check_asr_update(before_config, new_config): + if new_config["selected_module"].get("ASR") is None: + return False + update_asr = False + current_asr_module = before_config["selected_module"]["ASR"] + new_asr_module = new_config["selected_module"]["ASR"] + current_asr_type = ( + current_asr_module + if "type" not in before_config["ASR"][current_asr_module] + else before_config["ASR"][current_asr_module]["type"] + ) + new_asr_type = ( + new_asr_module + if "type" not in new_config["ASR"][new_asr_module] + else new_config["ASR"][new_asr_module]["type"] + ) + print(f"前asr:{current_asr_type}，后asr:{new_asr_type}") + update_asr = current_asr_type != new_asr_type + return update_asr diff --git a/main/xiaozhi-server/core/websocket_server.py b/main/xiaozhi-server/core/websocket_server.py index 9e84aaec..441fd4e1 100644 --- a/main/xiaozhi-server/core/websocket_server.py +++ b/main/xiaozhi-server/core/websocket_server.py @@ -2,7 +2,7 @@ import asyncio import websockets from config.logger import setup_logging from core.connection import ConnectionHandler -from core.utils.util import initialize_modules +from core.utils.util import initialize_modules, check_vad_update, check_asr_update from config.config_loader import get_config_from_api TAG = __name__ @@ -84,38 +84,8 @@ class WebSocketServer: return False # 检查 VAD 和 ASR 类型是否需要更新 - update_vad = False - update_asr = False - - # 获取当前和新的 VAD 类型 - current_vad_module = self.config["selected_module"]["VAD"] - new_vad_module = new_config["selected_module"]["VAD"] - current_vad_type = ( - current_vad_module - if "type" not in self.config["VAD"][current_vad_module] - else self.config["VAD"][current_vad_module]["type"] - ) - new_vad_type = ( - new_vad_module - if "type" not in new_config["VAD"][new_vad_module] - else new_config["VAD"][new_vad_module]["type"] - ) - update_vad = current_vad_type != new_vad_type - - # 获取当前和新的 ASR 类型 - current_asr_module = self.config["selected_module"]["ASR"] - new_asr_module = new_config["selected_module"]["ASR"] - current_asr_type = ( - current_asr_module - if "type" not in self.config["ASR"][current_asr_module] - else self.config["ASR"][current_asr_module]["type"] - ) - new_asr_type = ( - new_asr_module - if "type" not in new_config["ASR"][new_asr_module] - else new_config["ASR"][new_asr_module]["type"] - ) - update_asr = current_asr_type != new_asr_type + update_vad = check_vad_update(self.config, new_config) + update_asr = check_asr_update(self.config, new_config) # 更新配置 self.config = new_config @@ -132,12 +102,18 @@ class WebSocketServer: ) # 更新组件实例 - self._vad = modules["vad"] if "vad" in modules else None - self._asr = modules["asr"] if "asr" in modules else None - self._tts = modules["tts"] if "tts" in modules else None - self._llm = modules["llm"] if "llm" in modules else None - self._intent = modules["intent"] if "intent" in modules else None - self._memory = modules["memory"] if "memory" in modules else None + if "vad" in modules: + self._vad = modules["vad"] + if "asr" in modules: + self._asr = modules["asr"] + if "tts" in modules: + self._tts = modules["tts"] + if "llm" in modules: + self._llm = modules["llm"] + if "intent" in modules: + self._intent = modules["intent"] + if "memory" in modules: + self._memory = modules["memory"] return True except Exception as e: diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 0b283c8e..b9fc3aa9 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -11,9 +11,7 @@ from core.utils import p3 from core.handle.sendAudioHandle import send_stt_message from plugins_func.register import register_function, ToolType, ActionResponse, Action - TAG = __name__ -logger = setup_logging() MUSIC_CACHE = {} @@ -45,7 +43,7 @@ def play_music(conn, song_name: str): # 检查事件循环状态 if not conn.loop.is_running(): - logger.bind(tag=TAG).error("事件循环未运行，无法提交任务") + conn.logger.bind(tag=TAG).error("事件循环未运行，无法提交任务") return ActionResponse( action=Action.RESPONSE, result="系统繁忙", response="请稍后再试" ) @@ -59,9 +57,9 @@ def play_music(conn, song_name: str): def handle_done(f): try: f.result() # 可在此处理成功逻辑 - logger.bind(tag=TAG).info("播放完成") + conn.logger.bind(tag=TAG).info("播放完成") except Exception as e: - logger.bind(tag=TAG).error(f"播放失败: {e}") + conn.logger.bind(tag=TAG).error(f"播放失败: {e}") future.add_done_callback(handle_done) @@ -69,7 +67,7 @@ def play_music(conn, song_name: str): action=Action.NONE, result="指令已接收", response="正在为您播放音乐" ) except Exception as e: - logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}") + conn.logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}") return ActionResponse( action=Action.RESPONSE, result=str(e), response="播放音乐时出错了" ) @@ -150,7 +148,7 @@ async def handle_music_command(conn, text): """处理音乐播放指令""" clean_text = re.sub(r"[^\w\s]", "", text).strip() - logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}") + conn.logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}") # 尝试匹配具体歌名 if os.path.exists(MUSIC_CACHE["music_dir"]): @@ -165,7 +163,7 @@ async def handle_music_command(conn, text): if potential_song: best_match = _find_best_match(potential_song, MUSIC_CACHE["music_files"]) if best_match: - logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}") + conn.logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}") await play_local_music(conn, specific_file=best_match) return True # 检查是否是通用播放音乐命令 @@ -195,7 +193,9 @@ async def play_local_music(conn, specific_file=None): """播放本地音乐文件""" try: if not os.path.exists(MUSIC_CACHE["music_dir"]): - logger.bind(tag=TAG).error(f"音乐目录不存在: " + MUSIC_CACHE["music_dir"]) + conn.logger.bind(tag=TAG).error( + f"音乐目录不存在: " + MUSIC_CACHE["music_dir"] + ) return # 确保路径正确性 @@ -204,13 +204,13 @@ async def play_local_music(conn, specific_file=None): music_path = os.path.join(MUSIC_CACHE["music_dir"], specific_file) else: if not MUSIC_CACHE["music_files"]: - logger.bind(tag=TAG).error("未找到MP3音乐文件") + conn.logger.bind(tag=TAG).error("未找到MP3音乐文件") return selected_music = random.choice(MUSIC_CACHE["music_files"]) music_path = os.path.join(MUSIC_CACHE["music_dir"], selected_music) if not os.path.exists(music_path): - logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}") + conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}") return text = _get_random_play_prompt(selected_music) await send_stt_message(conn, text) @@ -233,5 +233,5 @@ async def play_local_music(conn, specific_file=None): conn.audio_play_queue.put((opus_packets, None, conn.tts_last_text_index)) except Exception as e: - logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") - logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}") + conn.logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") + conn.logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}") diff --git a/main/xiaozhi-server/plugins_func/functions/plugin_loader.py b/main/xiaozhi-server/plugins_func/functions/plugin_loader.py index 4747d997..7041d7a0 100644 --- a/main/xiaozhi-server/plugins_func/functions/plugin_loader.py +++ b/main/xiaozhi-server/plugins_func/functions/plugin_loader.py @@ -1,51 +1,56 @@ -from plugins_func.register import register_function,ToolType, ActionResponse, Action -from config.logger import setup_logging - -TAG = __name__ -logger = setup_logging() +from plugins_func.register import register_function, ToolType, ActionResponse, Action plugin_loader_function_desc = { - "type": "function", - "function": { - "name": "plugin_loader", - "description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]", - "parameters": { - "type": "object", - "properties": { - "oper": { - "type": "string", - "description": "load or unload" - }, - "name":{ - "type": "string", - "description": "要加载或卸载的插件名字" - } - }, - "required": ["oper","name"] - } - } - } + "type": "function", + "function": { + "name": "plugin_loader", + "description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]", + "parameters": { + "type": "object", + "properties": { + "oper": {"type": "string", "description": "load or unload"}, + "name": {"type": "string", "description": "要加载或卸载的插件名字"}, + }, + "required": ["oper", "name"], + }, + }, +} -@register_function('plugin_loader', plugin_loader_function_desc, ToolType.SYSTEM_CTL) + +@register_function("plugin_loader", plugin_loader_function_desc, ToolType.SYSTEM_CTL) def plugin_loader(conn, oper: str, name: str): """插件加载""" if oper not in ["load", "unload"]: - return ActionResponse(action=Action.RESPONSE, result="插件操作失败", response="不支持的操作") - + return ActionResponse( + action=Action.RESPONSE, result="插件操作失败", response="不支持的操作" + ) + cur_support = conn.func_handler.current_support_functions() if oper == "load": if name in cur_support: - return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response=f"{name}插件已加载,无需重复加载") + return ActionResponse( + action=Action.RESPONSE, + result="插件加载失败", + response=f"{name}插件已加载,无需重复加载", + ) func = conn.func_handler.function_registry.register_function(name) if not func: - return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response="插件未找到") + return ActionResponse( + action=Action.RESPONSE, result="插件加载失败", response="插件未找到" + ) res = f"{name}插件加载成功" else: if name not in cur_support: - return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response=f"{name}插件未加载") + return ActionResponse( + action=Action.RESPONSE, + result="插件卸载失败", + response=f"{name}插件未加载", + ) bOK = conn.func_handler.function_registry.unregister_function(name) if not bOK: - return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response="插件未找到") + return ActionResponse( + action=Action.RESPONSE, result="插件卸载失败", response="插件未找到" + ) res = f"{name}插件卸载成功" conn.func_handler.upload_functions_desc() return ActionResponse(action=Action.RESPONSE, result="插件操作成功", response=res) From 571080c1d623d299ddf483254363661f528fd005 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 7 May 2025 22:56:51 +0800 Subject: [PATCH 15/24] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=9C=80?= =?UTF-8?q?=E8=BF=91=E5=AF=B9=E8=AF=9D=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/AgentChatHistoryBizServiceImpl.java | 6 +++++ .../agent/service/impl/AgentServiceImpl.java | 1 + .../xiaozhi/modules/device/DeviceTest.java | 23 +++++++++---------- .../manager-web/src/components/DeviceItem.vue | 21 ++++++++++++++++- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java index b839e9e4..8b405e49 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java @@ -1,12 +1,15 @@ package xiaozhi.modules.agent.service.biz.impl; import java.util.Base64; +import java.util.Date; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import xiaozhi.common.redis.RedisKeys; +import xiaozhi.common.redis.RedisUtils; import xiaozhi.modules.agent.dto.AgentChatHistoryReportDTO; import xiaozhi.modules.agent.entity.AgentChatHistoryEntity; import xiaozhi.modules.agent.entity.AgentEntity; @@ -29,6 +32,7 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic private final AgentService agentService; private final AgentChatHistoryService agentChatHistoryService; private final AgentChatAudioService agentChatAudioService; + private final RedisUtils redisUtils; /** * 处理聊天记录上报，包括文件上传和相关信息记录 @@ -77,6 +81,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic // 3. 保存数据 agentChatHistoryService.save(entity); + // 4. 更新设备最后对话时间 + redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), new Date(), 60 * 2); return Boolean.TRUE; } } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java index 8fb85f06..ae650c85 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java @@ -34,6 +34,7 @@ public class AgentServiceImpl extends BaseServiceImpl imp private final TimbreService timbreModelService; private final ModelConfigService modelConfigService; private final RedisUtils redisUtils; + private final DeviceService deviceService; @Override public PageData adminAgentList(Map params) { diff --git a/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java b/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java index 4c13bcdb..37f67cab 100644 --- a/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java +++ b/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java @@ -1,5 +1,8 @@ package xiaozhi.modules.device; +import java.util.HashMap; +import java.util.UUID; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -8,13 +11,10 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import lombok.extern.slf4j.Slf4j; -import xiaozhi.common.redis.RedisKeys; import xiaozhi.common.redis.RedisUtils; import xiaozhi.modules.sys.dto.SysUserDTO; import xiaozhi.modules.sys.service.SysUserService; -import java.util.HashMap; - @Slf4j @SpringBootTest @ActiveProfiles("dev") @@ -29,8 +29,8 @@ public class DeviceTest { @Test public void testSaveUser() { SysUserDTO userDTO = new SysUserDTO(); - userDTO.setUsername("13536468486"); - userDTO.setPassword("0218jianyuQ!"); + userDTO.setUsername("test"); + userDTO.setPassword(UUID.randomUUID().toString()); sysUserService.save(userDTO); } @@ -43,18 +43,17 @@ public class DeviceTest { // 模拟设备验证码 String deviceCode = "123456"; - HashMap map = new HashMap<>(); - map.put("mac_address",macAddress); - map.put("activation_code",deviceCode); - map.put("board","硬件型号"); - map.put("app_version","0.3.13"); + HashMap map = new HashMap<>(); + map.put("mac_address", macAddress); + map.put("activation_code", deviceCode); + map.put("board", "硬件型号"); + map.put("app_version", "0.3.13"); String safeDeviceId = macAddress.replace(":", "_").toLowerCase(); String cacheDeviceKey = String.format("ota:activation:data:%s", safeDeviceId); redisUtils.set(cacheDeviceKey, map, 300); - - String redisKey = "ota:activation:code:"+deviceCode; + String redisKey = "ota:activation:code:" + deviceCode; log.info("Redis Key: {}", redisKey); // 将设备信息写入Redis diff --git a/main/manager-web/src/components/DeviceItem.vue b/main/manager-web/src/components/DeviceItem.vue index fa81a04d..c8acfde2 100644 --- a/main/manager-web/src/components/DeviceItem.vue +++ b/main/manager-web/src/components/DeviceItem.vue @@ -31,7 +31,7 @@
-
最近对话：{{ device.lastConnectedAt }}
+
最近对话：{{ formattedLastConnectedTime }}
@@ -45,6 +45,25 @@ export default { data() { return { switchValue: false } }, + computed: { + formattedLastConnectedTime() { + if (!this.device.lastConnectedAt) return '暂未对话'; + + const lastTime = new Date(this.device.lastConnectedAt); + const now = new Date(); + const diffMinutes = Math.floor((now - lastTime) / (1000 * 60)); + + if (diffMinutes < 60) { + return `${diffMinutes}分钟前`; + } else if (diffMinutes < 24 * 60) { + const hours = Math.floor(diffMinutes / 60); + const minutes = diffMinutes % 60; + return `${hours}小时${minutes > 0 ? minutes + '分钟' : ''}前`; + } else { + return this.device.lastConnectedAt; + } + } + }, methods: { handleDelete() { this.$emit('delete', this.device.agentId) From e6d63a811ea1bca9319fd154ea07b803291e1f08 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 7 May 2025 23:21:47 +0800 Subject: [PATCH 16/24] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=98=BE=E7=A4=BA=20(#1139)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/xiaozhi/common/constant/Constant.java | 2 +- main/manager-web/src/components/DeviceItem.vue | 4 +++- main/xiaozhi-server/config/logger.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 779fea83..a9b08d5c 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -177,5 +177,5 @@ public interface Constant { /** * 版本号 */ - public static final String VERSION = "0.3.14"; + public static final String VERSION = "0.4.1"; } \ No newline at end of file diff --git a/main/manager-web/src/components/DeviceItem.vue b/main/manager-web/src/components/DeviceItem.vue index c8acfde2..60f54b9d 100644 --- a/main/manager-web/src/components/DeviceItem.vue +++ b/main/manager-web/src/components/DeviceItem.vue @@ -53,7 +53,9 @@ export default { const now = new Date(); const diffMinutes = Math.floor((now - lastTime) / (1000 * 60)); - if (diffMinutes < 60) { + if (diffMinutes <= 1) { + return '刚刚'; + } else if (diffMinutes < 60) { return `${diffMinutes}分钟前`; } else if (diffMinutes < 24 * 60) { const hours = Math.floor(diffMinutes / 60); diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 03768989..ce04c31c 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -4,7 +4,7 @@ from loguru import logger from config.config_loader import load_config from config.settings import check_config_file -SERVER_VERSION = "0.3.14" +SERVER_VERSION = "0.4.1" def get_module_abbreviation(module_name, module_dict): From 919c2ffd46640278c334c2effe44160e70c869a8 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 7 May 2025 23:37:38 +0800 Subject: [PATCH 17/24] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=98=BE=E7=A4=BA=20(#1143)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update:优化时间显示 * update:优化时间显示 --- .../service/biz/impl/AgentChatHistoryBizServiceImpl.java | 2 +- .../modules/device/service/impl/DeviceServiceImpl.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java index 8b405e49..21e645de 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java @@ -82,7 +82,7 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic // 3. 保存数据 agentChatHistoryService.save(entity); // 4. 更新设备最后对话时间 - redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), new Date(), 60 * 2); + redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), new Date()); return Boolean.TRUE; } } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index 59e329fa..515aea7f 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -1,7 +1,12 @@ package xiaozhi.modules.device.service.impl; import java.time.Instant; -import java.util.*; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -262,7 +267,7 @@ public class DeviceServiceImpl extends BaseServiceImpl } Date maxDate = deviceDao.getAllLastConnectedAtByAgentId(agentId); if (maxDate != null) { - redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), maxDate, 60 * 2); + redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), maxDate); } return maxDate; } From b246d9e5676fd45b7ca42af278b8bb85a6e9920e Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 09:35:46 +0800 Subject: [PATCH 18/24] =?UTF-8?q?fix:selected=5Fmodule=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E4=B8=BA=E7=A9=BA=E7=9A=84bug=20(#1144)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update:优化时间显示 * update:优化时间显示 * fix:selected_module可能为空的bug --- main/xiaozhi-server/core/utils/util.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index b75a24da..2ed67b19 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -910,7 +910,10 @@ def audio_to_opus_data(audio_file_path): def check_vad_update(before_config, new_config): - if new_config["selected_module"].get("VAD") is None: + if ( + new_config.get("selected_module") is None + or new_config["selected_module"].get("VAD") is None + ): return False update_vad = False current_vad_module = before_config["selected_module"]["VAD"] @@ -931,7 +934,10 @@ def check_vad_update(before_config, new_config): def check_asr_update(before_config, new_config): - if new_config["selected_module"].get("ASR") is None: + if ( + new_config.get("selected_module") is None + or new_config["selected_module"].get("ASR") is None + ): return False update_asr = False current_asr_module = before_config["selected_module"]["ASR"] From c2e000f93705e1a72589327d142cf33aaff98673 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 11:11:28 +0800 Subject: [PATCH 19/24] =?UTF-8?q?add:asr=E8=B5=8B=E5=80=BCaudio=5Fformat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 3 +- .../core/providers/asr/aliyun.py | 106 ++++++++++-------- .../core/providers/asr/doubao.py | 2 +- .../core/providers/asr/fun_local.py | 14 ++- .../core/providers/asr/sherpa_onnx_local.py | 2 +- 5 files changed, 72 insertions(+), 55 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 5da1fa53..2aca4a3e 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -28,7 +28,8 @@ async def handleHelloMessage(conn, msg_json): format = audio_params.get("format") logger.bind(tag=TAG).info(f"客户端音频格式: {format}") conn.audio_format = format - conn.welcome_msg['audio_params'] = audio_params + conn.asr.set_audio_format(format) + conn.welcome_msg["audio_params"] = audio_params await conn.websocket.send(json.dumps(conn.welcome_msg)) diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index be5541ee..14421d7d 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -20,62 +20,75 @@ from core.providers.asr.base import ASRProviderBase TAG = __name__ logger = setup_logging() + class AccessToken: @staticmethod def _encode_text(text): encoded_text = parse.quote_plus(text) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") @staticmethod def _encode_dict(dic): keys = dic.keys() dic_sorted = [(key, dic[key]) for key in sorted(keys)] encoded_text = parse.urlencode(dic_sorted) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") @staticmethod def create_token(access_key_id, access_key_secret): - parameters = {'AccessKeyId': access_key_id, - 'Action': 'CreateToken', - 'Format': 'JSON', - 'RegionId': 'cn-shanghai', - 'SignatureMethod': 'HMAC-SHA1', - 'SignatureNonce': str(uuid.uuid1()), - 'SignatureVersion': '1.0', - 'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - 'Version': '2019-02-28'} + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } # 构造规范化的请求字符串 query_string = AccessToken._encode_dict(parameters) # print('规范化的请求字符串: %s' % query_string) # 构造待签名字符串 - string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string) + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) # print('待签名的字符串: %s' % string_to_sign) # 计算签名 - secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'), - bytes(string_to_sign, encoding='utf-8'), - hashlib.sha1).digest() + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() signature = base64.b64encode(secreted_string) # print('签名: %s' % signature) # 进行URL编码 signature = AccessToken._encode_text(signature) # print('URL编码后的签名: %s' % signature) # 调用服务 - full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) # print('url: %s' % full_url) # 提交HTTP GET请求 response = requests.get(full_url) if response.ok: root_obj = response.json() - key = 'Token' + key = "Token" if key in root_obj: - token = root_obj[key]['Id'] - expire_time = root_obj[key]['ExpireTime'] + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] return token, expire_time # print(response.text) return None, None - class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): """阿里云ASR初始化""" @@ -102,28 +115,23 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def _refresh_token(self): """刷新Token并记录过期时间""" if self.access_key_id and self.access_key_secret: self.token, expire_time_str = AccessToken.create_token( - self.access_key_id, - self.access_key_secret + self.access_key_id, self.access_key_secret ) if not expire_time_str: raise ValueError("无法获取有效的Token过期时间") try: - #统一转换为字符串处理 + # 统一转换为字符串处理 expire_str = str(expire_time_str).strip() if expire_str.isdigit(): expire_time = datetime.fromtimestamp(int(expire_str)) else: - expire_time = datetime.strptime( - expire_str, - "%Y-%m-%dT%H:%M:%SZ" - ) + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") self.expire_time = expire_time.timestamp() - 60 except Exception as e: raise ValueError(f"无效的过期时间格式: {expire_str}") from e @@ -145,9 +153,12 @@ class ASRProvider(ASRProviderBase): # f"过期时间 {datetime.fromtimestamp(self.expire_time)} | " # f"剩余 {remaining:.2f}秒") return time.time() > self.expire_time - def generate_filename(self, extension=".wav"): - return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + def generate_filename(self, extension=".wav"): + return os.path.join( + self.output_file, + f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) def _construct_request_url(self) -> str: """构造请求URL，包含参数""" @@ -158,7 +169,6 @@ class ASRProvider(ASRProviderBase): request += "&enable_inverse_text_normalization=true" request += "&enable_voice_detection=false" return request - def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" @@ -170,7 +180,7 @@ class ASRProvider(ASRProviderBase): wf.setnchannels(1) # 单声道 wf.setsampwidth(2) # 16-bit wf.setframerate(self.sample_rate) - wf.writeframes(b''.join(pcm_data)) + wf.writeframes(b"".join(pcm_data)) logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}") return file_path @@ -180,9 +190,9 @@ class ASRProvider(ASRProviderBase): try: # 设置HTTP头 headers = { - 'X-NLS-Token': self.token, - 'Content-type': 'application/octet-stream', - 'Content-Length': str(len(pcm_data)) + "X-NLS-Token": self.token, + "Content-type": "application/octet-stream", + "Content-Length": str(len(pcm_data)), } # 创建连接并发送请求 @@ -190,12 +200,12 @@ class ASRProvider(ASRProviderBase): request_url = self._construct_request_url() loop = asyncio.get_event_loop() - await loop.run_in_executor(None, lambda: conn.request( - method='POST', - url=request_url, - body=pcm_data, - headers=headers - )) + await loop.run_in_executor( + None, + lambda: conn.request( + method="POST", url=request_url, body=pcm_data, headers=headers + ), + ) # 获取响应 response = await loop.run_in_executor(None, conn.getresponse) @@ -205,10 +215,10 @@ class ASRProvider(ASRProviderBase): # 解析响应 try: body_json = json.loads(body) - status = body_json.get('status') + status = body_json.get("status") if status == 20000000: - result = body_json.get('result', '') + result = body_json.get("result", "") logger.bind(tag=TAG).debug(f"ASR结果: {result}") return result else: @@ -223,7 +233,9 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True) return None - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if self._is_token_expired(): logger.warning("Token已过期，正在自动刷新...") @@ -235,8 +247,8 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) - combined_pcm_data = b''.join(pcm_data) + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 if self.delete_audio_file: @@ -254,4 +266,4 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", file_path \ No newline at end of file + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index 0ca089c9..0775c2ab 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -253,7 +253,7 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 373d9c6a..107417b9 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -44,7 +44,7 @@ class ASRProvider(ASRProviderBase): model=self.model_dir, vad_kwargs={"max_single_segment_time": 30000}, disable_update=True, - hub="hf" + hub="hf", # device="cuda:0", # 启用GPU加速 ) @@ -62,7 +62,9 @@ class ASRProvider(ASRProviderBase): return file_path - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: @@ -70,8 +72,8 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) - + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 @@ -90,7 +92,9 @@ class ASRProvider(ASRProviderBase): batch_size_s=60, ) text = rich_transcription_postprocess(result[0]["text"]) - logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) return text, file_path diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 45e22336..2b86f6d7 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -132,7 +132,7 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) + pcm_data = self.decode_opus(opus_data) file_path = self.save_audio_to_file(pcm_data, session_id) logger.bind(tag=TAG).debug( f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" From 64f10b28e7ca815a83fff865f5bfcf8857d48870 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 11:31:12 +0800 Subject: [PATCH 20/24] =?UTF-8?q?update:=E5=90=88=E5=B9=B6main=E5=88=86?= =?UTF-8?q?=E6=94=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 3 +- .../core/handle/receiveAudioHandle.py | 10 ++-- .../core/handle/sendAudioHandle.py | 2 +- .../xiaozhi-server/core/providers/tts/base.py | 51 +------------------ main/xiaozhi-server/core/utils/util.py | 20 ++++---- 5 files changed, 19 insertions(+), 67 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index d40a0533..f902a64f 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,5 +1,4 @@ import json -from config.logger import setup_logging from core.handle.sendAudioHandle import send_stt_message from core.utils.util import remove_punctuation_and_length import shutil @@ -25,7 +24,7 @@ async def handleHelloMessage(conn, msg_json): audio_params = msg_json.get("audio_params") if audio_params: format = audio_params.get("format") - logger.bind(tag=TAG).info(f"客户端音频格式: {format}") + conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}") conn.audio_format = format conn.asr.set_audio_format(format) conn.welcome_msg["audio_params"] = audio_params diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 383616c3..992ae96f 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -5,7 +5,7 @@ from core.handle.sendAudioHandle import send_stt_message from core.handle.intentHandler import handle_user_intent from core.utils.output_counter import check_device_output_limit from core.handle.ttsReportHandle import enqueue_tts_report -from core.providers.tts.base import audio_to_opus_data +from core.utils.util import audio_to_data TAG = __name__ @@ -111,7 +111,7 @@ async def max_out_size(conn): conn.tts_last_text_index = 0 conn.llm_finish_task = True file_path = "config/assets/max_output_size.wav" - opus_packets, _ = audio_to_opus_data(file_path) + opus_packets, _ = audio_to_data(file_path) conn.audio_play_queue.put((opus_packets, text, 0)) conn.close_after_chat = True @@ -133,7 +133,7 @@ async def check_bind_device(conn): # 播放提示音 music_path = "config/assets/bind_code.wav" - opus_packets, _ = audio_to_opus_data(music_path) + opus_packets, _ = audio_to_data(music_path) conn.audio_play_queue.put((opus_packets, text, 0)) # 逐个播放数字 @@ -141,7 +141,7 @@ async def check_bind_device(conn): try: digit = conn.bind_code[i] num_path = f"config/assets/bind_code/{digit}.wav" - num_packets, _ = audio_to_opus_data(num_path) + num_packets, _ = audio_to_data(num_path) conn.audio_play_queue.put((num_packets, None, i + 1)) except Exception as e: conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") @@ -153,5 +153,5 @@ async def check_bind_device(conn): conn.tts_last_text_index = 0 conn.llm_finish_task = True music_path = "config/assets/bind_not_found.wav" - opus_packets, _ = audio_to_opus_data(music_path) + opus_packets, _ = audio_to_data(music_path) conn.audio_play_queue.put((opus_packets, text, 0)) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 78fe743c..89f16426 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -115,7 +115,7 @@ async def send_tts_message(conn, state, text=None): stop_tts_notify_voice = conn.config.get( "stop_tts_notify_voice", "config/assets/tts_notify.mp3" ) - audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice) + audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice) await sendAudio(conn, audios) # 清除服务端讲话状态 conn.clearSpeakStatus() diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 9bc693b0..f2632c02 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -56,56 +56,7 @@ class TTSProviderBase(ABC): def audio_to_pcm_data(self, audio_file_path): """音频文件转换为PCM编码""" return audio_to_data(audio_file_path, is_opus=False) - + def audio_to_opus_data(self, audio_file_path): """音频文件转换为Opus编码""" return audio_to_data(audio_file_path, is_opus=True) - - def audio_to_data(self, audio_file_path, is_opus=True): - # 获取文件后缀名 - file_type = os.path.splitext(audio_file_path)[1] - if file_type: - file_type = file_type.lstrip(".") - # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞 - audio = AudioSegment.from_file( - audio_file_path, format=file_type, parameters=["-nostdin"] - ) - - # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配） - audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) - - # 音频时长(秒) - duration = len(audio) / 1000.0 - - # 获取原始PCM数据（16位小端） - raw_data = audio.raw_data - - # 初始化Opus编码器 - encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) - - # 编码参数 - frame_duration = 60 # 60ms per frame - frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame - - datas = [] - # 按帧处理所有音频数据（包括最后一帧可能补零） - for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample - # 获取当前帧的二进制数据 - chunk = raw_data[i : i + frame_size * 2] - - # 如果最后一帧不足，补零 - if len(chunk) < frame_size * 2: - chunk += b"\x00" * (frame_size * 2 - len(chunk)) - - if is_opus: - # 转换为numpy数组处理 - np_frame = np.frombuffer(chunk, dtype=np.int16) - # 编码Opus数据 - frame_data = encoder.encode(np_frame.tobytes(), frame_size) - else: - frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) - - - datas.append(frame_data) - - return datas, duration diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 2ed67b19..effbf38e 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -862,8 +862,7 @@ def analyze_emotion(text): return top_emotions[0] # 如果都不在优先级列表里，返回第一个 -def audio_to_opus_data(audio_file_path): - """音频文件转换为Opus编码""" +def audio_to_data(audio_file_path, is_opus=True): # 获取文件后缀名 file_type = os.path.splitext(audio_file_path)[1] if file_type: @@ -889,7 +888,7 @@ def audio_to_opus_data(audio_file_path): frame_duration = 60 # 60ms per frame frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame - opus_datas = [] + datas = [] # 按帧处理所有音频数据（包括最后一帧可能补零） for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample # 获取当前帧的二进制数据 @@ -899,14 +898,17 @@ def audio_to_opus_data(audio_file_path): if len(chunk) < frame_size * 2: chunk += b"\x00" * (frame_size * 2 - len(chunk)) - # 转换为numpy数组处理 - np_frame = np.frombuffer(chunk, dtype=np.int16) + if is_opus: + # 转换为numpy数组处理 + np_frame = np.frombuffer(chunk, dtype=np.int16) + # 编码Opus数据 + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + else: + frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) - # 编码Opus数据 - opus_data = encoder.encode(np_frame.tobytes(), frame_size) - opus_datas.append(opus_data) + datas.append(frame_data) - return opus_datas, duration + return datas, duration def check_vad_update(before_config, new_config): From 4dad5ea6c1fb9c1ec338d8063f98ca091d5fd1ab Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 12:07:50 +0800 Subject: [PATCH 21/24] =?UTF-8?q?update:=E6=99=BA=E6=8E=A7=E5=8F=B0?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=99=BE=E5=BA=A6ASR=20=E4=BF=AE=E5=A4=8D:?= =?UTF-8?q?=E6=99=BA=E6=8E=A7=E5=8F=B0=E8=B1=86=E5=8C=85ASR=E7=BC=BA?= =?UTF-8?q?=E5=B0=91=E7=83=AD=E8=AF=8D=E5=BC=95=E5=8F=91=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202505081146.sql | 43 +++++++++++++++++++ .../db/changelog/db.changelog-master.yaml | 9 +++- .../core/providers/asr/doubao.py | 4 +- 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202505081146.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202505081146.sql b/main/manager-api/src/main/resources/db/changelog/202505081146.sql new file mode 100644 index 00000000..fe7582f3 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202505081146.sql @@ -0,0 +1,43 @@ +-- 添加百度ASR模型配置 +delete from `ai_model_config` where `id` = 'ASR_BaiduASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_BaiduASR', 'ASR', 'BaiduASR', '百度语音识别', 0, 1, '{\"type\": \"baidu\", \"app_id\": \"\", \"api_key\": \"\", \"secret_key\": \"\", \"dev_pid\": 1537, \"output_dir\": \"tmp/\"}', NULL, NULL, 7, NULL, NULL, NULL, NULL); + + +-- 添加百度ASR供应器 +delete from `ai_model_provider` where `id` = 'SYSTEM_ASR_BaiduASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_BaiduASR', 'ASR', 'baidu', '百度语音识别', '[{"key":"app_id","label":"应用AppID","type":"string"},{"key":"api_key","label":"API Key","type":"string"},{"key":"secret_key","label":"Secret Key","type":"string"},{"key":"dev_pid","label":"语言参数","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 7, 1, NOW(), 1, NOW()); + + +-- 更新百度ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list', +`remark` = '百度ASR配置说明： +1. 访问 https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list +2. 创建新应用 +3. 获取AppID、API Key和Secret Key +4. 填入配置文件中 +查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list +语言参数说明：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b +' WHERE `id` = 'ASR_BaiduASR'; + +-- 更新豆包供应器字段 +update `ai_model_provider` set `fields` = +'[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"boosting_table_name","label":"热词文件名称","type":"string"},{"key":"correct_table_name","label":"替换词文件名称","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]' +where `id` = 'SYSTEM_ASR_DoubaoASR'; + +-- 更新豆包ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.volcengine.com/speech/app', +`remark` = '豆包ASR配置说明： +1. 需要在火山引擎控制台创建应用并获取appid和access_token +2. 支持中文语音识别 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://console.volcengine.com/speech/app +2. 创建新应用 +3. 获取appid和access_token +4. 填入配置文件中 +如需设置热词，请参考：https://www.volcengine.com/docs/6561/155738 +' WHERE `id` = 'ASR_DoubaoASR'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index ffbef2ce..2936aed7 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -99,4 +99,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505022134.sql \ No newline at end of file + path: classpath:db/changelog/202505022134.sql + - changeSet: + id: 202505081146 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202505081146.sql \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index 0775c2ab..18e64a9b 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -88,8 +88,8 @@ class ASRProvider(ASRProviderBase): self.appid = config.get("appid") self.cluster = config.get("cluster") self.access_token = config.get("access_token") - self.boosting_table_name = config.get("boosting_table_name") - self.correct_table_name = config.get("correct_table_name") + self.boosting_table_name = config.get("boosting_table_name", "") + self.correct_table_name = config.get("correct_table_name", "") self.output_dir = config.get("output_dir") self.delete_audio_file = delete_audio_file From af3d00662e1e2b56c77fa37812e8f1b2d01edd79 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 12:10:39 +0800 Subject: [PATCH 22/24] Merge pull request #1147 from xinnan-tech/fix-web-doubaoasr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit update:更新版本号 --- .../src/main/java/xiaozhi/common/constant/Constant.java | 2 +- main/xiaozhi-server/config/logger.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index a9b08d5c..78154b5c 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -177,5 +177,5 @@ public interface Constant { /** * 版本号 */ - public static final String VERSION = "0.4.1"; + public static final String VERSION = "0.4.2"; } \ No newline at end of file diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index ce04c31c..48323c7a 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -4,7 +4,7 @@ from loguru import logger from config.config_loader import load_config from config.settings import check_config_file -SERVER_VERSION = "0.4.1" +SERVER_VERSION = "0.4.2" def get_module_abbreviation(module_name, module_dict): From 98bfe863fe248a892e97c0286b27320477a2e526 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 15:08:11 +0800 Subject: [PATCH 23/24] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dtest=5Fpage.html?= =?UTF-8?q?=E6=B2=A1=E6=9C=89hello=E6=B6=88=E6=81=AF=EF=BC=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=BC=BA=E5=A4=B1audio=5Fformat=E7=9A=84bug=20(#1151)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 9 ++- .../core/providers/asr/aliyun.py | 1 + .../core/providers/asr/baidu.py | 1 + .../xiaozhi-server/core/providers/asr/base.py | 11 ++- .../core/providers/asr/doubao.py | 1 + .../core/providers/asr/fun_local.py | 1 + .../core/providers/asr/fun_server.py | 81 ++++++++++++------- .../core/providers/asr/sherpa_onnx_local.py | 1 + .../core/providers/asr/tencent.py | 74 ++++++++++------- 9 files changed, 116 insertions(+), 64 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 4c355b5b..ff8c8b4f 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -241,9 +241,12 @@ class ConnectionHandler: def _initialize_components(self): """初始化组件""" - self.prompt = self.config["prompt"] - self.change_system_prompt(self.prompt) - self.logger.bind(tag=TAG).info(f"初始化组件: prompt成功 {self.prompt[:50]}...") + if self.config.get("prompt") is not None: + self.prompt = self.config["prompt"] + self.change_system_prompt(self.prompt) + self.logger.bind(tag=TAG).info( + f"初始化组件: prompt成功 {self.prompt[:50]}..." + ) """初始化本地组件""" if self.vad is None: diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index 14421d7d..6606168c 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -91,6 +91,7 @@ class AccessToken: class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() """阿里云ASR初始化""" # 新增空值判断逻辑 self.access_key_id = config.get("access_key_id") diff --git a/main/xiaozhi-server/core/providers/asr/baidu.py b/main/xiaozhi-server/core/providers/asr/baidu.py index 58b26375..42e14d8b 100644 --- a/main/xiaozhi-server/core/providers/asr/baidu.py +++ b/main/xiaozhi-server/core/providers/asr/baidu.py @@ -20,6 +20,7 @@ logger = setup_logging() class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool = True): + super().__init__() self.app_id = config.get("app_id") self.api_key = config.get("api_key") self.secret_key = config.get("secret_key") diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 61c1b754..227a906d 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -8,16 +8,21 @@ logger = setup_logging() class ASRProviderBase(ABC): + def __init__(self): + self.audio_format = "opus" + @abstractmethod def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" pass @abstractmethod - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" pass - + def set_audio_format(self, format: str) -> None: """设置音频格式""" self.audio_format = format @@ -36,4 +41,4 @@ class ASRProviderBase(ABC): except opuslib_next.OpusError as e: logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - return pcm_data \ No newline at end of file + return pcm_data diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index 18e64a9b..403c181f 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -85,6 +85,7 @@ def parse_response(res): class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() self.appid = config.get("appid") self.cluster = config.get("cluster") self.access_token = config.get("access_token") diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 107417b9..ec3df17f 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -33,6 +33,7 @@ class CaptureOutput: class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() self.model_dir = config.get("model_dir") self.output_dir = config.get("output_dir") # 修正配置键名 self.delete_audio_file = delete_audio_file diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index f20d5be7..53eb4e7f 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -9,22 +9,29 @@ import wave import websockets from config.logger import setup_logging import asyncio + TAG = __name__ logger = setup_logging() + class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): - ''' + """ Initialize the ASRProvider with server configuration. :param config: Dictionary containing 'host', 'port', and 'is_ssl'. :param delete_audio_file: Boolean to indicate whether to delete audio files after processing. - ''' - self.host = config.get('host', 'localhost') - self.port = config.get('port', 10095) - self.is_ssl = config.get('is_ssl', True) + """ + super().__init__() + self.host = config.get("host", "localhost") + self.port = config.get("port", 10095) + self.is_ssl = config.get("is_ssl", True) self.output_dir = config.get("output_dir") self.delete_audio_file = delete_audio_file - self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}" + self.uri = ( + f"wss://{self.host}:{self.port}" + if self.is_ssl + else f"ws://{self.host}:{self.port}" + ) self.ssl_context = ssl.SSLContext() if self.is_ssl else None if self.ssl_context: self.ssl_context.check_hostname = False @@ -45,10 +52,10 @@ class ASRProvider(ASRProviderBase): return file_path async def _receive_responses(self, ws) -> None: - ''' + """ Asynchronous generator to receive messages from the WebSocket. Yields each message as it is received. - ''' + """ text = "" while True: try: @@ -61,30 +68,35 @@ class ASRProvider(ASRProviderBase): else: text += response_data.get("text", "") except asyncio.TimeoutError: - logger.bind(tag=TAG).error("Timeout while waiting for response from WebSocket.") + logger.bind(tag=TAG).error( + "Timeout while waiting for response from WebSocket." + ) break except websockets.exceptions.ConnectionClosed as e: logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") break return text + async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple: - ''' + """ Internal method to handle WebSocket communication. Reuses the persistent WebSocket connection if available. :param pcm_data: PCM audio data to send. :param session_id: Unique session identifier. :return: Tuple containing recognized text and optional timestamp. - ''' + """ # Send initial configuration message - config_message = json.dumps({ - "mode": "offline", - "chunk_size": [5, 10, 5], - "chunk_interval": 10, - "wav_name": session_id, - "is_speaking": True, - "itn": False - }) + config_message = json.dumps( + { + "mode": "offline", + "chunk_size": [5, 10, 5], + "chunk_interval": 10, + "wav_name": session_id, + "is_speaking": True, + "itn": False, + } + ) await ws.send(config_message) logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}") @@ -97,14 +109,15 @@ class ASRProvider(ASRProviderBase): await ws.send(end_message) logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") - - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: - ''' + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: + """ Convert speech data to text using FunASR. :param opus_data: List of Opus-encoded audio data chunks. :param session_id: Unique session identifier. :return: Tuple containing recognized text and optional timestamp. - ''' + """ file_path = None if self.audio_format == "pcm": pcm_data = opus_data @@ -118,16 +131,19 @@ class ASRProvider(ASRProviderBase): else: file_path = self.save_audio_to_file(pcm_data, session_id) - async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws: + async with websockets.connect( + self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context + ) as ws: try: # Use asyncio to handle WebSocket communication - send_task = asyncio.create_task(self._send_data(ws, combined_pcm_data, session_id)) + send_task = asyncio.create_task( + self._send_data(ws, combined_pcm_data, session_id) + ) receive_task = asyncio.create_task(self._receive_responses(ws)) # Gather tasks with error handling done, pending = await asyncio.wait( - [send_task, receive_task], - return_when=asyncio.FIRST_EXCEPTION + [send_task, receive_task], return_when=asyncio.FIRST_EXCEPTION ) # Cancel any pending tasks @@ -141,11 +157,16 @@ class ASRProvider(ASRProviderBase): # Get the result from the receive task result = receive_task.result() - return result, file_path # Return the recognized text and timestamp (if any) + return ( + result, + file_path, + ) # Return the recognized text and timestamp (if any) except websockets.exceptions.ConnectionClosed as e: logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") return "", file_path except Exception as e: - logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True) - return "", file_path \ No newline at end of file + logger.bind(tag=TAG).error( + f"Error during speech-to-text conversion: {e}", exc_info=True + ) + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 2b86f6d7..667e1606 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -37,6 +37,7 @@ class CaptureOutput: class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() self.model_dir = config.get("model_dir") self.output_dir = config.get("output_dir") self.delete_audio_file = delete_audio_file diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index 2aa7bec5..d2db875e 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -17,12 +17,14 @@ from config.logger import setup_logging TAG = __name__ logger = setup_logging() + class ASRProvider(ASRProviderBase): API_URL = "https://asr.tencentcloudapi.com" API_VERSION = "2019-06-14" FORMAT = "pcm" # 支持的音频格式：pcm, wav, mp3 def __init__(self, config: dict, delete_audio_file: bool = True): + super().__init__() self.secret_id = config.get("secret_id") self.secret_key = config.get("secret_key") self.output_dir = config.get("output_dir") @@ -45,7 +47,9 @@ class ASRProvider(ASRProviderBase): return file_path - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: logger.bind(tag=TAG).warn("音频数据为空！") @@ -72,7 +76,7 @@ class ASRProvider(ASRProviderBase): self.save_audio_to_file(pcm_data, session_id) # 将音频数据转换为Base64编码 - base64_audio = base64.b64encode(combined_pcm_data).decode('utf-8') + base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8") # 构建请求体 request_body = self._build_request_body(base64_audio) @@ -85,7 +89,9 @@ class ASRProvider(ASRProviderBase): result = self._send_request(request_body, timestamp, authorization) if result: - logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}") + logger.bind(tag=TAG).debug( + f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}" + ) return result, file_path @@ -102,7 +108,7 @@ class ASRProvider(ASRProviderBase): "SourceType": 1, # 音频数据来源为语音文件 "VoiceFormat": self.FORMAT, # 音频格式 "Data": base64_audio, # Base64编码的音频数据 - "DataLen": len(base64_audio) # 数据长度 + "DataLen": len(base64_audio), # 数据长度 } return json.dumps(request_map) @@ -135,9 +141,11 @@ class ASRProvider(ASRProviderBase): action = "SentenceRecognition" # 接口名称 # 构建规范头部信息，注意顺序和格式 - canonical_headers = f"content-type:{content_type.lower()}\n" + \ - f"host:{host.lower()}\n" + \ - f"x-tc-action:{action.lower()}\n" + canonical_headers = ( + f"content-type:{content_type.lower()}\n" + + f"host:{host.lower()}\n" + + f"x-tc-action:{action.lower()}\n" + ) signed_headers = "content-type;host;x-tc-action" @@ -145,21 +153,25 @@ class ASRProvider(ASRProviderBase): payload_hash = self._sha256_hex(request_body) # 构建规范请求字符串 - canonical_request = f"{http_request_method}\n" + \ - f"{canonical_uri}\n" + \ - f"{canonical_query_string}\n" + \ - f"{canonical_headers}\n" + \ - f"{signed_headers}\n" + \ - f"{payload_hash}" + canonical_request = ( + f"{http_request_method}\n" + + f"{canonical_uri}\n" + + f"{canonical_query_string}\n" + + f"{canonical_headers}\n" + + f"{signed_headers}\n" + + f"{payload_hash}" + ) # 计算规范请求的哈希值 hashed_canonical_request = self._sha256_hex(canonical_request) # 构建待签名字符串 - string_to_sign = f"{algorithm}\n" + \ - f"{timestamp}\n" + \ - f"{credential_scope}\n" + \ - f"{hashed_canonical_request}" + string_to_sign = ( + f"{algorithm}\n" + + f"{timestamp}\n" + + f"{credential_scope}\n" + + f"{hashed_canonical_request}" + ) # 计算签名密钥 secret_date = self._hmac_sha256(f"TC3{self.secret_key}", date) @@ -167,13 +179,17 @@ class ASRProvider(ASRProviderBase): secret_signing = self._hmac_sha256(secret_service, "tc3_request") # 计算签名 - signature = self._bytes_to_hex(self._hmac_sha256(secret_signing, string_to_sign)) + signature = self._bytes_to_hex( + self._hmac_sha256(secret_signing, string_to_sign) + ) # 构建授权头 - authorization = f"{algorithm} " + \ - f"Credential={self.secret_id}/{credential_scope}, " + \ - f"SignedHeaders={signed_headers}, " + \ - f"Signature={signature}" + authorization = ( + f"{algorithm} " + + f"Credential={self.secret_id}/{credential_scope}, " + + f"SignedHeaders={signed_headers}, " + + f"Signature={signature}" + ) return timestamp, authorization @@ -181,7 +197,9 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"生成认证头失败: {e}", exc_info=True) raise RuntimeError(f"生成认证头失败: {e}") - def _send_request(self, request_body: str, timestamp: str, authorization: str) -> Optional[str]: + def _send_request( + self, request_body: str, timestamp: str, authorization: str + ) -> Optional[str]: """发送请求到腾讯云API""" headers = { "Content-Type": "application/json; charset=utf-8", @@ -190,7 +208,7 @@ class ASRProvider(ASRProviderBase): "X-TC-Action": "SentenceRecognition", "X-TC-Version": self.API_VERSION, "X-TC-Timestamp": timestamp, - "X-TC-Region": "ap-shanghai" + "X-TC-Region": "ap-shanghai", } try: @@ -221,16 +239,16 @@ class ASRProvider(ASRProviderBase): def _sha256_hex(self, data: str) -> str: """计算字符串的SHA256哈希值""" - digest = hashlib.sha256(data.encode('utf-8')).digest() + digest = hashlib.sha256(data.encode("utf-8")).digest() return self._bytes_to_hex(digest) def _hmac_sha256(self, key, data: str) -> bytes: """计算HMAC-SHA256""" if isinstance(key, str): - key = key.encode('utf-8') + key = key.encode("utf-8") - return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest() + return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest() def _bytes_to_hex(self, bytes_data: bytes) -> str: """字节数组转十六进制字符串""" - return ''.join(f"{b:02x}" for b in bytes_data) \ No newline at end of file + return "".join(f"{b:02x}" for b in bytes_data) From ec95917b4dfd19f8e2562c09d13e5abdb67739a4 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 8 May 2025 17:57:11 +0800 Subject: [PATCH 24/24] =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/AddDeviceDialog.vue | 2 +- .../src/components/AddModelDialog.vue | 6 +-- .../src/components/AddWisdomBodyDialog.vue | 4 +- .../src/components/ChangePasswordDialog.vue | 13 ++++- .../src/components/DictDataDialog.vue | 17 +++++-- .../src/components/DictTypeDialog.vue | 15 +++++- .../src/components/FirmwareDialog.vue | 20 ++++++-- .../src/components/ModelEditDialog.vue | 13 ++--- main/manager-web/src/components/TtsModel.vue | 9 +--- main/manager-web/src/views/DictManagement.vue | 48 ++++++++++++------- main/manager-web/src/views/roleConfig.vue | 13 +++-- 11 files changed, 104 insertions(+), 56 deletions(-) diff --git a/main/manager-web/src/components/AddDeviceDialog.vue b/main/manager-web/src/components/AddDeviceDialog.vue index 51f5426b..b273a8dc 100644 --- a/main/manager-web/src/components/AddDeviceDialog.vue +++ b/main/manager-web/src/components/AddDeviceDialog.vue @@ -1,5 +1,5 @@