update:优化时间显示 (#1143)

* update:优化时间显示

* update:优化时间显示

docs/Deployment_all.md

@@ -213,7 +213,7 @@ CREATE DATABASE xiaozhi_esp32_server CHARACTER SET utf8mb4 COLLATE utf8mb4_unico
 如果还没有MySQL，你可以通过docker安装mysql
 
 ```
-docker run --name xiaozhi-esp32-server-db -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -e MYSQL_DATABASE=xiaozhi_esp32_server -e MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" -d mysql:latest
+docker run --name xiaozhi-esp32-server-db -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -e MYSQL_DATABASE=xiaozhi_esp32_server -e MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" -e TZ=Asia/Shanghai -d mysql:latest
 ```
 
 ## 2.安装redis

main/manager-api/pom.xml

@@ -197,6 +197,10 @@
             <artifactId>lombok</artifactId>
             <optional>true</optional>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+        </dependency>
     </dependencies>
 
     <!-- 阿里云maven仓库 -->

main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java

@@ -177,5 +177,5 @@ public interface Constant {
     /**
      * 版本号
      */
-    public static final String VERSION = "0.3.13";
+    public static final String VERSION = "0.4.1";
 }

main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java

@@ -62,6 +62,13 @@ public class RedisKeys {
         return "agent:device:count:" + id;
     }
 
+    /**
+     * 获取智能体最后连接时间缓存key
+     */
+    public static String getAgentDeviceLastConnectedAtById(String id) {
+        return "agent:device:lastConnected:" + id;
+    }
+
     /**
      * 获取系统配置缓存key
      */
@@ -103,4 +110,11 @@ public class RedisKeys {
     public static String getDictDataByTypeKey(String dictType) {
         return "sys:dict:data:" + dictType;
     }
+
+    /**
+     * 获取智能体音频ID的缓存key
+     */
+    public static String getAgentAudioIdKey(String uuid) {
+        return "agent:audio:id:" + uuid;
+    }
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java

@@ -3,8 +3,13 @@ package xiaozhi.modules.agent.controller;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.shiro.authz.annotation.RequiresPermissions;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -25,14 +30,20 @@ import jakarta.validation.Valid;
 import lombok.AllArgsConstructor;
 import xiaozhi.common.constant.Constant;
 import xiaozhi.common.page.PageData;
+import xiaozhi.common.redis.RedisKeys;
+import xiaozhi.common.redis.RedisUtils;
 import xiaozhi.common.user.UserDetail;
 import xiaozhi.common.utils.ConvertUtils;
 import xiaozhi.common.utils.Result;
+import xiaozhi.modules.agent.dto.AgentChatHistoryDTO;
+import xiaozhi.modules.agent.dto.AgentChatSessionDTO;
 import xiaozhi.modules.agent.dto.AgentCreateDTO;
 import xiaozhi.modules.agent.dto.AgentDTO;
 import xiaozhi.modules.agent.dto.AgentUpdateDTO;
 import xiaozhi.modules.agent.entity.AgentEntity;
 import xiaozhi.modules.agent.entity.AgentTemplateEntity;
+import xiaozhi.modules.agent.service.AgentChatAudioService;
+import xiaozhi.modules.agent.service.AgentChatHistoryService;
 import xiaozhi.modules.agent.service.AgentService;
 import xiaozhi.modules.agent.service.AgentTemplateService;
 import xiaozhi.modules.device.service.DeviceService;
@@ -46,6 +57,9 @@ public class AgentController {
     private final AgentService agentService;
     private final AgentTemplateService agentTemplateService;
     private final DeviceService deviceService;
+    private final AgentChatHistoryService agentChatHistoryService;
+    private final AgentChatAudioService agentChatAudioService;
+    private final RedisUtils redisUtils;
 
     @GetMapping("/list")
     @Operation(summary = "获取用户智能体列表")
@@ -80,7 +94,7 @@ public class AgentController {
     @PostMapping
     @Operation(summary = "创建智能体")
     @RequiresPermissions("sys:role:normal")
-    public Result<Void> save(@RequestBody @Valid AgentCreateDTO dto) {
+    public Result<String> save(@RequestBody @Valid AgentCreateDTO dto) {
         AgentEntity entity = ConvertUtils.sourceToTarget(dto, AgentEntity.class);
 
         // 获取默认模板
@@ -108,7 +122,7 @@ public class AgentController {
         // ID、智能体编码和排序会在Service层自动生成
         agentService.insert(entity);
 
-        return new Result<>();
+        return new Result<String>().ok(entity.getId());
     }
 
     @PutMapping("/{id}")
@@ -178,6 +192,8 @@ public class AgentController {
     public Result<Void> delete(@PathVariable String id) {
         // 先删除关联的设备
         deviceService.deleteByAgentId(id);
+        // 删除关联的聊天记录
+        agentChatHistoryService.deleteByAgentId(id);
         // 再删除智能体
         agentService.deleteById(id);
         return new Result<>();
@@ -192,4 +208,71 @@ public class AgentController {
         return new Result<List<AgentTemplateEntity>>().ok(list);
     }
 
+    @GetMapping("/{id}/sessions")
+    @Operation(summary = "获取智能体会话列表")
+    @RequiresPermissions("sys:role:normal")
+    @Parameters({
+            @Parameter(name = Constant.PAGE, description = "当前页码，从1开始", required = true),
+            @Parameter(name = Constant.LIMIT, description = "每页显示记录数", required = true),
+    })
+    public Result<PageData<AgentChatSessionDTO>> getAgentSessions(
+            @PathVariable("id") String id,
+            @Parameter(hidden = true) @RequestParam Map<String, Object> params) {
+        params.put("agentId", id);
+        PageData<AgentChatSessionDTO> page = agentChatHistoryService.getSessionListByAgentId(params);
+        return new Result<PageData<AgentChatSessionDTO>>().ok(page);
+    }
+
+    @GetMapping("/{id}/chat-history/{sessionId}")
+    @Operation(summary = "获取智能体聊天记录")
+    @RequiresPermissions("sys:role:normal")
+    public Result<List<AgentChatHistoryDTO>> getAgentChatHistory(
+            @PathVariable("id") String id,
+            @PathVariable("sessionId") String sessionId) {
+        // 获取当前用户
+        UserDetail user = SecurityUser.getUser();
+
+        // 检查权限
+        if (!agentService.checkAgentPermission(id, user.getId())) {
+            return new Result<List<AgentChatHistoryDTO>>().error("没有权限查看该智能体的聊天记录");
+        }
+
+        // 查询聊天记录
+        List<AgentChatHistoryDTO> result = agentChatHistoryService.getChatHistoryBySessionId(id, sessionId);
+        return new Result<List<AgentChatHistoryDTO>>().ok(result);
+    }
+
+    @PostMapping("/audio/{audioId}")
+    @Operation(summary = "获取音频下载ID")
+    @RequiresPermissions("sys:role:normal")
+    public Result<String> getAudioId(@PathVariable("audioId") String audioId) {
+        byte[] audioData = agentChatAudioService.getAudio(audioId);
+        if (audioData == null) {
+            return new Result<String>().error("音频不存在");
+        }
+        String uuid = UUID.randomUUID().toString();
+        redisUtils.set(RedisKeys.getAgentAudioIdKey(uuid), audioId);
+        return new Result<String>().ok(uuid);
+    }
+
+    @GetMapping("/play/{uuid}")
+    @Operation(summary = "播放音频")
+    public ResponseEntity<byte[]> playAudio(@PathVariable("uuid") String uuid) {
+
+        String audioId = (String) redisUtils.get(RedisKeys.getAgentAudioIdKey(uuid));
+        if (StringUtils.isBlank(audioId)) {
+            return ResponseEntity.notFound().build();
+        }
+
+        byte[] audioData = agentChatAudioService.getAudio(audioId);
+        if (audioData == null) {
+            return ResponseEntity.notFound().build();
+        }
+        redisUtils.delete(RedisKeys.getAgentAudioIdKey(uuid));
+        return ResponseEntity.ok()
+                .contentType(MediaType.APPLICATION_OCTET_STREAM)
+                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"play.wav\"")
+                .body(audioData);
+    }
+
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AiAgentChatHistoryDao.java

@@ -1,7 +1,9 @@
 package xiaozhi.modules.agent.dao;
 
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
 import xiaozhi.modules.agent.entity.AgentChatHistoryEntity;
 
 /**
@@ -13,4 +15,17 @@ import xiaozhi.modules.agent.entity.AgentChatHistoryEntity;
  */
 @Mapper
 public interface AiAgentChatHistoryDao extends BaseMapper<AgentChatHistoryEntity> {
+    /**
+     * 根据智能体ID删除音频
+     *
+     * @param agentId 智能体ID
+     */
+    void deleteAudioByAgentId(String agentId);
+
+    /**
+     * 根据智能体ID删除聊天历史记录
+     *
+     * @param agentId 智能体ID
+     */
+    void deleteHistoryByAgentId(String agentId);
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryDTO.java

@@ -0,0 +1,28 @@
+package xiaozhi.modules.agent.dto;
+
+import java.util.Date;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.Data;
+
+/**
+ * 智能体聊天记录DTO
+ */
+@Data
+@Schema(description = "智能体聊天记录")
+public class AgentChatHistoryDTO {
+    @Schema(description = "创建时间")
+    private Date createdAt;
+
+    @Schema(description = "消息类型: 1-用户, 2-智能体")
+    private Byte chatType;
+
+    @Schema(description = "聊天内容")
+    private String content;
+
+    @Schema(description = "音频ID")
+    private String audioId;
+
+    @Schema(description = "MAC地址")
+    private String macAddress;
+}

main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java

@@ -26,6 +26,6 @@ public class AgentChatHistoryReportDTO {
     @Schema(description = "聊天内容", example = "你好呀")
     @NotBlank
     private String content;
-    @Schema(description = "文件数据（opus编码）", example = "")
-    private String opusDataBase64;
+    @Schema(description = "base64编码的opus音频数据", example = "")
+    private String audioBase64;
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatSessionDTO.java

@@ -0,0 +1,26 @@
+package xiaozhi.modules.agent.dto;
+
+import java.time.LocalDateTime;
+
+import lombok.Data;
+
+/**
+ * 智能体会话列表DTO
+ */
+@Data
+public class AgentChatSessionDTO {
+    /**
+     * 会话ID
+     */
+    private String sessionId;
+
+    /**
+     * 会话时间
+     */
+    private LocalDateTime createdAt;
+
+    /**
+     * 聊天条数
+     */
+    private Integer chatCount;
+}

main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentChatHistoryEntity.java

@@ -67,12 +67,6 @@ public class AgentChatHistoryEntity {
     @TableField(value = "audio_id")
     private String audioId;
 
-    /**
-     * 音频URL
-     */
-    @TableField(value = "audio_url")
-    private String audioUrl;
-
     /**
      * 创建时间
      */

main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java

@@ -19,4 +19,12 @@ public interface AgentChatAudioService extends IService<AgentChatAudioEntity> {
      * @return 音频ID
      */
     String saveAudio(byte[] audioData);
+
+    /**
+     * 获取音频数据
+     *
+     * @param audioId 音频ID
+     * @return 音频数据
+     */
+    byte[] getAudio(String audioId);
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java

@@ -1,6 +1,13 @@
 package xiaozhi.modules.agent.service;
 
+import java.util.List;
+import java.util.Map;
+
 import com.baomidou.mybatisplus.extension.service.IService;
+
+import xiaozhi.common.page.PageData;
+import xiaozhi.modules.agent.dto.AgentChatHistoryDTO;
+import xiaozhi.modules.agent.dto.AgentChatSessionDTO;
 import xiaozhi.modules.agent.entity.AgentChatHistoryEntity;
 
 /**
@@ -11,4 +18,28 @@ import xiaozhi.modules.agent.entity.AgentChatHistoryEntity;
  * @since 1.0.0
  */
 public interface AgentChatHistoryService extends IService<AgentChatHistoryEntity> {
+
+    /**
+     * 根据智能体ID获取会话列表
+     *
+     * @param params 查询参数，包含agentId、page、limit
+     * @return 分页的会话列表
+     */
+    PageData<AgentChatSessionDTO> getSessionListByAgentId(Map<String, Object> params);
+
+    /**
+     * 根据会话ID获取聊天记录列表
+     *
+     * @param agentId   智能体ID
+     * @param sessionId 会话ID
+     * @return 聊天记录列表
+     */
+    List<AgentChatHistoryDTO> getChatHistoryBySessionId(String agentId, String sessionId);
+
+    /**
+     * 根据智能体ID删除聊天记录
+     *
+     * @param agentId 智能体ID
+     */
+    void deleteByAgentId(String agentId);
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentService.java

@@ -8,36 +8,56 @@ import xiaozhi.common.service.BaseService;
 import xiaozhi.modules.agent.dto.AgentDTO;
 import xiaozhi.modules.agent.entity.AgentEntity;
 
+/**
+ * 智能体表处理service
+ *
+ * @author Goody
+ * @version 1.0, 2025/4/30
+ * @since 1.0.0
+ */
 public interface AgentService extends BaseService<AgentEntity> {
-
     /**
-     * 管理员获取所有智能体列表（分页）
+     * 获取管理员智能体列表
+     *
+     * @param params 查询参数
+     * @return 分页数据
      */
     PageData<AgentEntity> adminAgentList(Map<String, Object> params);
 
     /**
-     * 获取智能体详情
+     * 根据ID获取智能体
+     *
+     * @param id 智能体ID
+     * @return 智能体实体
      */
     AgentEntity getAgentById(String id);
 
     /**
-     * 删除这个用户的所有
-     * 
-     * @param userId
+     * 插入智能体
+     *
+     * @param entity 智能体实体
+     * @return 是否成功
+     */
+    boolean insert(AgentEntity entity);
+
+    /**
+     * 根据用户ID删除智能体
+     *
+     * @param userId 用户ID
      */
     void deleteAgentByUserId(Long userId);
 
     /**
      * 获取用户智能体列表
-     * 
-     * @param userId
-     * @return
+     *
+     * @param userId 用户ID
+     * @return 智能体列表
      */
     List<AgentDTO> getUserAgents(Long userId);
 
     /**
-     * 获取智能体的设备数量
-     * 
+     * 根据智能体ID获取设备数量
+     *
      * @param agentId 智能体ID
      * @return 设备数量
      */
@@ -50,4 +70,13 @@ public interface AgentService extends BaseService<AgentEntity> {
      * @return 默认智能体信息，不存在时返回null
      */
     AgentEntity getDefaultAgentByMacAddress(String macAddress);
+
+    /**
+     * 检查用户是否有权限访问智能体
+     *
+     * @param agentId 智能体ID
+     * @param userId  用户ID
+     * @return 是否有权限
+     */
+    boolean checkAgentPermission(String agentId, Long userId);
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java

@@ -1,10 +1,15 @@
 package xiaozhi.modules.agent.service.biz.impl;
 
+import java.util.Base64;
+import java.util.Date;
+
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import xiaozhi.common.redis.RedisKeys;
+import xiaozhi.common.redis.RedisUtils;
 import xiaozhi.modules.agent.dto.AgentChatHistoryReportDTO;
 import xiaozhi.modules.agent.entity.AgentChatHistoryEntity;
 import xiaozhi.modules.agent.entity.AgentEntity;
@@ -27,6 +32,7 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic
     private final AgentService agentService;
     private final AgentChatHistoryService agentChatHistoryService;
     private final AgentChatAudioService agentChatAudioService;
+    private final RedisUtils redisUtils;
 
     /**
      * 处理聊天记录上报，包括文件上传和相关信息记录
@@ -43,12 +49,11 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic
 
         // 1. base64解码report.getOpusDataBase64(),存入ai_agent_chat_audio表
         String audioId = null;
-        if (report.getOpusDataBase64() != null && !report.getOpusDataBase64().isEmpty()) {
+        if (report.getAudioBase64() != null && !report.getAudioBase64().isEmpty()) {
             try {
-                // TODO: 需要考虑保留什么格式的音频数据，比如是opus还是wave
-                // byte[] audioData = Base64.getDecoder().decode(report.getOpusDataBase64());
-                // audioId = agentChatAudioService.saveAudio(audioData);
-                // log.info("音频数据保存成功，audioId={}", audioId);
+                byte[] audioData = Base64.getDecoder().decode(report.getAudioBase64());
+                audioId = agentChatAudioService.saveAudio(audioData);
+                log.info("音频数据保存成功，audioId={}", audioId);
             } catch (Exception e) {
                 log.error("音频数据保存失败", e);
                 return false;
@@ -76,6 +81,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic
 
         // 3. 保存数据
         agentChatHistoryService.save(entity);
+        // 4. 更新设备最后对话时间
+        redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), new Date());
         return Boolean.TRUE;
     }
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java

@@ -25,4 +25,10 @@ public class AgentChatAudioServiceImpl extends ServiceImpl<AiAgentChatAudioDao,
         save(entity);
         return entity.getId();
     }
+
+    @Override
+    public byte[] getAudio(String audioId) {
+        AgentChatAudioEntity entity = getById(audioId);
+        return entity != null ? entity.getAudio() : null;
+    }
 }

main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java

@@ -1,19 +1,85 @@
-package xiaozhi.modules.agent.service.impl;
-
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import org.springframework.stereotype.Service;
-import xiaozhi.modules.agent.dao.AiAgentChatHistoryDao;
-import xiaozhi.modules.agent.entity.AgentChatHistoryEntity;
-import xiaozhi.modules.agent.service.AgentChatHistoryService;
-
-/**
- * 智能体聊天记录表处理service {@link AgentChatHistoryService} impl
- *
- * @author Goody
- * @version 1.0, 2025/4/30
- * @since 1.0.0
- */
-@Service
-public class AgentChatHistoryServiceImpl extends ServiceImpl<AiAgentChatHistoryDao, AgentChatHistoryEntity> implements AgentChatHistoryService {
-
-}
+package xiaozhi.modules.agent.service.impl;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+
+import xiaozhi.common.constant.Constant;
+import xiaozhi.common.page.PageData;
+import xiaozhi.common.utils.ConvertUtils;
+import xiaozhi.modules.agent.dao.AiAgentChatHistoryDao;
+import xiaozhi.modules.agent.dto.AgentChatHistoryDTO;
+import xiaozhi.modules.agent.dto.AgentChatSessionDTO;
+import xiaozhi.modules.agent.entity.AgentChatHistoryEntity;
+import xiaozhi.modules.agent.service.AgentChatHistoryService;
+
+/**
+ * 智能体聊天记录表处理service {@link AgentChatHistoryService} impl
+ *
+ * @author Goody
+ * @version 1.0, 2025/4/30
+ * @since 1.0.0
+ */
+@Service
+public class AgentChatHistoryServiceImpl extends ServiceImpl<AiAgentChatHistoryDao, AgentChatHistoryEntity>
+        implements AgentChatHistoryService {
+
+    @Override
+    public PageData<AgentChatSessionDTO> getSessionListByAgentId(Map<String, Object> params) {
+        String agentId = (String) params.get("agentId");
+        int page = Integer.parseInt(params.get(Constant.PAGE).toString());
+        int limit = Integer.parseInt(params.get(Constant.LIMIT).toString());
+
+        // 构建查询条件
+        QueryWrapper<AgentChatHistoryEntity> wrapper = new QueryWrapper<>();
+        wrapper.select("session_id", "MAX(created_at) as created_at", "COUNT(*) as chat_count")
+                .eq("agent_id", agentId)
+                .groupBy("session_id")
+                .orderByDesc("created_at");
+
+        // 执行分页查询
+        Page<Map<String, Object>> pageParam = new Page<>(page, limit);
+        IPage<Map<String, Object>> result = this.baseMapper.selectMapsPage(pageParam, wrapper);
+
+        List<AgentChatSessionDTO> records = result.getRecords().stream().map(map -> {
+            AgentChatSessionDTO dto = new AgentChatSessionDTO();
+            dto.setSessionId((String) map.get("session_id"));
+            dto.setCreatedAt((LocalDateTime) map.get("created_at"));
+            dto.setChatCount(((Number) map.get("chat_count")).intValue());
+            return dto;
+        }).collect(Collectors.toList());
+
+        return new PageData<>(records, result.getTotal());
+    }
+
+    @Override
+    public List<AgentChatHistoryDTO> getChatHistoryBySessionId(String agentId, String sessionId) {
+        // 构建查询条件
+        QueryWrapper<AgentChatHistoryEntity> wrapper = new QueryWrapper<>();
+        wrapper.eq("agent_id", agentId)
+                .eq("session_id", sessionId)
+                .orderByAsc("created_at");
+
+        // 查询聊天记录
+        List<AgentChatHistoryEntity> historyList = list(wrapper);
+
+        // 转换为DTO
+        return ConvertUtils.sourceToTarget(historyList, AgentChatHistoryDTO.class);
+    }
+
+    @Override
+    @Transactional(rollbackFor = Exception.class)
+    public void deleteByAgentId(String agentId) {
+        baseMapper.deleteAudioByAgentId(agentId);
+        baseMapper.deleteHistoryByAgentId(agentId);
+    }
+}

main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java

@@ -6,13 +6,13 @@ import java.util.UUID;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 
+import lombok.AllArgsConstructor;
 import xiaozhi.common.page.PageData;
 import xiaozhi.common.redis.RedisKeys;
 import xiaozhi.common.redis.RedisUtils;
@@ -21,25 +21,20 @@ import xiaozhi.modules.agent.dao.AgentDao;
 import xiaozhi.modules.agent.dto.AgentDTO;
 import xiaozhi.modules.agent.entity.AgentEntity;
 import xiaozhi.modules.agent.service.AgentService;
+import xiaozhi.modules.device.service.DeviceService;
 import xiaozhi.modules.model.service.ModelConfigService;
+import xiaozhi.modules.security.user.SecurityUser;
+import xiaozhi.modules.sys.enums.SuperAdminEnum;
 import xiaozhi.modules.timbre.service.TimbreService;
 
 @Service
+@AllArgsConstructor
 public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> implements AgentService {
     private final AgentDao agentDao;
-
-    @Autowired
-    private TimbreService timbreModelService;
-
-    @Autowired
-    private ModelConfigService modelConfigService;
-
-    @Autowired
-    private RedisUtils redisUtils;
-
-    public AgentServiceImpl(AgentDao agentDao) {
-        this.agentDao = agentDao;
-    }
+    private final TimbreService timbreModelService;
+    private final ModelConfigService modelConfigService;
+    private final RedisUtils redisUtils;
+    private final DeviceService deviceService;
 
     @Override
     public PageData<AgentEntity> adminAgentList(Map<String, Object> params) {
@@ -101,9 +96,11 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
             // 获取 TTS 音色名称
             dto.setTtsVoiceName(timbreModelService.getTimbreNameById(agent.getTtsVoiceId()));
 
+            // 获取智能体最近的最后连接时长
+            dto.setLastConnectedAt(deviceService.getLatestLastConnectionTime(agent.getId()));
+
             // 获取设备数量
             dto.setDeviceCount(getDeviceCountByAgentId(agent.getId()));
-
             return dto;
         }).collect(Collectors.toList());
     }
@@ -138,4 +135,21 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
         }
         return agentDao.getDefaultAgentByMacAddress(macAddress);
     }
+
+    @Override
+    public boolean checkAgentPermission(String agentId, Long userId) {
+        // 获取智能体信息
+        AgentEntity agent = getAgentById(agentId);
+        if (agent == null) {
+            return false;
+        }
+
+        // 如果是超级管理员，直接返回true
+        if (SecurityUser.getUser().getSuperAdmin() == SuperAdminEnum.YES.value()) {
+            return true;
+        }
+
+        // 检查是否是智能体的所有者
+        return userId.equals(agent.getUserId());
+    }
 }

main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java

@@ -61,15 +61,15 @@ public class ConfigServiceImpl implements ConfigService {
 
         // 构建模块配置
         buildModuleConfig(
-                agent.getAgentName(),
+                null,
                 null,
                 null,
                 agent.getVadModelId(),
                 agent.getAsrModelId(),
-                agent.getLlmModelId(),
-                agent.getTtsModelId(),
-                agent.getMemModelId(),
-                agent.getIntentModelId(),
+                null,
+                null,
+                null,
+                null,
                 result,
                 isCache);
 
@@ -117,18 +117,6 @@ public class ConfigServiceImpl implements ConfigService {
         if (alreadySelectedAsrModelId != null && alreadySelectedAsrModelId.equals(agent.getAsrModelId())) {
             agent.setAsrModelId(null);
         }
-        String alreadySelectedLlmModelId = (String) selectedModule.get("LLM");
-        if (alreadySelectedLlmModelId != null && alreadySelectedLlmModelId.equals(agent.getLlmModelId())) {
-            agent.setLlmModelId(null);
-        }
-        String alreadySelectedMemModelId = (String) selectedModule.get("Memory");
-        if (alreadySelectedMemModelId != null && alreadySelectedMemModelId.equals(agent.getMemModelId())) {
-            agent.setMemModelId(null);
-        }
-        String alreadySelectedIntentModelId = (String) selectedModule.get("Intent");
-        if (alreadySelectedIntentModelId != null && alreadySelectedIntentModelId.equals(agent.getIntentModelId())) {
-            agent.setIntentModelId(null);
-        }
 
         // 构建模块配置
         buildModuleConfig(

main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java

@@ -6,6 +6,16 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 
 import xiaozhi.modules.device.entity.DeviceEntity;
 
+import java.util.Date;
+import java.util.List;
+
 @Mapper
 public interface DeviceDao extends BaseMapper<DeviceEntity> {
+    /**
+     * 获取此智能体全部设备的最后连接时间
+     * @param agentId 智能体id
+     * @return
+     */
+    Date getAllLastConnectedAtByAgentId(String agentId);
+
 }

main/manager-api/src/main/java/xiaozhi/modules/device/service/DeviceService.java

@@ -1,5 +1,6 @@
 package xiaozhi.modules.device.service;
 
+import java.util.Date;
 import java.util.List;
 
 import xiaozhi.common.page.PageData;
@@ -78,4 +79,13 @@ public interface DeviceService extends BaseService<DeviceEntity> {
      * @return 激活码
      */
     String geCodeByDeviceId(String deviceId);
+
+    /**
+     * 获取这个智能体设备理的最近的最后连接时间
+     * @param agentId 智能体id
+     * @return 返回设备最近的最后连接时间
+     */
+    Date getLatestLastConnectionTime(String agentId);
+
+
 }

main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java

@@ -258,6 +258,20 @@ public class DeviceServiceImpl extends BaseServiceImpl<DeviceDao, DeviceEntity>
         return null;
     }
 
+    @Override
+    public Date getLatestLastConnectionTime(String agentId) {
+        // 查询是否有缓存时间，有则返回
+        Date cachedDate = (Date) redisUtils.get(RedisKeys.getAgentDeviceLastConnectedAtById(agentId));
+        if (cachedDate != null) {
+            return cachedDate;
+        }
+        Date maxDate = deviceDao.getAllLastConnectedAtByAgentId(agentId);
+        if (maxDate != null) {
+            redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), maxDate);
+        }
+        return maxDate;
+    }
+
     private String getDeviceCacheKey(String deviceId) {
         String safeDeviceId = deviceId.replace(":", "_").toLowerCase();
         String dataKey = String.format("ota:activation:data:%s", safeDeviceId);

main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java

@@ -1,6 +1,9 @@
 package xiaozhi.modules.security.config;
 
-import jakarta.servlet.Filter;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 import org.apache.shiro.mgt.SecurityManager;
 import org.apache.shiro.session.mgt.SessionManager;
 import org.apache.shiro.spring.LifecycleBeanPostProcessor;
@@ -11,15 +14,13 @@ import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
 import org.apache.shiro.web.session.mgt.DefaultWebSessionManager;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+
+import jakarta.servlet.Filter;
 import xiaozhi.modules.security.oauth2.Oauth2Filter;
 import xiaozhi.modules.security.oauth2.Oauth2Realm;
 import xiaozhi.modules.security.secret.ServerSecretFilter;
 import xiaozhi.modules.sys.service.SysParamsService;
 
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
 /**
  * Shiro的配置文件
  * Copyright (c) 人人开源 All rights reserved.
@@ -85,6 +86,7 @@ public class ShiroConfig {
         // 将config路径使用server服务过滤器
         filterMap.put("/config/**", "server");
         filterMap.put("/agent/chat-history/report", "server");
+        filterMap.put("/agent/play/**", "anon");
         filterMap.put("/**", "oauth2");
         shiroFilter.setFilterChainDefinitionMap(filterMap);
 

main/manager-api/src/main/java/xiaozhi/modules/security/config/WebMvcConfig.java

@@ -1,5 +1,6 @@
 package xiaozhi.modules.security.config;
 
+import java.text.SimpleDateFormat;
 import java.util.List;
 import java.util.TimeZone;
 
@@ -18,6 +19,15 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer;
+import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
+import com.fasterxml.jackson.datatype.jsr310.deser.LocalTimeDeserializer;
+import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateSerializer;
+import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
+import com.fasterxml.jackson.datatype.jsr310.ser.LocalTimeSerializer;
+
+import xiaozhi.common.utils.DateUtils;
 
 @Configuration
 public class WebMvcConfig implements WebMvcConfigurer {
@@ -53,10 +63,29 @@ public class WebMvcConfig implements WebMvcConfigurer {
         // 忽略未知属性
         mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
-        // 日期格式转换
-        // mapper.setDateFormat(new SimpleDateFormat(DateUtils.DATE_TIME_PATTERN));
+        // 设置时区
         mapper.setTimeZone(TimeZone.getTimeZone("GMT+8"));
 
+        // 配置Java8日期时间序列化
+        JavaTimeModule javaTimeModule = new JavaTimeModule();
+        javaTimeModule.addSerializer(java.time.LocalDateTime.class, new LocalDateTimeSerializer(
+                java.time.format.DateTimeFormatter.ofPattern(DateUtils.DATE_TIME_PATTERN)));
+        javaTimeModule.addSerializer(java.time.LocalDate.class, new LocalDateSerializer(
+                java.time.format.DateTimeFormatter.ofPattern(DateUtils.DATE_PATTERN)));
+        javaTimeModule.addSerializer(java.time.LocalTime.class,
+                new LocalTimeSerializer(java.time.format.DateTimeFormatter.ofPattern("HH:mm:ss")));
+        javaTimeModule.addDeserializer(java.time.LocalDateTime.class, new LocalDateTimeDeserializer(
+                java.time.format.DateTimeFormatter.ofPattern(DateUtils.DATE_TIME_PATTERN)));
+        javaTimeModule.addDeserializer(java.time.LocalDate.class, new LocalDateDeserializer(
+                java.time.format.DateTimeFormatter.ofPattern(DateUtils.DATE_PATTERN)));
+        javaTimeModule.addDeserializer(java.time.LocalTime.class,
+                new LocalTimeDeserializer(java.time.format.DateTimeFormatter.ofPattern("HH:mm:ss")));
+        mapper.registerModule(javaTimeModule);
+
+        // 配置java.util.Date的序列化和反序列化
+        SimpleDateFormat dateFormat = new SimpleDateFormat(DateUtils.DATE_TIME_PATTERN);
+        mapper.setDateFormat(dateFormat);
+
         // Long类型转String类型
         SimpleModule simpleModule = new SimpleModule();
         simpleModule.addSerializer(Long.class, ToStringSerializer.instance);

main/manager-api/src/main/java/xiaozhi/modules/sys/controller/SysDictDataController.java

@@ -96,7 +96,7 @@ public class SysDictDataController {
 
     @GetMapping("/type/{dictType}")
     @Operation(summary = "获取字典数据列表")
-    @RequiresPermissions("sys:role:superAdmin")
+    @RequiresPermissions("sys:role:normal")
     public Result<List<SysDictDataItem>> getDictDataByType(@PathVariable("dictType") String dictType) {
         List<SysDictDataItem> list = sysDictDataService.getDictDataByType(dictType);
         return new Result<List<SysDictDataItem>>().ok(list);

main/manager-api/src/main/java/xiaozhi/modules/sys/utils/WebSocketValidator.java

@@ -8,6 +8,7 @@ import java.util.regex.Pattern;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.web.socket.WebSocketHttpHeaders;
 import org.springframework.web.socket.client.WebSocketClient;
 import org.springframework.web.socket.client.standard.StandardWebSocketClient;
 
@@ -45,8 +46,9 @@ public class WebSocketValidator {
         try {
             WebSocketClient client = new StandardWebSocketClient();
             CompletableFuture<Boolean> future = new CompletableFuture<>();
+            WebSocketHttpHeaders headers = new WebSocketHttpHeaders();
 
-            client.doHandshake(new WebSocketTestHandler(future), null, URI.create(url));
+            client.execute(new WebSocketTestHandler(future), headers, URI.create(url));
 
             // 等待最多5秒获取连接结果
             return future.get(5, TimeUnit.SECONDS);

main/manager-api/src/main/resources/db/changelog/202505022134.sql

@@ -1,5 +1,6 @@
 -- 初始化智能体聊天记录
 DROP TABLE IF EXISTS ai_chat_history;
+DROP TABLE IF EXISTS ai_chat_message;
 DROP TABLE IF EXISTS ai_agent_chat_history;
 CREATE TABLE ai_agent_chat_history
 (
@@ -13,7 +14,9 @@ CREATE TABLE ai_agent_chat_history
     created_at  DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL COMMENT '创建时间',
     updated_at  DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL ON UPDATE CURRENT_TIMESTAMP(3) COMMENT '更新时间',
     INDEX idx_ai_agent_chat_history_mac (mac_address),
-    INDEX idx_ai_agent_chat_history_agent_id (agent_id)
+    INDEX idx_ai_agent_chat_history_session_id (session_id),
+    INDEX idx_ai_agent_chat_history_agent_id (agent_id),
+    INDEX idx_ai_agent_chat_history_agent_session_created (agent_id, session_id, created_at)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT '智能体聊天记录表';
 
 DROP TABLE IF EXISTS ai_agent_chat_audio;

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -94,9 +94,9 @@ databaseChangeLog:
             encoding: utf8
             path: classpath:db/changelog/202504301341.sql
   - changeSet:
-      id: 202505012207
+      id: 202505022134
       author: Goody
       changes:
         - sqlFile:
             encoding: utf8
-            path: classpath:db/changelog/202505012207.sql
+            path: classpath:db/changelog/202505022134.sql

main/manager-api/src/main/resources/mapper/agent/AiAgentChatHistoryDao.xml

@@ -21,4 +21,18 @@
     id, mac_address, agent_id, session_id, sort, chat_type, content, audio, audio_url, 
     created_at, updated_at
   </sql>
+
+  <delete id="deleteAudioByAgentId">
+    DELETE FROM ai_agent_chat_audio 
+    WHERE id IN (
+      SELECT audio_id 
+      FROM ai_agent_chat_history 
+      WHERE agent_id = #{agentId}
+    );
+  </delete>
+
+  <delete id="deleteHistoryByAgentId">
+    DELETE FROM ai_agent_chat_history 
+    WHERE agent_id = #{agentId};
+  </delete>
 </mapper>

main/manager-api/src/main/resources/mapper/device/DeviceDao.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="xiaozhi.modules.device.dao.DeviceDao">
+    <!-- 获取此智能体全部设备的最后连接时间 -->
+    <select id="getAllLastConnectedAtByAgentId" resultType="java.util.Date">
+        SELECT last_connected_at FROM ai_device
+        WHERE
+            agent_id = #{agentId}
+        order by
+            last_connected_at desc limit 0,1
+    </select>
+</mapper> 

main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java

@@ -1,5 +1,8 @@
 package xiaozhi.modules.device;
 
+import java.util.HashMap;
+import java.util.UUID;
+
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
@@ -8,8 +11,9 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 
 import lombok.extern.slf4j.Slf4j;
-import xiaozhi.common.redis.RedisKeys;
 import xiaozhi.common.redis.RedisUtils;
+import xiaozhi.modules.sys.dto.SysUserDTO;
+import xiaozhi.modules.sys.service.SysUserService;
 
 @Slf4j
 @SpringBootTest
@@ -19,18 +23,37 @@ public class DeviceTest {
 
     @Autowired
     private RedisUtils redisUtils;
+    @Autowired
+    private SysUserService sysUserService;
+
+    @Test
+    public void testSaveUser() {
+        SysUserDTO userDTO = new SysUserDTO();
+        userDTO.setUsername("test");
+        userDTO.setPassword(UUID.randomUUID().toString());
+        sysUserService.save(userDTO);
+    }
 
     @Test
     @DisplayName("测试写入设备信息")
     public void testWriteDeviceInfo() {
         log.info("开始测试写入设备信息...");
-
         // 模拟设备MAC地址
-        String macAddress = "00:11:22:33:44:55";
+        String macAddress = "00:11:22:33:44:66";
         // 模拟设备验证码
         String deviceCode = "123456";
 
-        String redisKey = RedisKeys.getDeviceCaptchaKey(deviceCode);
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("mac_address", macAddress);
+        map.put("activation_code", deviceCode);
+        map.put("board", "硬件型号");
+        map.put("app_version", "0.3.13");
+
+        String safeDeviceId = macAddress.replace(":", "_").toLowerCase();
+        String cacheDeviceKey = String.format("ota:activation:data:%s", safeDeviceId);
+        redisUtils.set(cacheDeviceKey, map, 300);
+
+        String redisKey = "ota:activation:code:" + deviceCode;
         log.info("Redis Key: {}", redisKey);
 
         // 将设备信息写入Redis

main/manager-web/src/apis/module/agent.js

@@ -97,4 +97,50 @@ export default {
                 });
             }).send();
     },
+    // 获取智能体会话列表
+    getAgentSessions(agentId, params, callback) {
+        RequestService.sendRequest()
+            .url(`${getServiceUrl()}/agent/${agentId}/sessions`)
+            .method('GET')
+            .data(params)
+            .success((res) => {
+                RequestService.clearRequestTime();
+                callback(res);
+            })
+            .fail(() => {
+                RequestService.reAjaxFun(() => {
+                    this.getAgentSessions(agentId, params, callback);
+                });
+            }).send();
+    },
+    // 获取智能体聊天记录
+    getAgentChatHistory(agentId, sessionId, callback) {
+        RequestService.sendRequest()
+            .url(`${getServiceUrl()}/agent/${agentId}/chat-history/${sessionId}`)
+            .method('GET')
+            .success((res) => {
+                RequestService.clearRequestTime();
+                callback(res);
+            })
+            .fail(() => {
+                RequestService.reAjaxFun(() => {
+                    this.getAgentChatHistory(agentId, sessionId, callback);
+                });
+            }).send();
+    },
+    // 获取音频下载ID
+    getAudioId(audioId, callback) {
+        RequestService.sendRequest()
+            .url(`${getServiceUrl()}/agent/audio/${audioId}`)
+            .method('POST')
+            .success((res) => {
+                RequestService.clearRequestTime();
+                callback(res);
+            })
+            .fail(() => {
+                RequestService.reAjaxFun(() => {
+                    this.getAudioId(audioId, callback);
+                });
+            }).send();
+    },
 }

main/manager-web/src/components/ChatHistoryDialog.vue

@@ -0,0 +1,447 @@
+<template>
+    <el-dialog :title="'与' + agentName + '的聊天记录' + (currentMacAddress ? '[' + currentMacAddress + ']' : '')"
+        :visible.sync="dialogVisible" width="80%" :before-close="handleClose" custom-class="chat-history-dialog">
+        <div class="chat-container">
+            <div class="session-list" @scroll="handleScroll">
+                <div v-for="session in sessions" :key="session.sessionId" class="session-item"
+                    :class="{ active: currentSessionId === session.sessionId }" @click="selectSession(session)">
+                    <img :src="getUserAvatar(session.sessionId)" class="avatar" />
+                    <div class="session-info">
+                        <div class="session-time">{{ formatTime(session.createdAt) }}</div>
+                        <div class="message-count">{{ session.chatCount > 99 ? '99' : session.chatCount }}</div>
+                    </div>
+                </div>
+                <div v-if="loading" class="loading">加载中...</div>
+                <div v-if="!hasMore" class="no-more">没有更多记录了</div>
+            </div>
+            <div class="chat-content">
+                <div v-if="currentSessionId" class="messages">
+                    <div v-for="(message, index) in messagesWithTime" :key="message.id">
+                        <div v-if="message.type === 'time'" class="time-divider">
+                            {{ message.content }}
+                        </div>
+                        <div v-else class="message-item" :class="{ 'user-message': message.chatType === 1 }">
+                            <img :src="message.chatType === 1 ? getUserAvatar(currentSessionId) : require('@/assets/xiaozhi-logo.png')"
+                                class="avatar" />
+                            <div class="message-content">
+                                {{ message.content }}
+                                <i v-if="message.audioId" :class="getAudioIconClass(message)"
+                                    @click="playAudio(message)" class="audio-icon"></i>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                <div v-else class="no-session-selected">
+                    请选择会话查看聊天记录
+                </div>
+            </div>
+        </div>
+    </el-dialog>
+</template>
+
+<script>
+import Api from '@/apis/api';
+
+export default {
+    name: 'ChatHistoryDialog',
+    props: {
+        visible: {
+            type: Boolean,
+            default: false
+        },
+        agentId: {
+            type: String,
+            required: true
+        },
+        agentName: {
+            type: String,
+            required: true
+        }
+    },
+    data() {
+        return {
+            dialogVisible: false,
+            sessions: [],
+            messages: [],
+            currentSessionId: '',
+            currentMacAddress: '',
+            page: 1,
+            limit: 20,
+            loading: false,
+            hasMore: true,
+            scrollTimer: null,
+            isFirstLoad: true,
+            playingAudioId: null,
+            audioElement: null
+        };
+    },
+    watch: {
+        visible(val) {
+            this.dialogVisible = val;
+            if (val) {
+                this.resetData();
+                this.loadSessions();
+            }
+        },
+        dialogVisible(val) {
+            if (!val) {
+                this.$emit('update:visible', false);
+            }
+        }
+    },
+    computed: {
+        messagesWithTime() {
+            if (!this.messages || this.messages.length === 0) return [];
+
+            const result = [];
+            const TIME_INTERVAL = 60 * 1000; // 1分钟的时间间隔（毫秒）
+
+            // 添加第一条消息的时间标记
+            if (this.messages[0]) {
+                result.push({
+                    type: 'time',
+                    content: this.formatTime(this.messages[0].createdAt),
+                    id: `time-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`
+                });
+            }
+
+            // 处理消息列表
+            for (let i = 0; i < this.messages.length; i++) {
+                const currentMessage = this.messages[i];
+                result.push(currentMessage);
+
+                // 检查是否需要添加时间标记
+                if (i < this.messages.length - 1) {
+                    const currentTime = new Date(currentMessage.createdAt).getTime();
+                    const nextTime = new Date(this.messages[i + 1].createdAt).getTime();
+
+                    if (nextTime - currentTime > TIME_INTERVAL) {
+                        result.push({
+                            type: 'time',
+                            content: this.formatTime(this.messages[i + 1].createdAt),
+                            id: `time-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`
+                        });
+                    }
+                }
+            }
+
+            return result;
+        }
+    },
+    methods: {
+        resetData() {
+            this.sessions = [];
+            this.messages = [];
+            this.currentSessionId = '';
+            this.currentMacAddress = '';
+            this.page = 1;
+            this.loading = false;
+            this.hasMore = true;
+            this.isFirstLoad = true;
+        },
+        handleClose() {
+            this.dialogVisible = false;
+        },
+        loadSessions() {
+            if (this.loading || (!this.isFirstLoad && !this.hasMore)) {
+                return;
+            }
+
+            this.loading = true;
+            const params = {
+                page: this.page,
+                limit: this.limit
+            };
+
+            Api.agent.getAgentSessions(this.agentId, params, (res) => {
+                if (res.data && res.data.data && Array.isArray(res.data.data.list)) {
+                    const list = res.data.data.list;
+                    this.hasMore = list.length === this.limit;
+
+                    this.sessions = [...this.sessions, ...list];
+                    this.page++;
+
+                    if (this.sessions.length > 0 && !this.currentSessionId) {
+                        this.selectSession(this.sessions[0]);
+                    }
+                }
+                this.loading = false;
+                this.isFirstLoad = false;
+            });
+        },
+        selectSession(session) {
+            this.currentSessionId = session.sessionId;
+            Api.agent.getAgentChatHistory(this.agentId, session.sessionId, (res) => {
+                if (res.data && res.data.data) {
+                    this.messages = res.data.data;
+                    if (this.messages.length > 0 && this.messages[0].macAddress) {
+                        this.currentMacAddress = this.messages[0].macAddress;
+                    }
+                }
+            });
+        },
+        handleScroll(e) {
+            if (this.scrollTimer) {
+                clearTimeout(this.scrollTimer);
+            }
+
+            this.scrollTimer = setTimeout(() => {
+                const { scrollTop, scrollHeight, clientHeight } = e.target;
+                // 当滚动到底部时加载更多
+                if (scrollHeight - scrollTop <= clientHeight + 50) {
+                    this.loadSessions();
+                }
+            }, 200);
+        },
+        formatTime(timestamp) {
+            const date = new Date(timestamp);
+            const now = new Date();
+            const today = new Date(now.getFullYear(), now.getMonth(), now.getDate());
+            const yesterday = new Date(today);
+            yesterday.setDate(yesterday.getDate() - 1);
+
+            const hours = date.getHours().toString().padStart(2, '0');
+            const minutes = date.getMinutes().toString().padStart(2, '0');
+
+            if (date >= today) {
+                return `今天 ${hours}:${minutes}`;
+            } else if (date >= yesterday) {
+                return `昨天 ${hours}:${minutes}`;
+            } else {
+                const year = date.getFullYear();
+                const month = (date.getMonth() + 1).toString().padStart(2, '0');
+                const day = date.getDate().toString().padStart(2, '0');
+                return `${year}-${month}-${day} ${hours}:${minutes}`;
+            }
+        },
+        getAudioIconClass(message) {
+            if (this.playingAudioId === message.audioId) {
+                return 'el-icon-loading';
+            }
+            return 'el-icon-video-play';
+        },
+        playAudio(message) {
+            if (this.playingAudioId === message.audioId) {
+                // 如果正在播放当前音频，则停止播放
+                if (this.audioElement) {
+                    this.audioElement.pause();
+                    this.audioElement = null;
+                }
+                this.playingAudioId = null;
+                return;
+            }
+
+            // 停止当前正在播放的音频
+            if (this.audioElement) {
+                this.audioElement.pause();
+                this.audioElement = null;
+            }
+
+            // 先获取音频下载ID
+            this.playingAudioId = message.audioId;
+            Api.agent.getAudioId(message.audioId, (res) => {
+                if (res.data && res.data.data) {
+                    // 使用获取到的下载ID播放音频
+                    this.audioElement = new Audio(Api.getServiceUrl() + `/agent/play/${res.data.data}`);
+
+                    this.audioElement.onended = () => {
+                        this.playingAudioId = null;
+                        this.audioElement = null;
+                    };
+
+                    this.audioElement.play();
+                }
+            });
+        },
+        getUserAvatar(sessionId) {
+            // 从 sessionId 中提取所有数字
+            const numbers = sessionId.match(/\d+/g);
+            if (!numbers) return require('@/assets/user-avatar1.png');
+
+            // 将所有数字相加
+            const sum = numbers.reduce((acc, num) => acc + parseInt(num), 0);
+
+            // 计算模5并加1，得到1-5之间的数字
+            const avatarIndex = (sum % 5) + 1;
+
+            // 返回对应的头像图片
+            return require(`@/assets/user-avatar${avatarIndex}.png`);
+        }
+    }
+};
+</script>
+
+<style scoped>
+.chat-container {
+    display: flex;
+    height: 100%;
+}
+
+.session-list {
+    width: 250px;
+    border-right: 1px solid #eee;
+    overflow-y: auto;
+    padding: 10px;
+}
+
+.session-item {
+    display: flex;
+    align-items: center;
+    padding: 10px;
+    cursor: pointer;
+    border-radius: 8px;
+    margin-bottom: 10px;
+}
+
+.session-item:hover {
+    background-color: #f5f5f5;
+}
+
+.session-item.active {
+    background-color: #e6f7ff;
+}
+
+.avatar {
+    width: 40px;
+    height: 40px;
+    border-radius: 50%;
+    margin-right: 10px;
+}
+
+.session-info {
+    flex: 1;
+}
+
+.session-time {
+    font-size: 14px;
+    color: #272727;
+    float: left;
+    height: 30px;
+    line-height: 30px;
+    width: calc(100% - 30px);
+    /* 为消息数量留出空间 */
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+}
+
+.message-count {
+    font-size: 14px;
+    color: #fff;
+    background-color: #b4b4b4;
+    border-radius: 20px;
+    float: left;
+    width: 20px;
+    height: 20px;
+    line-height: 20px;
+    margin-top: 5px;
+    margin-left: 5px;
+}
+
+.chat-content {
+    flex: 1;
+    padding: 20px;
+    overflow-y: auto;
+}
+
+.message-item {
+    display: flex;
+    margin-bottom: 20px;
+}
+
+.message-item.user-message {
+    flex-direction: row-reverse;
+}
+
+.message-content {
+    max-width: 60%;
+    padding: 10px 15px;
+    border-radius: 8px;
+    background-color: #f0f0f0;
+    margin: 0 10px;
+    text-align: left;
+    line-height: 20px;
+    position: relative;
+    display: flex;
+    align-items: center;
+}
+
+.audio-icon {
+    font-size: 20px;
+    cursor: pointer;
+    margin: 0 5px;
+    color: #1890ff;
+}
+
+.user-message .message-content {
+    background-color: #1890ff;
+    color: white;
+    flex-direction: row-reverse;
+}
+
+.user-message .audio-icon {
+    color: white;
+}
+
+.loading,
+.no-more {
+    text-align: center;
+    padding: 10px 10px 30px 10px;
+    color: #999;
+}
+
+.no-session-selected {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    height: 100%;
+    color: #999;
+}
+
+.time-divider {
+    text-align: center;
+    margin: 10px 0;
+    color: #999;
+    font-size: 12px;
+}
+
+.time-divider::before,
+.time-divider::after {
+    content: '';
+    display: inline-block;
+    width: 30%;
+    height: 1px;
+    background-color: #eee;
+    vertical-align: middle;
+    margin: 0 10px;
+}
+</style>
+
+<style>
+.chat-history-dialog {
+    display: flex;
+    flex-direction: column;
+    min-width: 700px;
+    margin: 0 !important;
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    height: 90vh;
+    max-width: 85vw;
+    border-radius: 12px;
+    overflow: hidden;
+}
+
+.chat-history-dialog .el-dialog__header {
+    background-color: #e6f7ff;
+    padding: 15px 20px;
+}
+
+.chat-history-dialog .el-dialog__body {
+    padding: 0;
+    overflow: hidden;
+    height: calc(90vh - 54px);
+    /* 减去标题栏的高度 */
+}
+</style>

main/manager-web/src/components/DeviceItem.vue

@@ -26,9 +26,12 @@
       <div class="settings-btn" @click="handleDeviceManage">
         设备管理({{ device.deviceCount }})
       </div>
+      <div class="settings-btn" @click="handleChatHistory">
+        聊天记录
+      </div>
     </div>
     <div class="version-info">
-      <div>最近对话：{{ device.lastConnectedAt }}</div>
+      <div>最近对话：{{ formattedLastConnectedTime }}</div>
     </div>
   </div>
 </template>
@@ -42,6 +45,27 @@ export default {
   data() {
     return { switchValue: false }
   },
+  computed: {
+    formattedLastConnectedTime() {
+      if (!this.device.lastConnectedAt) return '暂未对话';
+
+      const lastTime = new Date(this.device.lastConnectedAt);
+      const now = new Date();
+      const diffMinutes = Math.floor((now - lastTime) / (1000 * 60));
+
+      if (diffMinutes <= 1) {
+        return '刚刚';
+      } else if (diffMinutes < 60) {
+        return `${diffMinutes}分钟前`;
+      } else if (diffMinutes < 24 * 60) {
+        const hours = Math.floor(diffMinutes / 60);
+        const minutes = diffMinutes % 60;
+        return `${hours}小时${minutes > 0 ? minutes + '分钟' : ''}前`;
+      } else {
+        return this.device.lastConnectedAt;
+      }
+    }
+  },
   methods: {
     handleDelete() {
       this.$emit('delete', this.device.agentId)
@@ -51,6 +75,9 @@ export default {
     },
     handleDeviceManage() {
       this.$router.push({ path: '/device-management', query: { agentId: this.device.agentId } });
+    },
+    handleChatHistory() {
+      this.$emit('chat-history', { agentId: this.device.agentId, agentName: this.device.agentName })
     }
   }
 }

main/manager-web/src/views/DeviceManagement.vue

@@ -294,20 +294,13 @@ export default {
         this.loading = false;
         if (data.code === 0) {
           this.deviceList = data.data.map(device => {
-            const bindDate = new Date(device.createDate);
-            const formattedBindTime = `${bindDate.getFullYear()}-${(bindDate.getMonth() + 1).toString().padStart(2, '0')}-${bindDate.getDate().toString().padStart(2, '0')} ${bindDate.getHours().toString().padStart(2, '0')}:${bindDate.getMinutes().toString().padStart(2, '0')}:${bindDate.getSeconds().toString().padStart(2, '0')}`;
-            let formattedLastConversation = '';
-            if (device.lastConnectedAt) {
-              const lastConvoDate = new Date(device.lastConnectedAt);
-              formattedLastConversation = `${lastConvoDate.getFullYear()}-${(lastConvoDate.getMonth() + 1).toString().padStart(2, '0')}-${lastConvoDate.getDate().toString().padStart(2, '0')} ${lastConvoDate.getHours().toString().padStart(2, '0')}:${lastConvoDate.getMinutes().toString().padStart(2, '0')}:${lastConvoDate.getSeconds().toString().padStart(2, '0')}`;
-            }
             return {
               device_id: device.id,
               model: device.board,
               firmwareVersion: device.appVersion,
               macAddress: device.macAddress,
-              bindTime: formattedBindTime,
-              lastConversation: formattedLastConversation,
+              bindTime: device.createDate,
+              lastConversation: device.lastConnectedAt,
               remark: device.alias,
               isEdit: false,
               otaSwitch: device.autoUpdate === 1,

main/manager-web/src/views/ParamsManagement.vue

@@ -25,8 +25,19 @@
                                 </template>
                             </el-table-column>
                             <el-table-column label="参数编码" prop="paramCode" align="center"></el-table-column>
-                            <el-table-column label="参数值" prop="paramValue" align="center"
-                                show-overflow-tooltip></el-table-column>
+                            <el-table-column label="参数值" prop="paramValue" align="center" show-overflow-tooltip>
+                                <template slot-scope="scope">
+                                    <div v-if="isSensitiveParam(scope.row.paramCode)">
+                                        <span v-if="!scope.row.showValue">{{ maskSensitiveValue(scope.row.paramValue)
+                                        }}</span>
+                                        <span v-else>{{ scope.row.paramValue }}</span>
+                                        <el-button size="mini" type="text" @click="toggleSensitiveValue(scope.row)">
+                                            {{ scope.row.showValue ? '隐藏' : '查看' }}
+                                        </el-button>
+                                    </div>
+                                    <span v-else>{{ scope.row.paramValue }}</span>
+                                </template>
+                            </el-table-column>
                             <el-table-column label="备注" prop="remark" align="center"></el-table-column>
                             <el-table-column label="操作" align="center">
                                 <template slot-scope="scope">
@@ -100,6 +111,7 @@ export default {
             dialogVisible: false,
             dialogTitle: "新增参数",
             isAllSelected: false,
+            sensitive_keys: ["api_key", "personal_access_token", "access_token", "token", "secret", "access_key_secret", "secret_key"],
             paramForm: {
                 id: null,
                 paramCode: "",
@@ -152,7 +164,8 @@ export default {
                     if (data.code === 0) {
                         this.paramsList = data.data.list.map(item => ({
                             ...item,
-                            selected: false
+                            selected: false,
+                            showValue: false
                         }));
                         this.total = data.data.total;
                     } else {
@@ -314,7 +327,18 @@ export default {
         goToPage(page) {
             this.currentPage = page;
             this.fetchParams();
-        }
+        },
+        isSensitiveParam(paramCode) {
+            return this.sensitive_keys.some(key => paramCode.toLowerCase().includes(key.toLowerCase()));
+        },
+        maskSensitiveValue(value) {
+            if (!value) return '';
+            if (value.length <= 8) return '****';
+            return value.substring(0, 4) + '****' + value.substring(value.length - 4);
+        },
+        toggleSensitiveValue(row) {
+            this.$set(row, 'showValue', !row.showValue);
+        },
     },
 };
 </script>

main/manager-web/src/views/UserManagement.vue

@@ -26,11 +26,7 @@
               <el-table-column label="用户Id" prop="userid" align="center"></el-table-column>
               <el-table-column label="手机号码" prop="mobile" align="center"></el-table-column>
               <el-table-column label="设备数量" prop="deviceCount" align="center"></el-table-column>
-              <el-table-column label="注册时间" prop="createDate" align="center">
-                <template slot-scope="scope">
-                  {{ formatDate(scope.row.createDate) }}
-                </template>
-              </el-table-column>
+              <el-table-column label="注册时间" prop="createDate" align="center"></el-table-column>
               <el-table-column label="状态" prop="status" align="center">
                 <template slot-scope="scope">
                   <el-tag v-if="scope.row.status === 1" type="success">正常</el-tag>
@@ -342,11 +338,6 @@ export default {
         // 用户取消操作
       });
     },
-    formatDate(dateString) {
-      if (!dateString) return '';
-      const date = new Date(dateString);
-      return `${date.getFullYear()}-${(date.getMonth() + 1).toString().padStart(2, '0')}-${date.getDate().toString().padStart(2, '0')} ${date.getHours().toString().padStart(2, '0')}:${date.getMinutes().toString().padStart(2, '0')}:${date.getSeconds().toString().padStart(2, '0')}`;
-    },
   },
 };
 </script>

main/manager-web/src/views/home.vue

@@ -32,11 +32,7 @@
         </div>
         <div class="device-list-container">
           <template v-if="isLoading">
-            <div
-              v-for="i in skeletonCount"
-              :key="'skeleton-'+i"
-              class="skeleton-item"
-            >
+            <div v-for="i in skeletonCount" :key="'skeleton-' + i" class="skeleton-item">
               <div class="skeleton-image"></div>
               <div class="skeleton-content">
                 <div class="skeleton-line"></div>
@@ -46,14 +42,8 @@
           </template>
 
           <template v-else>
-            <DeviceItem
-              v-for="(item, index) in devices"
-              :key="index"
-              :device="item"
-              @configure="goToRoleConfig"
-              @deviceManage="handleDeviceManage"
-              @delete="handleDeleteAgent"
-            />
+            <DeviceItem v-for="(item, index) in devices" :key="index" :device="item" @configure="goToRoleConfig"
+              @deviceManage="handleDeviceManage" @delete="handleDeleteAgent" @chat-history="handleShowChatHistory" />
           </template>
         </div>
       </div>
@@ -62,6 +52,7 @@
     <el-footer>
       <version-footer />
     </el-footer>
+    <chat-history-dialog :visible.sync="showChatHistory" :agent-id="currentAgentId" :agent-name="currentAgentName" />
   </div>
 
 </template>
@@ -69,13 +60,14 @@
 <script>
 import Api from '@/apis/api';
 import AddWisdomBodyDialog from '@/components/AddWisdomBodyDialog.vue';
+import ChatHistoryDialog from '@/components/ChatHistoryDialog.vue';
 import DeviceItem from '@/components/DeviceItem.vue';
 import HeaderBar from '@/components/HeaderBar.vue';
 import VersionFooter from '@/components/VersionFooter.vue';
 
 export default {
   name: 'HomePage',
-  components: { DeviceItem, AddWisdomBodyDialog, HeaderBar, VersionFooter },
+  components: { DeviceItem, AddWisdomBodyDialog, HeaderBar, VersionFooter, ChatHistoryDialog },
   data() {
     return {
       addDeviceDialogVisible: false,
@@ -85,6 +77,9 @@ export default {
       searchRegex: null,
       isLoading: true,
       skeletonCount: localStorage.getItem('skeletonCount') || 8,
+      showChatHistory: false,
+      currentAgentId: '',
+      currentAgentName: ''
     }
   },
 
@@ -177,6 +172,11 @@ export default {
           }
         });
       }).catch(() => { });
+    },
+    handleShowChatHistory({ agentId, agentName }) {
+      this.currentAgentId = agentId;
+      this.currentAgentName = agentName;
+      this.showChatHistory = true;
     }
   }
 }
@@ -302,7 +302,9 @@ export default {
 
 /* 骨架屏动画 */
 @keyframes shimmer {
-  100% { transform: translateX(100%); }
+  100% {
+    transform: translateX(100%);
+  }
 }
 
 .skeleton-item {
@@ -353,12 +355,10 @@ export default {
   left: 0;
   width: 50%;
   height: 100%;
-  background: linear-gradient(
-    90deg,
-    rgba(255,255,255,0),
-    rgba(255,255,255,0.3),
-    rgba(255,255,255,0)
-  );
+  background: linear-gradient(90deg,
+      rgba(255, 255, 255, 0),
+      rgba(255, 255, 255, 0.3),
+      rgba(255, 255, 255, 0));
   animation: shimmer 1.5s infinite;
 }
 </style>

main/xiaozhi-server/app.py

@@ -12,22 +12,26 @@ TAG = __name__
 logger = setup_logging()
 
 
-async def wait_for_exit():
-    """Windows 和 Linux 兼容的退出监听"""
+async def wait_for_exit() -> None:
+    """
+    阻塞直到收到 Ctrl‑C / SIGTERM。
+    - Unix: 使用 add_signal_handler
+    - Windows: 依赖 KeyboardInterrupt
+    """
     loop = asyncio.get_running_loop()
     stop_event = asyncio.Event()
 
-    if sys.platform == "win32":
-        # Windows: 用 sys.stdin.read() 监听 Ctrl + C
-        await loop.run_in_executor(None, sys.stdin.read)
-    else:
-        # Linux/macOS: 用 signal 监听 Ctrl + C
-        def stop():
-            stop_event.set()
-
-        loop.add_signal_handler(signal.SIGINT, stop)
-        loop.add_signal_handler(signal.SIGTERM, stop)  # 支持 kill 进程
+    if sys.platform != "win32":  # Unix / macOS
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            loop.add_signal_handler(sig, stop_event.set)
         await stop_event.wait()
+    else:
+        # Windows：await一个永远pending的fut，
+        # 让 KeyboardInterrupt 冒泡到 asyncio.run，以此消除遗留普通线程导致进程退出阻塞的问题
+        try:
+            await asyncio.Future()
+        except KeyboardInterrupt:  # Ctrl‑C
+            pass
 
 
 async def main():

main/xiaozhi-server/config.yaml

@@ -221,6 +221,8 @@ ASR:
     type: fun_server
     host: 127.0.0.1
     port: 10096
+    is_ssl: true
+    output_dir: tmp/
   SherpaASR:
     type: sherpa_onnx_local
     model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
@@ -253,9 +255,19 @@ ASR:
     type: aliyun
     appkey: 你的阿里云智能语音交互服务项目Appkey
     token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
-    access_key_id: 的阿里云账号access_key_id
+    access_key_id: 你的阿里云账号access_key_id
     access_key_secret: 你的阿里云账号access_key_secret
     output_dir: tmp/
+  BaiduASR:
+    # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list
+    # 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list
+    type: baidu
+    app_id: 你的百度语音技术AppID
+    api_key: 你的百度语音技术APIKey
+    secret_key: 你的百度语音技术SecretKey
+    # 语言参数，1537为普通话，具体参考：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b
+    dev_pid: 1537
+    output_dir: tmp/
 
 VAD:
   SileroVAD:

main/xiaozhi-server/config/logger.py

@@ -4,14 +4,20 @@ from loguru import logger
 from config.config_loader import load_config
 from config.settings import check_config_file
 
-SERVER_VERSION = "0.3.13"
+SERVER_VERSION = "0.4.1"
 
 
 def get_module_abbreviation(module_name, module_dict):
-    """获取模块名称的缩写，如果为空则返回00"""
-    return (
-        module_dict.get(module_name, "")[:2] if module_dict.get(module_name) else "00"
-    )
+    """获取模块名称的缩写，如果为空则返回00
+    如果名称中包含下划线，则返回下划线后面的前两个字符
+    """
+    module_value = module_dict.get(module_name, "")
+    if not module_value:
+        return "00"
+    if "_" in module_value:
+        parts = module_value.split("_")
+        return parts[-1][:2] if parts[-1] else "00"
+    return module_value[:2]
 
 
 def build_module_string(selected_module):

main/xiaozhi-server/config/manage_api_client.py

@@ -146,24 +146,12 @@ def get_agent_models(
 
 
 def report(
-    mac_address: str, session_id: str, chat_type: int, content: str, opus_data
+    mac_address: str, session_id: str, chat_type: int, content: str, audio
 ) -> Optional[Dict]:
     """带熔断的业务方法示例"""
     if not content or not ManageApiClient._instance:
         return None
     try:
-        # 处理opus_data为列表的情况
-        if isinstance(opus_data, list):
-            # 将列表中的所有bytes数据合并
-            combined_data = b"".join(opus_data)
-        else:
-            combined_data = opus_data
-
-        # 将二进制数据转换为Base64编码的字符串
-        opus_data_base64 = (
-            base64.b64encode(combined_data).decode("utf-8") if combined_data else None
-        )
-
         return ManageApiClient._instance._execute_request(
             "POST",
             f"/agent/chat-history/report",
@@ -172,7 +160,9 @@ def report(
                 "sessionId": session_id,
                 "chatType": chat_type,
                 "content": content,
-                "opusDataBase64": opus_data_base64,
+                "audioBase64": (
+                    base64.b64encode(audio).decode("utf-8") if audio else None
+                ),
             },
         )
     except Exception as e:

main/xiaozhi-server/core/connection.py

@@ -18,6 +18,8 @@ from core.utils.util import (
     get_string_no_punctuation_or_emoji,
     extract_json_from_string,
     initialize_modules,
+    check_vad_update,
+    check_asr_update,
 )
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage
@@ -52,10 +54,12 @@ class ConnectionHandler:
         _intent,
         server=None,
     ):
-        self.config = config
-        self.server = server
+        self.common_config = config
+        self.config = copy.deepcopy(config)
+        self.session_id = str(uuid.uuid4())
         self.logger = setup_logging()
         self.auth = AuthMiddleware(config)
+        self.server = server  # 保存server实例的引用
 
         self.need_bind = False
         self.bind_code = None
@@ -66,7 +70,6 @@ class ConnectionHandler:
         self.device_id = None
         self.client_ip = None
         self.client_ip_info = {}
-        self.session_id = None
         self.prompt = None
         self.welcome_msg = None
         self.max_output_size = 0
@@ -87,8 +90,10 @@ class ConnectionHandler:
         self.tts_report_thread = None
 
         # 依赖的组件
-        self.vad = _vad
-        self.asr = _asr
+        self.vad = None
+        self.asr = None
+        self._asr = _asr
+        self._vad = _vad
         self.llm = _llm
         self.tts = _tts
         self.memory = _memory
@@ -151,11 +156,9 @@ class ConnectionHandler:
                     self.headers["device-id"] = query_params["device-id"][0]
                     self.headers["client-id"] = query_params["client-id"][0]
                 else:
-                    self.logger.bind(tag=TAG).error(
-                        "无法从请求头和URL查询参数中获取device-id"
-                    )
+                    await ws.send("端口正常，如需测试连接，请使用test_page.html")
+                    await self.close(ws)
                     return
-
             # 获取客户端ip地址
             self.client_ip = ws.remote_address[0]
             self.logger.bind(tag=TAG).info(
@@ -168,7 +171,6 @@ class ConnectionHandler:
             # 认证通过,继续处理
             self.websocket = ws
             self.device_id = self.headers.get("device-id", None)
-            self.session_id = str(uuid.uuid4())
 
             # 启动超时检查任务
             self.timeout_task = asyncio.create_task(self._check_timeout())
@@ -178,9 +180,9 @@ class ConnectionHandler:
             await self.websocket.send(json.dumps(self.welcome_msg))
 
             # 获取差异化配置
-            private_config = self._initialize_private_config()
+            self._initialize_private_config()
             # 异步初始化
-            self.executor.submit(self._initialize_components, private_config)
+            self.executor.submit(self._initialize_components)
             # tts 消化线程
             self.tts_priority_thread = threading.Thread(
                 target=self._tts_priority_thread, daemon=True
@@ -212,7 +214,8 @@ class ConnectionHandler:
     async def _save_and_close(self, ws):
         """保存记忆并关闭连接"""
         try:
-            await self.memory.save_memory(self.dialogue.dialogue)
+            if self.memory:
+                await self.memory.save_memory(self.dialogue.dialogue)
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
         finally:
@@ -234,81 +237,17 @@ class ConnectionHandler:
         elif isinstance(message, bytes):
             await handleAudioMessage(self, message)
 
-    async def handle_config_update(self, message):
-        """处理配置更新请求"""
-        content = message.get("content", {})
-        new_config = content
-
-        # 遍历所有支持的配置模块
-        updated_modules = []
-        for config_model in ["tts", "llm", "vad", "asr", "memory", "intent"]:
-            if config_model not in new_config:
-                continue
-
-            new_content = new_config[config_model]
-            old_content = self.config.get(config_model, {})
-
-            # 记录配置变更
-            self.logger.bind(tag=TAG).info(
-                f"配置更新: {config_model} 旧值: {json.dumps(old_content, ensure_ascii=False)} "
-                f"新值: {json.dumps(new_content, ensure_ascii=False)}"
-            )
-
-            # 深度合并配置
-            if isinstance(old_content, dict) and isinstance(new_content, dict):
-                merged = {**old_content, **new_content}
-                self.config[config_model] = merged
-            else:
-                self.config[config_model] = new_content
-
-            # 标记需要重新初始化的模块
-            if config_model in ["llm", "tts", "asr", "vad", "intent", "memory"]:
-                updated_modules.append(config_model)
-
-        # 同步更新 WebSocketServer 的配置
-        if self.server:
-            async with self.server.config_lock:  # 使用锁确保线程安全
-                for config_model in updated_modules:
-                    self.server.config[config_model].update(new_config[config_model])
-
-        # 批量初始化模块
-        if updated_modules:
-            try:
-                self._initialize_components(self.config)
-                self.logger.bind(tag=TAG).info(
-                    f"已重新初始化模块: {', '.join(updated_modules)}"
-                )
-            except Exception as e:
-                self.logger.bind(tag=TAG).error(f"模块初始化失败: {str(e)}")
-                await self.websocket.send(
-                    json.dumps(
-                        {
-                            "type": "config_update_response",
-                            "status": "error",
-                            "message": f"模块初始化失败: {str(e)}",
-                        }
-                    )
-                )
-                return
-
-        # 返回成功响应
-        await self.websocket.send(
-            json.dumps(
-                {
-                    "type": "config_update_response",
-                    "status": "success",
-                    "message": f"已更新配置: {', '.join(updated_modules)}",
-                }
-            )
-        )
-
-    def _initialize_components(self, private_config):
+    def _initialize_components(self):
         """初始化组件"""
-        if private_config is not None:
-            self._initialize_models(private_config)
-        else:
-            self.prompt = self.config["prompt"]
-            self.change_system_prompt(self.prompt)
+        self.prompt = self.config["prompt"]
+        self.change_system_prompt(self.prompt)
+        self.logger.bind(tag=TAG).info(f"初始化组件: prompt成功 {self.prompt[:50]}...")
+
+        """初始化本地组件"""
+        if self.vad is None:
+            self.vad = self._vad
+        if self.asr is None:
+            self.asr = self._asr
         """加载记忆"""
         self._initialize_memory()
         """加载意图识别"""
@@ -318,7 +257,7 @@ class ConnectionHandler:
 
     def _init_report_threads(self):
         """初始化ASR和TTS上报线程"""
-        if not self.read_config_from_api:
+        if not self.read_config_from_api or self.need_bind:
             return
         if self.tts_report_thread is None or not self.tts_report_thread.is_alive():
             self.tts_report_thread = threading.Thread(
@@ -355,54 +294,21 @@ class ConnectionHandler:
             self.logger.bind(tag=TAG).error(f"获取差异化配置失败: {e}")
             private_config = {}
 
-        init_tts = False
-        if private_config.get("TTS", None) is not None:
-            init_tts = True
-            self.config["TTS"] = private_config["TTS"]
-            self.config["selected_module"]["TTS"] = private_config["selected_module"][
-                "TTS"
-            ]
-
-        try:
-            modules = initialize_modules(
-                self.logger,
-                private_config,
-                False,
-                False,
-                False,
-                init_tts,
-                False,
-                False,
-            )
-        except Exception as e:
-            self.logger.bind(tag=TAG).error(f"初始化组件失败: {e}")
-            modules = {}
-        if modules.get("tts", None) is not None:
-            self.tts = modules["tts"]
-        if modules.get("prompt", None) is not None:
-            self.change_system_prompt(modules["prompt"])
-            private_config["prompt"] = None
-        return private_config
-
-    def _initialize_models(self, private_config):
-        init_vad, init_asr, init_llm, init_memory, init_intent = (
-            False,
+        init_llm, init_tts, init_memory, init_intent = (
             False,
             False,
             False,
             False,
         )
-        if private_config.get("VAD", None) is not None:
-            init_vad = True
-            self.config["VAD"] = private_config["VAD"]
-            self.config["selected_module"]["VAD"] = private_config["selected_module"][
-                "VAD"
-            ]
-        if private_config.get("ASR", None) is not None:
-            init_asr = True
-            self.config["ASR"] = private_config["ASR"]
-            self.config["selected_module"]["ASR"] = private_config["selected_module"][
-                "ASR"
+
+        init_vad = check_vad_update(self.common_config, private_config)
+        init_asr = check_asr_update(self.common_config, private_config)
+
+        if private_config.get("TTS", None) is not None:
+            init_tts = True
+            self.config["TTS"] = private_config["TTS"]
+            self.config["selected_module"]["TTS"] = private_config["selected_module"][
+                "TTS"
             ]
         if private_config.get("LLM", None) is not None:
             init_llm = True
@@ -422,8 +328,11 @@ class ConnectionHandler:
             self.config["selected_module"]["Intent"] = private_config[
                 "selected_module"
             ]["Intent"]
+        if private_config.get("prompt", None) is not None:
+            self.config["prompt"] = private_config["prompt"]
         if private_config.get("device_max_output_size", None) is not None:
             self.max_output_size = int(private_config["device_max_output_size"])
+
         try:
             modules = initialize_modules(
                 self.logger,
@@ -431,13 +340,15 @@ class ConnectionHandler:
                 init_vad,
                 init_asr,
                 init_llm,
-                False,
+                init_tts,
                 init_memory,
                 init_intent,
             )
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"初始化组件失败: {e}")
             modules = {}
+        if modules.get("tts", None) is not None:
+            self.tts = modules["tts"]
         if modules.get("vad", None) is not None:
             self.vad = modules["vad"]
         if modules.get("asr", None) is not None:
@@ -515,10 +426,12 @@ class ConnectionHandler:
         processed_chars = 0  # 跟踪已处理的字符位置
         try:
             # 使用带记忆的对话
-            future = asyncio.run_coroutine_threadsafe(
-                self.memory.query_memory(query), self.loop
-            )
-            memory_str = future.result()
+            memory_str = None
+            if self.memory is not None:
+                future = asyncio.run_coroutine_threadsafe(
+                    self.memory.query_memory(query), self.loop
+                )
+                memory_str = future.result()
 
             self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
             llm_responses = self.llm.response(
@@ -565,7 +478,7 @@ class ConnectionHandler:
                     future = self.executor.submit(
                         self.speak_and_play, segment_text, text_index
                     )
-                    self.tts_queue.put(future)
+                    self.tts_queue.put((future, text_index))
                     processed_chars += len(segment_text_raw)  # 更新已处理字符位置
 
         # 处理最后剩余的文本
@@ -579,7 +492,7 @@ class ConnectionHandler:
                 future = self.executor.submit(
                     self.speak_and_play, segment_text, text_index
                 )
-                self.tts_queue.put(future)
+                self.tts_queue.put((future, text_index))
 
         self.llm_finish_task = True
         self.dialogue.put(Message(role="assistant", content="".join(response_message)))
@@ -606,10 +519,12 @@ class ConnectionHandler:
             start_time = time.time()
 
             # 使用带记忆的对话
-            future = asyncio.run_coroutine_threadsafe(
-                self.memory.query_memory(query), self.loop
-            )
-            memory_str = future.result()
+            memory_str = None
+            if self.memory is not None:
+                future = asyncio.run_coroutine_threadsafe(
+                    self.memory.query_memory(query), self.loop
+                )
+                memory_str = future.result()
 
             # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
 
@@ -695,7 +610,7 @@ class ConnectionHandler:
                             future = self.executor.submit(
                                 self.speak_and_play, segment_text, text_index
                             )
-                            self.tts_queue.put(future)
+                            self.tts_queue.put((future, text_index))
                             # 更新已处理字符位置
                             processed_chars += len(segment_text_raw)
 
@@ -754,7 +669,7 @@ class ConnectionHandler:
                 future = self.executor.submit(
                     self.speak_and_play, segment_text, text_index
                 )
-                self.tts_queue.put(future)
+                self.tts_queue.put((future, text_index))
 
         # 存储对话内容
         if len(response_message) > 0:
@@ -816,7 +731,7 @@ class ConnectionHandler:
             text = result.response
             self.recode_first_last_text(text, text_index)
             future = self.executor.submit(self.speak_and_play, text, text_index)
-            self.tts_queue.put(future)
+            self.tts_queue.put((future, text_index))
             self.dialogue.put(Message(role="assistant", content=text))
         elif result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
             text = result.result
@@ -849,7 +764,7 @@ class ConnectionHandler:
             text = result.result
             self.recode_first_last_text(text, text_index)
             future = self.executor.submit(self.speak_and_play, text, text_index)
-            self.tts_queue.put(future)
+            self.tts_queue.put((future, text_index))
             self.dialogue.put(Message(role="assistant", content=text))
         else:
             pass
@@ -859,7 +774,10 @@ class ConnectionHandler:
             text = None
             try:
                 try:
-                    future = self.tts_queue.get(timeout=1)
+                    item = self.tts_queue.get(timeout=1)
+                    if item is None:
+                        continue
+                    future, text_index = item  # 解包获取 Future 和 text_index
                 except queue.Empty:
                     if self.stop_event.is_set():
                         break
@@ -867,11 +785,11 @@ class ConnectionHandler:
                 if future is None:
                     continue
                 text = None
-                opus_datas, text_index, tts_file = [], 0, None
+                opus_datas, tts_file = [], None
                 try:
                     self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
                     tts_timeout = int(self.config.get("tts_timeout", 10))
-                    tts_file, text, text_index = future.result(timeout=tts_timeout)
+                    tts_file, text, _ = future.result(timeout=tts_timeout)
                     if text is None or len(text) <= 0:
                         self.logger.bind(tag=TAG).error(
                             f"TTS出错：{text_index}: tts text is empty"
@@ -1091,6 +1009,7 @@ def filter_sensitive_info(config: dict) -> dict:
         "personal_access_token",
         "access_token",
         "token",
+        "secret",
         "access_key_secret",
         "secret_key",
     ]

main/xiaozhi-server/core/handle/abortHandle.py

@@ -3,15 +3,16 @@ import queue
 from config.logger import setup_logging
 
 TAG = __name__
-logger = setup_logging()
 
 
 async def handleAbortMessage(conn):
-    logger.bind(tag=TAG).info("Abort message received")
+    conn.logger.bind(tag=TAG).info("Abort message received")
     # 设置成打断状态，会自动打断llm、tts任务
     conn.client_abort = True
     conn.clear_queues()
     # 打断客户端说话状态
-    await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
+    await conn.websocket.send(
+        json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})
+    )
     conn.clearSpeakStatus()
-    logger.bind(tag=TAG).info("Abort message received-end")
+    conn.logger.bind(tag=TAG).info("Abort message received-end")

main/xiaozhi-server/core/handle/functionHandler.py

@@ -4,7 +4,6 @@ from plugins_func.register import FunctionRegistry, ActionResponse, Action, Tool
 from plugins_func.functions.hass_init import append_devices_to_prompt
 
 TAG = __name__
-logger = setup_logging()
 
 
 class FunctionHandler:
@@ -40,7 +39,9 @@ class FunctionHandler:
         for func in self.functions_desc:
             func_names.append(func["function"]["name"])
         # 打印当前支持的函数列表
-        logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
+        self.conn.logger.bind(tag=TAG, session_id=self.conn.session_id).info(
+            f"当前支持的函数列表: {func_names}"
+        )
         return func_names
 
     def get_functions(self):
@@ -79,7 +80,9 @@ class FunctionHandler:
             func = funcItem.func
             arguments = function_call_data["arguments"]
             arguments = json.loads(arguments) if arguments else {}
-            logger.bind(tag=TAG).debug(f"调用函数: {function_name}, 参数: {arguments}")
+            self.conn.logger.bind(tag=TAG).debug(
+                f"调用函数: {function_name}, 参数: {arguments}"
+            )
             if (
                 funcItem.type == ToolType.SYSTEM_CTL
                 or funcItem.type == ToolType.IOT_CTL
@@ -94,6 +97,6 @@ class FunctionHandler:
                     action=Action.NOTFOUND, result="没有找到对应的函数", response=""
                 )
         except Exception as e:
-            logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+            self.conn.logger.bind(tag=TAG).error(f"处理function call错误: {e}")
 
         return None

main/xiaozhi-server/core/handle/helloHandle.py

@@ -9,7 +9,6 @@ import random
 import time
 
 TAG = __name__
-logger = setup_logging()
 
 WAKEUP_CONFIG = {
     "dir": "config/assets/",
@@ -44,7 +43,7 @@ async def checkWakeupWords(conn, text):
         if file is None:
             asyncio.create_task(wakeupWordsResponse(conn))
             return False
-        opus_packets, duration = conn.tts.audio_to_opus_data(file)
+        opus_packets, _ = conn.tts.audio_to_opus_data(file)
         text_hello = WAKEUP_CONFIG["text"]
         if not text_hello:
             text_hello = text
@@ -75,7 +74,7 @@ async def wakeupWordsResponse(conn):
         await asyncio.sleep(1)
         wait_max_time -= 1
         if wait_max_time <= 0:
-            logger.bind(tag=TAG).error("连接对象没有llm")
+            conn.logger.bind(tag=TAG).error("连接对象没有llm")
             return
 
     """唤醒词响应"""

main/xiaozhi-server/core/handle/intentHandler.py

@@ -5,10 +5,8 @@ from core.handle.sendAudioHandle import send_stt_message
 from core.handle.helloHandle import checkWakeupWords
 from core.utils.util import remove_punctuation_and_length
 from core.utils.dialogue import Message
-from loguru import logger
 
 TAG = __name__
-logger = setup_logging()
 
 
 async def handle_user_intent(conn, text):
@@ -36,7 +34,7 @@ async def check_direct_exit(conn, text):
     cmd_exit = conn.cmd_exit
     for cmd in cmd_exit:
         if text == cmd:
-            logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
+            conn.logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
             await send_stt_message(conn, text)
             await conn.close()
             return True
@@ -46,7 +44,7 @@ async def check_direct_exit(conn, text):
 async def analyze_intent_with_llm(conn, text):
     """使用LLM分析用户意图"""
     if not hasattr(conn, "intent") or not conn.intent:
-        logger.bind(tag=TAG).warning("意图识别服务未初始化")
+        conn.logger.bind(tag=TAG).warning("意图识别服务未初始化")
         return None
 
     # 对话历史记录
@@ -55,7 +53,7 @@ async def analyze_intent_with_llm(conn, text):
         intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text)
         return intent_result
     except Exception as e:
-        logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
+        conn.logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
 
     return None
 
@@ -69,7 +67,7 @@ async def process_intent_result(conn, intent_result, original_text):
         # 检查是否有function_call
         if "function_call" in intent_data:
             # 直接从意图识别获取了function_call
-            logger.bind(tag=TAG).debug(
+            conn.logger.bind(tag=TAG).debug(
                 f"检测到function_call格式的意图结果: {intent_data['function_call']['name']}"
             )
             function_name = intent_data["function_call"]["name"]
@@ -118,7 +116,7 @@ async def process_intent_result(conn, intent_result, original_text):
                             conn.speak_and_play, text, text_index
                         )
                         conn.llm_finish_task = True
-                        conn.tts_queue.put(future)
+                        conn.tts_queue.put((future, text_index))
                         conn.dialogue.put(Message(role="assistant", content=text))
 
             # 将函数执行放在线程池中
@@ -126,7 +124,7 @@ async def process_intent_result(conn, intent_result, original_text):
             return True
         return False
     except json.JSONDecodeError as e:
-        logger.bind(tag=TAG).error(f"处理意图结果时出错: {e}")
+        conn.logger.bind(tag=TAG).error(f"处理意图结果时出错: {e}")
         return False
 
 

main/xiaozhi-server/core/handle/iotHandle.py

@@ -10,7 +10,6 @@ from plugins_func.register import (
 )
 
 TAG = __name__
-logger = setup_logging()
 
 
 def wrap_async_function(async_func):
@@ -21,7 +20,7 @@ def wrap_async_function(async_func):
             # 获取连接对象（第一个参数）
             conn = args[0]
             if not hasattr(conn, "loop"):
-                logger.bind(tag=TAG).error("Connection对象没有loop属性")
+                conn.logger.bind(tag=TAG).error("Connection对象没有loop属性")
                 return ActionResponse(
                     Action.ERROR,
                     "Connection对象没有loop属性",
@@ -35,7 +34,7 @@ def wrap_async_function(async_func):
             # 等待结果返回
             return future.result()
         except Exception as e:
-            logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
+            conn.logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
             return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
 
     return wrapper
@@ -57,7 +56,7 @@ def create_iot_function(device_name, method_name, method_info):
                 response_failure = "操作失败"
 
             # 打印响应参数
-            logger.bind(tag=TAG).debug(
+            conn.logger.bind(tag=TAG).debug(
                 f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
             )
 
@@ -86,7 +85,9 @@ def create_iot_function(device_name, method_name, method_info):
 
             return ActionResponse(Action.RESPONSE, result, response)
         except Exception as e:
-            logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
+            conn.logger.bind(tag=TAG).error(
+                f"执行{device_name}的{method_name}操作失败: {e}"
+            )
 
             # 操作失败时使用大模型提供的失败响应
             response = response_failure
@@ -104,7 +105,7 @@ def create_iot_query_function(device_name, prop_name, prop_info):
     async def iot_query_function(conn, response_success=None, response_failure=None):
         try:
             # 打印响应参数
-            logger.bind(tag=TAG).info(
+            conn.logger.bind(tag=TAG).info(
                 f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
             )
 
@@ -122,7 +123,9 @@ def create_iot_query_function(device_name, prop_name, prop_info):
 
                 return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
         except Exception as e:
-            logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
+            conn.logger.bind(tag=TAG).error(
+                f"查询{device_name}的{prop_name}时出错: {e}"
+            )
 
             # 查询出错时使用大模型提供的失败响应
             response = response_failure
@@ -280,7 +283,7 @@ async def handleIotDescriptors(conn, descriptors):
         await asyncio.sleep(1)
         wait_max_time -= 1
         if wait_max_time <= 0:
-            logger.bind(tag=TAG).debug("连接对象没有func_handler")
+            conn.logger.bind(tag=TAG).debug("连接对象没有func_handler")
             return
     """处理物联网描述"""
     functions_changed = False
@@ -323,7 +326,7 @@ async def handleIotDescriptors(conn, descriptors):
             if hasattr(conn, "func_handler"):
                 for func_name in device_functions:
                     conn.func_handler.function_registry.register_function(func_name)
-                    logger.bind(tag=TAG).info(
+                    conn.logger.bind(tag=TAG).info(
                         f"注册IOT函数到function handler: {func_name}"
                     )
                     functions_changed = True
@@ -332,8 +335,8 @@ async def handleIotDescriptors(conn, descriptors):
     if functions_changed and hasattr(conn, "func_handler"):
         conn.func_handler.upload_functions_desc()
         func_names = conn.func_handler.current_support_functions()
-        logger.bind(tag=TAG).info(f"设备类型: {type_id}")
-        logger.bind(tag=TAG).info(
+        conn.logger.bind(tag=TAG).info(f"设备类型: {type_id}")
+        conn.logger.bind(tag=TAG).info(
             f"更新function描述列表完成，当前支持的函数: {func_names}"
         )
 
@@ -347,13 +350,13 @@ async def handleIotStatus(conn, states):
                     for k, v in state["state"].items():
                         if property_item["name"] == k:
                             if type(v) != type(property_item["value"]):
-                                logger.bind(tag=TAG).error(
+                                conn.logger.bind(tag=TAG).error(
                                     f"属性{property_item['name']}的值类型不匹配"
                                 )
                                 break
                             else:
                                 property_item["value"] = v
-                                logger.bind(tag=TAG).info(
+                                conn.logger.bind(tag=TAG).info(
                                     f"物联网状态更新: {key} , {property_item['name']} = {v}"
                                 )
                             break
@@ -367,7 +370,7 @@ async def get_iot_status(conn, name, property_name):
             for property_item in value.properties:
                 if property_item["name"] == property_name:
                     return property_item["value"]
-    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
+    conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
     return None
 
 
@@ -378,16 +381,16 @@ async def set_iot_status(conn, name, property_name, value):
             for property_item in iot_descriptor.properties:
                 if property_item["name"] == property_name:
                     if type(value) != type(property_item["value"]):
-                        logger.bind(tag=TAG).error(
+                        conn.logger.bind(tag=TAG).error(
                             f"属性{property_item['name']}的值类型不匹配"
                         )
                         return
                     property_item["value"] = value
-                    logger.bind(tag=TAG).info(
+                    conn.logger.bind(tag=TAG).info(
                         f"物联网状态更新: {name} , {property_name} = {value}"
                     )
                     return
-    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
+    conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
 
 
 async def send_iot_conn(conn, name, method_name, parameters):
@@ -409,6 +412,6 @@ async def send_iot_conn(conn, name, method_name, parameters):
                         command["parameters"] = parameters
                     send_message = json.dumps({"type": "iot", "commands": [command]})
                     await conn.websocket.send(send_message)
-                    logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}")
+                    conn.logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}")
                     return
-    logger.bind(tag=TAG).error(f"未找到方法{method_name}")
+    conn.logger.bind(tag=TAG).error(f"未找到方法{method_name}")

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -1,4 +1,3 @@
-from config.logger import setup_logging
 import time
 import copy
 from core.utils.util import remove_punctuation_and_length
@@ -6,14 +5,16 @@ from core.handle.sendAudioHandle import send_stt_message
 from core.handle.intentHandler import handle_user_intent
 from core.utils.output_counter import check_device_output_limit
 from core.handle.ttsReportHandle import enqueue_tts_report
+from core.providers.tts.base import audio_to_opus_data
 
 TAG = __name__
-logger = setup_logging()
 
 
 async def handleAudioMessage(conn, audio):
+    if conn.vad is None:
+        return
     if not conn.asr_server_receive:
-        logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
+        conn.logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
         return
     if conn.client_listen_mode == "auto":
         have_voice = conn.vad.is_vad(conn, audio)
@@ -39,7 +40,7 @@ async def handleAudioMessage(conn, audio):
             conn.asr_server_receive = True
         else:
             text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
-            logger.bind(tag=TAG).info(f"识别文本: {text}")
+            conn.logger.bind(tag=TAG).info(f"识别文本: {text}")
             text_len, _ = remove_punctuation_and_length(text)
             if text_len > 0:
                 # 使用自定义模块进行上报
@@ -110,7 +111,7 @@ async def max_out_size(conn):
     conn.tts_last_text_index = 0
     conn.llm_finish_task = True
     file_path = "config/assets/max_output_size.wav"
-    opus_packets, _ = conn.tts.audio_to_opus_data(file_path)
+    opus_packets, _ = audio_to_opus_data(file_path)
     conn.audio_play_queue.put((opus_packets, text, 0))
     conn.close_after_chat = True
 
@@ -119,7 +120,7 @@ async def check_bind_device(conn):
     if conn.bind_code:
         # 确保bind_code是6位数字
         if len(conn.bind_code) != 6:
-            logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}")
+            conn.logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}")
             text = "绑定码格式错误，请检查配置。"
             await send_stt_message(conn, text)
             return
@@ -132,7 +133,7 @@ async def check_bind_device(conn):
 
         # 播放提示音
         music_path = "config/assets/bind_code.wav"
-        opus_packets, _ = conn.tts.audio_to_opus_data(music_path)
+        opus_packets, _ = audio_to_opus_data(music_path)
         conn.audio_play_queue.put((opus_packets, text, 0))
 
         # 逐个播放数字
@@ -140,10 +141,10 @@ async def check_bind_device(conn):
             try:
                 digit = conn.bind_code[i]
                 num_path = f"config/assets/bind_code/{digit}.wav"
-                num_packets, _ = conn.tts.audio_to_opus_data(num_path)
+                num_packets, _ = audio_to_opus_data(num_path)
                 conn.audio_play_queue.put((num_packets, None, i + 1))
             except Exception as e:
-                logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
+                conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
                 continue
     else:
         text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
@@ -152,5 +153,5 @@ async def check_bind_device(conn):
         conn.tts_last_text_index = 0
         conn.llm_finish_task = True
         music_path = "config/assets/bind_not_found.wav"
-        opus_packets, _ = conn.tts.audio_to_opus_data(music_path)
+        opus_packets, _ = audio_to_opus_data(music_path)
         conn.audio_play_queue.put((opus_packets, text, 0))

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -1,11 +1,9 @@
-from config.logger import setup_logging
 import json
 import asyncio
 import time
 from core.utils.util import get_string_no_punctuation_or_emoji, analyze_emotion
 
 TAG = __name__
-logger = setup_logging()
 
 emoji_map = {
     "neutral": "😶",
@@ -49,10 +47,10 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
         )
 
     if text_index == conn.tts_first_text_index:
-        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+        conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
     await send_tts_message(conn, "sentence_start", text)
 
-    is_first_audio = (text_index == conn.tts_first_text_index)
+    is_first_audio = text_index == conn.tts_first_text_index
     await sendAudio(conn, audios, pre_buffer=is_first_audio)
 
     await send_tts_message(conn, "sentence_end", text)

main/xiaozhi-server/core/handle/textHandle.py

@@ -1,4 +1,3 @@
-from config.logger import setup_logging
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
@@ -10,12 +9,11 @@ from core.handle.ttsReportHandle import enqueue_tts_report
 import asyncio
 
 TAG = __name__
-logger = setup_logging()
 
 
 async def handleTextMessage(conn, message):
     """处理文本消息"""
-    logger.bind(tag=TAG).info(f"收到文本消息：{message}")
+    conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}")
     try:
         msg_json = json.loads(message)
         if isinstance(msg_json, int):
@@ -28,7 +26,9 @@ async def handleTextMessage(conn, message):
         elif msg_json["type"] == "listen":
             if "mode" in msg_json:
                 conn.client_listen_mode = msg_json["mode"]
-                logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
+                conn.logger.bind(tag=TAG).debug(
+                    f"客户端拾音模式：{conn.client_listen_mode}"
+                )
             if msg_json["state"] == "start":
                 conn.client_have_voice = True
                 conn.client_voice_stop = False
@@ -80,7 +80,7 @@ async def handleTextMessage(conn, message):
                 await conn.websocket.send(
                     json.dumps(
                         {
-                            "type": "config_update_response",
+                            "type": "server",
                             "status": "error",
                             "message": "服务器密钥验证失败",
                         }
@@ -89,6 +89,52 @@ async def handleTextMessage(conn, message):
                 return
             # 动态更新配置
             if msg_json["action"] == "update_config":
-                await conn.handle_config_update(msg_json)
+                try:
+                    # 更新WebSocketServer的配置
+                    if not conn.server:
+                        await conn.websocket.send(
+                            json.dumps(
+                                {
+                                    "type": "config_update_response",
+                                    "status": "error",
+                                    "message": "无法获取服务器实例",
+                                }
+                            )
+                        )
+                        return
+
+                    if not await conn.server.update_config():
+                        await conn.websocket.send(
+                            json.dumps(
+                                {
+                                    "type": "config_update_response",
+                                    "status": "error",
+                                    "message": "更新服务器配置失败",
+                                }
+                            )
+                        )
+                        return
+
+                    # 发送成功响应
+                    await conn.websocket.send(
+                        json.dumps(
+                            {
+                                "type": "config_update_response",
+                                "status": "success",
+                                "message": "配置更新成功",
+                            }
+                        )
+                    )
+                except Exception as e:
+                    conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
+                    await conn.websocket.send(
+                        json.dumps(
+                            {
+                                "type": "config_update_response",
+                                "status": "error",
+                                "message": f"更新配置失败: {str(e)}",
+                            }
+                        )
+                    )
     except json.JSONDecodeError:
         await conn.websocket.send(message)

main/xiaozhi-server/core/handle/ttsReportHandle.py

@@ -9,11 +9,11 @@ TTS上报功能已集成到ConnectionHandler类中。
 具体实现请参考core/connection.py中的相关代码。
 """
 
-from config.logger import setup_logging
+import opuslib_next
+
 from config.manage_api_client import report
 
 TAG = __name__
-logger = setup_logging()
 
 
 def report_tts(conn, type, text, opus_data):
@@ -26,20 +26,71 @@ def report_tts(conn, type, text, opus_data):
         opus_data: opus音频数据
     """
     try:
+        if opus_data:
+            audio_data = opus_to_wav(conn, opus_data)
+        else:
+            audio_data = None
         # 执行上报
         report(
             mac_address=conn.device_id,
             session_id=conn.session_id,
             chat_type=type,
             content=text,
-            opus_data=opus_data,
+            audio=audio_data,
         )
     except Exception as e:
-        logger.bind(tag=TAG).error(f"TTS上报失败: {e}")
+        conn.logger.bind(tag=TAG).error(f"TTS上报失败: {e}")
+
+
+def opus_to_wav(conn, opus_data):
+    """将Opus数据转换为WAV格式的字节流
+
+    Args:
+        output_dir: 输出目录（保留参数以保持接口兼容）
+        opus_data: opus音频数据
+
+    Returns:
+        bytes: WAV格式的音频数据
+    """
+    decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+    pcm_data = []
+
+    for opus_packet in opus_data:
+        try:
+            pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+            pcm_data.append(pcm_frame)
+        except opuslib_next.OpusError as e:
+            conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+    if not pcm_data:
+        raise ValueError("没有有效的PCM数据")
+
+    # 创建WAV文件头
+    pcm_data_bytes = b"".join(pcm_data)
+    num_samples = len(pcm_data_bytes) // 2  # 16-bit samples
+
+    # WAV文件头
+    wav_header = bytearray()
+    wav_header.extend(b"RIFF")  # ChunkID
+    wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little"))  # ChunkSize
+    wav_header.extend(b"WAVE")  # Format
+    wav_header.extend(b"fmt ")  # Subchunk1ID
+    wav_header.extend((16).to_bytes(4, "little"))  # Subchunk1Size
+    wav_header.extend((1).to_bytes(2, "little"))  # AudioFormat (PCM)
+    wav_header.extend((1).to_bytes(2, "little"))  # NumChannels
+    wav_header.extend((16000).to_bytes(4, "little"))  # SampleRate
+    wav_header.extend((32000).to_bytes(4, "little"))  # ByteRate
+    wav_header.extend((2).to_bytes(2, "little"))  # BlockAlign
+    wav_header.extend((16).to_bytes(2, "little"))  # BitsPerSample
+    wav_header.extend(b"data")  # Subchunk2ID
+    wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little"))  # Subchunk2Size
+
+    # 返回完整的WAV数据
+    return bytes(wav_header) + pcm_data_bytes
 
 
 def enqueue_tts_report(conn, type, text, opus_data):
-    if not conn.read_config_from_api:
+    if not conn.read_config_from_api or conn.need_bind:
         return
     """将TTS数据加入上报队列
 
@@ -52,8 +103,8 @@ def enqueue_tts_report(conn, type, text, opus_data):
         # 使用连接对象的队列，传入文本和二进制数据而非文件路径
         conn.tts_report_queue.put((type, text, opus_data))
 
-        logger.bind(tag=TAG).info(
+        conn.logger.bind(tag=TAG).debug(
             f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
         )
     except Exception as e:
-        logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
+        conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")

main/xiaozhi-server/core/mcp/MCPClient.py

@@ -1,86 +1,125 @@
+from __future__ import annotations
+
 from datetime import timedelta
-from typing import Optional
+import asyncio, os, shutil, concurrent.futures
 from contextlib import AsyncExitStack
-import os, shutil
+from typing import Optional, List, Dict, Any
+
 from mcp import ClientSession, StdioServerParameters
 from mcp.client.stdio import stdio_client
-
+from mcp.client.sse import sse_client
 from config.logger import setup_logging
 
 TAG = __name__
 
+
 class MCPClient:
-    def __init__(self, config):
-        # Initialize session and client objects
-        self.session: Optional[ClientSession] = None
-        self.exit_stack = AsyncExitStack()
+    def __init__(self, config: Dict[str, Any]):
         self.logger = setup_logging()
         self.config = config
-        self.tolls = []
+
+        self._worker_task: Optional[asyncio.Task] = None
+        self._ready_evt = asyncio.Event()
+        self._shutdown_evt = asyncio.Event()
+
+        self.session: Optional[ClientSession] = None
+        self.tools: List = []
 
     async def initialize(self):
-        args = self.config.get("args", [])
+        if self._worker_task:
+            return
+        self._worker_task = asyncio.create_task(self._worker(), name="MCPClientWorker")
+        await self._ready_evt.wait()
 
-        command = (
-            shutil.which("npx")
-            if self.config["command"] == "npx"
-            else self.config["command"]
+        self.logger.bind(tag=TAG).info(
+            f"Connected, tools = {[t.name for t in self.tools]}"
         )
-        
-        env={**os.environ}
-        if self.config.get("env"):
-            env.update(self.config["env"])
-        
-        server_params = StdioServerParameters(
-            command=command,
-            args=args,
-            env=env
-        )
-        
-        stdio_transport = await self.exit_stack.enter_async_context(stdio_client(server_params))
-        self.stdio, self.write = stdio_transport
-        time_out_delta =  timedelta(seconds=15)
-        self.session = await self.exit_stack.enter_async_context(ClientSession(read_stream=self.stdio, write_stream=self.write, read_timeout_seconds=time_out_delta))
-        
-        await self.session.initialize()
-        
-        # List available tools
-        response = await self.session.list_tools()
-        tools = response.tools
-        self.tools = tools
-        self.logger.bind(tag=TAG).info(f"Connected to server with tools:{[tool.name for tool in tools]}")
-    
-    def has_tool(self, tool_name):
-        return any(tool.name == tool_name for tool in self.tools)
-    
-    def get_available_tools(self):
-        available_tools = [{"type": "function", "function":{ 
-            "name": tool.name,
-            "description": tool.description,
-            "parameters": tool.inputSchema
-        } } for tool in self.tools]
-
-        return available_tools
-    
-    async def call_tool(self, tool_name: str, tool_args: dict):
-        self.logger.bind(tag=TAG).info(f"MCPClient Calling tool {tool_name} with args: {tool_args}")
-        try:
-            response = await self.session.call_tool(tool_name, tool_args)
-        except Exception as e:
-            self.logger.bind(tag=TAG).error(f"Error calling tool {tool_name}: {e}")
-            from types import SimpleNamespace
-            error_content = SimpleNamespace(
-                type='text',
-                text=f"Error calling tool {tool_name}: {e}"
-            )
-            error_response = SimpleNamespace(
-                content=[error_content],
-                isError=True
-            )
-            return error_response
-        self.logger.bind(tag=TAG).info(f"MCPClient Response from tool {tool_name}: {response}")
-        return response
 
     async def cleanup(self):
-        """Clean up resources"""
-        await self.exit_stack.aclose()
+        if not self._worker_task:
+            return
+
+        self._shutdown_evt.set()
+        try:
+            await asyncio.wait_for(self._worker_task, timeout=20)
+        except (asyncio.TimeoutError, Exception) as e:
+            self.logger.bind(tag=TAG).error(f"worker shutdown err: {e}")
+        finally:
+            self._worker_task = None
+
+    def has_tool(self, name: str) -> bool:
+        return any(t.name == name for t in self.tools)
+
+    def get_available_tools(self):
+        return [
+            {
+                "type": "function",
+                "function": {
+                    "name": t.name,
+                    "description": t.description,
+                    "parameters": t.inputSchema,
+                },
+            }
+            for t in self.tools
+        ]
+
+    async def call_tool(self, name: str, args: dict):
+        if not self.session:
+            raise RuntimeError("MCPClient not initialized")
+
+        loop = self._worker_task.get_loop()
+        coro = self.session.call_tool(name, args)
+
+        if loop is asyncio.get_running_loop():
+            return await coro
+
+        fut: concurrent.futures.Future = asyncio.run_coroutine_threadsafe(coro, loop)
+        return await asyncio.wrap_future(fut)
+
+    async def _worker(self):
+        async with AsyncExitStack() as stack:
+            try:
+                # 建立 StdioClient
+                if "command" in self.config:
+                    cmd = (
+                        shutil.which("npx")
+                        if self.config["command"] == "npx"
+                        else self.config["command"]
+                    )
+                    env = {**os.environ, **self.config.get("env", {})}
+                    params = StdioServerParameters(
+                        command=cmd,
+                        args=self.config.get("args", []),
+                        env=env,
+                    )
+                    stdio_r, stdio_w = await stack.enter_async_context(stdio_client(params))
+                    read_stream, write_stream = stdio_r, stdio_w
+                # 建立SSEClient
+                elif "url" in self.config:
+                    sse_r, sse_w = await stack.enter_async_context(sse_client(self.config["url"]))
+                    read_stream, write_stream = sse_r, sse_w
+
+                else:
+                    raise ValueError("MCPClient config must include 'command' or 'url'")
+
+                self.session = await stack.enter_async_context(
+                    ClientSession(
+                        read_stream=read_stream,
+                        write_stream=write_stream,
+                        read_timeout_seconds=timedelta(seconds=15),
+                    )
+                )
+                await self.session.initialize()
+
+                # 获取工具
+                self.tools = (await self.session.list_tools()).tools
+
+                self._ready_evt.set()
+
+                # 挂起等待关闭
+                await self._shutdown_evt.wait()
+
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"worker error: {e}")
+                self._ready_evt.set()
+                raise

main/xiaozhi-server/core/mcp/manager.py

@@ -1,9 +1,9 @@
 """MCP服务管理器"""
 
+import asyncio
 import os, json
 from typing import Dict, Any, List
 from .MCPClient import MCPClient
-from config.logger import setup_logging
 from plugins_func.register import register_function, ToolType
 from config.config_loader import get_project_dir
 
@@ -18,11 +18,10 @@ class MCPManager:
         初始化MCP管理器
         """
         self.conn = conn
-        self.logger = setup_logging()
         self.config_path = get_project_dir() + "data/.mcp_server_settings.json"
         if os.path.exists(self.config_path) == False:
             self.config_path = ""
-            self.logger.bind(tag=TAG).warning(
+            self.conn.logger.bind(tag=TAG).warning(
                 f"请检查mcp服务配置文件：data/.mcp_server_settings.json"
             )
         self.client: Dict[str, MCPClient] = {}
@@ -41,7 +40,7 @@ class MCPManager:
                 config = json.load(f)
             return config.get("mcpServers", {})
         except Exception as e:
-            self.logger.bind(tag=TAG).error(
+            self.conn.logger.bind(tag=TAG).error(
                 f"Error loading MCP config from {self.config_path}: {e}"
             )
             return {}
@@ -50,9 +49,9 @@ class MCPManager:
         """初始化所有MCP服务"""
         config = self.load_config()
         for name, srv_config in config.items():
-            if not srv_config.get("command"):
-                self.logger.bind(tag=TAG).warning(
-                    f"Skipping server {name}: command not specified"
+            if not srv_config.get("command") and not srv_config.get("url"):
+                self.conn.logger.bind(tag=TAG).warning(
+                    f"Skipping server {name}: neither command nor url specified"
                 )
                 continue
 
@@ -60,7 +59,7 @@ class MCPManager:
                 client = MCPClient(srv_config)
                 await client.initialize()
                 self.client[name] = client
-                self.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}")
+                self.conn.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}")
                 client_tools = client.get_available_tools()
                 self.tools.extend(client_tools)
                 for tool in client_tools:
@@ -73,7 +72,7 @@ class MCPManager:
                     )
 
             except Exception as e:
-                self.logger.bind(tag=TAG).error(
+                self.conn.logger.bind(tag=TAG).error(
                     f"Failed to initialize MCP server {name}: {e}"
                 )
         self.conn.func_handler.upload_functions_desc()
@@ -110,7 +109,7 @@ class MCPManager:
         Raises:
             ValueError: 工具未找到时抛出
         """
-        self.logger.bind(tag=TAG).info(
+        self.conn.logger.bind(tag=TAG).info(
             f"Executing tool {tool_name} with arguments: {arguments}"
         )
         for client in self.client.values():
@@ -120,12 +119,13 @@ class MCPManager:
         raise ValueError(f"Tool {tool_name} not found in any MCP server")
 
     async def cleanup_all(self) -> None:
-        for name, client in self.client.items():
+        """依次关闭所有 MCPClient，不让异常阻断整体流程。"""
+        for name, client in list(self.client.items()):
             try:
-                await client.cleanup()
-                self.logger.bind(tag=TAG).info(f"Cleaned up MCP client: {name}")
-            except Exception as e:
-                self.logger.bind(tag=TAG).error(
-                    f"Error cleaning up MCP client {name}: {e}"
+                await asyncio.wait_for(client.cleanup(), timeout=20)
+                self.conn.logger.bind(tag=TAG).info(f"MCP client closed: {name}")
+            except (asyncio.TimeoutError, Exception) as e:
+                self.conn.logger.bind(tag=TAG).error(
+                    f"Error closing MCP client {name}: {e}"
                 )
         self.client.clear()

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -25,14 +25,14 @@ class AccessToken:
     def _encode_text(text):
         encoded_text = parse.quote_plus(text)
         return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
-    
+
     @staticmethod
     def _encode_dict(dic):
         keys = dic.keys()
         dic_sorted = [(key, dic[key]) for key in sorted(keys)]
         encoded_text = parse.urlencode(dic_sorted)
         return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
-    
+
     @staticmethod
     def create_token(access_key_id, access_key_secret):
         parameters = {'AccessKeyId': access_key_id,
@@ -98,7 +98,7 @@ class ASRProvider(ASRProviderBase):
             # 直接使用预生成的长期token
             self.token = config.get("token")
             self.expire_time = None
-        
+
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
@@ -107,7 +107,7 @@ class ASRProvider(ASRProviderBase):
         """刷新Token并记录过期时间"""
         if self.access_key_id and self.access_key_secret:
             self.token, expire_time_str = AccessToken.create_token(
-                self.access_key_id, 
+                self.access_key_id,
                 self.access_key_secret
             )
             if not expire_time_str:
@@ -121,16 +121,16 @@ class ASRProvider(ASRProviderBase):
                     expire_time = datetime.fromtimestamp(int(expire_str))
                 else:
                     expire_time = datetime.strptime(
-                        expire_str, 
+                        expire_str,
                         "%Y-%m-%dT%H:%M:%SZ"
                     )
                 self.expire_time = expire_time.timestamp() - 60
             except Exception as e:
                 raise ValueError(f"无效的过期时间格式: {expire_str}") from e
-        
+
         else:
             self.expire_time = None
-            
+
         if not self.token:
             raise ValueError("无法获取有效的访问Token")
 
@@ -173,13 +173,12 @@ class ASRProvider(ASRProviderBase):
 
         return pcm_data
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        pcm_data = self.decode_opus(opus_data, session_id)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)  # 单声道
             wf.setsampwidth(2)  # 16-bit
@@ -202,7 +201,7 @@ class ASRProvider(ASRProviderBase):
             # 创建连接并发送请求
             conn = http.client.HTTPSConnection(self.host)
             request_url = self._construct_request_url()
-            
+
             loop = asyncio.get_event_loop()
             await loop.run_in_executor(None, lambda: conn.request(
                 method='POST',
@@ -220,7 +219,7 @@ class ASRProvider(ASRProviderBase):
             try:
                 body_json = json.loads(body)
                 status = body_json.get('status')
-                
+
                 if status == 20000000:
                     result = body_json.get('result', '')
                     logger.bind(tag=TAG).debug(f"ASR结果: {result}")
@@ -228,7 +227,7 @@ class ASRProvider(ASRProviderBase):
                 else:
                     logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
                     return None
-                    
+
             except ValueError:
                 logger.bind(tag=TAG).error("响应不是JSON格式")
                 return None
@@ -242,24 +241,27 @@ class ASRProvider(ASRProviderBase):
         if self._is_token_expired():
             logger.warning("Token已过期，正在自动刷新...")
             self._refresh_token()
-            
+
+        file_path = None
         try:
             # 解码Opus为PCM
-            pcm_data_list = self.decode_opus(opus_data, session_id)
-            combined_pcm_data = b''.join(pcm_data_list)
+            pcm_data = self.decode_opus(opus_data, session_id)
+            combined_pcm_data = b''.join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 发送请求并获取文本
             text = await self._send_request(combined_pcm_data)
-            
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            if self.delete_audio_file:
-                os.remove(file_path)
-                logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}")
 
             if text:
-                return text, None
-            return "", None
+                return text, file_path
+
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -0,0 +1,118 @@
+import base64
+import hashlib
+import hmac
+import json
+import time
+from datetime import datetime, timezone
+import os
+import uuid
+from typing import Optional, Tuple, List
+import wave
+import opuslib_next
+
+from aip import AipSpeech
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.secret_key = config.get("secret_key")
+
+        dev_pid = config.get("dev_pid", "1537")
+        self.dev_pid = int(dev_pid) if dev_pid else 1537
+
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        self.client = AipSpeech(str(self.app_id), self.api_key, self.secret_key)
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes]) -> bytes:
+        """将Opus音频数据解码为PCM数据"""
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if not opus_data:
+            logger.bind(tag=TAG).warn("音频数据为空！")
+            return None, None
+
+        file_path = None
+        try:
+            # 检查配置是否已设置
+            if not self.app_id or not self.api_key or not self.secret_key:
+                logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
+                return None, file_path
+
+            # 将Opus音频数据解码为PCM
+            pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                self.save_audio_to_file(pcm_data, session_id)
+
+            start_time = time.time()
+            # 识别本地文件
+            result = self.client.asr(
+                combined_pcm_data,
+                "pcm",
+                16000,
+                {
+                    "dev_pid": str(self.dev_pid),
+                },
+            )
+
+            if result and result["err_no"] == 0:
+                logger.bind(tag=TAG).debug(
+                    f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+                result = result["result"][0]
+                return result, file_path
+            else:
+                raise Exception(
+                    f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
+                )
+                return None, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
+            return None, file_path

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -103,7 +103,8 @@ class ASRProvider(ASRProviderBase):
 
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
         with wave.open(file_path, "wb") as wf:
@@ -139,8 +140,8 @@ class ASRProvider(ASRProviderBase):
                 "uid": str(uuid.uuid4()),
             },
             "request": {
-                "reqid": reqid, 
-                "show_utterances": False, 
+                "reqid": reqid,
+                "show_utterances": False,
                 "sequence": 1,
                 "boosting_table_name": self.boosting_table_name,
                 "correct_table_name": self.correct_table_name,
@@ -260,6 +261,8 @@ class ASRProvider(ASRProviderBase):
         self, opus_data: List[bytes], session_id: str
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
+
+        file_path = None
         try:
             # 合并所有opus数据包
             pcm_data = self.decode_opus(opus_data, session_id)
@@ -269,7 +272,7 @@ class ASRProvider(ASRProviderBase):
             if self.delete_audio_file:
                 pass
             else:
-                self.save_audio_to_file(pcm_data, session_id)
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 直接使用PCM数据
             # 计算分段大小 (单声道, 16bit, 16kHz采样率)
@@ -283,9 +286,9 @@ class ASRProvider(ASRProviderBase):
                 logger.bind(tag=TAG).debug(
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
                 )
-                return text, None
-            return "", None
+                return text, file_path
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -50,21 +50,12 @@ class ASRProvider(ASRProviderBase):
                 # device="cuda:0",  # 启用GPU加速
             )
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -72,7 +63,7 @@ class ASRProvider(ASRProviderBase):
             wf.writeframes(b"".join(pcm_data))
 
         return file_path
-    
+
     @staticmethod
     def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
 
@@ -100,7 +91,7 @@ class ASRProvider(ASRProviderBase):
             if self.delete_audio_file:
                 pass
             else:
-                self.save_audio_to_file(pcm_data, session_id)
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 语音识别
             start_time = time.time()
@@ -118,13 +109,13 @@ class ASRProvider(ASRProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
 
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
+        # finally:
+        #     # 文件清理逻辑
+        #     if self.delete_audio_file and file_path and os.path.exists(file_path):
+        #         try:
+        #             os.remove(file_path)
+        #             logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+        #         except Exception as e:
+        #             logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -1,8 +1,11 @@
 from typing import Optional, Tuple, List
 import opuslib_next
 from core.providers.asr.base import ASRProviderBase
-import  ssl
+import os
+import ssl
 import json
+import uuid
+import wave
 import websockets
 from config.logger import setup_logging
 import asyncio
@@ -19,6 +22,7 @@ class ASRProvider(ASRProviderBase):
         self.host = config.get('host', 'localhost')
         self.port = config.get('port', 10095)
         self.is_ssl = config.get('is_ssl', True)
+        self.output_dir = config.get("output_dir")
         self.delete_audio_file = delete_audio_file
         self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}"
         self.ssl_context = ssl.SSLContext() if self.is_ssl else None
@@ -26,25 +30,35 @@ class ASRProvider(ASRProviderBase):
             self.ssl_context.check_hostname = False
             self.ssl_context.verify_mode = ssl.CERT_NONE
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """解码Opus数据并保存为WAV文件"""
-        pass
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
 
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
-        """将Opus音频数据解码为PCM数据"""        
+        """将Opus音频数据解码为PCM数据"""
         decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
         pcm_data = []
-        
+
         for opus_packet in opus_data:
             try:
                 pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
                 pcm_data.append(pcm_frame)
             except opuslib_next.OpusError as e:
                 logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
-        return b"".join(pcm_data)
-    
+
+        return pcm_data
+
 
 
     async def _receive_responses(self, ws) -> None:
@@ -90,15 +104,15 @@ class ASRProvider(ASRProviderBase):
         })
         await ws.send(config_message)
         logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
-        
+
         # Send PCM data
         await ws.send(pcm_data)
         logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
-        
+
         # Indicate end of speech
         end_message = json.dumps({"is_speaking": False})
         await ws.send(end_message)
-        logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")    
+        logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
 
 
     async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
@@ -108,13 +122,20 @@ class ASRProvider(ASRProviderBase):
         :param session_id: Unique session identifier.
         :return: Tuple containing recognized text and optional timestamp.
         '''
-
+        file_path = None
         pcm_data = self.decode_opus(opus_data)
+        combined_pcm_data = b"".join(pcm_data)
+
+        # 判断是否保存为WAV文件
+        if self.delete_audio_file:
+            pass
+        else:
+            file_path = self.save_audio_to_file(pcm_data, session_id)
 
         async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws:
             try:
                 # Use asyncio to handle WebSocket communication
-                send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id))
+                send_task = asyncio.create_task(self._send_data(ws, combined_pcm_data, session_id))
                 receive_task = asyncio.create_task(self._receive_responses(ws))
 
                 # Gather tasks with error handling
@@ -134,11 +155,11 @@ class ASRProvider(ASRProviderBase):
 
                 # Get the result from the receive task
                 result = receive_task.result()
-                return result, None  # Return the recognized text and timestamp (if any)
+                return result, file_path  # Return the recognized text and timestamp (if any)
 
             except websockets.exceptions.ConnectionClosed as e:
                 logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
-                return "", None
+                return "", file_path
             except Exception as e:
                 logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True)
-                return "", None
+                return "", file_path

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -85,7 +85,8 @@ class ASRProvider(ASRProviderBase):
 
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
         with wave.open(file_path, "wb") as wf:
@@ -164,7 +165,7 @@ class ASRProvider(ASRProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
         finally:
             # 文件清理逻辑
             if self.delete_audio_file and file_path and os.path.exists(file_path):

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -26,40 +26,40 @@ class ASRProvider(ASRProviderBase):
         self.secret_id = config.get("secret_id")
         self.secret_key = config.get("secret_key")
         self.output_dir = config.get("output_dir")
-        
+        self.delete_audio_file = delete_audio_file
+
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
-
-        file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
-        
+
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
             wf.setframerate(16000)
             wf.writeframes(b"".join(pcm_data))
-        
+
         return file_path
 
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
         """将Opus音频数据解码为PCM数据"""
-        import opuslib_next
-        
+
         decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
         pcm_data = []
-        
+
         for opus_packet in opus_data:
             try:
                 pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
                 pcm_data.append(pcm_frame)
             except opuslib_next.OpusError as e:
                 logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
-        return b"".join(pcm_data)
+
+        return pcm_data
 
     async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
@@ -67,23 +67,25 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).warn("音频数据为空！")
             return None, None
 
+        file_path = None
         try:
             # 检查配置是否已设置
             if not self.secret_id or not self.secret_key:
                 logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
-                return None, None
+                return None, file_path
 
             # 将Opus音频数据解码为PCM
             pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
 
             # 判断是否保存为WAV文件
             if self.delete_audio_file:
                 pass
             else:
                 self.save_audio_to_file(pcm_data, session_id)
-            
+
             # 将音频数据转换为Base64编码
-            base64_audio = base64.b64encode(pcm_data).decode('utf-8')
+            base64_audio = base64.b64encode(combined_pcm_data).decode('utf-8')
 
             # 构建请求体
             request_body = self._build_request_body(base64_audio)
@@ -94,15 +96,15 @@ class ASRProvider(ASRProviderBase):
             # 发送请求
             start_time = time.time()
             result = self._send_request(request_body, timestamp, authorization)
-            
+
             if result:
                 logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
-            
-            return result, None
+
+            return result, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
-            return None, None
+            return None, file_path
 
     def _build_request_body(self, base64_audio: str) -> str:
         """构建请求体"""
@@ -206,26 +208,26 @@ class ASRProvider(ASRProviderBase):
 
         try:
             response = requests.post(self.API_URL, headers=headers, data=request_body)
-            
+
             if not response.ok:
                 raise IOError(f"请求失败: {response.status_code} {response.reason}")
-            
+
             response_json = response.json()
-            
+
             # 检查是否有错误
             if "Response" in response_json and "Error" in response_json["Response"]:
                 error = response_json["Response"]["Error"]
                 error_code = error["Code"]
                 error_message = error["Message"]
                 raise IOError(f"API返回错误: {error_code}: {error_message}")
-            
+
             # 提取识别结果
             if "Response" in response_json and "Result" in response_json["Response"]:
                 return response_json["Response"]["Result"]
             else:
                 logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
                 return ""
-                
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
             return None
@@ -239,7 +241,7 @@ class ASRProvider(ASRProviderBase):
         """计算HMAC-SHA256"""
         if isinstance(key, str):
             key = key.encode('utf-8')
-        
+
         return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest()
 
     def _bytes_to_hex(self, bytes_data: bytes) -> str:

main/xiaozhi-server/core/providers/tts/base.py

@@ -1,11 +1,9 @@
 import asyncio
 from config.logger import setup_logging
 import os
-import numpy as np
-import opuslib_next
-from pydub import AudioSegment
 from abc import ABC, abstractmethod
 from core.utils.tts import MarkdownCleaner
+from core.utils.util import audio_to_opus_data
 
 TAG = __name__
 logger = setup_logging()
@@ -29,7 +27,9 @@ class TTSProviderBase(ABC):
                 try:
                     asyncio.run(self.text_to_speak(text, tmp_file))
                 except Exception as e:
-                    logger.bind(tag=TAG).warning(f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}")
+                    logger.bind(tag=TAG).warning(
+                        f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
+                    )
                     # 未执行成功，删除文件
                     if os.path.exists(tmp_file):
                         os.remove(tmp_file)
@@ -54,47 +54,4 @@ class TTSProviderBase(ABC):
         pass
 
     def audio_to_opus_data(self, audio_file_path):
-        """音频文件转换为Opus编码"""
-        # 获取文件后缀名
-        file_type = os.path.splitext(audio_file_path)[1]
-        if file_type:
-            file_type = file_type.lstrip(".")
-        # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
-        audio = AudioSegment.from_file(
-            audio_file_path, format=file_type, parameters=["-nostdin"]
-        )
-
-        # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
-        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
-
-        # 音频时长(秒)
-        duration = len(audio) / 1000.0
-
-        # 获取原始PCM数据（16位小端）
-        raw_data = audio.raw_data
-
-        # 初始化Opus编码器
-        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
-
-        # 编码参数
-        frame_duration = 60  # 60ms per frame
-        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
-
-        opus_datas = []
-        # 按帧处理所有音频数据（包括最后一帧可能补零）
-        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
-            # 获取当前帧的二进制数据
-            chunk = raw_data[i : i + frame_size * 2]
-
-            # 如果最后一帧不足，补零
-            if len(chunk) < frame_size * 2:
-                chunk += b"\x00" * (frame_size * 2 - len(chunk))
-
-            # 转换为numpy数组处理
-            np_frame = np.frombuffer(chunk, dtype=np.int16)
-
-            # 编码Opus数据
-            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
-            opus_datas.append(opus_data)
-
-        return opus_datas, duration
+        return audio_to_opus_data(audio_file_path)

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -51,7 +51,7 @@ class TTSProvider(TTSProviderBase):
         request_json = {
             "app": {
                 "appid": f"{self.appid}",
-                "token": "access_token",
+                "token": self.access_token,
                 "cluster": self.cluster,
             },
             "user": {"uid": "1"},

main/xiaozhi-server/core/utils/dialogue.py

@@ -4,7 +4,14 @@ from datetime import datetime
 
 
 class Message:
-    def __init__(self, role: str, content: str = None, uniq_id: str = None, tool_calls = None, tool_call_id=None):
+    def __init__(
+        self,
+        role: str,
+        content: str = None,
+        uniq_id: str = None,
+        tool_calls=None,
+        tool_call_id=None,
+    ):
         self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
         self.role = role
         self.content = content
@@ -16,7 +23,7 @@ class Dialogue:
     def __init__(self):
         self.dialogue: List[Message] = []
         # 获取当前时间
-        self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        self.current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
     def put(self, message: Message):
         self.dialogue.append(message)
@@ -25,7 +32,9 @@ class Dialogue:
         if m.tool_calls is not None:
             dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
         elif m.role == "tool":
-            dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content})
+            dialogue.append(
+                {"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content}
+            )
         else:
             dialogue.append({"role": m.role, "content": m.content})
 
@@ -44,23 +53,23 @@ class Dialogue:
         else:
             self.put(Message(role="system", content=new_content))
 
-    def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
+    def get_llm_dialogue_with_memory(
+        self, memory_str: str = None
+    ) -> List[Dict[str, str]]:
         if memory_str is None or len(memory_str) == 0:
             return self.get_llm_dialogue()
-        
+
         # 构建带记忆的对话
         dialogue = []
-        
+
         # 添加系统提示和记忆
         system_message = next(
             (msg for msg in self.dialogue if msg.role == "system"), None
         )
 
-
         if system_message:
             enhanced_system_prompt = (
-                f"{system_message.content}\n\n"
-                f"相关记忆：\n{memory_str}"
+                f"{system_message.content}\n\n" f"相关记忆：\n{memory_str}"
             )
             dialogue.append({"role": "system", "content": enhanced_system_prompt})
 

main/xiaozhi-server/core/utils/util.py

@@ -2,35 +2,40 @@ import json
 import socket
 import subprocess
 import re
+import os
+import numpy as np
 import requests
+import opuslib_next
+from pydub import AudioSegment
 from typing import Dict, Any
 from core.utils import tts, llm, intent, memory, vad, asr
 
 TAG = __name__
 emoji_map = {
-    'neutral': '😶',
-    'happy': '🙂',
-    'laughing': '😆',
-    'funny': '😂',
-    'sad': '😔',
-    'angry': '😠',
-    'crying': '😭',
-    'loving': '😍',
-    'embarrassed': '😳',
-    'surprised': '😲',
-    'shocked': '😱',
-    'thinking': '🤔',
-    'winking': '😉',
-    'cool': '😎',
-    'relaxed': '😌',
-    'delicious': '🤤',
-    'kissy': '😘',
-    'confident': '😏',
-    'sleepy': '😴',
-    'silly': '😜',
-    'confused': '🙄'
+    "neutral": "😶",
+    "happy": "🙂",
+    "laughing": "😆",
+    "funny": "😂",
+    "sad": "😔",
+    "angry": "😠",
+    "crying": "😭",
+    "loving": "😍",
+    "embarrassed": "😳",
+    "surprised": "😲",
+    "shocked": "😱",
+    "thinking": "🤔",
+    "winking": "😉",
+    "cool": "😎",
+    "relaxed": "😌",
+    "delicious": "🤤",
+    "kissy": "😘",
+    "confident": "😏",
+    "sleepy": "😴",
+    "silly": "😜",
+    "confused": "🙄",
 }
 
+
 def get_local_ip():
     try:
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -117,9 +122,9 @@ def is_punctuation_or_emoji(char):
         "、",  # 中文顿号
         "“",
         "”",
-        "\"",  # 中文双引号 + 英文引号
+        '"',  # 中文双引号 + 英文引号
         "：",
-        ":",   # 中文冒号 + 英文冒号
+        ":",  # 中文冒号 + 英文冒号
     }
     if char.isspace() or char in punctuation_set:
         return True
@@ -345,20 +350,15 @@ def initialize_modules(
             str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
         )
         logger.bind(tag=TAG).info(f"初始化组件: asr成功 {select_asr_module}")
-
-    # 初始化自定义prompt
-    if config.get("prompt", None) is not None:
-        modules["prompt"] = config["prompt"]
-        logger.bind(tag=TAG).info(f"初始化组件: prompt成功 {modules['prompt'][:50]}...")
-
     return modules
 
+
 def analyze_emotion(text):
     """
     分析文本情感并返回对应的emoji名称（支持中英文）
     """
     if not text or not isinstance(text, str):
-        return 'neutral'
+        return "neutral"
 
     original_text = text
     text = text.lower().strip()
@@ -369,84 +369,444 @@ def analyze_emotion(text):
             return emotion
 
     # 标点符号分析
-    has_exclamation = '!' in original_text or '！' in original_text
-    has_question = '?' in original_text or '？' in original_text
-    has_ellipsis = '...' in original_text or '…' in original_text
+    has_exclamation = "!" in original_text or "！" in original_text
+    has_question = "?" in original_text or "？" in original_text
+    has_ellipsis = "..." in original_text or "…" in original_text
 
     # 定义情感关键词映射（中英文扩展版）
     emotion_keywords = {
-        'happy': ['开心', '高兴', '快乐', '愉快', '幸福', '满意', '棒', '好', '不错', '完美', '棒极了', '太好了',
-                  '好呀', '好的', 'happy', 'joy', 'great', 'good', 'nice', 'awesome', 'fantastic', 'wonderful'],
-        'laughing': ['哈哈', '哈哈哈', '呵呵', '嘿嘿', '嘻嘻', '笑死', '太好笑了', '笑死我了', 'lol', 'lmao', 'haha',
-                     'hahaha', 'hehe', 'rofl', 'funny', 'laugh'],
-        'funny': ['搞笑', '滑稽', '逗', '幽默', '笑点', '段子', '笑话', '太逗了', 'hilarious', 'joke', 'comedy'],
-        'sad': ['伤心', '难过', '悲哀', '悲伤', '忧郁', '郁闷', '沮丧', '失望', '想哭', '难受', '不开心', '唉', '呜呜',
-                'sad', 'upset', 'unhappy', 'depressed', 'sorrow', 'gloomy'],
-        'angry': ['生气', '愤怒', '气死', '讨厌', '烦人', '可恶', '烦死了', '恼火', '暴躁', '火大', '愤怒', '气炸了',
-                  'angry', 'mad', 'annoyed', 'furious', 'pissed', 'hate'],
-        'crying': ['哭泣', '泪流', '大哭', '伤心欲绝', '泪目', '流泪', '哭死', '哭晕', '想哭', '泪崩',
-                   'cry', 'crying', 'tears', 'sob', 'weep'],
-        'loving': ['爱你', '喜欢', '爱', '亲爱的', '宝贝', '么么哒', '抱抱', '想你', '思念', '最爱', '亲亲', '喜欢你',
-                   'love', 'like', 'adore', 'darling', 'sweetie', 'honey', 'miss you', 'heart'],
-        'embarrassed': ['尴尬', '不好意思', '害羞', '脸红', '难为情', '社死', '丢脸', '出丑',
-                        'embarrassed', 'awkward', 'shy', 'blush'],
-        'surprised': ['惊讶', '吃惊', '天啊', '哇塞', '哇', '居然', '竟然', '没想到', '出乎意料',
-                      'surprise', 'wow', 'omg', 'oh my god', 'amazing', 'unbelievable'],
-        'shocked': ['震惊', '吓到', '惊呆了', '不敢相信', '震撼', '吓死', '恐怖', '害怕', '吓人',
-                    'shocked', 'shocking', 'scared', 'frightened', 'terrified', 'horror'],
-        'thinking': ['思考', '考虑', '想一下', '琢磨', '沉思', '冥想', '想', '思考中', '在想',
-                      'think', 'thinking', 'consider', 'ponder', 'meditate'],
-        'winking': ['调皮', '眨眼', '你懂的', '坏笑', '邪恶', '奸笑', '使眼色',
-                    'wink', 'teasing', 'naughty', 'mischievous'],
-        'cool': ['酷', '帅', '厉害', '棒极了', '真棒', '牛逼', '强', '优秀', '杰出', '出色', '完美',
-                 'cool', 'awesome', 'amazing', 'great', 'impressive', 'perfect'],
-        'relaxed': ['放松', '舒服', '惬意', '悠闲', '轻松', '舒适', '安逸', '自在',
-                     'relax', 'relaxed', 'comfortable', 'cozy', 'chill', 'peaceful'],
-        'delicious': ['好吃', '美味', '香', '馋', '可口', '香甜', '大餐', '大快朵颐', '流口水', '垂涎',
-                      'delicious', 'yummy', 'tasty', 'yum', 'appetizing', 'mouthwatering'],
-        'kissy': ['亲亲', '么么', '吻', 'mua', 'muah', '亲一下', '飞吻',
-                  'kiss', 'xoxo', 'hug', 'muah', 'smooch'],
-        'confident': ['自信', '肯定', '确定', '毫无疑问', '当然', '必须的', '毫无疑问', '确信', '坚信',
-                      'confident', 'sure', 'certain', 'definitely', 'positive'],
-        'sleepy': ['困', '睡觉', '晚安', '想睡', '好累', '疲惫', '疲倦', '困了', '想休息', '睡意',
-                   'sleep', 'sleepy', 'tired', 'exhausted', 'bedtime', 'good night'],
-        'silly': ['傻', '笨', '呆', '憨', '蠢', '二', '憨憨', '傻乎乎', '呆萌',
-                  'silly', 'stupid', 'dumb', 'foolish', 'goofy', 'ridiculous'],
-        'confused': ['疑惑', '不明白', '不懂', '困惑', '疑问', '为什么', '怎么回事', '啥意思', '不清楚',
-                     'confused', 'puzzled', 'doubt', 'question', 'what', 'why', 'how']
+        "happy": [
+            "开心",
+            "高兴",
+            "快乐",
+            "愉快",
+            "幸福",
+            "满意",
+            "棒",
+            "好",
+            "不错",
+            "完美",
+            "棒极了",
+            "太好了",
+            "好呀",
+            "好的",
+            "happy",
+            "joy",
+            "great",
+            "good",
+            "nice",
+            "awesome",
+            "fantastic",
+            "wonderful",
+        ],
+        "laughing": [
+            "哈哈",
+            "哈哈哈",
+            "呵呵",
+            "嘿嘿",
+            "嘻嘻",
+            "笑死",
+            "太好笑了",
+            "笑死我了",
+            "lol",
+            "lmao",
+            "haha",
+            "hahaha",
+            "hehe",
+            "rofl",
+            "funny",
+            "laugh",
+        ],
+        "funny": [
+            "搞笑",
+            "滑稽",
+            "逗",
+            "幽默",
+            "笑点",
+            "段子",
+            "笑话",
+            "太逗了",
+            "hilarious",
+            "joke",
+            "comedy",
+        ],
+        "sad": [
+            "伤心",
+            "难过",
+            "悲哀",
+            "悲伤",
+            "忧郁",
+            "郁闷",
+            "沮丧",
+            "失望",
+            "想哭",
+            "难受",
+            "不开心",
+            "唉",
+            "呜呜",
+            "sad",
+            "upset",
+            "unhappy",
+            "depressed",
+            "sorrow",
+            "gloomy",
+        ],
+        "angry": [
+            "生气",
+            "愤怒",
+            "气死",
+            "讨厌",
+            "烦人",
+            "可恶",
+            "烦死了",
+            "恼火",
+            "暴躁",
+            "火大",
+            "愤怒",
+            "气炸了",
+            "angry",
+            "mad",
+            "annoyed",
+            "furious",
+            "pissed",
+            "hate",
+        ],
+        "crying": [
+            "哭泣",
+            "泪流",
+            "大哭",
+            "伤心欲绝",
+            "泪目",
+            "流泪",
+            "哭死",
+            "哭晕",
+            "想哭",
+            "泪崩",
+            "cry",
+            "crying",
+            "tears",
+            "sob",
+            "weep",
+        ],
+        "loving": [
+            "爱你",
+            "喜欢",
+            "爱",
+            "亲爱的",
+            "宝贝",
+            "么么哒",
+            "抱抱",
+            "想你",
+            "思念",
+            "最爱",
+            "亲亲",
+            "喜欢你",
+            "love",
+            "like",
+            "adore",
+            "darling",
+            "sweetie",
+            "honey",
+            "miss you",
+            "heart",
+        ],
+        "embarrassed": [
+            "尴尬",
+            "不好意思",
+            "害羞",
+            "脸红",
+            "难为情",
+            "社死",
+            "丢脸",
+            "出丑",
+            "embarrassed",
+            "awkward",
+            "shy",
+            "blush",
+        ],
+        "surprised": [
+            "惊讶",
+            "吃惊",
+            "天啊",
+            "哇塞",
+            "哇",
+            "居然",
+            "竟然",
+            "没想到",
+            "出乎意料",
+            "surprise",
+            "wow",
+            "omg",
+            "oh my god",
+            "amazing",
+            "unbelievable",
+        ],
+        "shocked": [
+            "震惊",
+            "吓到",
+            "惊呆了",
+            "不敢相信",
+            "震撼",
+            "吓死",
+            "恐怖",
+            "害怕",
+            "吓人",
+            "shocked",
+            "shocking",
+            "scared",
+            "frightened",
+            "terrified",
+            "horror",
+        ],
+        "thinking": [
+            "思考",
+            "考虑",
+            "想一下",
+            "琢磨",
+            "沉思",
+            "冥想",
+            "想",
+            "思考中",
+            "在想",
+            "think",
+            "thinking",
+            "consider",
+            "ponder",
+            "meditate",
+        ],
+        "winking": [
+            "调皮",
+            "眨眼",
+            "你懂的",
+            "坏笑",
+            "邪恶",
+            "奸笑",
+            "使眼色",
+            "wink",
+            "teasing",
+            "naughty",
+            "mischievous",
+        ],
+        "cool": [
+            "酷",
+            "帅",
+            "厉害",
+            "棒极了",
+            "真棒",
+            "牛逼",
+            "强",
+            "优秀",
+            "杰出",
+            "出色",
+            "完美",
+            "cool",
+            "awesome",
+            "amazing",
+            "great",
+            "impressive",
+            "perfect",
+        ],
+        "relaxed": [
+            "放松",
+            "舒服",
+            "惬意",
+            "悠闲",
+            "轻松",
+            "舒适",
+            "安逸",
+            "自在",
+            "relax",
+            "relaxed",
+            "comfortable",
+            "cozy",
+            "chill",
+            "peaceful",
+        ],
+        "delicious": [
+            "好吃",
+            "美味",
+            "香",
+            "馋",
+            "可口",
+            "香甜",
+            "大餐",
+            "大快朵颐",
+            "流口水",
+            "垂涎",
+            "delicious",
+            "yummy",
+            "tasty",
+            "yum",
+            "appetizing",
+            "mouthwatering",
+        ],
+        "kissy": [
+            "亲亲",
+            "么么",
+            "吻",
+            "mua",
+            "muah",
+            "亲一下",
+            "飞吻",
+            "kiss",
+            "xoxo",
+            "hug",
+            "muah",
+            "smooch",
+        ],
+        "confident": [
+            "自信",
+            "肯定",
+            "确定",
+            "毫无疑问",
+            "当然",
+            "必须的",
+            "毫无疑问",
+            "确信",
+            "坚信",
+            "confident",
+            "sure",
+            "certain",
+            "definitely",
+            "positive",
+        ],
+        "sleepy": [
+            "困",
+            "睡觉",
+            "晚安",
+            "想睡",
+            "好累",
+            "疲惫",
+            "疲倦",
+            "困了",
+            "想休息",
+            "睡意",
+            "sleep",
+            "sleepy",
+            "tired",
+            "exhausted",
+            "bedtime",
+            "good night",
+        ],
+        "silly": [
+            "傻",
+            "笨",
+            "呆",
+            "憨",
+            "蠢",
+            "二",
+            "憨憨",
+            "傻乎乎",
+            "呆萌",
+            "silly",
+            "stupid",
+            "dumb",
+            "foolish",
+            "goofy",
+            "ridiculous",
+        ],
+        "confused": [
+            "疑惑",
+            "不明白",
+            "不懂",
+            "困惑",
+            "疑问",
+            "为什么",
+            "怎么回事",
+            "啥意思",
+            "不清楚",
+            "confused",
+            "puzzled",
+            "doubt",
+            "question",
+            "what",
+            "why",
+            "how",
+        ],
     }
 
     # 特殊句型判断（中英文）
     # 赞美他人
-    if any(phrase in text for phrase in
-           ['你真', '你好', '您真', '你真棒', '你好厉害', '你太强了', '你真好', '你真聪明',
-            'you are', 'you\'re', 'you look', 'you seem', 'so smart', 'so kind']):
-        return 'loving'
+    if any(
+        phrase in text
+        for phrase in [
+            "你真",
+            "你好",
+            "您真",
+            "你真棒",
+            "你好厉害",
+            "你太强了",
+            "你真好",
+            "你真聪明",
+            "you are",
+            "you're",
+            "you look",
+            "you seem",
+            "so smart",
+            "so kind",
+        ]
+    ):
+        return "loving"
     # 自我赞美
-    if any(phrase in text for phrase in ['我真', '我最', '我太棒了', '我厉害', '我聪明', '我优秀',
-                                        'i am', 'i\'m', 'i feel', 'so good', 'so happy']):
-        return 'cool'
+    if any(
+        phrase in text
+        for phrase in [
+            "我真",
+            "我最",
+            "我太棒了",
+            "我厉害",
+            "我聪明",
+            "我优秀",
+            "i am",
+            "i'm",
+            "i feel",
+            "so good",
+            "so happy",
+        ]
+    ):
+        return "cool"
     # 晚安/睡觉相关
-    if any(phrase in text for phrase in ['睡觉', '晚安', '睡了', '好梦', '休息了', '去睡了',
-                                         'sleep', 'good night', 'bedtime', 'go to bed']):
-        return 'sleepy'
+    if any(
+        phrase in text
+        for phrase in [
+            "睡觉",
+            "晚安",
+            "睡了",
+            "好梦",
+            "休息了",
+            "去睡了",
+            "sleep",
+            "good night",
+            "bedtime",
+            "go to bed",
+        ]
+    ):
+        return "sleepy"
     # 疑问句
     if has_question and not has_exclamation:
-        return 'thinking'
+        return "thinking"
     # 强烈情感（感叹号）
     if has_exclamation and not has_question:
         # 检查是否是积极内容
-        positive_words = emotion_keywords['happy'] + emotion_keywords['laughing'] + emotion_keywords['cool']
+        positive_words = (
+            emotion_keywords["happy"]
+            + emotion_keywords["laughing"]
+            + emotion_keywords["cool"]
+        )
         if any(word in text for word in positive_words):
-            return 'laughing'
+            return "laughing"
         # 检查是否是消极内容
-        negative_words = emotion_keywords['angry'] + emotion_keywords['sad'] + emotion_keywords['crying']
+        negative_words = (
+            emotion_keywords["angry"]
+            + emotion_keywords["sad"]
+            + emotion_keywords["crying"]
+        )
         if any(word in text for word in negative_words):
-            return 'angry'
-        return 'surprised'
+            return "angry"
+        return "surprised"
     # 省略号（表示犹豫或思考）
     if has_ellipsis:
-        return 'thinking'
+        return "thinking"
 
     # 关键词匹配（带权重）
     emotion_scores = {emotion: 0 for emotion in emoji_map.keys()}
@@ -466,18 +826,33 @@ def analyze_emotion(text):
     # 根据分数选择最可能的情感
     max_score = max(emotion_scores.values())
     if max_score == 0:
-        return 'happy'  # 默认
+        return "happy"  # 默认
 
     # 可能有多个情感同分，根据上下文选择最合适的
     top_emotions = [e for e, s in emotion_scores.items() if s == max_score]
 
     # 如果多个情感同分，使用以下优先级
     priority_order = [
-        'laughing', 'crying', 'angry', 'surprised', 'shocked',  # 强烈情感优先
-        'loving', 'happy', 'funny', 'cool',  # 积极情感
-        'sad', 'embarrassed', 'confused',  # 消极情感
-        'thinking', 'winking', 'relaxed',  # 中性情感
-        'delicious', 'kissy', 'confident', 'sleepy', 'silly'  # 特殊场景
+        "laughing",
+        "crying",
+        "angry",
+        "surprised",
+        "shocked",  # 强烈情感优先
+        "loving",
+        "happy",
+        "funny",
+        "cool",  # 积极情感
+        "sad",
+        "embarrassed",
+        "confused",  # 消极情感
+        "thinking",
+        "winking",
+        "relaxed",  # 中性情感
+        "delicious",
+        "kissy",
+        "confident",
+        "sleepy",
+        "silly",  # 特殊场景
     ]
 
     for emotion in priority_order:
@@ -485,3 +860,92 @@ def analyze_emotion(text):
             return emotion
 
     return top_emotions[0]  # 如果都不在优先级列表里，返回第一个
+
+
+def audio_to_opus_data(audio_file_path):
+    """音频文件转换为Opus编码"""
+    # 获取文件后缀名
+    file_type = os.path.splitext(audio_file_path)[1]
+    if file_type:
+        file_type = file_type.lstrip(".")
+    # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
+    audio = AudioSegment.from_file(
+        audio_file_path, format=file_type, parameters=["-nostdin"]
+    )
+
+    # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
+    audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+
+    # 音频时长(秒)
+    duration = len(audio) / 1000.0
+
+    # 获取原始PCM数据（16位小端）
+    raw_data = audio.raw_data
+
+    # 初始化Opus编码器
+    encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+    # 编码参数
+    frame_duration = 60  # 60ms per frame
+    frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+    opus_datas = []
+    # 按帧处理所有音频数据（包括最后一帧可能补零）
+    for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+        # 获取当前帧的二进制数据
+        chunk = raw_data[i : i + frame_size * 2]
+
+        # 如果最后一帧不足，补零
+        if len(chunk) < frame_size * 2:
+            chunk += b"\x00" * (frame_size * 2 - len(chunk))
+
+        # 转换为numpy数组处理
+        np_frame = np.frombuffer(chunk, dtype=np.int16)
+
+        # 编码Opus数据
+        opus_data = encoder.encode(np_frame.tobytes(), frame_size)
+        opus_datas.append(opus_data)
+
+    return opus_datas, duration
+
+
+def check_vad_update(before_config, new_config):
+    if new_config["selected_module"].get("VAD") is None:
+        return False
+    update_vad = False
+    current_vad_module = before_config["selected_module"]["VAD"]
+    new_vad_module = new_config["selected_module"]["VAD"]
+    current_vad_type = (
+        current_vad_module
+        if "type" not in before_config["VAD"][current_vad_module]
+        else before_config["VAD"][current_vad_module]["type"]
+    )
+    new_vad_type = (
+        new_vad_module
+        if "type" not in new_config["VAD"][new_vad_module]
+        else new_config["VAD"][new_vad_module]["type"]
+    )
+    print(f"前vad:{current_vad_type}，后vad:{new_vad_type}")
+    update_vad = current_vad_type != new_vad_type
+    return update_vad
+
+
+def check_asr_update(before_config, new_config):
+    if new_config["selected_module"].get("ASR") is None:
+        return False
+    update_asr = False
+    current_asr_module = before_config["selected_module"]["ASR"]
+    new_asr_module = new_config["selected_module"]["ASR"]
+    current_asr_type = (
+        current_asr_module
+        if "type" not in before_config["ASR"][current_asr_module]
+        else before_config["ASR"][current_asr_module]["type"]
+    )
+    new_asr_type = (
+        new_asr_module
+        if "type" not in new_config["ASR"][new_asr_module]
+        else new_config["ASR"][new_asr_module]["type"]
+    )
+    print(f"前asr:{current_asr_type}，后asr:{new_asr_type}")
+    update_asr = current_asr_type != new_asr_type
+    return update_asr

main/xiaozhi-server/core/websocket_server.py

@@ -2,7 +2,8 @@ import asyncio
 import websockets
 from config.logger import setup_logging
 from core.connection import ConnectionHandler
-from core.utils.util import initialize_modules
+from core.utils.util import initialize_modules, check_vad_update, check_asr_update
+from config.config_loader import get_config_from_api
 
 TAG = __name__
 
@@ -13,14 +14,21 @@ class WebSocketServer:
         self.logger = setup_logging()
         self.config_lock = asyncio.Lock()
         modules = initialize_modules(
-            self.logger, self.config, True, True, True, True, True, True
+            self.logger,
+            self.config,
+            "VAD" in self.config["selected_module"],
+            "ASR" in self.config["selected_module"],
+            "LLM" in self.config["selected_module"],
+            "TTS" in self.config["selected_module"],
+            "Memory" in self.config["selected_module"],
+            "Intent" in self.config["selected_module"],
         )
-        self._vad = modules["vad"]
-        self._asr = modules["asr"]
-        self._tts = modules["tts"]
-        self._llm = modules["llm"]
-        self._intent = modules["intent"]
-        self._memory = modules["memory"]
+        self._vad = modules["vad"] if "vad" in modules else None
+        self._asr = modules["asr"] if "asr" in modules else None
+        self._tts = modules["tts"] if "tts" in modules else None
+        self._llm = modules["llm"] if "llm" in modules else None
+        self._intent = modules["intent"] if "intent" in modules else None
+        self._memory = modules["memory"] if "memory" in modules else None
         self.active_connections = set()
 
     async def start(self):
@@ -44,7 +52,7 @@ class WebSocketServer:
             self._tts,
             self._memory,
             self._intent,
-            self  # 传入当前 WebSocketServer 实例
+            self,  # 传入server实例
         )
         self.active_connections.add(handler)
         try:
@@ -60,3 +68,54 @@ class WebSocketServer:
         else:
             # 如果是普通 HTTP 请求，返回 "server is running"
             return websocket.respond(200, "Server is running\n")
+
+    async def update_config(self) -> bool:
+        """更新服务器配置并重新初始化组件
+
+        Returns:
+            bool: 更新是否成功
+        """
+        try:
+            async with self.config_lock:
+                # 重新获取配置
+                new_config = get_config_from_api(self.config)
+                if new_config is None:
+                    self.logger.bind(tag=TAG).error("获取新配置失败")
+                    return False
+
+                # 检查 VAD 和 ASR 类型是否需要更新
+                update_vad = check_vad_update(self.config, new_config)
+                update_asr = check_asr_update(self.config, new_config)
+
+                # 更新配置
+                self.config = new_config
+                # 重新初始化组件
+                modules = initialize_modules(
+                    self.logger,
+                    new_config,
+                    update_vad,
+                    update_asr,
+                    "LLM" in new_config["selected_module"],
+                    "TTS" in new_config["selected_module"],
+                    "Memory" in new_config["selected_module"],
+                    "Intent" in new_config["selected_module"],
+                )
+
+                # 更新组件实例
+                if "vad" in modules:
+                    self._vad = modules["vad"]
+                if "asr" in modules:
+                    self._asr = modules["asr"]
+                if "tts" in modules:
+                    self._tts = modules["tts"]
+                if "llm" in modules:
+                    self._llm = modules["llm"]
+                if "intent" in modules:
+                    self._intent = modules["intent"]
+                if "memory" in modules:
+                    self._memory = modules["memory"]
+
+                return True
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")
+            return False

main/xiaozhi-server/mcp_server_settings.json

@@ -3,7 +3,8 @@
     "在data目录下创建.mcp_server_settings.json文件，可以选择下面的MCP服务，也可以自行添加新的MCP服务。",
     "后面不断测试补充好用的mcp服务，欢迎大家一起补充。",
     "记得删除注释行,des属性仅为说明,不会被解析。",
-    "des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。"
+    "des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。",
+    "当前支持stdio/sse两种模式。"
   ],
   "mcpServers": {
     "filesystem": {

main/xiaozhi-server/plugins_func/functions/play_music.py

@@ -10,10 +10,9 @@ from pathlib import Path
 from core.utils import p3
 from core.handle.sendAudioHandle import send_stt_message
 from plugins_func.register import register_function, ToolType, ActionResponse, Action
-
+from core.utils.dialogue import Message
 
 TAG = __name__
-logger = setup_logging()
 
 MUSIC_CACHE = {}
 
@@ -45,7 +44,7 @@ def play_music(conn, song_name: str):
 
         # 检查事件循环状态
         if not conn.loop.is_running():
-            logger.bind(tag=TAG).error("事件循环未运行，无法提交任务")
+            conn.logger.bind(tag=TAG).error("事件循环未运行，无法提交任务")
             return ActionResponse(
                 action=Action.RESPONSE, result="系统繁忙", response="请稍后再试"
             )
@@ -59,9 +58,9 @@ def play_music(conn, song_name: str):
         def handle_done(f):
             try:
                 f.result()  # 可在此处理成功逻辑
-                logger.bind(tag=TAG).info("播放完成")
+                conn.logger.bind(tag=TAG).info("播放完成")
             except Exception as e:
-                logger.bind(tag=TAG).error(f"播放失败: {e}")
+                conn.logger.bind(tag=TAG).error(f"播放失败: {e}")
 
         future.add_done_callback(handle_done)
 
@@ -69,7 +68,7 @@ def play_music(conn, song_name: str):
             action=Action.NONE, result="指令已接收", response="正在为您播放音乐"
         )
     except Exception as e:
-        logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+        conn.logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
         return ActionResponse(
             action=Action.RESPONSE, result=str(e), response="播放音乐时出错了"
         )
@@ -150,7 +149,7 @@ async def handle_music_command(conn, text):
 
     """处理音乐播放指令"""
     clean_text = re.sub(r"[^\w\s]", "", text).strip()
-    logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
+    conn.logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
 
     # 尝试匹配具体歌名
     if os.path.exists(MUSIC_CACHE["music_dir"]):
@@ -165,7 +164,7 @@ async def handle_music_command(conn, text):
         if potential_song:
             best_match = _find_best_match(potential_song, MUSIC_CACHE["music_files"])
             if best_match:
-                logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
+                conn.logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
                 await play_local_music(conn, specific_file=best_match)
                 return True
     # 检查是否是通用播放音乐命令
@@ -195,7 +194,9 @@ async def play_local_music(conn, specific_file=None):
     """播放本地音乐文件"""
     try:
         if not os.path.exists(MUSIC_CACHE["music_dir"]):
-            logger.bind(tag=TAG).error(f"音乐目录不存在: " + MUSIC_CACHE["music_dir"])
+            conn.logger.bind(tag=TAG).error(
+                f"音乐目录不存在: " + MUSIC_CACHE["music_dir"]
+            )
             return
 
         # 确保路径正确性
@@ -204,16 +205,17 @@ async def play_local_music(conn, specific_file=None):
             music_path = os.path.join(MUSIC_CACHE["music_dir"], specific_file)
         else:
             if not MUSIC_CACHE["music_files"]:
-                logger.bind(tag=TAG).error("未找到MP3音乐文件")
+                conn.logger.bind(tag=TAG).error("未找到MP3音乐文件")
                 return
             selected_music = random.choice(MUSIC_CACHE["music_files"])
             music_path = os.path.join(MUSIC_CACHE["music_dir"], selected_music)
 
         if not os.path.exists(music_path):
-            logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
+            conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
             return
         text = _get_random_play_prompt(selected_music)
         await send_stt_message(conn, text)
+        conn.dialogue.put(Message(role="assistant", content=text))
         conn.tts_first_text_index = 0
         conn.tts_last_text_index = 0
 
@@ -233,5 +235,5 @@ async def play_local_music(conn, specific_file=None):
         conn.audio_play_queue.put((opus_packets, None, conn.tts_last_text_index))
 
     except Exception as e:
-        logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
-        logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
+        conn.logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
+        conn.logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/plugins_func/functions/plugin_loader.py

@@ -1,51 +1,56 @@
-from plugins_func.register import register_function,ToolType, ActionResponse, Action
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
 
 plugin_loader_function_desc = {
-                "type": "function",
-                "function": {
-                    "name": "plugin_loader",
-                    "description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "oper": {
-                                "type": "string",
-                                "description": "load or unload"
-                            },
-                            "name":{
-                                "type": "string",
-                                "description": "要加载或卸载的插件名字"
-                            }
-                        },
-                        "required": ["oper","name"]
-                    }
-                }
-            }
+    "type": "function",
+    "function": {
+        "name": "plugin_loader",
+        "description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "oper": {"type": "string", "description": "load or unload"},
+                "name": {"type": "string", "description": "要加载或卸载的插件名字"},
+            },
+            "required": ["oper", "name"],
+        },
+    },
+}
 
-@register_function('plugin_loader', plugin_loader_function_desc, ToolType.SYSTEM_CTL)
+
+@register_function("plugin_loader", plugin_loader_function_desc, ToolType.SYSTEM_CTL)
 def plugin_loader(conn, oper: str, name: str):
     """插件加载"""
     if oper not in ["load", "unload"]:
-        return ActionResponse(action=Action.RESPONSE, result="插件操作失败", response="不支持的操作")
-    
+        return ActionResponse(
+            action=Action.RESPONSE, result="插件操作失败", response="不支持的操作"
+        )
+
     cur_support = conn.func_handler.current_support_functions()
     if oper == "load":
         if name in cur_support:
-            return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response=f"{name}插件已加载,无需重复加载")
+            return ActionResponse(
+                action=Action.RESPONSE,
+                result="插件加载失败",
+                response=f"{name}插件已加载,无需重复加载",
+            )
         func = conn.func_handler.function_registry.register_function(name)
         if not func:
-            return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response="插件未找到")
+            return ActionResponse(
+                action=Action.RESPONSE, result="插件加载失败", response="插件未找到"
+            )
         res = f"{name}插件加载成功"
     else:
         if name not in cur_support:
-            return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response=f"{name}插件未加载")
+            return ActionResponse(
+                action=Action.RESPONSE,
+                result="插件卸载失败",
+                response=f"{name}插件未加载",
+            )
         bOK = conn.func_handler.function_registry.unregister_function(name)
         if not bOK:
-            return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response="插件未找到")
+            return ActionResponse(
+                action=Action.RESPONSE, result="插件卸载失败", response="插件未找到"
+            )
         res = f"{name}插件卸载成功"
     conn.func_handler.upload_functions_desc()
     return ActionResponse(action=Action.RESPONSE, result="插件操作成功", response=res)

main/xiaozhi-server/requirements.txt

@@ -22,7 +22,9 @@ mem0ai==0.1.62
 bs4==0.0.2
 modelscope==1.23.2
 sherpa_onnx==1.11.0
-mcp==1.4.1
+mcp==1.7.1
 cnlunar==0.2.0
 PySocks==1.7.1
-dashscope==1.23.1
+dashscope==1.23.1
+baidu-aip==4.16.13
+chardet==5.2.0