Merge branch 'py_test_typing' into feature/python-typing

.github/workflows/build-base-image.yml

@@ -37,7 +37,7 @@ jobs:
           file: Dockerfile-server-base
           push: true
           tags: ghcr.io/${{ github.repository }}:server-base
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           cache-from: type=gha,scope=server-base
           cache-to: type=gha,mode=max,scope=server-base
           build-args: |

.github/workflows/docker-image.yml

@@ -66,7 +66,7 @@ jobs:
           push: true
           tags: |
             ${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:server_{1},ghcr.io/{0}:server_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:server_latest', github.repository) }}
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           cache-from: type=gha
           cache-to: type=gha,mode=max
           build-args: |
@@ -81,7 +81,7 @@ jobs:
           push: true
           tags: |
             ${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:web_{1},ghcr.io/{0}:web_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:web_latest', github.repository) }}
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           cache-from: type=gha
           cache-to: type=gha,mode=max
           build-args: |

Dockerfile-web

@@ -1,5 +1,5 @@
 # 第一阶段：构建Vue前端
-FROM node:18 as web-builder
+FROM node:18 AS web-builder
 WORKDIR /app
 COPY main/manager-web/package*.json ./
 RUN npm install
@@ -7,7 +7,7 @@ COPY main/manager-web .
 RUN npm run build
 
 # 第二阶段：构建Java后端
-FROM maven:3.9.4-eclipse-temurin-21 as api-builder
+FROM maven:3.9.4-eclipse-temurin-21 AS api-builder
 WORKDIR /app
 COPY main/manager-api/pom.xml .
 COPY main/manager-api/src ./src
@@ -18,18 +18,19 @@ FROM bellsoft/liberica-runtime-container:jre-21-glibc
 
 # 安装Nginx和字体库
 RUN apk update && \
-    apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
+    apk add --no-cache --no-scripts \
         nginx \
         bash \
         fontconfig \
         ttf-dejavu \
-        msttcorefonts-installer \
+    && rm -rf /var/cache/apk/* \
-    && ACCEPT_EULA=Y apk add --no-cache msttcorefonts-installer \
+    && mkdir -p /run/nginx /var/log/nginx /var/tmp/nginx /etc/nginx/conf.d
-    && fc-cache -f -v \
+
-    && rm -rf /var/cache/apk/*
+# 复制项目自带的中文字体
+COPY main/manager-web/public/generator/static/fonts/*.ttf /usr/share/fonts/
 
 # 更新字体缓存
-RUN (printf 'YES\n' | update-ms-fonts || true) && fc-cache -f -v
+RUN fc-cache -f -v
 
 # 配置Nginx
 COPY docs/docker/nginx.conf /etc/nginx/nginx.conf

main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java

@@ -304,7 +304,7 @@ public interface Constant {
     /**
      * 版本号
      */
-    public static final String VERSION = "0.8.11";
+    public static final String VERSION = "0.9.1";
 
     /**
      * 无效固件URL

main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java

@@ -1,6 +1,12 @@
 package xiaozhi.modules.agent.service.impl;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -81,9 +87,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
 
         if (agent.getMemModelId() != null && agent.getMemModelId().equals(Constant.MEMORY_NO_MEM)) {
             agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.IGNORE.getCode());
-            if (agent.getChatHistoryConf() == null) {
+        }
-                agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
+        if (agent.getChatHistoryConf() == null) {
-            }
+            agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
         }
 
         // 查询上下文源配置
@@ -132,7 +138,8 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
         if (StringUtils.isNotBlank(keyword)) {
             if ("mac".equals(searchType)) {
                 // 按MAC地址搜索：先搜索设备，再获取对应的智能体
-                List<DeviceEntity> devices = Optional.ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
+                List<DeviceEntity> devices = Optional
+                        .ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
                 // 获取设备对应的智能体ID列表
                 List<String> agentIds = devices.stream()
                         .map(DeviceEntity::getAgentId)

main/manager-api/src/main/java/xiaozhi/modules/model/controller/ModelController.java

@@ -129,6 +129,10 @@ public class ModelController {
         if (entity == null) {
             return new Result<Void>().error("模型配置不存在");
         }
+        // 不能关闭默认模型
+        if (status == 0 && entity.getIsDefault() > 0) {
+            return new Result<Void>().error("默认模型配置不允许关闭");
+        }
         // 不更新ConfigJson字段
         entity.setConfigJson(null);
         entity.setIsEnabled(status);

main/manager-api/src/main/resources/db/changelog/202601141645.sql

@@ -0,0 +1,31 @@
+-- 批量清理 ai_model_provider 中的 sample_rate 字段定义
+UPDATE `ai_model_provider` ap
+JOIN (
+    SELECT 
+        id,
+        JSON_ARRAYAGG(
+            JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t)
+        ) AS new_fields
+    FROM `ai_model_provider`,
+         JSON_TABLE(`fields`, '$[*]' COLUMNS (
+             k VARCHAR(50) PATH '$.key',
+             l VARCHAR(100) PATH '$.label',
+             t VARCHAR(20) PATH '$.type'
+         )) AS jt
+    WHERE `model_type` = 'TTS' 
+      AND jt.k != 'sample_rate'
+    GROUP BY id
+) filtered ON ap.id = filtered.id
+SET ap.fields = filtered.new_fields;
+
+-- 清理 config_json 顶层的 sample_rate
+UPDATE `ai_model_config`
+SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate')
+WHERE `model_type` = 'TTS'
+  AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL;
+
+-- 清理Minimax流式TTS的sample_rate参数（位于audio_setting内部）
+UPDATE `ai_model_config` SET 
+`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate'))
+WHERE `id` = 'TTS_MinimaxStreamTTS'
+AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL;

main/manager-api/src/main/resources/db/changelog/202601231530.sql

@@ -0,0 +1,87 @@
+-- 更新HuoshanDoubleStreamTTS供应器配置，将分散的参数改为JSON字典配置
+-- 将 speech_rate, loudness_rate, pitch, emotion, emotion_scale 等参数整合为 audio_params, additions, mix_speaker 三个JSON字典
+
+UPDATE `ai_model_provider`
+SET `fields` = '[
+  {"key": "ws_url", "type": "string", "label": "WebSocket地址"},
+  {"key": "appid", "type": "string", "label": "应用ID"},
+  {"key": "access_token", "type": "string", "label": "访问令牌"},
+  {"key": "resource_id", "type": "string", "label": "资源ID"},
+  {"key": "speaker", "type": "string", "label": "默认音色"},
+  {"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true},
+  {"key": "audio_params", "type": "dict", "label": "音频输出配置"},
+  {"key": "additions", "type": "dict", "label": "高级文本处理配置"},
+  {"key": "mix_speaker", "type": "dict", "label": "混音控制配置"}
+]'
+WHERE `id` = 'SYSTEM_TTS_HSDSTTS';
+
+-- 更新现有配置，将旧的分散参数迁移到新的JSON字典结构
+UPDATE `ai_model_config`
+SET `config_json` = JSON_SET(
+    `config_json`,
+    '$.audio_params', JSON_OBJECT(
+        'speech_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.speech_rate')), ''), '0') AS SIGNED),
+        'loudness_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.loudness_rate')), ''), '0') AS SIGNED)
+    ),
+    '$.additions', JSON_OBJECT(
+        'aigc_metadata', JSON_OBJECT(),
+        'cache_config', JSON_OBJECT(),
+        'post_process', JSON_OBJECT(
+            'pitch', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.pitch')), ''), '0') AS SIGNED)
+        )
+    ),
+    '$.mix_speaker', JSON_OBJECT()
+)
+WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
+
+-- 删除旧的分散参数字段
+UPDATE `ai_model_config`
+SET `config_json` = JSON_REMOVE(
+    `config_json`,
+    '$.speech_rate',
+    '$.loudness_rate',
+    '$.pitch',
+    '$.emotion',
+    '$.emotion_scale'
+)
+WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
+
+-- 更新文档链接和备注说明
+UPDATE `ai_model_config` SET
+`doc_link` = 'https://www.volcengine.com/docs/6561/1329505',
+`remark` = '火山引擎双向流式TTS配置说明：
+1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号
+2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型，购买音色
+3. 在页面底部获取appid和access_token
+4. 资源ID固定为：volc.service_type.10029（大模型语音合成及混音）
+5. 链接复用：开启WebSocket连接复用，默认true减少链接损耗（注意：复用后设备处于聆听状态时空闲链接会占并发数）
+
+详细参数文档：https://www.volcengine.com/docs/6561/1329505
+【audio_params】音频输出配置 - 用户可自定义添加火山引擎支持的任何音频参数
+  - speech_rate: 语速(-50~100)，默认0
+  - loudness_rate: 音量(-50~100)，默认0
+  - emotion: 情感类型（仅部分音色支持），可选值：neutral、happy、sad、angry、fearful、disgusted、surprised
+  - emotion_scale: 情感强度(1~5)，默认4
+  示例：{"speech_rate": 10, "loudness_rate": 5, "emotion": "happy", "emotion_scale": 4}
+
+【additions】高级文本处理配置 - 用户可自定义添加火山引擎支持的任何高级参数
+  - post_process.pitch: 音高(-12~12)，默认0
+  - aigc_metadata: AIGC元数据配置
+  - cache_config: 缓存配置
+  示例：{"post_process": {"pitch": 2}, "aigc_metadata": {}, "cache_config": {}}
+
+【mix_speaker】混音控制配置 - 多音色混合（仅 TTS 1.0）
+  示例：
+    {"speakers": [
+      {"source_speaker": "zh_male_bvlazysheep","mix_factor": 0.3}, 
+      {"source_speaker": "BV120_streaming","mix_factor": 0.3}, 
+      {"source_speaker": "zh_male_ahu_conversation_wvae_bigtts","mix_factor": 0.4}
+    ]}
+
+注意：
+- 多情感音色参数（emotion、emotion_scale）仅部分音色支持
+- 相关音色列表：https://www.volcengine.com/docs/6561/1257544
+- 用户可根据火山引擎API文档自行添加更多参数
+- 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts
+'
+WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';

main/manager-api/src/main/resources/db/changelog/202601261730.sql

@@ -0,0 +1,14 @@
+-- 更新小智参数中的默认采样率从 16000 改为 24000
+UPDATE `sys_params`
+SET `param_value` = '{
+  "type": "hello",
+  "version": 1,
+  "transport": "websocket",
+  "audio_params": {
+    "format": "opus",
+    "sample_rate": 24000,
+    "channels": 1,
+    "frame_duration": 60
+  }
+}'
+WHERE `id` = 309 AND `param_code` = 'xiaozhi';

main/manager-api/src/main/resources/db/changelog/202602021555.sql

@@ -0,0 +1 @@
+INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (502, 'system_error_response', '主人，小智现在有点忙，我们稍后再试吧。', 'string', 1, '系统错误时的回复');

main/manager-api/src/main/resources/db/changelog/202602051125.sql

@@ -0,0 +1,2 @@
+-- 删除无用业务表ai_voiceprint
+DROP TABLE IF EXISTS ai_voiceprint;

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -487,3 +487,38 @@ databaseChangeLog:
         - sqlFile:
             encoding: utf8
             path: classpath:db/changelog/202601051433.sql
+  - changeSet:
+      id: 202601141645
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202601141645.sql
+  - changeSet:
+      id: 202601231530
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202601231530.sql
+  - changeSet:
+      id: 202601261730
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202601261730.sql
+  - changeSet:
+      id: 202602021555
+      author: shengzhou1216
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202602021555.sql
+  - changeSet:
+      id: 202602051125
+      author: DaGou12138
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202602051125.sql

main/manager-mobile/src/pages/settings/index.vue

@@ -235,7 +235,7 @@ function showAbout() {
     title: t('settings.aboutApp', { appName: import.meta.env.VITE_APP_TITLE }),
     content: t('settings.aboutContent', {
       appName: import.meta.env.VITE_APP_TITLE,
-      version: '0.8.11'
+      version: '0.9.1'
     }),
     showCancel: false,
     confirmText: t('common.confirm'),

main/manager-web/public/generator/index.html

@@ -4,9 +4,10 @@
     <meta charset="UTF-8" />
     <link rel="icon" type="image/svg+xml" href="/vite.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="icon" href="./favicon.ico">
     <title>Xiaozhi AI Customization</title>
-    <script type="module" crossorigin src="./assets/index-FKVSBRAB.js"></script>
+    <script type="module" crossorigin src="./assets/index-B8r0c7xg.js"></script>
-    <link rel="stylesheet" crossorigin href="./assets/index-NXxBVrod.css">
+    <link rel="stylesheet" crossorigin href="./assets/index-CrIJdTCK.css">
   </head>
   <body>
     <div id="app"></div>

main/manager-web/src/i18n/de.js

@@ -836,6 +836,7 @@ export default {
   'modelConfig.unknown': 'Unbekannt',
   'modelConfig.isEnabled': 'Aktiviert',
   'modelConfig.isDefault': 'Standard',
+  'modelConfig.defaultModelCannotDisable': 'Standardmodellkonfiguration kann nicht deaktiviert werden',
   'modelConfig.action': 'Aktion',
   'modelConfig.voiceManagement': 'Stimmverwaltung',
   'modelConfig.edit': 'Bearbeiten',

main/manager-web/src/i18n/en.js

@@ -836,6 +836,7 @@ export default {
   'modelConfig.unknown': 'Unknown',
   'modelConfig.isEnabled': 'Enabled',
   'modelConfig.isDefault': 'Default',
+  'modelConfig.defaultModelCannotDisable': 'Default model configuration cannot be disabled',
   'modelConfig.action': 'Action',
   'modelConfig.voiceManagement': 'Voice Management',
   'modelConfig.edit': 'Edit',

main/manager-web/src/i18n/vi.js

@@ -836,6 +836,7 @@ export default {
   'modelConfig.unknown': 'Không xác định',
   'modelConfig.isEnabled': 'Đã bật',
   'modelConfig.isDefault': 'Mặc định',
+  'modelConfig.defaultModelCannotDisable': 'Cấu hình mô hình mặc định không thể tắt',
   'modelConfig.action': 'Hành động',
   'modelConfig.voiceManagement': 'Quản lý giọng nói',
   'modelConfig.edit': 'Chỉnh sửa',

main/manager-web/src/i18n/zh_CN.js

@@ -836,6 +836,7 @@ export default {
   'modelConfig.unknown': '未知',
   'modelConfig.isEnabled': '是否启用',
   'modelConfig.isDefault': '是否默认',
+  'modelConfig.defaultModelCannotDisable': '默认模型配置不允许关闭',
   'modelConfig.action': '操作',
   'modelConfig.voiceManagement': '音色管理',
   'modelConfig.edit': '修改',

main/manager-web/src/i18n/zh_TW.js

@@ -836,6 +836,7 @@ export default {
   'modelConfig.unknown': '未知',
   'modelConfig.isEnabled': '是否啟用',
   'modelConfig.isDefault': '是否默認',
+  'modelConfig.defaultModelCannotDisable': '預設模型配置不允許關閉',
   'modelConfig.action': '操作',
   'modelConfig.voiceManagement': '音色管理',
   'modelConfig.edit': '修改',

main/manager-web/src/views/DeviceManagement.vue

@@ -64,7 +64,7 @@
                   <el-button size="mini" type="text" @click="handleUnbind(scope.row.device_id)">
                     {{ $t('device.unbind') }}
                   </el-button>
-                  <el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor">
+                  <el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor(scope.row)">
                     {{ $t('device.deviceThemeGeneration') }}
                   </el-button>
                 </template>
@@ -337,10 +337,10 @@ export default {
         });
       });
     },
-    handleGenertor() {
+    handleGenertor(row) {
       const pathname = window.location.pathname;
       const basePath = pathname.split('/').slice(0, -1).join('/');
-      const url = `${window.location.origin}${basePath}/generator/`;
+      const url = `${window.location.origin}${basePath}/generator/?deviceId=${row.device_id}`;
       sessionStorage.setItem('devicePath', window.location.href);
       window.location.href = url;
     },

main/manager-web/src/views/ModelConfig.vue

@@ -97,7 +97,23 @@
               </el-table-column>
               <el-table-column :label="$t('modelConfig.isEnabled')" align="center">
                 <template slot-scope="scope">
+                  <el-tooltip
+                    v-if="scope.row.isDefault === 1 && scope.row.isEnabled === 1"
+                    :content="$t('modelConfig.defaultModelCannotDisable')"
+                    placement="top"
+                    effect="light"
+                  > 
+                    <el-switch
+                      v-model="scope.row.isEnabled"
+                      class="custom-switch"
+                      :active-value="1"
+                      :inactive-value="0"
+                      disabled
+                      @change="handleStatusChange(scope.row)"
+                    />
+                  </el-tooltip>
                   <el-switch
+                    v-else
                     v-model="scope.row.isEnabled"
                     class="custom-switch"
                     :active-value="1"

main/xiaozhi-server/config.yaml

@@ -89,7 +89,8 @@ xiaozhi:
   transport: websocket
   audio_params:
     format: opus
-    sample_rate: 16000
+    # Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000]
+    sample_rate: 24000
     channels: 1
     frame_duration: 60
 
@@ -202,6 +203,9 @@ prompt: |
 # 默认系统提示词模板文件
 prompt_template: agent-base-prompt.txt
 
+# 系统错误时的回复
+system_error_response: "主人，小智现在有点忙，我们稍后再试吧。"
+
 # 结束语prompt
 end_prompt:
   enable: true # 是否开启结束语
@@ -714,13 +718,31 @@ TTS:
     speaker: zh_female_wanwanxiaohe_moon_bigtts
     # 开启WebSocket连接复用，默认复用（注意：复用后设备处于聆听状态时空闲链接会占并发数）
     enable_ws_reuse: True
-    speech_rate: 0
+    # 相关参数文档：https://www.volcengine.com/docs/6561/1329505
-    loudness_rate: 0
+    # 音频输出配置（audio_params）- 用户可自定义添加火山引擎支持的任何音频参数
-    pitch: 0
+    audio_params:
-    # 多情感音色参数，注意：当前仅部分音色支持设置情感。
+      speech_rate: 0  # 语速(-50~100)
-    # 相关音色列表：https://www.volcengine.com/docs/6561/1257544
+      loudness_rate: 0  # 音量(-50~100)
-    emotion: "neutral"  # 情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised
+      # 多情感音色参数，注意：当前仅部分音色支持设置情感。
-    emotion_scale: 4  # 情感强度，可选值为：1~5，默认值为4
+      # 相关音色列表：https://www.volcengine.com/docs/6561/1257544
+      # emotion: "neutral"  # 情感类型（仅部分音色支持）：neutral、happy、sad、angry、fearful、disgusted、surprised
+      # emotion_scale: 4  # 情感强度(1~5)
+    # 高级文本处理配置（additions）- 用户可自定义添加火山引擎支持的任何高级参数
+    additions:
+      post_process:
+        pitch: 0  # 音高(-12~12)
+      # aigc_metadata: {}  # AIGC元数据配置
+      # cache_config: {}  # 缓存配置
+    # 混音控制配置（mix_speaker）- 多音色混合（仅 TTS 1.0）
+    # 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts
+    # mix_speaker:
+    #   speakers:
+    #     - source_speaker: zh_male_bvlazysheep
+    #       mix_factor: 0.3
+    #     - source_speaker: BV120_streaming
+    #       mix_factor: 0.3
+    #     - source_speaker: zh_male_ahu_conversation_wvae_bigtts
+    #       mix_factor: 0.4
   CosyVoiceSiliconflow:
     type: siliconflow
     # 硅基流动TTS
@@ -837,7 +859,6 @@ TTS:
     #       - "处理/(chu3)(li3)"
     #       - "危险/dangerous"
     # audio_setting:
-    #     sample_rate: 24000
     #     bitrate: 128000
     #     format: "mp3"
     #     channel: 1
@@ -865,7 +886,6 @@ TTS:
 
     # 以下可不用设置，使用默认设置
     # format: wav
-    # sample_rate: 16000
     # volume: 50
     # speech_rate: 0
     # pitch_rate: 0
@@ -889,7 +909,6 @@ TTS:
     host: nls-gateway-cn-beijing.aliyuncs.com
     # 以下可不用设置，使用默认设置
     # format: pcm  # 音频格式：pcm、wav、mp3
-    # sample_rate: 16000  # 采样率：8000、16000、24000
     # volume: 50  # 音量：0-100
     # speech_rate: 0  # 语速：-500到500
     # pitch_rate: 0  # 语调：-500到500
@@ -1004,7 +1023,6 @@ TTS:
     protocol: websocket # protocol choices = ['websocket', 'http']
     url: ws://127.0.0.1:8092/paddlespeech/tts/streaming  # TTS 服务的 URL 地址，指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming，http默认http://127.0.0.1:8090/paddlespeech/tts]
     spk_id: 0  # 发音人 ID，0 通常表示默认的发音人
-    sample_rate: 24000  # 采样率 [websocket默认24000，http默认0 自动选择]
     speed: 1.0  # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢
     volume: 1.0  # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小
     save_path:   # 保存路径
@@ -1028,7 +1046,6 @@ TTS:
     output_dir: tmp/
     # 以下可不用设置，使用默认设置
     # format: pcm  # 音频格式：pcm、wav、mp3、opus
-    # sample_rate: 24000  # 采样率：16000, 24000, 48000
     # volume: 50  # 音量：0-100
     # rate: 1  # 语速：0.5~2
     # pitch: 1  # 语调：0.5~2
@@ -1050,7 +1067,6 @@ TTS:
     # stop_split: 0  # 关闭服务端拆句 不关闭：0，关闭：1
     # remain: 0  # 是否保留原书面语的样子	保留:1, 不保留:0
     # format: raw  # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
-    # sample_rate: 24000  # 采样率：16000, 8000, 24000
     # volume: 50  # 音量：0-100
     # speed: 50  # 语速：0-100
     # pitch: 50  # 语调：0-100

main/xiaozhi-server/config/logger.py

@@ -5,7 +5,7 @@ from config.config_loader import load_config
 from config.settings import check_config_file
 from datetime import datetime
 
-SERVER_VERSION = "0.8.11"
+SERVER_VERSION = "0.9.1"
 _logger_initialized = False
 
 

main/xiaozhi-server/core/api/vision_handler.py

@@ -29,7 +29,15 @@ class VisionHandler(BaseHandler):
 
     def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]:
         """验证认证token"""
+        # 测试模式：允许特定测试令牌或跳过验证
         auth_header = request.headers.get("Authorization", "")
+        client_id = request.headers.get("Client-Id", "")
+
+        # 允许测试客户端跳过认证
+        if client_id == "web_test_client":
+            device_id = request.headers.get("Device-Id", "test_device")
+            return True, device_id
+
         if not auth_header.startswith("Bearer "):
             return False, None
 

main/xiaozhi-server/core/connection.py

@@ -40,8 +40,10 @@ from config.logger import setup_logging, build_module_string, create_connection_
 from config.manage_api_client import DeviceNotFoundException, DeviceBindException
 from core.utils.prompt_manager import PromptManager
 from core.utils.voiceprint_provider import VoiceprintProvider
+from core.utils.util import get_system_error_response
 from core.utils import textUtils
 
+
 TAG = __name__
 
 auto_import_modules("plugins_func.functions")
@@ -85,6 +87,7 @@ class ConnectionHandler:
         self.max_output_size = 0
         self.chat_history_conf = 0
         self.audio_format = "opus"
+        self.sample_rate = 24000  # 默认采样率，从客户端 hello 消息中动态更新
 
         # 客户端状态相关
         self.client_abort = False
@@ -134,7 +137,6 @@ class ConnectionHandler:
         self.current_language_tag = None  # 存储当前ASR识别的语言标签
 
         # llm相关变量
-        self.llm_finish_task = True
         self.dialogue = Dialogue()
 
         # tts相关变量
@@ -206,6 +208,10 @@ class ConnectionHandler:
             self.welcome_msg = self.config["xiaozhi"]
             self.welcome_msg["session_id"] = self.session_id
 
+            # 从配置中读取采样率
+            self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
+            self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
+
             # 在后台初始化配置和组件（完全不阻塞主循环）
             asyncio.create_task(self._background_initialize())
 
@@ -794,7 +800,6 @@ class ConnectionHandler:
 
         # 为最顶层时新建会话ID和发送FIRST请求
         if depth == 0:
-            self.llm_finish_task = False
             self.sentence_id = str(uuid.uuid4().hex)
             self.dialogue.put(Message(role="user", content=query))
             self.tts.tts_text_queue.put(
@@ -869,46 +874,66 @@ class ConnectionHandler:
         content_arguments = ""
         self.client_abort = False
         emotion_flag = True
-        for response in llm_responses:
+        try:
-            if self.client_abort:
+            for response in llm_responses:
-                break
+                if self.client_abort:
-            if self.intent_type == "function_call" and functions is not None:
+                    break
-                content, tools_call = response
+                if self.intent_type == "function_call" and functions is not None:
-                if "content" in response:
+                    content, tools_call = response
-                    content = response["content"]
+                    if "content" in response:
-                    tools_call = None
+                        content = response["content"]
-                if content is not None and len(content) > 0:
+                        tools_call = None
-                    content_arguments += content
+                    if content is not None and len(content) > 0:
+                        content_arguments += content
 
-                if not tool_call_flag and content_arguments.startswith("<tool_call>"):
+                    if not tool_call_flag and content_arguments.startswith("<tool_call>"):
-                    # print("content_arguments", content_arguments)
+                        # print("content_arguments", content_arguments)
-                    tool_call_flag = True
+                        tool_call_flag = True
 
-                if tools_call is not None and len(tools_call) > 0:
+                    if tools_call is not None and len(tools_call) > 0:
-                    tool_call_flag = True
+                        tool_call_flag = True
-                    self._merge_tool_calls(tool_calls_list, tools_call)
+                        self._merge_tool_calls(tool_calls_list, tools_call)
-            else:
+                else:
-                content = response
+                    content = response
 
-            # 在llm回复中获取情绪表情，一轮对话只在开头获取一次
+                # 在llm回复中获取情绪表情，一轮对话只在开头获取一次
-            if emotion_flag and content is not None and content.strip():
+                if emotion_flag and content is not None and content.strip():
-                asyncio.run_coroutine_threadsafe(
+                    asyncio.run_coroutine_threadsafe(
-                    textUtils.get_emotion(self, content),
+                        textUtils.get_emotion(self, content),
-                    self.loop,
+                        self.loop,
-                )
-                emotion_flag = False
-
-            if content is not None and len(content) > 0:
-                if not tool_call_flag:
-                    response_message.append(content)
-                    self.tts.tts_text_queue.put(
-                        TTSMessageDTO(
-                            sentence_id=self.sentence_id,
-                            sentence_type=SentenceType.MIDDLE,
-                            content_type=ContentType.TEXT,
-                            content_detail=content,
-                        )
                     )
+                    emotion_flag = False
+
+                if content is not None and len(content) > 0:
+                    if not tool_call_flag:
+                        response_message.append(content)
+                        self.tts.tts_text_queue.put(
+                            TTSMessageDTO(
+                                sentence_id=self.sentence_id,
+                                sentence_type=SentenceType.MIDDLE,
+                                content_type=ContentType.TEXT,
+                                content_detail=content,
+                            )
+                        )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"LLM stream processing error: {e}")
+            self.tts.tts_text_queue.put(
+                TTSMessageDTO(
+                    sentence_id=self.sentence_id,
+                    sentence_type=SentenceType.MIDDLE,
+                    content_type=ContentType.TEXT,
+                    content_detail=get_system_error_response(self.config),
+                )
+            )
+            if depth == 0:
+                self.tts.tts_text_queue.put(
+                    TTSMessageDTO(
+                        sentence_id=self.sentence_id,
+                        sentence_type=SentenceType.LAST,
+                        content_type=ContentType.ACTION,
+                    )
+                )
+            return
         # 处理function call
         if tool_call_flag:
             bHasError = False
@@ -989,7 +1014,6 @@ class ConnectionHandler:
                     content_type=ContentType.ACTION,
                 )
             )
-            self.llm_finish_task = True
             # 使用lambda延迟计算，只有在DEBUG级别时才执行get_llm_dialogue()
             self.logger.bind(tag=TAG).debug(
                 lambda: json.dumps(
@@ -1161,6 +1185,8 @@ class ConnectionHandler:
 
             if self.tts:
                 await self.tts.close()
+            if self.asr:
+                await self.asr.close()
 
             # 最后关闭线程池（避免阻塞）
             if self.executor:
@@ -1209,11 +1235,21 @@ class ConnectionHandler:
                 f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
             )
 
-    def reset_vad_states(self):
+    def reset_audio_states(self):
-        self.client_audio_buffer = bytearray()
+        """
+        重置所有音频相关状态(VAD + ASR)
+        """
+        # Reset VAD states
+        self.client_audio_buffer.clear()
         self.client_have_voice = False
         self.client_voice_stop = False
-        self.logger.bind(tag=TAG).debug("VAD states reset.")
+        self.client_voice_window.clear()
+        self.last_is_voice = False
+
+        # Clear ASR buffers
+        self.asr_audio.clear()
+
+        self.logger.bind(tag=TAG).debug("All audio states reset.")
 
     def chat_and_close(self, text):
         """Chat with the user and then close the connection"""

main/xiaozhi-server/core/handle/helloHandle.py

@@ -143,7 +143,8 @@ async def wakeupWordsResponse(conn: "ConnectionHandler"):
         # 获取当前音色
         voice = getattr(conn.tts, "voice", "default")
 
-        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
+        # 使用链接的sample_rate
+        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
         file_path = wakeup_words_config.generate_file_path(voice)
         with open(file_path, "wb") as f:
             f.write(wav_bytes)

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -21,7 +21,6 @@ async def handleAudioMessage(conn: "ConnectionHandler", audio):
     if hasattr(conn, "just_woken_up") and conn.just_woken_up:
         have_voice = False
         # 设置一个短暂延迟后恢复VAD检测
-        conn.asr_audio.clear()
         if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
             conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
         return

main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py

@@ -29,10 +29,9 @@ class ListenTextMessageHandler(TextMessageHandler):
                 f"客户端拾音模式：{conn.client_listen_mode}"
             )
         if msg_json["state"] == "start":
-            conn.client_have_voice = True
+            # 设备从播放模式切回录音模式,清除所有音频状态和缓冲区
-            conn.client_voice_stop = False
+            conn.reset_audio_states()
         elif msg_json["state"] == "stop":
-            conn.client_have_voice = True
             conn.client_voice_stop = True
             if conn.asr.interface_type == InterfaceType.STREAM:
                 # 流式模式下，发送结束请求
@@ -41,14 +40,13 @@ class ListenTextMessageHandler(TextMessageHandler):
                 # 非流式模式：直接触发ASR识别
                 if len(conn.asr_audio) > 0:
                     asr_audio_task = conn.asr_audio.copy()
-                    conn.asr_audio.clear()
+                    conn.reset_audio_states()
-                    conn.reset_vad_states()
 
                     if len(asr_audio_task) > 0:
                         await conn.asr.handle_voice_stop(conn, asr_audio_task)
         elif msg_json["state"] == "detect":
             conn.client_have_voice = False
-            conn.asr_audio.clear()
+            conn.reset_audio_states()
             if "text" in msg_json:
                 conn.last_activity_time = time.time() * 1000
                 original_text = msg_json["text"]  # 保留原始文本

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -213,36 +213,24 @@ class ASRProvider(ASRProviderBase):
             return None
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if self._is_token_expired():
             logger.warning("Token已过期，正在自动刷新...")
             self._refresh_token()
 
-        file_path = None
         try:
-            # 解码Opus为PCM
+            if artifacts is None:
-            if audio_format == "pcm":
+                return "", None
-                pcm_data = opus_data
-            else:
-                pcm_data = self.decode_opus(opus_data)
-            combined_pcm_data = b"".join(pcm_data)
-
-            # 判断是否保存为WAV文件
-            if self.delete_audio_file:
-                pass
-            else:
-                file_path = self.save_audio_to_file(pcm_data, session_id)
-
             # 发送请求并获取文本
-            text = await self._send_request(combined_pcm_data)
+            text = await self._send_request(artifacts.pcm_bytes)
 
             if text:
-                return text, file_path
+                return text, artifacts.file_path
 
-            return "", file_path
+            return "", artifacts.file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", file_path
+            return "", None

main/xiaozhi-server/core/providers/asr/aliyun_stream.py

@@ -129,17 +129,9 @@ class ASRProvider(ASRProviderBase):
     async def open_audio_channels(self, conn):
         await super().open_audio_channels(conn)
 
-    async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
+    async def receive_audio(self, conn, audio, audio_have_voice):
-        # 初始化音频缓存
+        # 先调用父类方法处理基础逻辑
-        if not hasattr(conn, 'asr_audio_for_voiceprint'):
+        await super().receive_audio(conn, audio, audio_have_voice)
-            conn.asr_audio_for_voiceprint = []
-        
-        # 存储音频数据
-        if audio:
-            conn.asr_audio_for_voiceprint.append(audio)
-        
-        conn.asr_audio.append(audio)
-        conn.asr_audio = conn.asr_audio[-10:]
 
         # 只在有声音且没有连接时建立连接（排除正在停止的情况）
         if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -156,7 +148,7 @@ class ASRProvider(ASRProviderBase):
                 await self.asr_ws.send(pcm_frame)
             except Exception as e:
                 logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}")
-                await self._cleanup(conn)
+                await self._cleanup()
 
     async def _start_recognition(self, conn: "ConnectionHandler"):
         """开始识别会话"""
@@ -208,8 +200,10 @@ class ASRProvider(ASRProviderBase):
         """转发识别结果"""
         try:
             while not conn.stop_event.is_set():
+                # 获取当前连接的音频数据
+                audio_data = conn.asr_audio
                 try:
-                    response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
+                    response = await self.asr_ws.recv()
                     result = json.loads(response)
 
                     header = result.get("header", {})
@@ -261,19 +255,12 @@ class ASRProvider(ASRProviderBase):
 
                                 # 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
                                 if conn.client_voice_stop:
-                                    audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                    logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
-                                    if len(audio_data) > 0:
+                                    await self.handle_voice_stop(conn, audio_data)
-                                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
-                                        await self.handle_voice_stop(conn, audio_data)
-                                        # 清理音频缓存
-                                        conn.asr_audio.clear()
-                                        conn.reset_vad_states()
                                     break
                             else:
                                 # 自动模式下直接覆盖
                                 self.text = text
-                                conn.reset_vad_states()
-                                audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
                                 await self.handle_voice_stop(conn, audio_data)
                                 break
 
@@ -293,11 +280,7 @@ class ASRProvider(ASRProviderBase):
         finally:
             # 清理连接的音频缓存
             await self._cleanup()
-            if conn:
+            conn.reset_audio_states()
-                if hasattr(conn, 'asr_audio_for_voiceprint'):
-                    conn.asr_audio_for_voiceprint = []
-                if hasattr(conn, 'asr_audio'):
-                    conn.asr_audio = []
 
     async def _send_stop_request(self):
         """发送停止识别请求（不关闭连接）"""
@@ -345,7 +328,7 @@ class ASRProvider(ASRProviderBase):
 
         logger.bind(tag=TAG).debug("ASR会话清理完成")
 
-    async def speech_to_text(self, opus_data, session_id, audio_format):
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
         """获取识别结果"""
         result = self.text
         self.text = ""
@@ -353,7 +336,7 @@ class ASRProvider(ASRProviderBase):
 
     async def close(self):
         """关闭资源"""
-        await self._cleanup(None)
+        await self._cleanup()
         if hasattr(self, 'decoder') and self.decoder is not None:
             try:
                 del self.decoder

main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py

@@ -55,17 +55,9 @@ class ASRProvider(ASRProviderBase):
     async def open_audio_channels(self, conn):
         await super().open_audio_channels(conn)
 
-    async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
+    async def receive_audio(self, conn, audio, audio_have_voice):
-        # 初始化音频缓存
+        # 先调用父类方法处理基础逻辑
-        if not hasattr(conn, 'asr_audio_for_voiceprint'):
+        await super().receive_audio(conn, audio, audio_have_voice)
-            conn.asr_audio_for_voiceprint = []
-
-        # 存储音频数据
-        if audio:
-            conn.asr_audio_for_voiceprint.append(audio)
-
-        conn.asr_audio.append(audio)
-        conn.asr_audio = conn.asr_audio[-10:]
 
         # 只在有声音且没有连接时建立连接
         if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -170,6 +162,8 @@ class ASRProvider(ASRProviderBase):
         """转发识别结果"""
         try:
             while not conn.stop_event.is_set():
+                # 获取当前连接的音频数据
+                audio_data = conn.asr_audio
                 try:
                     response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
                     result = json.loads(response)
@@ -218,19 +212,12 @@ class ASRProvider(ASRProviderBase):
 
                                 # 手动模式下,只有在收到stop信号后才触发处理
                                 if conn.client_voice_stop:
-                                    audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                    logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
-                                    if len(audio_data) > 0:
+                                    await self.handle_voice_stop(conn, audio_data)
-                                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
-                                        await self.handle_voice_stop(conn, audio_data)
-                                        # 清理音频缓存
-                                        conn.asr_audio.clear()
-                                        conn.reset_vad_states()
                                     break
                             else:
                                 # 自动模式下直接覆盖
                                 self.text = text
-                                conn.reset_vad_states()
-                                audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
                                 await self.handle_voice_stop(conn, audio_data)
                                 break
 
@@ -261,11 +248,7 @@ class ASRProvider(ASRProviderBase):
         finally:
             # 清理连接的音频缓存
             await self._cleanup()
-            if conn:
+            conn.reset_audio_states()
-                if hasattr(conn, 'asr_audio_for_voiceprint'):
-                    conn.asr_audio_for_voiceprint = []
-                if hasattr(conn, 'asr_audio'):
-                    conn.asr_audio = []
 
     async def _send_stop_request(self):
         """发送停止请求(用于手动模式停止录音)"""
@@ -329,7 +312,7 @@ class ASRProvider(ASRProviderBase):
 
         logger.bind(tag=TAG).debug("ASR会话清理完成")
 
-    async def speech_to_text(self, opus_data, session_id, audio_format):
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
         """获取识别结果"""
         result = self.text
         self.text = ""

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -30,37 +30,26 @@ class ASRProvider(ASRProviderBase):
         os.makedirs(self.output_dir, exist_ok=True)
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
             logger.bind(tag=TAG).warning("音频数据为空！")
             return None, None
 
-        file_path = None
         try:
             # 检查配置是否已设置
             if not self.app_id or not self.api_key or not self.secret_key:
                 logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
-                return None, file_path
+                return None, None
 
-            # 将Opus音频数据解码为PCM
+            if artifacts is None:
-            if audio_format == "pcm":
+                return "", None
-                pcm_data = opus_data
-            else:
-                pcm_data = self.decode_opus(opus_data)
-            combined_pcm_data = b"".join(pcm_data)
-
-            # 判断是否保存为WAV文件
-            if self.delete_audio_file:
-                pass
-            else:
-                self.save_audio_to_file(pcm_data, session_id)
 
             start_time = time.time()
             # 识别本地文件
             result = self.client.asr(
-                combined_pcm_data,
+                artifacts.pcm_bytes,
                 "pcm",
                 16000,
                 {
@@ -73,13 +62,13 @@ class ASRProvider(ASRProviderBase):
                     f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
                 )
                 result = result["result"][0]
-                return result, file_path
+                return result, artifacts.file_path
             else:
                 raise Exception(
                     f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
                 )
-                return None, file_path
+                return None, artifacts.file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
-            return None, file_path
+            return None, None

main/xiaozhi-server/core/providers/asr/base.py

@@ -5,20 +5,25 @@ import uuid
 import json
 import time
 import queue
+import shutil
 import asyncio
+import tempfile
 import traceback
 import threading
 import opuslib_next
+
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
-from typing import Optional, Tuple, List, TYPE_CHECKING
+from core.providers.asr.dto.dto import InterfaceType
-
-if TYPE_CHECKING:
-    from core.connection import ConnectionHandler
 from core.handle.receiveAudioHandle import startToChat
 from core.handle.reportHandle import enqueue_asr_report
 from core.utils.util import remove_punctuation_and_length
 from core.handle.receiveAudioHandle import handleAudioMessage
+from typing import Optional, Tuple, List, NamedTuple, TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
 
 TAG = __name__
 logger = setup_logging()
@@ -60,18 +65,17 @@ class ASRProviderBase(ABC):
             conn.asr_audio.append(audio)
         else:
             # 自动/实时模式：使用VAD检测
-            have_voice = audio_have_voice
-
             conn.asr_audio.append(audio)
-            if not have_voice and not conn.client_have_voice:
+
+            # 如果没有语音，且之前也没有声音，缓存部分音频
+            if not audio_have_voice and not conn.client_have_voice:
                 conn.asr_audio = conn.asr_audio[-10:]
                 return
 
             # 自动模式下通过VAD检测到语音停止时触发识别
-            if conn.client_voice_stop:
+            if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
                 asr_audio_task = conn.asr_audio.copy()
-                conn.asr_audio.clear()
+                conn.reset_audio_states()
-                conn.reset_vad_states()
 
                 if len(asr_audio_task) > 15:
                     await self.handle_voice_stop(conn, asr_audio_task)
@@ -96,10 +100,14 @@ class ASRProviderBase(ABC):
                 wav_data = self._pcm_to_wav(combined_pcm_data)
 
             # 定义ASR任务
-            asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
+            asr_task = self.speech_to_text_wrapper(
+                asr_audio_task, conn.session_id, conn.audio_format
+            )
 
             if conn.voiceprint_provider and wav_data:
-                voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
+                voiceprint_task = conn.voiceprint_provider.identify_speaker(
+                    wav_data, conn.session_id
+                )
                 # 并发等待两个结果
                 asr_result, voiceprint_result = await asyncio.gather(
                     asr_task, voiceprint_task, return_exceptions=True
@@ -162,20 +170,20 @@ class ASRProviderBase(ABC):
             if text_len > 0:
                 # 使用自定义模块进行上报
                 await startToChat(conn, enhanced_text)
-                enqueue_asr_report(conn, enhanced_text, asr_audio_task)
+                audio_snapshot = asr_audio_task.copy()
-                
+                enqueue_asr_report(conn, enhanced_text, audio_snapshot)
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
             import traceback
+
             logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
 
     def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
         """构建包含说话人信息的文本（仅用于纯文本ASR）"""
         if speaker_name and speaker_name.strip():
-            return json.dumps({
+            return json.dumps(
-                "speaker": speaker_name,
+                {"speaker": speaker_name, "content": text}, ensure_ascii=False
-                "content": text
+            )
-            }, ensure_ascii=False)
         else:
             return text
 
@@ -184,23 +192,23 @@ class ASRProviderBase(ABC):
         if len(pcm_data) == 0:
             logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV")
             return b""
-        
+
         # 确保数据长度是偶数（16位音频）
         if len(pcm_data) % 2 != 0:
             pcm_data = pcm_data[:-1]
-        
+
         # 创建WAV文件头
         wav_buffer = io.BytesIO()
         try:
-            with wave.open(wav_buffer, 'wb') as wav_file:
+            with wave.open(wav_buffer, "wb") as wav_file:
-                wav_file.setnchannels(1)      # 单声道
+                wav_file.setnchannels(1)  # 单声道
-                wav_file.setsampwidth(2)      # 16位
+                wav_file.setsampwidth(2)  # 16位
                 wav_file.setframerate(16000)  # 16kHz采样率
                 wav_file.writeframes(pcm_data)
-            
+
             wav_buffer.seek(0)
             wav_data = wav_buffer.read()
-            
+
             return wav_data
         except Exception as e:
             logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
@@ -209,6 +217,44 @@ class ASRProviderBase(ABC):
     def stop_ws_connection(self):
         pass
 
+    async def close(self):
+        pass
+
+    class AudioArtifacts(NamedTuple):
+        pcm_frames: List[bytes]
+        """PCM音频帧列表"""
+        pcm_bytes: bytes
+        """合并后的PCM音频字节数据"""
+        file_path: Optional[str]
+        """WAV文件路径"""
+        temp_path: Optional[str]
+        """临时WAV文件路径"""
+
+    def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]:
+        return self._current_artifacts
+
+    def requires_file(self) -> bool:
+        """是否需要文件输入"""
+        return False
+
+    def prefers_temp_file(self) -> bool:
+        """是否优先使用临时文件"""
+        return False
+
+    def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]:
+        try:
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
+                temp_path = temp_file.name
+            with wave.open(temp_path, "wb") as wav_file:
+                wav_file.setnchannels(1)
+                wav_file.setsampwidth(2)
+                wav_file.setframerate(16000)
+                wav_file.writeframes(pcm_bytes)
+            return temp_path
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}")
+            return None
+
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
         module_name = __name__.split(".")[-1]
@@ -223,11 +269,80 @@ class ASRProviderBase(ABC):
 
         return file_path
 
-    @abstractmethod
+    async def speech_to_text_wrapper(
-    async def speech_to_text(
         self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
-        """将语音数据转换为文本"""
+        file_path = None
+        temp_path = None
+        try:
+            if audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            free_space = shutil.disk_usage(self.output_dir).free
+            if free_space < len(combined_pcm_data) * 2:
+                raise OSError("磁盘空间不足")
+
+            if self.requires_file() and self.prefers_temp_file():
+                temp_path = self.build_temp_file(combined_pcm_data)
+
+            if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or (
+                self.requires_file() and not self.prefers_temp_file()
+            ):
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+
+            if len(combined_pcm_data) == 0:
+                artifacts = None
+            else:
+                artifacts = ASRProviderBase.AudioArtifacts(
+                    pcm_frames=pcm_data,
+                    pcm_bytes=combined_pcm_data,
+                    file_path=file_path,
+                    temp_path=temp_path,
+                )
+
+            text, _ = await self.speech_to_text(
+                opus_data, session_id, audio_format, artifacts
+            )
+            return text, file_path
+        except OSError as e:
+            logger.bind(tag=TAG).error(f"文件操作错误: {e}")
+            return None, None
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}")
+            return None, None
+        finally:
+            try:
+                if temp_path and os.path.exists(temp_path):
+                    os.unlink(temp_path)
+                if (
+                    hasattr(self, "delete_audio_file")
+                    and self.delete_audio_file
+                    and file_path
+                    and os.path.exists(file_path)
+                ):
+                    os.remove(file_path)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"文件清理失败: {e}")
+
+    @abstractmethod
+    async def speech_to_text(
+        self,
+        opus_data: List[bytes],
+        session_id: str,
+        audio_format="opus",
+        artifacts: Optional[AudioArtifacts] = None,
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本
+
+        :param opus_data: 输入的Opus音频数据
+        :param session_id: 会话ID
+        :param audio_format: 音频格式，默认"opus"
+        :param artifacts: 音频工件，包含PCM数据、文件路径等
+        :return: 识别结果文本和文件路径（如果有）
+        """
         pass
 
     @staticmethod
@@ -238,23 +353,23 @@ class ASRProviderBase(ABC):
             decoder = opuslib_next.Decoder(16000, 1)
             pcm_data = []
             buffer_size = 960  # 每次处理960个采样点 (60ms at 16kHz)
-            
+
             for i, opus_packet in enumerate(opus_data):
                 try:
                     if not opus_packet or len(opus_packet) == 0:
                         continue
-                    
+
                     pcm_frame = decoder.decode(opus_packet, buffer_size)
                     if pcm_frame and len(pcm_frame) > 0:
                         pcm_data.append(pcm_frame)
-                        
+
                 except opuslib_next.OpusError as e:
                     logger.bind(tag=TAG).warning(f"Opus解码错误，跳过数据包 {i}: {e}")
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"音频处理错误，数据包 {i}: {e}")
-            
+
             return pcm_data
-            
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
             return []

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -232,24 +232,13 @@ class ASRProvider(ASRProviderBase):
             yield data[offset:data_len], True
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
 
-        file_path = None
         try:
-            # 合并所有opus数据包
+            if artifacts is None:
-            if audio_format == "pcm":
+                return "", None
-                pcm_data = opus_data
-            else:
-                pcm_data = self.decode_opus(opus_data)
-            combined_pcm_data = b"".join(pcm_data)
-
-            # 判断是否保存为WAV文件
-            if self.delete_audio_file:
-                pass
-            else:
-                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 直接使用PCM数据
             # 计算分段大小 (单声道, 16bit, 16kHz采样率)
@@ -258,14 +247,14 @@ class ASRProvider(ASRProviderBase):
 
             # 语音识别
             start_time = time.time()
-            text = await self._send_request(combined_pcm_data, segment_size)
+            text = await self._send_request(artifacts.pcm_bytes, segment_size)
             if text:
                 logger.bind(tag=TAG).debug(
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
                 )
-                return text, file_path
+                return text, artifacts.file_path
-            return "", file_path
+            return "", artifacts.file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", file_path
+            return "", None

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -66,22 +66,9 @@ class ASRProvider(ASRProviderBase):
         await super().open_audio_channels(conn)
 
     async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
-        conn.asr_audio.append(audio)
+        # 先调用父类方法处理基础逻辑
-        conn.asr_audio = conn.asr_audio[-10:]
+        await super().receive_audio(conn, audio, audio_have_voice)
-        # 存储音频数据
+        
-        if not hasattr(conn, "asr_audio_for_voiceprint"):
-            conn.asr_audio_for_voiceprint = []
-        conn.asr_audio_for_voiceprint.append(audio)
-
-        # 当没有音频数据时处理完整语音片段
-        if (
-            conn.client_listen_mode != "manual"
-            and not audio
-            and len(conn.asr_audio_for_voiceprint) > 0
-        ):
-            await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
-            conn.asr_audio_for_voiceprint = []
-
         # 如果本次有声音，且之前没有建立连接
         if audio_have_voice and self.asr_ws is None and not self.is_processing:
             try:
@@ -174,7 +161,7 @@ class ASRProvider(ASRProviderBase):
         try:
             while self.asr_ws and not conn.stop_event.is_set():
                 # 获取当前连接的音频数据
-                audio_data = getattr(conn, "asr_audio_for_voiceprint", [])
+                audio_data = conn.asr_audio
                 try:
                     response = await self.asr_ws.recv()
                     result = self.parse_response(response)
@@ -200,7 +187,6 @@ class ASRProvider(ASRProviderBase):
                             ):
                                 logger.bind(tag=TAG).error(f"识别文本：空")
                                 self.text = ""
-                                conn.reset_vad_states()
                                 if len(audio_data) > 15:  # 确保有足够音频数据
                                     await self.handle_voice_stop(conn, audio_data)
                                 break
@@ -211,18 +197,9 @@ class ASRProvider(ASRProviderBase):
                                 if self.enable_multilingual:
                                     continue
 
-                                if (
+                                if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 15:
-                                    conn.client_listen_mode == "manual"
+                                    logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
-                                    and conn.client_voice_stop
-                                    and len(audio_data) > 0
-                                ):
-                                    logger.bind(tag=TAG).debug(
-                                        "消息结束收到停止信号，触发处理"
-                                    )
                                     await self.handle_voice_stop(conn, audio_data)
-                                    # 清理音频缓存
-                                    conn.asr_audio.clear()
-                                    conn.reset_vad_states()
                                     break
 
                             for utterance in utterances:
@@ -240,24 +217,13 @@ class ASRProvider(ASRProviderBase):
                                             self.text = current_text
 
                                         # 在接收消息中途时收到停止信号
-                                        if (
+                                        if conn.client_voice_stop and len(audio_data) > 0:
-                                            conn.client_voice_stop
+                                            logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
-                                            and len(audio_data) > 0
+                                            await self.handle_voice_stop(conn, audio_data)
-                                        ):
-                                            logger.bind(tag=TAG).debug(
-                                                "消息中途收到停止信号，触发处理"
-                                            )
-                                            await self.handle_voice_stop(
-                                                conn, audio_data
-                                            )
-                                            # 清理音频缓存
-                                            conn.asr_audio.clear()
-                                            conn.reset_vad_states()
                                         break
                                     else:
                                         # 自动模式下直接覆盖
                                         self.text = current_text
-                                        conn.reset_vad_states()
                                         if len(audio_data) > 15:  # 确保有足够音频数据
                                             await self.handle_voice_stop(
                                                 conn, audio_data
@@ -288,11 +254,8 @@ class ASRProvider(ASRProviderBase):
                 await self.asr_ws.close()
                 self.asr_ws = None
             self.is_processing = False
-            if conn:
+            # 重置所有音频相关状态
-                if hasattr(conn, "asr_audio_for_voiceprint"):
+            conn.reset_audio_states()
-                    conn.asr_audio_for_voiceprint = []
-                if hasattr(conn, "asr_audio"):
-                    conn.asr_audio = []
 
     def stop_ws_connection(self):
         if self.asr_ws:
@@ -436,7 +399,7 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).error(f"原始响应数据: {res.hex()}")
             raise
 
-    async def speech_to_text(self, opus_data, session_id, audio_format):
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
         result = self.text
         self.text = ""  # 清空text
         return result, None
@@ -463,11 +426,3 @@ class ASRProvider(ASRProviderBase):
                 logger.bind(tag=TAG).debug("Doubao decoder resources released")
             except Exception as e:
                 logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
-
-        # 清理所有连接的音频缓冲区
-        if hasattr(self, "_connections"):
-            for conn in self._connections.values():
-                if hasattr(conn, "asr_audio_for_voiceprint"):
-                    conn.asr_audio_for_voiceprint = []
-                if hasattr(conn, "asr_audio"):
-                    conn.asr_audio = []

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -64,39 +64,21 @@ class ASRProvider(ASRProviderBase):
             )
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
-        file_path = None
         retry_count = 0
-
+        
         while retry_count < MAX_RETRIES:
             try:
-                # 合并所有opus数据包
+                if artifacts is None:
-                if audio_format == "pcm":
+                    return "", None
-                    pcm_data = opus_data
-                else:
-                    pcm_data = self.decode_opus(opus_data)
-
-                combined_pcm_data = b"".join(pcm_data)
-
-                # 检查磁盘空间
-                if not self.delete_audio_file:
-                    free_space = shutil.disk_usage(self.output_dir).free
-                    if free_space < len(combined_pcm_data) * 2:  # 预留2倍空间
-                        raise OSError("磁盘空间不足")
-
-                # 判断是否保存为WAV文件
-                if self.delete_audio_file:
-                    pass
-                else:
-                    file_path = self.save_audio_to_file(pcm_data, session_id)
 
                 # 语音识别 - 使用线程池避免阻塞事件循环
                 start_time = time.time()
                 result = await asyncio.to_thread(
                     self.model.generate,
-                    input=combined_pcm_data,
+                    input=artifacts.pcm_bytes,
                     cache={},
                     language="auto",
                     use_itn=True,
@@ -107,7 +89,7 @@ class ASRProvider(ASRProviderBase):
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
                 )
 
-                return text, file_path
+                return text, artifacts.file_path
 
             except OSError as e:
                 retry_count += 1
@@ -115,7 +97,7 @@ class ASRProvider(ASRProviderBase):
                     logger.bind(tag=TAG).error(
                         f"语音识别失败（已重试{retry_count}次）: {e}", exc_info=True
                     )
-                    return "", file_path
+                    return "", None
                 logger.bind(tag=TAG).warning(
                     f"语音识别失败，正在重试（{retry_count}/{MAX_RETRIES}）: {e}"
                 )
@@ -123,15 +105,4 @@ class ASRProvider(ASRProviderBase):
 
             except Exception as e:
                 logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-                return "", file_path
+                return "", None
-
-            finally:
-                # 文件清理逻辑
-                if self.delete_audio_file and file_path and os.path.exists(file_path):
-                    try:
-                        os.remove(file_path)
-                        logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                    except Exception as e:
-                        logger.bind(tag=TAG).error(
-                            f"文件删除失败: {file_path} | 错误: {e}"
-                        )

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -101,7 +101,7 @@ class ASRProvider(ASRProviderBase):
         logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """
         Convert speech data to text using FunASR.
@@ -109,18 +109,9 @@ class ASRProvider(ASRProviderBase):
         :param session_id: Unique session identifier.
         :return: Tuple containing recognized text and optional timestamp.
         """
-        file_path = None
+        
-        if audio_format == "pcm":
+        if artifacts is None:
-            pcm_data = opus_data
+            return "", None
-        else:
-            pcm_data = self.decode_opus(opus_data)
-        combined_pcm_data = b"".join(pcm_data)
-
-        # 判断是否保存为WAV文件
-        if self.delete_audio_file:
-            pass
-        else:
-            file_path = self.save_audio_to_file(pcm_data, session_id)
         auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
         async with websockets.connect(
             self.uri,
@@ -132,7 +123,7 @@ class ASRProvider(ASRProviderBase):
             try:
                 # Use asyncio to handle WebSocket communication
                 send_task = asyncio.create_task(
-                    self._send_data(ws, combined_pcm_data, session_id)
+                    self._send_data(ws, artifacts.pcm_bytes, session_id)
                 )
                 receive_task = asyncio.create_task(self._receive_responses(ws))
 
@@ -161,14 +152,14 @@ class ASRProvider(ASRProviderBase):
                 result = lang_tag_filter(result)
                 return (
                     result,
-                    file_path,
+                    artifacts.file_path,
                 )  # Return the recognized text and timestamp (if any)
 
             except websockets.exceptions.ConnectionClosed as e:
                 logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
-                return "", file_path
+                return "", artifacts.file_path
             except Exception as e:
                 logger.bind(tag=TAG).error(
                     f"Error during speech-to-text conversion: {e}", exc_info=True
                 )
-                return "", file_path
+                return "", artifacts.file_path

main/xiaozhi-server/core/providers/asr/openai.py

@@ -21,20 +21,16 @@ class ASRProvider(ASRProviderBase):
 
         os.makedirs(self.output_dir, exist_ok=True)
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]:
+    def requires_file(self) -> bool:
+        return True
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None) -> Tuple[Optional[str], Optional[str]]:
         file_path = None
         try:
-            start_time = time.time()
+            if artifacts is None:
-            if audio_format == "pcm":
+                return "", None
-                pcm_data = opus_data
+            file_path = artifacts.file_path
-            else:
+                
-                pcm_data = self.decode_opus(opus_data)
-            file_path = self.save_audio_to_file(pcm_data, session_id)
-
-            logger.bind(tag=TAG).debug(
-                f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
-            )
-
             logger.bind(tag=TAG).info(f"file path: {file_path}")
             headers = {
                 "Authorization": f"Bearer {self.api_key}",
@@ -71,12 +67,4 @@ class ASRProvider(ASRProviderBase):
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}")
             return "", None
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
         

main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py

@@ -1,5 +1,4 @@
 import os
-import tempfile
 from typing import Optional, Tuple, List
 import dashscope
 from config.logger import setup_logging
@@ -35,56 +34,25 @@ class ASRProvider(ASRProviderBase):
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def _prepare_audio_file(self, pcm_data: bytes) -> str:
+    def prefers_temp_file(self) -> bool:
-        """将PCM数据转换为WAV文件并返回文件路径"""
+        return True
-        try:
+
-            import wave
+    def requires_file(self) -> bool:
-            
+        return True
-            # 创建临时WAV文件
-            with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
-                temp_path = temp_file.name
-                
-            # 写入WAV格式
-            with wave.open(temp_path, 'wb') as wav_file:
-                wav_file.setnchannels(1)      # 单声道
-                wav_file.setsampwidth(2)      # 16位
-                wav_file.setframerate(16000)  # 16kHz采样率
-                wav_file.writeframes(pcm_data)
-                
-            return temp_path
-            
-        except Exception as e:
-            logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
-            return None
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         temp_file_path = None
         file_path = None
-        
         try:
-            # 解码音频数据
+            if artifacts is None:
-            if audio_format == "pcm":
-                pcm_data = opus_data
-            else:
-                pcm_data = self.decode_opus(opus_data)
-            
-            combined_pcm_data = b"".join(pcm_data)
-            if len(combined_pcm_data) == 0:
-                logger.bind(tag=tag).warning("音频数据为空")
                 return "", None
-            
+            temp_file_path = artifacts.temp_path
-            # 准备音频文件
+            file_path = artifacts.file_path
-            temp_file_path = self._prepare_audio_file(combined_pcm_data)
             if not temp_file_path:
-                return "", None
+                return "", file_path
-            
-            # 保存音频文件（如果需要）
-            if not self.delete_audio_file:
-                file_path = self.save_audio_to_file(pcm_data, session_id)
-            
             # 构造请求消息
             messages = [
                 {
@@ -141,11 +109,3 @@ class ASRProvider(ASRProviderBase):
         except Exception as e:
             logger.bind(tag=tag).error(f"语音识别失败: {e}")
             return "", file_path
-            
-        finally:
-            # 清理临时文件
-            if temp_file_path and os.path.exists(temp_file_path):
-                try:
-                    os.unlink(temp_file_path)
-                except Exception as e:
-                    logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -120,24 +120,19 @@ class ASRProvider(ASRProviderBase):
             samples_float32 = samples_float32 / 32768
             return samples_float32, f.getframerate()
 
+    def requires_file(self) -> bool:
+        return True
+
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
         try:
-            # 保存音频文件
+            if artifacts is None:
-            start_time = time.time()
+                return "", None
-            if audio_format == "pcm":
+            file_path = artifacts.file_path
-                pcm_data = opus_data
-            else:
-                pcm_data = self.decode_opus(opus_data)
-            file_path = self.save_audio_to_file(pcm_data, session_id)
-            logger.bind(tag=TAG).debug(
-                f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
-            )
 
-            # 语音识别
             start_time = time.time()
             s = self.model.create_stream()
             samples, sample_rate = self.read_wave(file_path)
@@ -153,11 +148,3 @@ class ASRProvider(ASRProviderBase):
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
             return "", file_path
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -32,35 +32,24 @@ class ASRProvider(ASRProviderBase):
         os.makedirs(self.output_dir, exist_ok=True)
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
             logger.bind(tag=TAG).warning("音频数据为空！")
             return None, None
 
-        file_path = None
         try:
             # 检查配置是否已设置
             if not self.secret_id or not self.secret_key:
                 logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
-                return None, file_path
+                return None, None
 
-            # 将Opus音频数据解码为PCM
+            if artifacts is None:
-            if audio_format == "pcm":
+                return "", None
-                pcm_data = opus_data
-            else:
-                pcm_data = self.decode_opus(opus_data)
-            combined_pcm_data = b"".join(pcm_data)
-
-            # 判断是否保存为WAV文件
-            if self.delete_audio_file:
-                pass
-            else:
-                self.save_audio_to_file(pcm_data, session_id)
 
             # 将音频数据转换为Base64编码
-            base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8")
+            base64_audio = base64.b64encode(artifacts.pcm_bytes).decode("utf-8")
 
             # 构建请求体
             request_body = self._build_request_body(base64_audio)
@@ -77,11 +66,11 @@ class ASRProvider(ASRProviderBase):
                     f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
                 )
 
-            return result, file_path
+            return result, artifacts.file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
-            return None, file_path
+            return None, None
 
     def _build_request_body(self, base64_audio: str) -> str:
         """构建请求体"""

main/xiaozhi-server/core/providers/asr/vosk.py

@@ -44,36 +44,21 @@ class ASRProvider(ASRProviderBase):
             raise
 
     async def speech_to_text(
-        self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
-        file_path = None
         try:
             # 检查模型是否加载成功
             if not self.model:
                 logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别")
                 return "", None
-
+            
-            # 解码音频（如果原始格式是Opus）
+            if artifacts is None:
-            if audio_format == "pcm":
-                pcm_data = audio_data
-            else:
-                pcm_data = self.decode_opus(audio_data)
-                
-            if not pcm_data:
-                logger.bind(tag=TAG).warning("解码后的PCM数据为空，无法进行识别")
                 return "", None
-                
+            if not artifacts.pcm_bytes:
-            # 合并PCM数据
-            combined_pcm_data = b"".join(pcm_data)
-            if len(combined_pcm_data) == 0:
                 logger.bind(tag=TAG).warning("合并后的PCM数据为空")
                 return "", None
 
-            # 判断是否保存为WAV文件
-            if not self.delete_audio_file:
-                file_path = self.save_audio_to_file(pcm_data, session_id)
-
             start_time = time.time()
             
             
@@ -81,8 +66,8 @@ class ASRProvider(ASRProviderBase):
             chunk_size = 2000
             text_result = ""
             
-            for i in range(0, len(combined_pcm_data), chunk_size):
+            for i in range(0, len(artifacts.pcm_bytes), chunk_size):
-                chunk = combined_pcm_data[i:i+chunk_size]
+                chunk = artifacts.pcm_bytes[i:i+chunk_size]
                 if self.recognizer.AcceptWaveform(chunk):
                     result = json.loads(self.recognizer.Result())
                     text = result.get('text', '')
@@ -99,16 +84,8 @@ class ASRProvider(ASRProviderBase):
                 f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
             )
             
-            return text_result.strip(), file_path
+            return text_result.strip(), artifacts.file_path
             
         except Exception as e:
             logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
             return "", None
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/xunfei_stream.py

@@ -104,11 +104,6 @@ class ASRProvider(ASRProviderBase):
         # 先调用父类方法处理基础逻辑
         await super().receive_audio(conn, audio, audio_have_voice)
 
-        # 存储音频数据用于声纹识别
-        if not hasattr(conn, "asr_audio_for_voiceprint"):
-            conn.asr_audio_for_voiceprint = []
-        conn.asr_audio_for_voiceprint.append(audio)
-
         # 如果本次有声音，且之前没有建立连接
         if audio_have_voice and self.asr_ws is None and not self.is_processing:
             try:
@@ -234,14 +229,8 @@ class ASRProvider(ASRProviderBase):
                                     self.text += w
 
                     if status == 2:
-                        if conn.client_listen_mode == "manual":
+                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
-                            audio_data = getattr(conn, "asr_audio_for_voiceprint", [])
+                        await self.handle_voice_stop(conn, conn.asr_audio)
-                            if len(audio_data) > 0:
-                                logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
-                                await self.handle_voice_stop(conn, audio_data)
-                                # 清理音频缓存
-                                conn.asr_audio.clear()
-                        conn.reset_vad_states()
                         break
 
                 except asyncio.TimeoutError:
@@ -265,13 +254,7 @@ class ASRProvider(ASRProviderBase):
         finally:
             # 清理连接资源
             await self._cleanup()
-
+            conn.reset_audio_states()
-            # 清理连接的音频缓存
-            if conn:
-                if hasattr(conn, "asr_audio_for_voiceprint"):
-                    conn.asr_audio_for_voiceprint = []
-                if hasattr(conn, "asr_audio"):
-                    conn.asr_audio = []
 
     async def handle_voice_stop(
         self, conn: "ConnectionHandler", asr_audio_task: List[bytes]
@@ -339,7 +322,7 @@ class ASRProvider(ASRProviderBase):
 
         logger.bind(tag=TAG).debug("ASR会话清理完成")
 
-    async def speech_to_text(self, opus_data, session_id, audio_format):
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
         """获取识别结果"""
         result = self.text
         self.text = ""
@@ -368,10 +351,3 @@ class ASRProvider(ASRProviderBase):
             except Exception as e:
                 logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
 
-        # 清理所有连接的音频缓冲区
-        if hasattr(self, "_connections"):
-            for conn in self._connections.values():
-                if hasattr(conn, "asr_audio_for_voiceprint"):
-                    conn.asr_audio_for_voiceprint = []
-                if hasattr(conn, "asr_audio"):
-                    conn.asr_audio = []

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -5,11 +5,14 @@ if TYPE_CHECKING:
 from ..base import IntentProviderBase
 from plugins_func.functions.play_music import initialize_music_handler
 from config.logger import setup_logging
+from core.utils.util import get_system_error_response
 import re
 import json
 import hashlib
 import time
 
+
+
 TAG = __name__
 logger = setup_logging()
 
@@ -118,12 +121,16 @@ class IntentProvider(IntentProviderBase):
         return prompt
 
     def replyResult(self, text: str, original_text: str):
-        llm_result = self.llm.response_no_stream(
+        try:
-            system_prompt=text,
+            llm_result = self.llm.response_no_stream(
-            user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+                system_prompt=text,
-            + original_text,
+                user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
-        )
+                + original_text,
-        return llm_result
+            )
+            return llm_result
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in generating reply result: {e}")
+            return get_system_error_response(self.config)
 
     async def detect_intent(
         self, conn: "ConnectionHandler", dialogue_history: List[Dict], text: str
@@ -199,9 +206,13 @@ class IntentProvider(IntentProviderBase):
         llm_start_time = time.time()
         logger.bind(tag=TAG).debug(f"开始LLM意图识别调用, 模型: {model_info}")
 
-        intent = self.llm.response_no_stream(
+        try:
-            system_prompt=prompt_music, user_prompt=user_prompt
+            intent = self.llm.response_no_stream(
-        )
+                system_prompt=prompt_music, user_prompt=user_prompt
+            )
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in intent detection LLM call: {e}")
+            return '{"function_call": {"name": "continue_chat"}}'
 
         # 记录LLM调用完成时间
         llm_time = time.time() - llm_start_time

main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py

@@ -21,83 +21,78 @@ class LLMProvider(LLMProviderBase):
         check_model_key("AliBLLLM", self.api_key)
 
     def response(self, session_id, dialogue):
-        try:
+        # 处理dialogue
-            # 处理dialogue
+        if self.is_No_prompt:
-            if self.is_No_prompt:
+            dialogue.pop(0)
-                dialogue.pop(0)
-                logger.bind(tag=TAG).debug(
-                    f"【阿里百练API服务】处理后的dialogue: {dialogue}"
-                )
-
-            # 构造调用参数
-            call_params = {
-                "api_key": self.api_key,
-                "app_id": self.app_id,
-                "session_id": session_id,
-                "messages": dialogue,
-                # 开启SDK原生流式
-                "stream": True,
-            }
-            if self.memory_id != False:
-                # 百练memory需要prompt参数
-                prompt = dialogue[-1].get("content")
-                call_params["memory_id"] = self.memory_id
-                call_params["prompt"] = prompt
-                logger.bind(tag=TAG).debug(
-                    f"【阿里百练API服务】处理后的prompt: {prompt}"
-                )
-
-            # 可选地设置自定义API基地址（若配置为兼容模式URL则忽略）
-            if self.base_url and ("/api/" in self.base_url):
-                dashscope.base_http_api_url = self.base_url
-
-            responses = Application.call(**call_params)
-
-            # 流式处理（SDK在stream=True时返回可迭代对象；否则返回单次响应对象）
             logger.bind(tag=TAG).debug(
-                f"【阿里百练API服务】构造参数: {dict(call_params, api_key='***')}"
+                f"【阿里百练API服务】处理后的dialogue: {dialogue}"
             )
 
-            last_text = ""
+        # 构造调用参数
-            try:
+        call_params = {
-                for resp in responses:
+            "api_key": self.api_key,
-                    if resp.status_code != HTTPStatus.OK:
+            "app_id": self.app_id,
-                        logger.bind(tag=TAG).error(
+            "session_id": session_id,
-                            f"code={resp.status_code}, message={resp.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
+            "messages": dialogue,
-                        )
+            # 开启SDK原生流式
-                        continue
+            "stream": True,
-                    current_text = getattr(getattr(resp, "output", None), "text", None)
+        }
-                    if current_text is None:
+        if self.memory_id != False:
-                        continue
+            # 百练memory需要prompt参数
-                    # SDK流式为增量覆盖，计算差量输出
+            prompt = dialogue[-1].get("content")
-                    if len(current_text) >= len(last_text):
+            call_params["memory_id"] = self.memory_id
-                        delta = current_text[len(last_text):]
+            call_params["prompt"] = prompt
-                    else:
+            logger.bind(tag=TAG).debug(
-                        # 避免偶发回退
+                f"【阿里百练API服务】处理后的prompt: {prompt}"
-                        delta = current_text
+            )
-                    if delta:
-                        yield delta
-                    last_text = current_text
-            except TypeError:
-                # 非流式回落（一次性返回）
-                if responses.status_code != HTTPStatus.OK:
-                    logger.bind(tag=TAG).error(
-                        f"code={responses.status_code}, message={responses.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
-                    )
-                    yield "【阿里百练API服务响应异常】"
-                else:
-                    full_text = getattr(getattr(responses, "output", None), "text", "")
-                    logger.bind(tag=TAG).info(
-                        f"【阿里百练API服务】完整响应长度: {len(full_text)}"
-                    )
-                    for i in range(0, len(full_text), self.streaming_chunk_size):
-                        chunk = full_text[i:i + self.streaming_chunk_size]
-                        if chunk:
-                            yield chunk
 
-        except Exception as e:
+        # 可选地设置自定义API基地址（若配置为兼容模式URL则忽略）
-            logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
+        if self.base_url and ("/api/" in self.base_url):
-            yield "【LLM服务响应异常】"
+            dashscope.base_http_api_url = self.base_url
+
+        responses = Application.call(**call_params)
+
+        # 流式处理（SDK在stream=True时返回可迭代对象；否则返回单次响应对象）
+        logger.bind(tag=TAG).debug(
+            f"【阿里百练API服务】构造参数: {dict(call_params, api_key='***')}"
+        )
+
+        last_text = ""
+        try:
+            for resp in responses:
+                if resp.status_code != HTTPStatus.OK:
+                    logger.bind(tag=TAG).error(
+                        f"code={resp.status_code}, message={resp.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
+                    )
+                    continue
+                current_text = getattr(getattr(resp, "output", None), "text", None)
+                if current_text is None:
+                    continue
+                # SDK流式为增量覆盖，计算差量输出
+                if len(current_text) >= len(last_text):
+                    delta = current_text[len(last_text):]
+                else:
+                    # 避免偶发回退
+                    delta = current_text
+                if delta:
+                    yield delta
+                last_text = current_text
+        except TypeError:
+            # 非流式回落（一次性返回）
+            if responses.status_code != HTTPStatus.OK:
+                logger.bind(tag=TAG).error(
+                    f"code={responses.status_code}, message={responses.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
+                )
+                yield "【阿里百练API服务响应异常】"
+            else:
+                full_text = getattr(getattr(responses, "output", None), "text", "")
+                logger.bind(tag=TAG).info(
+                    f"【阿里百练API服务】完整响应长度: {len(full_text)}"
+                )
+                for i in range(0, len(full_text), self.streaming_chunk_size):
+                    chunk = full_text[i:i + self.streaming_chunk_size]
+                    if chunk:
+                        yield chunk
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         # 阿里百练当前未支持原生的 function call。为保持兼容，这里回退到普通文本流式输出。

main/xiaozhi-server/core/providers/llm/base.py

@@ -11,20 +11,15 @@ class LLMProviderBase(ABC):
         pass
 
     def response_no_stream(self, system_prompt, user_prompt, **kwargs):
-        try:
+        # 构造对话格式
-            # 构造对话格式
+        dialogue = [
-            dialogue = [
+            {"role": "system", "content": system_prompt},
-                {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
-                {"role": "user", "content": user_prompt}
+        ]
-            ]
+        result = ""
-            result = ""
+        for part in self.response("", dialogue, **kwargs):
-            for part in self.response("", dialogue, **kwargs):
+            result += part
-                result += part
+        return result
-            return result
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
-            return "【LLM服务响应异常】"
     
     def response_with_functions(self, session_id, dialogue, functions=None):
         """

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -20,76 +20,71 @@ class LLMProvider(LLMProviderBase):
             logger.bind(tag=TAG).error(model_key_msg)
 
     def response(self, session_id, dialogue, **kwargs):
-        try:
+        # 取最后一条用户消息
-            # 取最后一条用户消息
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+        conversation_id = self.session_conversation_map.get(session_id)
-            conversation_id = self.session_conversation_map.get(session_id)
 
-            # 发起流式请求
+        # 发起流式请求
+        if self.mode == "chat-messages":
+            request_json = {
+                "query": last_msg["content"],
+                "response_mode": "streaming",
+                "user": session_id,
+                "inputs": {},
+                "conversation_id": conversation_id,
+            }
+        elif self.mode == "workflows/run":
+            request_json = {
+                "inputs": {"query": last_msg["content"]},
+                "response_mode": "streaming",
+                "user": session_id,
+            }
+        elif self.mode == "completion-messages":
+            request_json = {
+                "inputs": {"query": last_msg["content"]},
+                "response_mode": "streaming",
+                "user": session_id,
+            }
+
+        with requests.post(
+            f"{self.base_url}/{self.mode}",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json=request_json,
+            stream=True,
+        ) as r:
             if self.mode == "chat-messages":
-                request_json = {
+                for line in r.iter_lines():
-                    "query": last_msg["content"],
+                    if line.startswith(b"data: "):
-                    "response_mode": "streaming",
+                        event = json.loads(line[6:])
-                    "user": session_id,
+                        # 如果没有找到conversation_id，则获取此次conversation_id
-                    "inputs": {},
+                        if not conversation_id:
-                    "conversation_id": conversation_id,
+                            conversation_id = event.get("conversation_id")
-                }
+                            self.session_conversation_map[session_id] = (
+                                conversation_id  # 更新映射
+                            )
+                        # 过滤 message_replace 事件，此事件会全量推一次
+                        if event.get("event") != "message_replace" and event.get(
+                            "answer"
+                        ):
+                            yield event["answer"]
             elif self.mode == "workflows/run":
-                request_json = {
+                for line in r.iter_lines():
-                    "inputs": {"query": last_msg["content"]},
+                    if line.startswith(b"data: "):
-                    "response_mode": "streaming",
+                        event = json.loads(line[6:])
-                    "user": session_id,
+                        if event.get("event") == "workflow_finished":
-                }
+                            if event["data"]["status"] == "succeeded":
+                                yield event["data"]["outputs"]["answer"]
+                            else:
+                                yield "【服务响应异常】"
             elif self.mode == "completion-messages":
-                request_json = {
+                for line in r.iter_lines():
-                    "inputs": {"query": last_msg["content"]},
+                    if line.startswith(b"data: "):
-                    "response_mode": "streaming",
+                        event = json.loads(line[6:])
-                    "user": session_id,
+                        # 过滤 message_replace 事件，此事件会全量推一次
-                }
+                        if event.get("event") != "message_replace" and event.get(
-
+                            "answer"
-            with requests.post(
+                        ):
-                f"{self.base_url}/{self.mode}",
+                            yield event["answer"]
-                headers={"Authorization": f"Bearer {self.api_key}"},
-                json=request_json,
-                stream=True,
-            ) as r:
-                if self.mode == "chat-messages":
-                    for line in r.iter_lines():
-                        if line.startswith(b"data: "):
-                            event = json.loads(line[6:])
-                            # 如果没有找到conversation_id，则获取此次conversation_id
-                            if not conversation_id:
-                                conversation_id = event.get("conversation_id")
-                                self.session_conversation_map[session_id] = (
-                                    conversation_id  # 更新映射
-                                )
-                            # 过滤 message_replace 事件，此事件会全量推一次
-                            if event.get("event") != "message_replace" and event.get(
-                                "answer"
-                            ):
-                                yield event["answer"]
-                elif self.mode == "workflows/run":
-                    for line in r.iter_lines():
-                        if line.startswith(b"data: "):
-                            event = json.loads(line[6:])
-                            if event.get("event") == "workflow_finished":
-                                if event["data"]["status"] == "succeeded":
-                                    yield event["data"]["outputs"]["answer"]
-                                else:
-                                    yield "【服务响应异常】"
-                elif self.mode == "completion-messages":
-                    for line in r.iter_lines():
-                        if line.startswith(b"data: "):
-                            event = json.loads(line[6:])
-                            # 过滤 message_replace 事件，此事件会全量推一次
-                            if event.get("event") != "message_replace" and event.get(
-                                "answer"
-                            ):
-                                yield event["answer"]
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
-            yield "【服务响应异常】"
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         if len(dialogue) == 2 and functions is not None and len(functions) > 0:

main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py

@@ -19,53 +19,48 @@ class LLMProvider(LLMProviderBase):
             logger.bind(tag=TAG).error(model_key_msg)
 
     def response(self, session_id, dialogue, **kwargs):
-        try:
+        # 取最后一条用户消息
-            # 取最后一条用户消息
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
 
-            # 发起流式请求
+        # 发起流式请求
-            with requests.post(
+        with requests.post(
-                f"{self.base_url}/chat/completions",
+            f"{self.base_url}/chat/completions",
-                headers={"Authorization": f"Bearer {self.api_key}"},
+            headers={"Authorization": f"Bearer {self.api_key}"},
-                json={
+            json={
-                    "stream": True,
+                "stream": True,
-                    "chatId": session_id,
+                "chatId": session_id,
-                    "detail": self.detail,
+                "detail": self.detail,
-                    "variables": self.variables,
+                "variables": self.variables,
-                    "messages": [{"role": "user", "content": last_msg["content"]}],
+                "messages": [{"role": "user", "content": last_msg["content"]}],
-                },
+            },
-                stream=True,
+            stream=True,
-            ) as r:
+        ) as r:
-                for line in r.iter_lines():
+            for line in r.iter_lines():
-                    if line:
+                if line:
-                        try:
+                    try:
-                            if line.startswith(b"data: "):
+                        if line.startswith(b"data: "):
-                                if line[6:].decode("utf-8") == "[DONE]":
+                            if line[6:].decode("utf-8") == "[DONE]":
-                                    break
+                                break
 
-                                data = json.loads(line[6:])
+                            data = json.loads(line[6:])
-                                if "choices" in data and len(data["choices"]) > 0:
+                            if "choices" in data and len(data["choices"]) > 0:
-                                    delta = data["choices"][0].get("delta", {})
+                                delta = data["choices"][0].get("delta", {})
-                                    if (
+                                if (
-                                        delta
+                                    delta
-                                        and "content" in delta
+                                    and "content" in delta
-                                        and delta["content"] is not None
+                                    and delta["content"] is not None
-                                    ):
+                                ):
-                                        content = delta["content"]
+                                    content = delta["content"]
-                                        if "<think>" in content:
+                                    if "<think>" in content:
-                                            continue
+                                        continue
-                                        if "</think>" in content:
+                                    if "</think>" in content:
-                                            continue
+                                        continue
-                                        yield content
+                                    yield content
 
-                        except json.JSONDecodeError as e:
+                    except json.JSONDecodeError as e:
-                            continue
+                        continue
-                        except Exception as e:
+                    except Exception as e:
-                            continue
+                        continue
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
-            yield "【服务响应异常】"
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         logger.bind(tag=TAG).error(

main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py

@@ -15,55 +15,49 @@ class LLMProvider(LLMProviderBase):
         self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
 
     def response(self, session_id, dialogue, **kwargs):
-        try:
+        # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
-            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
 
-            # 提取最后一个 role 为 'user' 的 content
+        # 提取最后一个 role 为 'user' 的 content
-            input_text = None
+        input_text = None
-            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
+        if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
-                # 逆序遍历，找到最后一个 role 为 'user' 的消息
+            # 逆序遍历，找到最后一个 role 为 'user' 的消息
-                for message in reversed(dialogue):
+            for message in reversed(dialogue):
-                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
+                if message.get("role") == "user":  # 找到 role 为 'user' 的消息
-                        input_text = message.get("content", "")
+                    input_text = message.get("content", "")
-                        break  # 找到后立即退出循环
+                    break  # 找到后立即退出循环
 
-            # 构造请求数据
+        # 构造请求数据
-            payload = {
+        payload = {
-                "text": input_text,
+            "text": input_text,
-                "agent_id": self.agent_id,
+            "agent_id": self.agent_id,
-                "conversation_id": session_id,  # 使用 session_id 作为 conversation_id
+            "conversation_id": session_id,  # 使用 session_id 作为 conversation_id
-            }
+        }
-            # 设置请求头
+        # 设置请求头
-            headers = {
+        headers = {
-                "Authorization": f"Bearer {self.api_key}",
+            "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json",
+            "Content-Type": "application/json",
-            }
+        }
 
-            # 发起 POST 请求
+        # 发起 POST 请求
-            response = requests.post(self.api_url, json=payload, headers=headers)
+        response = requests.post(self.api_url, json=payload, headers=headers)
 
-            # 检查请求是否成功
+        # 检查请求是否成功
-            response.raise_for_status()
+        response.raise_for_status()
 
-            # 解析返回数据
+        # 解析返回数据
-            data = response.json()
+        data = response.json()
-            speech = (
+        speech = (
-                data.get("response", {})
+            data.get("response", {})
-                .get("speech", {})
+            .get("speech", {})
-                .get("plain", {})
+            .get("plain", {})
-                .get("speech", "")
+            .get("speech", "")
-            )
+        )
 
-            # 返回生成的内容
+        # 返回生成的内容
-            if speech:
+        if speech:
-                yield speech
+            yield speech
-            else:
+        else:
-                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
+            logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
-
-        except RequestException as e:
-            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         logger.bind(tag=TAG).error(

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -25,151 +25,141 @@ class LLMProvider(LLMProviderBase):
         self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
 
     def response(self, session_id, dialogue, **kwargs):
-        try:
+        # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
-            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+        if self.is_qwen3:
-            if self.is_qwen3:
+            # 复制对话列表，避免修改原始对话
-                # 复制对话列表，避免修改原始对话
+            dialogue_copy = dialogue.copy()
-                dialogue_copy = dialogue.copy()
 
-                # 找到最后一条用户消息
+            # 找到最后一条用户消息
-                for i in range(len(dialogue_copy) - 1, -1, -1):
+            for i in range(len(dialogue_copy) - 1, -1, -1):
-                    if dialogue_copy[i]["role"] == "user":
+                if dialogue_copy[i]["role"] == "user":
-                        # 在用户消息前添加/no_think指令
+                    # 在用户消息前添加/no_think指令
-                        dialogue_copy[i]["content"] = (
+                    dialogue_copy[i]["content"] = (
-                            "/no_think " + dialogue_copy[i]["content"]
+                        "/no_think " + dialogue_copy[i]["content"]
-                        )
-                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
-                        break
-
-                # 使用修改后的对话
-                dialogue = dialogue_copy
-
-            responses = self.client.chat.completions.create(
-                model=self.model_name, messages=dialogue, stream=True
-            )
-            is_active = True
-            # 用于处理跨chunk的标签
-            buffer = ""
-
-            for chunk in responses:
-                try:
-                    delta = (
-                        chunk.choices[0].delta
-                        if getattr(chunk, "choices", None)
-                        else None
                     )
-                    content = delta.content if hasattr(delta, "content") else ""
+                    logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                    break
 
-                    if content:
+            # 使用修改后的对话
-                        # 将内容添加到缓冲区
+            dialogue = dialogue_copy
-                        buffer += content
 
-                        # 处理缓冲区中的标签
+        responses = self.client.chat.completions.create(
-                        while "<think>" in buffer and "</think>" in buffer:
+            model=self.model_name, messages=dialogue, stream=True
-                            # 找到完整的<think></think>标签并移除
+        )
-                            pre = buffer.split("<think>", 1)[0]
+        is_active = True
-                            post = buffer.split("</think>", 1)[1]
+        # 用于处理跨chunk的标签
-                            buffer = pre + post
+        buffer = ""
 
-                        # 处理只有开始标签的情况
+        for chunk in responses:
-                        if "<think>" in buffer:
+            try:
-                            is_active = False
+                delta = (
-                            buffer = buffer.split("<think>", 1)[0]
+                    chunk.choices[0].delta
+                    if getattr(chunk, "choices", None)
+                    else None
+                )
+                content = delta.content if hasattr(delta, "content") else ""
 
-                        # 处理只有结束标签的情况
+                if content:
-                        if "</think>" in buffer:
+                    # 将内容添加到缓冲区
-                            is_active = True
+                    buffer += content
-                            buffer = buffer.split("</think>", 1)[1]
 
-                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                    # 处理缓冲区中的标签
-                        if is_active and buffer:
+                    while "<think>" in buffer and "</think>" in buffer:
-                            yield buffer
+                        # 找到完整的<think></think>标签并移除
-                            buffer = ""  # 清空缓冲区
+                        pre = buffer.split("<think>", 1)[0]
+                        post = buffer.split("</think>", 1)[1]
+                        buffer = pre + post
 
-                except Exception as e:
+                    # 处理只有开始标签的情况
-                    logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
+                    if "<think>" in buffer:
+                        is_active = False
+                        buffer = buffer.split("<think>", 1)[0]
 
-        except Exception as e:
+                    # 处理只有结束标签的情况
-            logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
+                    if "</think>" in buffer:
-            yield "【Ollama服务响应异常】"
+                        is_active = True
+                        buffer = buffer.split("</think>", 1)[1]
+
+                    # 如果当前处于活动状态且缓冲区有内容，则输出
+                    if is_active and buffer:
+                        yield buffer
+                        buffer = ""  # 清空缓冲区
+
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
 
     def response_with_functions(self, session_id, dialogue, functions=None):
-        try:
+        # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
-            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+        if self.is_qwen3:
-            if self.is_qwen3:
+            # 复制对话列表，避免修改原始对话
-                # 复制对话列表，避免修改原始对话
+            dialogue_copy = dialogue.copy()
-                dialogue_copy = dialogue.copy()
 
-                # 找到最后一条用户消息
+            # 找到最后一条用户消息
-                for i in range(len(dialogue_copy) - 1, -1, -1):
+            for i in range(len(dialogue_copy) - 1, -1, -1):
-                    if dialogue_copy[i]["role"] == "user":
+                if dialogue_copy[i]["role"] == "user":
-                        # 在用户消息前添加/no_think指令
+                    # 在用户消息前添加/no_think指令
-                        dialogue_copy[i]["content"] = (
+                    dialogue_copy[i]["content"] = (
-                            "/no_think " + dialogue_copy[i]["content"]
+                        "/no_think " + dialogue_copy[i]["content"]
-                        )
-                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
-                        break
-
-                # 使用修改后的对话
-                dialogue = dialogue_copy
-
-            stream = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                tools=functions,
-            )
-
-            is_active = True
-            buffer = ""
-
-            for chunk in stream:
-                try:
-                    delta = (
-                        chunk.choices[0].delta
-                        if getattr(chunk, "choices", None)
-                        else None
-                    )
-                    content = delta.content if hasattr(delta, "content") else None
-                    tool_calls = (
-                        delta.tool_calls if hasattr(delta, "tool_calls") else None
                     )
+                    logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                    break
 
-                    # 如果是工具调用，直接传递
+            # 使用修改后的对话
-                    if tool_calls:
+            dialogue = dialogue_copy
-                        yield None, tool_calls
-                        continue
 
-                    # 处理文本内容
+        stream = self.client.chat.completions.create(
-                    if content:
+            model=self.model_name,
-                        # 将内容添加到缓冲区
+            messages=dialogue,
-                        buffer += content
+            stream=True,
+            tools=functions,
+        )
 
-                        # 处理缓冲区中的标签
+        is_active = True
-                        while "<think>" in buffer and "</think>" in buffer:
+        buffer = ""
-                            # 找到完整的<think></think>标签并移除
-                            pre = buffer.split("<think>", 1)[0]
-                            post = buffer.split("</think>", 1)[1]
-                            buffer = pre + post
 
-                        # 处理只有开始标签的情况
+        for chunk in stream:
-                        if "<think>" in buffer:
+            try:
-                            is_active = False
+                delta = (
-                            buffer = buffer.split("<think>", 1)[0]
+                    chunk.choices[0].delta
+                    if getattr(chunk, "choices", None)
+                    else None
+                )
+                content = delta.content if hasattr(delta, "content") else None
+                tool_calls = (
+                    delta.tool_calls if hasattr(delta, "tool_calls") else None
+                )
 
-                        # 处理只有结束标签的情况
+                # 如果是工具调用，直接传递
-                        if "</think>" in buffer:
+                if tool_calls:
-                            is_active = True
+                    yield None, tool_calls
-                            buffer = buffer.split("</think>", 1)[1]
-
-                        # 如果当前处于活动状态且缓冲区有内容，则输出
-                        if is_active and buffer:
-                            yield buffer, None
-                            buffer = ""  # 清空缓冲区
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
                     continue
 
-        except Exception as e:
+                # 处理文本内容
-            logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
+                if content:
-            yield f"【Ollama服务响应异常: {str(e)}】", None
+                    # 将内容添加到缓冲区
+                    buffer += content
+
+                    # 处理缓冲区中的标签
+                    while "<think>" in buffer and "</think>" in buffer:
+                        # 找到完整的<think></think>标签并移除
+                        pre = buffer.split("<think>", 1)[0]
+                        post = buffer.split("</think>", 1)[1]
+                        buffer = pre + post
+
+                    # 处理只有开始标签的情况
+                    if "<think>" in buffer:
+                        is_active = False
+                        buffer = buffer.split("<think>", 1)[0]
+
+                    # 处理只有结束标签的情况
+                    if "</think>" in buffer:
+                        is_active = True
+                        buffer = buffer.split("</think>", 1)[1]
+
+                    # 如果当前处于活动状态且缓冲区有内容，则输出
+                    if is_active and buffer:
+                        yield buffer, None
+                        buffer = ""  # 清空缓冲区
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
+                continue

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -56,87 +56,78 @@ class LLMProvider(LLMProviderBase):
         return dialogue
 
     def response(self, session_id, dialogue, **kwargs):
-        try:
+        dialogue = self.normalize_dialogue(dialogue)
-            dialogue = self.normalize_dialogue(dialogue)
 
-            request_params = {
+        request_params = {
-                "model": self.model_name,
+            "model": self.model_name,
-                "messages": dialogue,
+            "messages": dialogue,
-                "stream": True,
+            "stream": True,
-            }
+        }
 
-            # 添加可选参数,只有当参数不为None时才添加
+        # 添加可选参数,只有当参数不为None时才添加
-            optional_params = {
+        optional_params = {
-                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+            "max_tokens": kwargs.get("max_tokens", self.max_tokens),
-                "temperature": kwargs.get("temperature", self.temperature),
+            "temperature": kwargs.get("temperature", self.temperature),
-                "top_p": kwargs.get("top_p", self.top_p),
+            "top_p": kwargs.get("top_p", self.top_p),
-                "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+            "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
-            }
+        }
 
-            for key, value in optional_params.items():
+        for key, value in optional_params.items():
-                if value is not None:
+            if value is not None:
-                    request_params[key] = value
+                request_params[key] = value
 
-            responses = self.client.chat.completions.create(**request_params)
+        responses = self.client.chat.completions.create(**request_params)
 
-            is_active = True
+        is_active = True
-            for chunk in responses:
+        for chunk in responses:
-                try:
+            try:
-                    delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
+                delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
-                    content = getattr(delta, "content", "") if delta else ""
+                content = getattr(delta, "content", "") if delta else ""
-                except IndexError:
+            except IndexError:
-                    content = ""
+                content = ""
-                if content:
+            if content:
-                    if "<think>" in content:
+                if "<think>" in content:
-                        is_active = False
+                    is_active = False
-                        content = content.split("<think>")[0]
+                    content = content.split("<think>")[0]
-                    if "</think>" in content:
+                if "</think>" in content:
-                        is_active = True
+                    is_active = True
-                        content = content.split("</think>")[-1]
+                    content = content.split("</think>")[-1]
-                    if is_active:
+                if is_active:
-                        yield content
+                    yield content
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
 
     def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
-        try:
+        dialogue = self.normalize_dialogue(dialogue)
-            dialogue = self.normalize_dialogue(dialogue)
 
-            request_params = {
+        request_params = {
-                "model": self.model_name,
+            "model": self.model_name,
-                "messages": dialogue,
+            "messages": dialogue,
-                "stream": True,
+            "stream": True,
-                "tools": functions,
+            "tools": functions,
-            }
+        }
 
-            optional_params = {
+        optional_params = {
-                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+            "max_tokens": kwargs.get("max_tokens", self.max_tokens),
-                "temperature": kwargs.get("temperature", self.temperature),
+            "temperature": kwargs.get("temperature", self.temperature),
-                "top_p": kwargs.get("top_p", self.top_p),
+            "top_p": kwargs.get("top_p", self.top_p),
-                "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+            "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
-            }
+        }
 
-            for key, value in optional_params.items():
+        for key, value in optional_params.items():
-                if value is not None:
+            if value is not None:
-                    request_params[key] = value
+                request_params[key] = value
 
-            stream = self.client.chat.completions.create(**request_params)
+        stream = self.client.chat.completions.create(**request_params)
 
-            for chunk in stream:
+        for chunk in stream:
-                if getattr(chunk, "choices", None):
+            if getattr(chunk, "choices", None):
-                    delta = chunk.choices[0].delta
+                delta = chunk.choices[0].delta
-                    content = getattr(delta, "content", "")
+                content = getattr(delta, "content", "")
-                    tool_calls = getattr(delta, "tool_calls", None)
+                tool_calls = getattr(delta, "tool_calls", None)
-                    yield content, tool_calls
+                yield content, tool_calls
-                elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
+            elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
-                    usage_info = getattr(chunk, "usage", None)
+                usage_info = getattr(chunk, "usage", None)
-                    logger.bind(tag=TAG).info(
+                logger.bind(tag=TAG).info(
-                        f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，"
+                    f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，"
-                        f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
+                    f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
-                        f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
+                    f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
-                    )
+                )
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
-            yield f"【OpenAI服务响应异常: {e}】", None

main/xiaozhi-server/core/providers/llm/xinference/xinference.py

@@ -31,68 +31,55 @@ class LLMProvider(LLMProviderBase):
             raise
 
     def response(self, session_id, dialogue, **kwargs):
-        try:
+        logger.bind(tag=TAG).debug(
-            logger.bind(tag=TAG).debug(
+            f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
-                f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
+        )
-            )
+        responses = self.client.chat.completions.create(
-            responses = self.client.chat.completions.create(
+            model=self.model_name, messages=dialogue, stream=True
-                model=self.model_name, messages=dialogue, stream=True
+        )
-            )
+        is_active = True
-            is_active = True
+        for chunk in responses:
-            for chunk in responses:
+            try:
-                try:
+                delta = (
-                    delta = (
+                    chunk.choices[0].delta
-                        chunk.choices[0].delta
+                    if getattr(chunk, "choices", None)
-                        if getattr(chunk, "choices", None)
+                    else None
-                        else None
+                )
-                    )
+                content = delta.content if hasattr(delta, "content") else ""
-                    content = delta.content if hasattr(delta, "content") else ""
+                if content:
-                    if content:
+                    if "<think>" in content:
-                        if "<think>" in content:
+                        is_active = False
-                            is_active = False
+                        content = content.split("<think>")[0]
-                            content = content.split("<think>")[0]
+                    if "</think>" in content:
-                        if "</think>" in content:
+                        is_active = True
-                            is_active = True
+                        content = content.split("</think>")[-1]
-                            content = content.split("</think>")[-1]
+                    if is_active:
-                        if is_active:
+                        yield content
-                            yield content
+            except Exception as e:
-                except Exception as e:
+                logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
-                    logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in Xinference response generation: {e}")
-            yield "【Xinference服务响应异常】"
 
     def response_with_functions(self, session_id, dialogue, functions=None):
-        try:
+        logger.bind(tag=TAG).debug(
+            f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
+        )
+        if functions:
             logger.bind(tag=TAG).debug(
-                f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
+                f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}"
-            )
-            if functions:
-                logger.bind(tag=TAG).debug(
-                    f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}"
-                )
-
-            stream = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                tools=functions,
             )
 
-            for chunk in stream:
+        stream = self.client.chat.completions.create(
-                delta = chunk.choices[0].delta
+            model=self.model_name,
-                content = delta.content
+            messages=dialogue,
-                tool_calls = delta.tool_calls
+            stream=True,
+            tools=functions,
+        )
 
-                if content:
+        for chunk in stream:
-                    yield content, tool_calls
+            delta = chunk.choices[0].delta
-                elif tool_calls:
+            content = delta.content
-                    yield None, tool_calls
+            tool_calls = delta.tool_calls
 
-        except Exception as e:
+            if content:
-            logger.bind(tag=TAG).error(f"Error in Xinference function call: {e}")
+                yield content, tool_calls
-            yield {
+            elif tool_calls:
-                "type": "content",
+                yield None, tool_calls
-                "content": f"【Xinference服务响应异常: {str(e)}】",
-            }

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -162,19 +162,19 @@ class MemoryProvider(MemoryProviderBase):
         msgStr += f"当前时间：{time_str}"
 
         if self.save_to_file:
-            result = self.llm.response_no_stream(
-                short_term_memory_prompt,
-                msgStr,
-                max_tokens=2000,
-                temperature=0.2,
-            )
-            json_str = extract_json_data(result)
             try:
+                result = self.llm.response_no_stream(
+                    short_term_memory_prompt,
+                    msgStr,
+                    max_tokens=2000,
+                    temperature=0.2,
+                )
+                json_str = extract_json_data(result)
                 json.loads(json_str)  # 检查json格式是否正确
                 self.short_memory = json_str
                 self.save_memory_to_file()
             except Exception as e:
-                print("Error:", e)
+                logger.bind(tag=TAG).error(f"Error in saving memory: {e}")
         else:
             # 当save_to_file为False时，调用Java端的聊天记录总结接口
             summary_id = session_id if session_id else self.role_id

main/xiaozhi-server/core/providers/tts/alibl_stream.py

@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
 
         # 音频参数配置
         self.format = config.get("format", "pcm")
-        sample_rate = config.get("sample_rate", "24000")
-        self.sample_rate = int(sample_rate) if sample_rate else 24000
 
         volume = config.get("volume", "50")
         self.volume = int(volume) if volume else 50
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
             "X-DashScope-DataInspection": "enable",
         }
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
-        )
-
     async def _ensure_connection(self):
         """确保WebSocket连接可用，支持60秒内连接复用"""
         try:
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
                         "text_type": "PlainText",
                         "voice": self.voice,
                         "format": self.format,
-                        "sample_rate": self.sample_rate,
+                        "sample_rate": self.conn.sample_rate,
                         "volume": self.volume,
                         "rate": self.rate,
                         "pitch": self.pitch,
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
                                 "text_type": "PlainText",
                                 "voice": self.voice,
                                 "format": self.format,
-                                "sample_rate": self.sample_rate,
+                                "sample_rate": self.conn.sample_rate,
                                 "volume": self.volume,
                                 "rate": self.rate,
                                 "pitch": self.pitch,

main/xiaozhi-server/core/providers/tts/aliyun.py

@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
         self.appkey = config.get("appkey")
         self.format = config.get("format", "wav")
         self.audio_file_type = config.get("format", "wav")
-        sample_rate = config.get("sample_rate", "16000")
-        self.sample_rate = int(sample_rate) if sample_rate else 16000
 
         if config.get("private_voice"):
             self.voice = config.get("private_voice")
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
             "token": self.token,
             "text": text,
             "format": self.format,
-            "sample_rate": self.sample_rate,
+            "sample_rate": self.conn.sample_rate,
             "voice": self.voice,
             "volume": self.volume,
             "speech_rate": self.speech_rate,

main/xiaozhi-server/core/providers/tts/aliyun_stream.py

@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
         self.format = config.get("format", "pcm")
         self.audio_file_type = config.get("format", "pcm")
 
-        # 采样率配置
-        sample_rate = config.get("sample_rate", "16000")
-        self.sample_rate = int(sample_rate) if sample_rate else 16000
-
         # 音色配置 - CosyVoice大模型音色
         if config.get("private_voice"):
             self.voice = config.get("private_voice")
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
         # 专属tts设置
         self.task_id = uuid.uuid4().hex
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=16000, channels=1, frame_size_ms=60
-        )
-
         # Token管理
         if self.access_key_id and self.access_key_secret:
             self._refresh_token()
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
                 "payload": {
                     "voice": self.voice,
                     "format": self.format,
-                    "sample_rate": self.sample_rate,
+                    "sample_rate": self.conn.sample_rate,
                     "volume": self.volume,
                     "speech_rate": self.speech_rate,
                     "pitch_rate": self.pitch_rate,
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
                         "payload": {
                             "voice": self.voice,
                             "format": self.format,
-                            "sample_rate": self.sample_rate,
+                            "sample_rate": self.conn.sample_rate,
                             "volume": self.volume,
                             "speech_rate": self.speech_rate,
                             "pitch_rate": self.pitch_rate,

main/xiaozhi-server/core/providers/tts/base.py

@@ -1,17 +1,18 @@
 import os
 import re
-import time
 import uuid
 import queue
 import asyncio
 import threading
 import traceback
+
 from core.utils import p3
 from datetime import datetime
 from core.utils import textUtils
 from typing import Callable, Any
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
+from core.utils import opus_encoder_utils
 from core.utils.tts import MarkdownCleaner
 from core.utils.output_counter import add_device_output
 from core.handle.reportHandle import enqueue_tts_report
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
                             file_type=self.audio_file_type,
                             is_opus=True,
                             callback=opus_handler,
+                            sample_rate=self.conn.sample_rate,
+                            opus_encoder=self.opus_encoder,
                         )
                         break
                     else:
@@ -138,7 +141,7 @@ class TTSProviderBase(ABC):
                     logger.bind(tag=TAG).error(
                         f"语音生成失败: {text}，请检查网络或服务是否正常"
                     )
-                    self.tts_audio_queue.put((SentenceType.FIRST, None, text))
+                self.tts_audio_queue.put((SentenceType.FIRST, None, text))
                 self._process_audio_file_stream(tmp_file, callback=opus_handler)
             except Exception as e:
                 logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
                             audio_bytes,
                             file_type=self.audio_file_type,
                             is_opus=True,
-                            callback=lambda data: audio_datas.append(data)
+                            callback=lambda data: audio_datas.append(data),
+                            sample_rate=self.conn.sample_rate,
                         )
                         return audio_datas
                     else:
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
         self, audio_file_path, callback: Callable[[Any], Any] = None
     ):
         """音频文件转换为PCM编码"""
-        return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
+        return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
 
     def audio_to_opus_data_stream(
         self, audio_file_path, callback: Callable[[Any], Any] = None
     ):
         """音频文件转换为Opus编码"""
-        return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
+        return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
 
     def tts_one_sentence(
         self,
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
 
     async def open_audio_channels(self, conn):
         self.conn = conn
+
+        # 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理）
+        if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
+            self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+                sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
+            )
+
         # tts 消化线程
         self.tts_priority_thread = threading.Thread(
             target=self.tts_text_priority_thread, daemon=True

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("private_voice")
         else:
             self.voice = config.get("speaker")
-        speech_rate = config.get("speech_rate", "0")
+
-        loudness_rate = config.get("loudness_rate", "0")
+        # 默认 audio_params 配置
-        pitch = config.get("pitch", "0")
+        default_audio_params = {
-        self.speech_rate = int(speech_rate) if speech_rate else 0
+            "speech_rate": 0,
-        self.loudness_rate = int(loudness_rate) if loudness_rate else 0
+            "loudness_rate": 0
-        self.pitch = int(pitch) if pitch else 0
+        }
-        # 多情感音色参数
+
-        self.emotion = config.get("emotion", "neutral")  
+        # 默认 additions 配置
-        emotion_scale = config.get("emotion_scale", "4")
+        default_additions = {
-        self.emotion_scale = int(emotion_scale) if emotion_scale else 4
+            "aigc_metadata": {},
+            "cache_config": {},
+            "post_process": {
+                "pitch": 0
+            }
+        }
+
+        # 默认 mix_speaker 配置
+        default_mix_speaker = {}
+
+        # 合并用户配置
+        self.audio_params = {**default_audio_params, **config.get("audio_params", {})}
+        self.additions = {**default_additions, **config.get("additions", {})}
+        self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})}
 
         self.ws_url = config.get("ws_url")
         self.authorization = config.get("authorization")
@@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase):
         enable_ws_reuse_value = config.get("enable_ws_reuse", True)
         self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
         self.tts_text = ""
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+
-            sample_rate=16000, channels=1, frame_size_ms=60
-        )
         model_key_msg = check_model_key("TTS", self.access_token)
         if model_key_msg:
             logger.bind(tag=TAG).error(model_key_msg)
@@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase):
     async def open_audio_channels(self, conn):
         try:
             await super().open_audio_channels(conn)
+            # 更新 audio_params 中的采样率为实际的 conn.sample_rate
+            self.audio_params["sample_rate"] = conn.sample_rate
         except Exception as e:
             logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}")
             self.ws = None
@@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase):
         text="",
         speaker="",
         audio_format="pcm",
-        audio_sample_rate=16000,
     ):
-        audio_params = {
+        # 构建 req_params
-            "format": audio_format,
+        req_params = {
-            "sample_rate": audio_sample_rate,
+            "text": text,
-            "speech_rate": self.speech_rate,
+            "speaker": speaker,
-            "loudness_rate": self.loudness_rate
+            "audio_params": {**self.audio_params, "format": audio_format},
+            "additions": json.dumps(self.additions)
         }
-
+        
-        # 如果是多情感音色,添加情感参数
+        # 如果有 mix_speaker 配置，添加到 req_params
-        if '_emo_' in self.voice:
+        if self.mix_speaker:
-            if self.emotion:
+            req_params["mix_speaker"] = self.mix_speaker
-                audio_params["emotion"] = self.emotion
-                audio_params["emotion_scale"] = self.emotion_scale
 
         return str.encode(
             json.dumps(
@@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase):
                     "user": {"uid": uid},
                     "event": event,
                     "namespace": "BidirectionalTTS",
-                    "req_params": {
+                    "req_params": req_params
-                        "text": text,
-                        "speaker": speaker,
-                        "audio_params": audio_params,
-                        "additions": json.dumps({
-                            "post_process": {
-                                "pitch": self.pitch
-                            }
-                        })
-                    },
-
                 }
             )
         )

main/xiaozhi-server/core/providers/tts/index_stream.py

@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
             logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
             self.tts_audio_queue.put((SentenceType.LAST, [], None))
 
+    def audio_to_pcm_data_stream(
+        self, audio_file_path, callback=None
+    ):
+        """音频文件转换为PCM编码，使用24kHz采样率"""
+        from core.utils.util import audio_to_data_stream
+        return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
+
+    def audio_to_opus_data_stream(
+        self, audio_file_path, callback=None
+    ):
+        """音频文件转换为Opus编码，使用24kHz采样率和自己的编码器"""
+        from core.utils.util import audio_to_data_stream
+        return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
+
     async def close(self):
         """资源清理"""
         await super().close()

main/xiaozhi-server/core/providers/tts/linkerai.py

@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
         self.audio_format = "pcm"
         self.before_stop_play_files = []
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=16000, channels=1, frame_size_ms=60
-        )
-
         # PCM缓冲区
         self.pcm_buffer = bytearray()
 
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
             "spk_id": self.voice,
             "frame_durition": 60,
             "stream": "true",
-            "target_sr": 16000,
+            "target_sr": self.conn.sample_rate,
             "audio_format": "pcm",
             "instruct_text": "请生成一段自然流畅的语音",
         }
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
             "Content-Type": "application/json",
         }
 
-        # 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920
+        # 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B
         frame_bytes = int(
             self.opus_encoder.sample_rate
             * self.opus_encoder.channels  # 1
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
             "spk_id": self.voice,
             "frame_duration": 60,
             "stream": False,
-            "target_sr": 16000,
+            "target_sr": self.conn.sample_rate,
             "audio_format": self.audio_format,
             "instruct_text": "请生成一段自然流畅的语音",
         }

main/xiaozhi-server/core/providers/tts/minimax_httpstream.py

@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
         }
         self.audio_file_type = defult_audio_setting.get("format", "pcm")
 
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=24000, channels=1, frame_size_ms=60
-        )
-
         # PCM缓冲区
         self.pcm_buffer = bytearray()
 
+    async def open_audio_channels(self, conn):
+        """初始化音频通道,并根据conn.sample_rate更新配置"""
+        # 调用父类方法
+        await super().open_audio_channels(conn)
+
+        # 更新audio_setting中的采样率为实际的conn.sample_rate
+        self.audio_setting["sample_rate"] = conn.sample_rate
+
     def tts_text_priority_thread(self):
         """流式文本处理线程"""
         while not self.conn.stop_event.is_set():
@@ -212,6 +216,18 @@ class TTSProvider(TTSProviderBase):
 
                             try:
                                 data = json.loads(json_str)
+
+                                # 检查业务层错误
+                                base_resp = data.get("base_resp", {})
+                                status_code = base_resp.get("status_code", 0)
+                                if status_code != 0:
+                                    status_msg = base_resp.get("status_msg", "未知错误")
+                                    logger.bind(tag=TAG).error(
+                                        f"TTS请求失败, 错误码:{status_code}, 错误消息:{status_msg}"
+                                    )
+                                    self.tts_audio_queue.put((SentenceType.LAST, [], None))
+                                    return
+
                                 status = data.get("data", {}).get("status", 1)
                                 audio_hex = data.get("data", {}).get("audio")
 

main/xiaozhi-server/core/providers/tts/paddle_speech.py

@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
             self.spk_id = int(config.get("private_voice"))
         else:
             self.spk_id = int(config.get("spk_id", "0"))
-            
+
-        sample_rate = config.get("sample_rate", 24000)
-        self.sample_rate = float(sample_rate) if sample_rate else 24000
-        
         speed = config.get("speed", 1.0)
         self.speed = float(speed) if speed else 1.0
         

main/xiaozhi-server/core/providers/tts/siliconflow.py

@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("voice")
         self.response_format = config.get("response_format", "mp3")
         self.audio_file_type = config.get("response_format", "mp3")
-        self.sample_rate = config.get("sample_rate")
         self.speed = float(config.get("speed", 1.0))
         self.gain = config.get("gain")
 

main/xiaozhi-server/core/providers/tts/xunfei_stream.py

@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
         # 音频编码配置
         self.format = config.get("format", "raw")
 
-        sample_rate = config.get("sample_rate", "24000")
-        self.sample_rate = int(sample_rate) if sample_rate else 24000
-
         # 口语化配置
         self.oral_level = config.get("oral_level", "mid")
 
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
         # 序列号管理
         self.text_seq = 0
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
-        )
-
         # 验证必需参数
         if not all([self.app_id, self.api_key, self.api_secret]):
             raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
                     "rhy": 0,
                     "audio": {
                         "encoding": self.format,
-                        "sample_rate": self.sample_rate,
+                        "sample_rate": self.conn.sample_rate,
                         "channels": 1,
                         "bit_depth": 16,
                         "frame_size": 0

main/xiaozhi-server/core/utils/util.py

@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
 
 
 def audio_to_data_stream(
-    audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
+    audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
 ) -> None:
     # 获取文件后缀名
     file_type = os.path.splitext(audio_file_path)[1]
@@ -238,12 +238,12 @@ def audio_to_data_stream(
         audio_file_path, format=file_type, parameters=["-nostdin"]
     )
 
-    # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
+    # 转换为单声道/指定采样率/16位小端编码（确保与编码器匹配）
-    audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+    audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
 
     # 获取原始PCM数据（16位小端）
     raw_data = audio.raw_data
-    pcm_to_data_stream(raw_data, is_opus, callback)
+    pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
 
 
 async def audio_to_data(
@@ -325,7 +325,7 @@ async def audio_to_data(
 
 
 def audio_bytes_to_data_stream(
-    audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
+    audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
 ) -> None:
     """
     直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
         audio = AudioSegment.from_file(
             BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
         )
-        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+        audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
         raw_data = audio.raw_data
-        pcm_to_data_stream(raw_data, is_opus, callback)
+        pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
 
 
-def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
+def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
-    # 初始化Opus编码器
+    """
-    encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+    将PCM数据流式编码为Opus或直接输出PCM
+
+    Args:
+        raw_data: PCM原始数据
+        is_opus: 是否编码为Opus
+        callback: 回调函数
+        sample_rate: 采样率
+        opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
+    """
+    using_temp_encoder = False
+    if is_opus and opus_encoder is None:
+        encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
+        using_temp_encoder = True
 
     # 编码参数
     frame_duration = 60  # 60ms per frame
-    frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+    frame_size = int(sample_rate * frame_duration / 1000)  # samples/frame
 
     # 按帧处理所有音频数据（包括最后一帧可能补零）
     for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
             chunk += b"\x00" * (frame_size * 2 - len(chunk))
 
         if is_opus:
-            # 转换为numpy数组处理
+            if using_temp_encoder:
-            np_frame = np.frombuffer(chunk, dtype=np.int16)
+                # 使用临时编码器（仅用于独立音频场景）
-            # 编码Opus数据
+                np_frame = np.frombuffer(chunk, dtype=np.int16)
-            frame_data = encoder.encode(np_frame.tobytes(), frame_size)
+                frame_data = encoder.encode(np_frame.tobytes(), frame_size)
-            callback(frame_data)
+                callback(frame_data)
+            else:
+                # 使用外部编码器（TTS流式场景,保持状态连续）
+                is_last = (i + frame_size * 2 >= len(raw_data))
+                opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
         else:
+            # PCM模式,直接输出
             frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
             callback(frame_data)
 
@@ -573,3 +590,14 @@ def validate_mcp_endpoint(mcp_endpoint: str) -> bool:
         return False
 
     return True
+
+def get_system_error_response(config: dict) -> str:
+    """获取系统错误时的回复
+
+    Args:
+        config: 配置字典
+
+    Returns:
+        str: 系统错误时的回复
+    """
+    return config.get("system_error_response", "主人，小智现在有点忙，我们稍后再试吧。")

main/xiaozhi-server/test/css/test_page.css

@@ -108,26 +108,68 @@ body {
     animation: shadow-float 1.26s ease-in-out infinite;
 }
 
-.loading-char-float .main-char span:nth-child(1) { animation-delay: 0s; }
+.loading-char-float .main-char span:nth-child(1) {
-.loading-char-float .main-char span:nth-child(2) { animation-delay: 0.11s; }
+    animation-delay: 0s;
-.loading-char-float .main-char span:nth-child(3) { animation-delay: 0.22s; }
+}
-.loading-char-float .main-char span:nth-child(4) { animation-delay: 0.33s; }
+
-.loading-char-float .main-char span:nth-child(5) { animation-delay: 0.44s; }
+.loading-char-float .main-char span:nth-child(2) {
-.loading-char-float .main-char span:nth-child(6) { animation-delay: 0.55s; }
+    animation-delay: 0.11s;
+}
+
+.loading-char-float .main-char span:nth-child(3) {
+    animation-delay: 0.22s;
+}
+
+.loading-char-float .main-char span:nth-child(4) {
+    animation-delay: 0.33s;
+}
+
+.loading-char-float .main-char span:nth-child(5) {
+    animation-delay: 0.44s;
+}
+
+.loading-char-float .main-char span:nth-child(6) {
+    animation-delay: 0.55s;
+}
 
 @keyframes scan {
-    0% { background-position: 0 0; }
+    0% {
-    100% { background-position: 0 100px; }
+        background-position: 0 0;
+    }
+
+    100% {
+        background-position: 0 100px;
+    }
 }
 
 @keyframes char-float {
-    0%, 100% { transform: translateY(0); opacity: 1; text-shadow: 0 0 5px #5865f2, 0 0 10px rgba(88, 101, 242, 0.8); }
+
-    50% { transform: translateY(-5px); opacity: 0.9; text-shadow: 0 0 10px #5865f2, 0 0 20px rgba(88, 101, 242, 0.8); }
+    0%,
+    100% {
+        transform: translateY(0);
+        opacity: 1;
+        text-shadow: 0 0 5px #5865f2, 0 0 10px rgba(88, 101, 242, 0.8);
+    }
+
+    50% {
+        transform: translateY(-5px);
+        opacity: 0.9;
+        text-shadow: 0 0 10px #5865f2, 0 0 20px rgba(88, 101, 242, 0.8);
+    }
 }
 
 @keyframes shadow-float {
-    0%, 100% { transform: translateY(3px); opacity: 0.3; }
+
-    50% { transform: translateY(8px); opacity: 0.15; }
+    0%,
+    100% {
+        transform: translateY(3px);
+        opacity: 0.3;
+    }
+
+    50% {
+        transform: translateY(8px);
+        opacity: 0.15;
+    }
 }
 
 /* ==================== Live2D显示区域 ==================== */
@@ -402,6 +444,15 @@ body {
     background: rgba(237, 66, 69, 1);
 }
 
+/* 摄像头按钮特殊样式 */
+#cameraBtn.camera-active {
+    background: rgba(237, 66, 69, 0.9);
+}
+
+#cameraBtn.camera-active:hover {
+    background: rgba(237, 66, 69, 1);
+}
+
 /* 录音按钮脉冲动画 */
 @keyframes pulse {
     0% {
@@ -761,6 +812,72 @@ body {
     font-size: 14px;
 }
 
+.model-select {
+    width: 100%;
+    padding: 10px 40px 10px 14px;
+    border: 1px solid #40444b;
+    border-radius: 8px;
+    background: #40444b;
+    color: #ffffff;
+    font-size: 14px;
+    font-weight: 500;
+    cursor: pointer;
+    appearance: none;
+    -webkit-appearance: none;
+    -moz-appearance: none;
+    background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='%23ffffff'%3E%3Cpath d='M7 10l5 5 5-5z'/%3E%3C/svg%3E");
+    background-repeat: no-repeat;
+    background-position: right 12px center;
+    background-size: 16px 16px;
+}
+
+.model-select:hover {
+    border-color: #5865f2;
+}
+
+.model-select:focus {
+    outline: none;
+    border-color: #5865f2;
+}
+
+.model-select option {
+    background: #2f3136;
+    color: #ffffff;
+    padding: 12px;
+    font-size: 14px;
+}
+
+.background-btn {
+    width: 100%;
+    padding: 10px 14px;
+    border: 1px solid #40444b;
+    border-radius: 8px;
+    background: #40444b;
+    color: #ffffff;
+    font-size: 14px;
+    font-weight: 500;
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    gap: 8px;
+    transition: all 0.3s ease;
+}
+
+.background-btn:hover {
+    background: #4f545c;
+}
+
+.background-btn:active {
+    background: #40444b;
+}
+
+.background-btn .btn-icon {
+    width: 20px;
+    height: 20px;
+    fill: #ffffff;
+}
+
 /* ==================== MCP工具样式 ==================== */
 .mcp-tools-container {
     background: #2f3136;
@@ -911,35 +1028,174 @@ body {
     margin-bottom: 12px;
 }
 
-.property-item {
+.properties-container #addMcpPropertyBtn {
+    display: block;
+    margin: 0 auto;
+}
+
+.properties-container #addMcpPropertyBtn:hover {
+    background: #4752c4;
+}
+
+.mcp-checkbox-label {
     display: flex;
-    gap: 8px;
-    margin-bottom: 8px;
     align-items: center;
+    gap: 8px;
+    margin-top: 8px;
+    color: #b9bbbe;
+    font-size: 13px;
+    cursor: pointer;
+    justify-content: flex-start;
 }
 
-.property-item input {
+.mcp-checkbox-label input[type="checkbox"] {
-    flex: 1;
+    width: 16px;
-    padding: 6px 8px;
+    height: 16px;
-    border: 1px solid #40444b;
+    cursor: pointer;
-    border-radius: 4px;
+}
-    background: #40444b;
+
+.mcp-property-name {
     color: white;
-    font-size: 12px;
+    font-size: 14px;
+    font-weight: 500;
 }
 
-.remove-property {
+.mcp-property-header {
-    background: #ed4245;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 8px;
+}
+
+.mcp-properties-list {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 12px;
+    min-height: 60px;
+    padding: 8px;
+}
+
+.input-group-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 8px;
+}
+
+.input-group-header label {
+    margin-bottom: 0;
+}
+
+.input-group-header .properties-btn-primary {
+    margin: 0;
+}
+
+.mcp-empty-state {
+    text-align: center;
+    padding: 20px;
+    color: #999;
+    font-size: 14px;
+    width: 100%;
+    display: none;
+}
+
+.properties-btn-primary {
+    background: #2196f3;
     color: white;
     border: none;
-    padding: 4px 8px;
+    padding: 6px 12px;
     border-radius: 4px;
     cursor: pointer;
     font-size: 12px;
+    font-weight: 500;
+    transition: background-color 0.2s;
+    display: block;
+    margin: 0 auto;
 }
 
-.remove-property:hover {
+.properties-btn-primary:hover {
-    background: #c03537;
+    background: #2196f3;
+}
+
+.mcp-property-card {
+    background: #36393f;
+    border: 1px solid #40444b;
+    border-radius: 8px;
+    padding: 12px;
+    width: 100%;
+    cursor: pointer;
+    transition: all 0.2s ease;
+}
+
+.mcp-property-card:hover {
+    background: #40444b;
+    border-color: #5865f2;
+}
+
+.mcp-property-card:active {
+    transform: none;
+}
+
+.mcp-property-row-label {
+    display: flex;
+    align-items: flex-start;
+    margin-bottom: 6px;
+    gap: 8px;
+}
+
+.mcp-property-label {
+    color: #b9bbbe;
+    font-size: 13px;
+    width: 60px;
+    flex-shrink: 0;
+    text-align: left;
+}
+
+.mcp-property-value {
+    color: white;
+    font-size: 13px;
+    flex: 1;
+    word-break: break-all;
+}
+
+.mcp-property-required-badge {
+    color: #f44336;
+    font-size: 12px;
+}
+
+.mcp-property-row-action {
+    display: flex;
+    justify-content: flex-end;
+    margin-top: 6px;
+    padding-top: 6px;
+    border-top: 1px solid #40444b;
+}
+
+.mcp-property-delete-btn {
+    background: #f44336;
+    color: white;
+    border: none;
+    padding: 4px 12px;
+    border-radius: 4px;
+    cursor: pointer;
+    font-size: 11px;
+    transition: background-color 0.2s;
+}
+
+.mcp-property-delete-btn:hover {
+    background: #d32f2f;
+}
+
+.mcp-property-card-optional {
+    background: #4f545c;
+    color: #b9bbbe;
+    font-size: 11px;
+    padding: 2px 8px;
+    border-radius: 4px;
+}
+
+.property-modal {
+    max-width: 450px;
 }
 
 /* ==================== 音频可视化器样式 ==================== */
@@ -1125,4 +1381,41 @@ body {
 .modal-body::-webkit-scrollbar-thumb:hover,
 .mcp-tools-list::-webkit-scrollbar-thumb:hover {
     background: #4f545c;
+}
+
+/* ==================== 摄像头显示区域样式 ==================== */
+.camera-container {
+    position: fixed;
+    top: 20px;
+    left: 20px;
+    width: 240px;
+    height: 180px;
+    background: rgba(0, 0, 0, 0.5);
+    backdrop-filter: blur(10px);
+    border-radius: 8px;
+    border: 1px solid rgba(255, 255, 255, 0.2);
+    z-index: 1000;
+    display: none;
+    overflow: hidden;
+    box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
+    cursor: move;
+}
+
+.camera-container.active {
+    display: block;
+}
+
+.camera-container.dragging {
+    cursor: grabbing;
+    box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4);
+    border-color: rgba(88, 101, 242, 0.5);
+}
+
+#cameraVideo {
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+    background: #1a1a1a;
+    pointer-events: none;
+    transform: scaleX(-1);
 }

main/xiaozhi-server/test/js/app.js

@@ -1,9 +1,22 @@
 // 主应用入口
-import { log } from './utils/logger.js';
+import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js?v=0127';
-import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js';
+import { getAudioPlayer } from './core/audio/player.js?v=0127';
-import { uiController } from './ui/controller.js';
+import { checkMicrophoneAvailability, isHttpNonLocalhost } from './core/audio/recorder.js?v=0127';
-import { getAudioPlayer } from './core/audio/player.js';
+import { initMcpTools } from './core/mcp/tools.js?v=0127';
-import { initMcpTools } from './core/mcp/tools.js';
+import { uiController } from './ui/controller.js?v=0127';
+import { log } from './utils/logger.js?v=0127';
+
+// 辅助函数：将Base64数据转换为Blob
+function dataURItoBlob(dataURI) {
+    const byteString = atob(dataURI.split(',')[1]);
+    const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
+    const ab = new ArrayBuffer(byteString.length);
+    const ia = new Uint8Array(ab);
+    for (let i = 0; i < byteString.length; i++) {
+        ia[i] = byteString.charCodeAt(i);
+    }
+    return new Blob([ab], { type: mimeString });
+}
 
 // 应用类
 class App {
@@ -11,35 +24,34 @@ class App {
         this.uiController = null;
         this.audioPlayer = null;
         this.live2dManager = null;
+        this.cameraStream = null;
     }
 
     // 初始化应用
     async init() {
         log('正在初始化应用...', 'info');
-
         // 初始化UI控制器
         this.uiController = uiController;
         this.uiController.init();
-
         // 检查Opus库
         checkOpusLoaded();
-
         // 初始化Opus编码器
         initOpusEncoder();
-
         // 初始化音频播放器
         this.audioPlayer = getAudioPlayer();
         await this.audioPlayer.start();
-
         // 初始化MCP工具
         initMcpTools();
-
+        // 检查麦克风可用性
+        await this.checkMicrophoneAvailability();
+        // 检查摄像头可用性
+        this.checkCameraAvailability();
         // 初始化Live2D
         await this.initLive2D();
-
+        // 初始化摄像头
+        this.initCamera();
         // 关闭加载loading
         this.setModelLoadingStatus(false);
-
         log('应用初始化完成', 'success');
     }
 
@@ -50,21 +62,17 @@ class App {
             if (typeof window.Live2DManager === 'undefined') {
                 throw new Error('Live2DManager未加载，请检查脚本引入顺序');
             }
-
             this.live2dManager = new window.Live2DManager();
             await this.live2dManager.initializeLive2D();
-
             // 更新UI状态
             const live2dStatus = document.getElementById('live2dStatus');
             if (live2dStatus) {
                 live2dStatus.textContent = '● 已加载';
                 live2dStatus.className = 'status loaded';
             }
-
             log('Live2D初始化完成', 'success');
         } catch (error) {
             log(`Live2D初始化失败: ${error.message}`, 'error');
-
             // 更新UI状态
             const live2dStatus = document.getElementById('live2dStatus');
             if (live2dStatus) {
@@ -81,18 +89,239 @@ class App {
             modelLoading.style.display = isLoading ? 'flex' : 'none';
         }
     }
+
+    /**
+     * 检查麦克风可用性
+     * 在应用初始化时调用，检查麦克风是否可用并更新UI状态
+     */
+    async checkMicrophoneAvailability() {
+        try {
+            const isAvailable = await checkMicrophoneAvailability();
+            const isHttp = isHttpNonLocalhost();
+            // 保存可用性状态到全局变量
+            window.microphoneAvailable = isAvailable;
+            window.isHttpNonLocalhost = isHttp;
+            // 更新UI
+            if (this.uiController) {
+                this.uiController.updateMicrophoneAvailability(isAvailable, isHttp);
+            }
+            log(`麦克风可用性检查完成: ${isAvailable ? '可用' : '不可用'}`, isAvailable ? 'success' : 'warning');
+        } catch (error) {
+            log(`检查麦克风可用性失败: ${error.message}`, 'error');
+            // 默认设置为不可用
+            window.microphoneAvailable = false;
+            window.isHttpNonLocalhost = isHttpNonLocalhost();
+            if (this.uiController) {
+                this.uiController.updateMicrophoneAvailability(false, window.isHttpNonLocalhost);
+            }
+        }
+    }
+
+    // 检查摄像头可用性
+    checkCameraAvailability() {
+        window.cameraAvailable = true;
+        log('摄像头可用性检查完成: 默认已绑定验证码', 'success');
+    }
+
+    // 初始化摄像头
+    async initCamera() {
+        const cameraContainer = document.getElementById('cameraContainer');
+        const cameraVideo = document.getElementById('cameraVideo');
+
+        if (!cameraContainer || !cameraVideo) {
+            log('摄像头元素未找到，跳过初始化', 'warning');
+            return Promise.resolve(false);
+        }
+
+        let isDragging = false;
+        let currentX, currentY, initialX, initialY;
+        let xOffset = 0, yOffset = 0;
+
+        cameraContainer.addEventListener('mousedown', dragStart);
+        document.addEventListener('mousemove', drag);
+        document.addEventListener('mouseup', dragEnd);
+        cameraContainer.addEventListener('touchstart', dragStart, { passive: false });
+        document.addEventListener('touchmove', drag, { passive: false });
+        document.addEventListener('touchend', dragEnd);
+
+        function dragStart(e) {
+            if (e.type === 'touchstart') {
+                initialX = e.touches[0].clientX - xOffset;
+                initialY = e.touches[0].clientY - yOffset;
+            } else {
+                initialX = e.clientX - xOffset;
+                initialY = e.clientY - yOffset;
+            }
+            isDragging = true;
+            cameraContainer.classList.add('dragging');
+        }
+
+        function drag(e) {
+            if (isDragging) {
+                e.preventDefault();
+                if (e.type === 'touchmove') {
+                    currentX = e.touches[0].clientX - initialX;
+                    currentY = e.touches[0].clientY - initialY;
+                } else {
+                    currentX = e.clientX - initialX;
+                    currentY = e.clientY - initialY;
+                }
+                xOffset = currentX;
+                yOffset = currentY;
+                cameraContainer.style.transform = `translate3d(${currentX}px, ${currentY}px, 0)`;
+            }
+        }
+
+        function dragEnd() {
+            initialX = currentX;
+            initialY = currentY;
+            isDragging = false;
+            cameraContainer.classList.remove('dragging');
+        }
+
+        return new Promise((resolve) => {
+            window.startCamera = async () => {
+                try {
+                    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+                        log('浏览器不支持摄像头API', 'warning');
+                        return false;
+                    }
+                    log('正在请求摄像头权限...', 'info');
+                    this.cameraStream = await navigator.mediaDevices.getUserMedia({
+                        video: { width: 320, height: 240, facingMode: 'user' },
+                        audio: false
+                    });
+                    cameraVideo.srcObject = this.cameraStream;
+                    cameraContainer.classList.add('active');
+                    log('摄像头已启动', 'success');
+                    return true;
+                } catch (error) {
+                    log(`启动摄像头失败: ${error.name} - ${error.message}`, 'error');
+                    if (error.name === 'NotAllowedError') {
+                        log('摄像头权限被拒绝，请检查浏览器设置', 'warning');
+                    } else if (error.name === 'NotFoundError') {
+                        log('未找到摄像头设备', 'warning');
+                    } else if (error.name === 'NotReadableError') {
+                        log('摄像头已被其他程序占用', 'warning');
+                    }
+                    return false;
+                }
+            };
+
+            window.stopCamera = () => {
+                if (this.cameraStream) {
+                    this.cameraStream.getTracks().forEach(track => track.stop());
+                    this.cameraStream = null;
+                    cameraVideo.srcObject = null;
+                    log('摄像头已关闭', 'info');
+                }
+            };
+
+            window.takePhoto = (question = '描述一下看到的物品') => {
+                return new Promise(async (resolve) => {
+                    const canvas = document.createElement('canvas');
+                    const video = cameraVideo;
+
+                    if (!video || video.readyState !== video.HAVE_ENOUGH_DATA) {
+                        log('无法拍照：摄像头未就绪', 'warning');
+                        resolve({
+                            success: false,
+                            error: '摄像头未就绪，请确保已连接且摄像头已启动'
+                        });
+                        return;
+                    }
+
+                    canvas.width = video.videoWidth || 320;
+                    canvas.height = video.videoHeight || 240;
+                    const ctx = canvas.getContext('2d');
+                    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+                    const photoData = canvas.toDataURL('image/jpeg', 0.8);
+                    log(`拍照成功，图像数据长度: ${photoData.length}`, 'success');
+
+                    try {
+                        const xz_tester_vision = localStorage.getItem('xz_tester_vision');
+                        if (xz_tester_vision) {
+                            let visionInfo = null;
+
+                            try {
+                                visionInfo = JSON.parse(xz_tester_vision);
+                            } catch (err) {
+                                throw new Error(`视觉配置解析失败`);
+                            }
+
+                            const { url, token } = visionInfo || {};
+                            if (!url || !token) {
+                                throw new Error('视觉分析失败：配置缺少接口地址(url)或令牌(token)');
+                            }
+                            
+                            log(`正在发送图片到视觉分析接口: ${url}`, 'info');
+
+                            const deviceId = document.getElementById('deviceMac')?.value || '';
+                            const clientId = document.getElementById('clientId')?.value || 'web_test_client';
+
+                            const formData = new FormData();
+                            formData.append('question', question);
+                            formData.append('image', dataURItoBlob(photoData), 'photo.jpg');
+
+                            const response = await fetch(url, {
+                                method: 'POST',
+                                body: formData,
+                                headers: {
+                                    'Device-Id': deviceId,
+                                    'Client-Id': clientId,
+                                    'Authorization': `Bearer ${token}`
+                                }
+                            });
+
+                            if (!response.ok) {
+                                throw new Error(`HTTP error! status: ${response.status}`);
+                            }
+
+                            const analysisResult = await response.json();
+                            log(`视觉分析完成: ${JSON.stringify(analysisResult).substring(0, 200)}...`, 'success');
+
+                            resolve({
+                                success: true,
+                                message: question,
+                                photo_data: photoData,
+                                photo_width: canvas.width,
+                                photo_height: canvas.height,
+                                vision_analysis: analysisResult
+                            });
+                        } else {
+                            log('未配置视觉分析服务', 'warning');
+                        }
+                    } catch (error) {
+                        log(`视觉分析失败: ${error.message}`, 'error');
+                        resolve({
+                            success: true,
+                            message: question,
+                            photo_data: photoData,
+                            photo_width: canvas.width,
+                            photo_height: canvas.height,
+                            vision_analysis: {
+                                success: false,
+                                error: error.message,
+                                fallback: '无法连接到视觉分析服务'
+                            }
+                        });
+                    }
+                });
+            };
+
+            log('摄像头初始化完成', 'success');
+            resolve(true);
+        });
+    }
 }
 
 // 创建并启动应用
 const app = new App();
-
 // 将应用实例暴露到全局，供其他模块访问
 window.chatApp = app;
-
 document.addEventListener('DOMContentLoaded', () => {
     // 初始化应用
     app.init();
 });
-
-
 export default app;

main/xiaozhi-server/test/js/config/default-mcp-tools.json

@@ -1,4 +1,22 @@
 [
+    {
+        "name": "self_camera_take_photo",
+        "description": "Take a photo using the device's camera. This tool captures the current camera frame and returns the image data. You MUST call this tool when user asks to 'take a photo', 'what do you see', 'describe what you see', or similar requests. After getting the photo data, describe what you see in the image. Parameter 'question' is optional, defaults to 'describe objects seen'.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "question": {
+                    "type": "string",
+                    "description": "Question to guide the photo analysis, e.g., 'describe the objects seen'. Can be empty."
+                }
+            }
+        },
+        "mockResponse": {
+            "success": true,
+            "photo_data": "base64_image_data",
+            "response": "Please describe what you see based on the returned photo_data"
+        }
+    },
     {
         "name": "self.get_device_status",
         "description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",

main/xiaozhi-server/test/js/core/audio/opus-codec.js

@@ -1,4 +1,4 @@
-import { log } from '../../utils/logger.js';
+import { log } from '../../utils/logger.js?v=0127';
 
 
 // 检查Opus库是否已加载

main/xiaozhi-server/test/js/core/audio/player.js

@@ -1,7 +1,7 @@
 // 音频播放模块
-import { log } from '../../utils/logger.js';
+import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
-import BlockingQueue from '../../utils/blocking-queue.js';
+import { log } from '../../utils/logger.js?v=0127';
-import { createStreamingContext } from './stream-context.js';
+import { createStreamingContext } from './stream-context.js?v=0127';
 
 // 音频播放器类
 export class AudioPlayer {

main/xiaozhi-server/test/js/core/audio/recorder.js

@@ -1,9 +1,9 @@
-// 音频录制模块
+// Audio recording module
-import { log } from '../../utils/logger.js';
+import { log } from '../../utils/logger.js?v=0127';
-import { initOpusEncoder } from './opus-codec.js';
+import { initOpusEncoder } from './opus-codec.js?v=0127';
-import { getAudioPlayer } from './player.js';
+import { getAudioPlayer } from './player.js?v=0127';
 
-// 音频录制器类
+// Audio recorder class
 export class AudioRecorder {
     constructor() {
         this.isRecording = false;
@@ -19,25 +19,23 @@ export class AudioRecorder {
         this.visualizationRequest = null;
         this.recordingTimer = null;
         this.websocket = null;
-
+        // Callback functions
-        // 回调函数
         this.onRecordingStart = null;
         this.onRecordingStop = null;
         this.onVisualizerUpdate = null;
     }
 
-    // 设置WebSocket实例
+    // Set WebSocket instance
     setWebSocket(ws) {
         this.websocket = ws;
     }
 
-    // 获取AudioContext实例
+    // Get AudioContext instance
     getAudioContext() {
-        const audioPlayer = getAudioPlayer();
+        return getAudioPlayer().getAudioContext();
-        return audioPlayer.getAudioContext();
     }
 
-    // 初始化编码器
+    // Initialize encoder
     initEncoder() {
         if (!this.opusEncoder) {
             this.opusEncoder = initOpusEncoder();
@@ -45,7 +43,7 @@ export class AudioRecorder {
         return this.opusEncoder;
     }
 
-    // PCM处理器代码
+    // PCM processor code
     getAudioProcessorCode() {
         return `
             class AudioRecorderProcessor extends AudioWorkletProcessor {
@@ -56,166 +54,132 @@ export class AudioRecorder {
                     this.buffer = new Int16Array(this.frameSize);
                     this.bufferIndex = 0;
                     this.isRecording = false;
-
                     this.port.onmessage = (event) => {
                         if (event.data.command === 'start') {
                             this.isRecording = true;
                             this.port.postMessage({ type: 'status', status: 'started' });
                         } else if (event.data.command === 'stop') {
                             this.isRecording = false;
-
                             if (this.bufferIndex > 0) {
                                 const finalBuffer = this.buffer.slice(0, this.bufferIndex);
-                                this.port.postMessage({
+                                this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
-                                    type: 'buffer',
-                                    buffer: finalBuffer
-                                });
                                 this.bufferIndex = 0;
                             }
-
                             this.port.postMessage({ type: 'status', status: 'stopped' });
                         }
                     };
                 }
-
                 process(inputs, outputs, parameters) {
                     if (!this.isRecording) return true;
-
                     const input = inputs[0][0];
                     if (!input) return true;
-
                     for (let i = 0; i < input.length; i++) {
                         if (this.bufferIndex >= this.frameSize) {
-                            this.port.postMessage({
+                            this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
-                                type: 'buffer',
-                                buffer: this.buffer.slice(0)
-                            });
                             this.bufferIndex = 0;
                         }
-
                         this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
                     }
-
                     return true;
                 }
             }
-
             registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
         `;
     }
 
-    // 创建音频处理器
+    // Create audio processor
     async createAudioProcessor() {
         this.audioContext = this.getAudioContext();
-
         try {
             if (this.audioContext.audioWorklet) {
                 const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
                 const url = URL.createObjectURL(blob);
                 await this.audioContext.audioWorklet.addModule(url);
                 URL.revokeObjectURL(url);
-
                 const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
-
                 audioProcessor.port.onmessage = (event) => {
                     if (event.data.type === 'buffer') {
                         this.processPCMBuffer(event.data.buffer);
                     }
                 };
-
                 log('使用AudioWorklet处理音频', 'success');
-
                 const silent = this.audioContext.createGain();
                 silent.gain.value = 0;
                 audioProcessor.connect(silent);
                 silent.connect(this.audioContext.destination);
                 return { node: audioProcessor, type: 'worklet' };
             } else {
-                log('AudioWorklet不可用，使用ScriptProcessorNode作为回退方案', 'warning');
+                log('AudioWorklet不可用，使用ScriptProcessorNode作为后备方案', 'warning');
                 return this.createScriptProcessor();
             }
         } catch (error) {
-            log(`创建音频处理器失败: ${error.message}，尝试回退方案`, 'error');
+            log(`创建音频处理器失败: ${error.message}，尝试后备方案`, 'error');
             return this.createScriptProcessor();
         }
     }
 
-    // 创建ScriptProcessor作为回退
+    // Create ScriptProcessor as fallback
     createScriptProcessor() {
         try {
             const frameSize = 4096;
             const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
-
             scriptProcessor.onaudioprocess = (event) => {
                 if (!this.isRecording) return;
-
                 const input = event.inputBuffer.getChannelData(0);
                 const buffer = new Int16Array(input.length);
-
                 for (let i = 0; i < input.length; i++) {
                     buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
                 }
-
                 this.processPCMBuffer(buffer);
             };
-
             const silent = this.audioContext.createGain();
             silent.gain.value = 0;
             scriptProcessor.connect(silent);
             silent.connect(this.audioContext.destination);
-
+            log('使用ScriptProcessorNode作为后备方案成功', 'warning');
-            log('使用ScriptProcessorNode作为回退方案成功', 'warning');
             return { node: scriptProcessor, type: 'processor' };
         } catch (fallbackError) {
-            log(`回退方案也失败: ${fallbackError.message}`, 'error');
+            log(`后备方案也失败: ${fallbackError.message}`, 'error');
             return null;
         }
     }
 
-    // 处理PCM缓冲数据
+    // Process PCM buffer data
     processPCMBuffer(buffer) {
         if (!this.isRecording) return;
-
         const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
         newBuffer.set(this.pcmDataBuffer);
         newBuffer.set(buffer, this.pcmDataBuffer.length);
         this.pcmDataBuffer = newBuffer;
-
         const samplesPerFrame = 960;
-
         while (this.pcmDataBuffer.length >= samplesPerFrame) {
             const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
             this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
-
             this.encodeAndSendOpus(frameData);
         }
     }
 
-    // 编码并发送Opus数据
+    // Encode and send Opus data
     encodeAndSendOpus(pcmData = null) {
         if (!this.opusEncoder) {
             log('Opus编码器未初始化', 'error');
             return;
         }
-
         try {
             if (pcmData) {
                 const opusData = this.opusEncoder.encode(pcmData);
-
                 if (opusData && opusData.length > 0) {
                     this.audioBuffers.push(opusData.buffer);
                     this.totalAudioSize += opusData.length;
-
                     if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
                         try {
                             this.websocket.send(opusData.buffer);
-                            log(`发送Opus帧，大小：${opusData.length}字节`, 'debug');
                         } catch (error) {
                             log(`WebSocket发送错误: ${error.message}`, 'error');
                         }
                     }
                 } else {
-                    log('Opus编码失败，无有效数据返回', 'error');
+                    log('Opus编码失败，未返回有效数据', 'error');
                 }
             } else {
                 if (this.pcmDataBuffer.length > 0) {
@@ -235,96 +199,67 @@ export class AudioRecorder {
         }
     }
 
-    // 开始录音
+    // Start recording
     async start() {
         if (this.isRecording) return false;
-
         try {
-            // 检查是否有WebSocketHandler实例
+            // Check if WebSocketHandler instance exists
-            const { getWebSocketHandler } = await import('../network/websocket.js');
+            const { getWebSocketHandler } = await import('../network/websocket.js?v=0127');
             const wsHandler = getWebSocketHandler();
-
+            // If machine is speaking, send abort message
-            // 如果机器正在说话，发送打断消息
             if (wsHandler && wsHandler.isRemoteSpeaking && wsHandler.currentSessionId) {
-                const abortMessage = {
+                const abortMessage = { session_id: wsHandler.currentSessionId, type: 'abort', reason: 'wake_word_detected' };
-                    session_id: wsHandler.currentSessionId,
-                    type: 'abort',
-                    reason: 'wake_word_detected'
-                };
-
                 if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
                     this.websocket.send(JSON.stringify(abortMessage));
-                    log('发送打断消息', 'info');
+                    log('已发送中止消息', 'info');
                 }
             }
-
             if (!this.initEncoder()) {
-                log('无法启动录音: Opus编码器初始化失败', 'error');
+                log('无法开始录音: Opus编码器初始化失败', 'error');
                 return false;
             }
-
+            log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
-            log('请至少录制1-2秒钟的音频，确保采集到足够数据', 'info');
+            const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
-
-            const stream = await navigator.mediaDevices.getUserMedia({
-                audio: {
-                    echoCancellation: true,
-                    noiseSuppression: true,
-                    sampleRate: 16000,
-                    channelCount: 1
-                }
-            });
-
             this.audioContext = this.getAudioContext();
-
             if (this.audioContext.state === 'suspended') {
                 await this.audioContext.resume();
             }
-
             const processorResult = await this.createAudioProcessor();
             if (!processorResult) {
                 log('无法创建音频处理器', 'error');
                 return false;
             }
-
             this.audioProcessor = processorResult.node;
             this.audioProcessorType = processorResult.type;
-
             this.audioSource = this.audioContext.createMediaStreamSource(stream);
             this.analyser = this.audioContext.createAnalyser();
             this.analyser.fftSize = 2048;
-
             this.audioSource.connect(this.analyser);
             this.audioSource.connect(this.audioProcessor);
-
             this.pcmDataBuffer = new Int16Array();
             this.audioBuffers = [];
             this.totalAudioSize = 0;
             this.isRecording = true;
-
             if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
                 this.audioProcessor.port.postMessage({ command: 'start' });
             }
-
+            // Send listening start message
-            // 发送监听开始消息
             if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
-                log(`发送录音开始消息`, 'info');
+                log(`已发送录音开始消息`, 'info');
             } else {
                 log('WebSocket未连接，无法发送开始消息', 'error');
                 return false;
             }
-
+            // Start visualization
-            // 开始可视化
             if (this.onVisualizerUpdate) {
                 const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
                 this.startVisualization(dataArray);
             }
-
+            // Immediately notify recording start, update button state
-            // 立即通知录音开始，更新按钮状态
             if (this.onRecordingStart) {
                 this.onRecordingStart(0);
             }
-
+            // Start recording timer
-            // 启动录音计时器
             let recordingSeconds = 0;
             this.recordingTimer = setInterval(() => {
                 recordingSeconds += 0.1;
@@ -332,8 +267,7 @@ export class AudioRecorder {
                     this.onRecordingStart(recordingSeconds);
                 }
             }, 100);
-
+            log('已开始PCM直接录音', 'success');
-            log('开始PCM直接录音', 'success');
             return true;
         } catch (error) {
             log(`直接录音启动错误: ${error.message}`, 'error');
@@ -342,15 +276,12 @@ export class AudioRecorder {
         }
     }
 
-    // 开始可视化
+    // Start visualization
     startVisualization(dataArray) {
         const draw = () => {
             this.visualizationRequest = requestAnimationFrame(() => draw());
-
             if (!this.isRecording) return;
-
             this.analyser.getByteFrequencyData(dataArray);
-
             if (this.onVisualizerUpdate) {
                 this.onVisualizerUpdate(dataArray);
             }
@@ -358,52 +289,42 @@ export class AudioRecorder {
         draw();
     }
 
-    // 停止录音
+    // Stop recording
     stop() {
         if (!this.isRecording) return false;
-
         try {
             this.isRecording = false;
-
             if (this.audioProcessor) {
                 if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
                     this.audioProcessor.port.postMessage({ command: 'stop' });
                 }
-
                 this.audioProcessor.disconnect();
                 this.audioProcessor = null;
             }
-
             if (this.audioSource) {
                 this.audioSource.disconnect();
                 this.audioSource = null;
             }
-
             if (this.visualizationRequest) {
                 cancelAnimationFrame(this.visualizationRequest);
                 this.visualizationRequest = null;
             }
-
             if (this.recordingTimer) {
                 clearInterval(this.recordingTimer);
                 this.recordingTimer = null;
             }
-
+            // Encode and send remaining data
-            // 编码并发送剩余的数据
             this.encodeAndSendOpus();
-
+            // Send end signal
-            // 发送结束信号
             if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
                 const emptyOpusFrame = new Uint8Array(0);
                 this.websocket.send(emptyOpusFrame);
                 log('已发送录音停止信号', 'info');
             }
-
             if (this.onRecordingStop) {
                 this.onRecordingStop();
             }
-
+            log('已停止PCM直接录音', 'success');
-            log('停止PCM直接录音', 'success');
             return true;
         } catch (error) {
             log(`直接录音停止错误: ${error.message}`, 'error');
@@ -411,13 +332,13 @@ export class AudioRecorder {
         }
     }
 
-    // 获取分析器
+    // Get analyser
     getAnalyser() {
         return this.analyser;
     }
 }
 
-// 创建单例
+// Create singleton instance
 let audioRecorderInstance = null;
 
 export function getAudioRecorder() {
@@ -426,3 +347,49 @@ export function getAudioRecorder() {
     }
     return audioRecorderInstance;
 }
+
+/**
+ * Check if microphone is available
+ * @returns {Promise<boolean>} Returns true if available, false if not available
+ */
+export async function checkMicrophoneAvailability() {
+    // Check if browser supports getUserMedia API
+    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+        log('浏览器不支持getUserMedia API', 'warning');
+        return false;
+    }
+    try {
+        // Try to access microphone
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
+        // Immediately stop all tracks to release microphone
+        stream.getTracks().forEach(track => track.stop());
+        log('麦克风可用性检查成功', 'success');
+        return true;
+    } catch (error) {
+        log(`麦克风不可用: ${error.message}`, 'warning');
+        return false;
+    }
+}
+
+/**
+ * Check if it is HTTP non-localhost access
+ * @returns {boolean} Returns true if it is HTTP non-localhost access
+ */
+export function isHttpNonLocalhost() {
+    const protocol = window.location.protocol;
+    const hostname = window.location.hostname;
+    // Check if it is HTTP protocol
+    if (protocol !== 'http:') {
+        return false;
+    }
+    // localhost and 127.0.0.1 can use microphone
+    if (hostname === 'localhost' || hostname === '127.0.0.1') {
+        return false;
+    }
+    // Private IP addresses can also use microphone (browser allows)
+    if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
+        return false;
+    }
+    // Other HTTP access is considered non-localhost
+    return true;
+}

main/xiaozhi-server/test/js/core/audio/stream-context.js

@@ -1,5 +1,5 @@
-import BlockingQueue from '../../utils/blocking-queue.js';
+import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
-import { log } from '../../utils/logger.js';
+import { log } from '../../utils/logger.js?v=0127';
 
 // 音频流播放上下文类
 export class StreamingContext {

main/xiaozhi-server/test/js/core/mcp/tools.js

@@ -1,4 +1,4 @@
-import { log } from '../../utils/logger.js';
+import { log } from '../../utils/logger.js?v=0127';
 
 // ==========================================
 // MCP 工具管理逻辑
@@ -24,11 +24,19 @@ export function setWebSocket(ws) {
 export async function initMcpTools() {
     // 加载默认工具数据
     const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
-
     const savedTools = localStorage.getItem('mcpTools');
     if (savedTools) {
         try {
-            mcpTools = JSON.parse(savedTools);
+            const parsedTools = JSON.parse(savedTools);
+            // 合并默认工具和用户保存的工具，保留用户自定义的工具
+            const defaultToolNames = new Set(defaultMcpTools.map(t => t.name));
+            // 添加默认工具中不存在的新工具
+            parsedTools.forEach(tool => {
+                if (!defaultToolNames.has(tool.name)) {
+                    defaultMcpTools.push(tool);
+                }
+            });
+            mcpTools = defaultMcpTools;
         } catch (e) {
             log('加载MCP工具失败，使用默认工具', 'warning');
             mcpTools = [...defaultMcpTools];
@@ -36,9 +44,11 @@ export async function initMcpTools() {
     } else {
         mcpTools = [...defaultMcpTools];
     }
-
     renderMcpTools();
-    setupMcpEventListeners();
+    // Only setup event listeners if DOM elements exist
+    if (document.getElementById('toggleMcpTools')) {
+        setupMcpEventListeners();
+    }
 }
 
 /**
@@ -47,21 +57,20 @@ export async function initMcpTools() {
 function renderMcpTools() {
     const container = document.getElementById('mcpToolsContainer');
     const countSpan = document.getElementById('mcpToolsCount');
-
+    if (!container) {
+        return; // Container not found, skip rendering
+    }
     if (countSpan) {
         countSpan.textContent = `${mcpTools.length} 个工具`;
     }
-
     if (mcpTools.length === 0) {
         container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
         return;
     }
-
     container.innerHTML = mcpTools.map((tool, index) => {
         const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
         const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
         const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
-
         return `
             <div class="mcp-tool-card">
                 <div class="mcp-tool-header">
@@ -96,100 +105,167 @@ function renderMcpTools() {
  */
 function renderMcpProperties() {
     const container = document.getElementById('mcpPropertiesContainer');
-
+    const emptyState = document.getElementById('mcpEmptyState');
+    if (!container) {
+        return; // Container not found, skip rendering
+    }
     if (mcpProperties.length === 0) {
-        container.innerHTML = '<div style="text-align: center; padding: 20px; color: #999; font-size: 14px;">暂无参数，点击下方按钮添加参数</div>';
+        if (emptyState) {
+            emptyState.style.display = 'block';
+        }
+        container.innerHTML = '';
         return;
     }
-
+    if (emptyState) {
+        emptyState.style.display = 'none';
+    }
     container.innerHTML = mcpProperties.map((prop, index) => `
-        <div class="mcp-property-item">
+        <div class="mcp-property-card" onclick="window.mcpModule.editMcpProperty(${index})">
-            <div class="mcp-property-header">
+            <div class="mcp-property-row-label">
-                <span class="mcp-property-name">${prop.name}</span>
+                <span class="mcp-property-label">参数名称</span>
-                <button type="button" onclick="window.mcpModule.deleteMcpProperty(${index})"
+                <span class="mcp-property-value">${prop.name}${prop.required ? ' <span class="mcp-property-required-badge">[必填]</span>' : ''}</span>
-                    style="padding: 3px 8px; border: none; border-radius: 3px; background-color: #f44336; color: white; cursor: pointer; font-size: 11px;">
-                    删除
-                </button>
             </div>
-            <div class="mcp-property-row">
+            <div class="mcp-property-row-label">
-                <div>
+                <span class="mcp-property-label">数据类型</span>
-                    <label class="mcp-small-label">参数名称 *</label>
+                <span class="mcp-property-value">${getTypeLabel(prop.type)}</span>
-                    <input type="text" class="mcp-small-input" value="${prop.name}"
-                        onchange="window.mcpModule.updateMcpProperty(${index}, 'name', this.value)" required>
-                </div>
-                <div>
-                    <label class="mcp-small-label">数据类型 *</label>
-                    <select class="mcp-small-input" onchange="window.mcpModule.updateMcpProperty(${index}, 'type', this.value)">
-                        <option value="string" ${prop.type === 'string' ? 'selected' : ''}>字符串</option>
-                        <option value="integer" ${prop.type === 'integer' ? 'selected' : ''}>整数</option>
-                        <option value="number" ${prop.type === 'number' ? 'selected' : ''}>数字</option>
-                        <option value="boolean" ${prop.type === 'boolean' ? 'selected' : ''}>布尔值</option>
-                        <option value="array" ${prop.type === 'array' ? 'selected' : ''}>数组</option>
-                        <option value="object" ${prop.type === 'object' ? 'selected' : ''}>对象</option>
-                    </select>
-                </div>
             </div>
-            ${(prop.type === 'integer' || prop.type === 'number') ? `
+            <div class="mcp-property-row-label">
-            <div class="mcp-property-row">
+                <span class="mcp-property-label">描述</span>
-                <div>
+                <span class="mcp-property-value">${prop.description || '-'}</span>
-                    <label class="mcp-small-label">最小值</label>
-                    <input type="number" class="mcp-small-input" value="${prop.minimum !== undefined ? prop.minimum : ''}"
-                        placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'minimum', this.value ? parseFloat(this.value) : undefined)">
-                </div>
-                <div>
-                    <label class="mcp-small-label">最大值</label>
-                    <input type="number" class="mcp-small-input" value="${prop.maximum !== undefined ? prop.maximum : ''}"
-                        placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'maximum', this.value ? parseFloat(this.value) : undefined)">
-                </div>
             </div>
-            ` : ''}
+            <div class="mcp-property-row-action">
-            <div class="mcp-property-row-full">
+                <button class="mcp-property-delete-btn" onclick="event.stopPropagation(); window.mcpModule.deleteMcpProperty(${index})">删除</button>
-                <label class="mcp-small-label">参数描述</label>
-                <input type="text" class="mcp-small-input" value="${prop.description || ''}"
-                    placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'description', this.value)">
             </div>
-            <label class="mcp-checkbox-label">
-                <input type="checkbox" ${prop.required ? 'checked' : ''}
-                    onchange="window.mcpModule.updateMcpProperty(${index}, 'required', this.checked)">
-                必填参数
-            </label>
         </div>
     `).join('');
 }
 
 /**
- * 添加参数
+ * 获取数据类型标签
  */
-function addMcpProperty() {
+function getTypeLabel(type) {
-    mcpProperties.push({
+    const typeMap = {
-        name: `param_${mcpProperties.length + 1}`,
+        'string': '字符串',
-        type: 'string',
+        'integer': '整数',
-        required: false,
+        'number': '数字',
-        description: ''
+        'boolean': '布尔值',
-    });
+        'array': '数组',
-    renderMcpProperties();
+        'object': '对象'
+    };
+    return typeMap[type] || type;
 }
 
 /**
- * 更新参数
+ * 添加参数 - 打开参数编辑模态框
  */
-function updateMcpProperty(index, field, value) {
+function addMcpProperty() {
-    if (field === 'name') {
+    openPropertyModal();
-        const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === value);
+}
-        if (isDuplicate) {
+
-            alert('参数名称已存在，请使用不同的名称');
+/**
-            renderMcpProperties();
+ * 编辑参数 - 打开参数编辑模态框
-            return;
+ */
+function editMcpProperty(index) {
+    openPropertyModal(index);
+}
+
+/**
+ * 打开参数编辑模态框
+ */
+function openPropertyModal(index = null) {
+    const form = document.getElementById('mcpPropertyForm');
+    const title = document.getElementById('mcpPropertyModalTitle');
+    document.getElementById('mcpPropertyIndex').value = index !== null ? index : -1;
+
+    if (index !== null) {
+        const prop = mcpProperties[index];
+        title.textContent = '编辑参数';
+        document.getElementById('mcpPropertyName').value = prop.name;
+        document.getElementById('mcpPropertyType').value = prop.type || 'string';
+        document.getElementById('mcpPropertyMinimum').value = prop.minimum !== undefined ? prop.minimum : '';
+        document.getElementById('mcpPropertyMaximum').value = prop.maximum !== undefined ? prop.maximum : '';
+        document.getElementById('mcpPropertyDescription').value = prop.description || '';
+        document.getElementById('mcpPropertyRequired').checked = prop.required || false;
+    } else {
+        title.textContent = '添加参数';
+        form.reset();
+        document.getElementById('mcpPropertyName').value = `param_${mcpProperties.length + 1}`;
+        document.getElementById('mcpPropertyType').value = 'string';
+        document.getElementById('mcpPropertyMinimum').value = '';
+        document.getElementById('mcpPropertyMaximum').value = '';
+        document.getElementById('mcpPropertyDescription').value = '';
+        document.getElementById('mcpPropertyRequired').checked = false;
+    }
+
+    updatePropertyRangeVisibility();
+    document.getElementById('mcpPropertyModal').style.display = 'flex';
+}
+
+/**
+ * 关闭参数编辑模态框
+ */
+function closePropertyModal() {
+    document.getElementById('mcpPropertyModal').style.display = 'none';
+}
+
+/**
+ * 更新数值范围输入框的可见性
+ */
+function updatePropertyRangeVisibility() {
+    const type = document.getElementById('mcpPropertyType').value;
+    const rangeGroup = document.getElementById('mcpPropertyRangeGroup');
+    if (type === 'integer' || type === 'number') {
+        rangeGroup.style.display = 'block';
+    } else {
+        rangeGroup.style.display = 'none';
+    }
+}
+
+/**
+ * 处理参数表单提交
+ */
+function handlePropertySubmit(e) {
+    e.preventDefault();
+    const index = parseInt(document.getElementById('mcpPropertyIndex').value);
+    const name = document.getElementById('mcpPropertyName').value.trim();
+    const type = document.getElementById('mcpPropertyType').value;
+    const minimum = document.getElementById('mcpPropertyMinimum').value;
+    const maximum = document.getElementById('mcpPropertyMaximum').value;
+    const description = document.getElementById('mcpPropertyDescription').value.trim();
+    const required = document.getElementById('mcpPropertyRequired').checked;
+
+    // 检查名称重复
+    const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === name);
+    if (isDuplicate) {
+        alert('参数名称已存在，请使用不同的名称');
+        return;
+    }
+
+    const propData = {
+        name,
+        type,
+        description,
+        required
+    };
+
+    // 数值类型添加范围限制
+    if (type === 'integer' || type === 'number') {
+        if (minimum !== '') {
+            propData.minimum = parseFloat(minimum);
+        }
+        if (maximum !== '') {
+            propData.maximum = parseFloat(maximum);
         }
     }
 
-    mcpProperties[index][field] = value;
+    if (index >= 0) {
-
+        mcpProperties[index] = propData;
-    if (field === 'type' && value !== 'integer' && value !== 'number') {
+    } else {
-        delete mcpProperties[index].minimum;
+        mcpProperties.push(propData);
-        delete mcpProperties[index].maximum;
-        renderMcpProperties();
     }
+
+    renderMcpProperties();
+    closePropertyModal();
 }
 
 /**
@@ -213,25 +289,43 @@ function setupMcpEventListeners() {
     const form = document.getElementById('mcpToolForm');
     const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
 
+    // 参数编辑模态框相关元素
+    const propertyModal = document.getElementById('mcpPropertyModal');
+    const closePropertyBtn = document.getElementById('closeMcpPropertyModalBtn');
+    const cancelPropertyBtn = document.getElementById('cancelMcpPropertyBtn');
+    const propertyForm = document.getElementById('mcpPropertyForm');
+    const propertyTypeSelect = document.getElementById('mcpPropertyType');
+
+    // Return early if required elements don't exist (e.g., in test environment)
+    if (!toggleBtn || !panel || !addBtn || !modal || !closeBtn || !cancelBtn || !form || !addPropertyBtn) {
+        return;
+    }
     toggleBtn.addEventListener('click', () => {
         const isExpanded = panel.classList.contains('expanded');
         panel.classList.toggle('expanded');
-        toggleBtn.textContent = isExpanded ? '展开' : '收起';
+        toggleBtn.textContent = isExpanded ? '收起' : '展开';
     });
-    
     // 确保面板默认展开
     panel.classList.add('expanded');
-
     addBtn.addEventListener('click', () => openMcpModal());
     closeBtn.addEventListener('click', closeMcpModal);
     cancelBtn.addEventListener('click', closeMcpModal);
     addPropertyBtn.addEventListener('click', addMcpProperty);
-
     modal.addEventListener('click', (e) => {
         if (e.target === modal) closeMcpModal();
     });
-
     form.addEventListener('submit', handleMcpSubmit);
+
+    // 参数编辑模态框事件
+    if (propertyModal && closePropertyBtn && cancelPropertyBtn && propertyForm && propertyTypeSelect) {
+        closePropertyBtn.addEventListener('click', closePropertyModal);
+        cancelPropertyBtn.addEventListener('click', closePropertyModal);
+        propertyModal.addEventListener('click', (e) => {
+            if (e.target === propertyModal) closePropertyModal();
+        });
+        propertyForm.addEventListener('submit', handlePropertySubmit);
+        propertyTypeSelect.addEventListener('change', updatePropertyRangeVisibility);
+    }
 }
 
 /**
@@ -243,18 +337,15 @@ function openMcpModal(index = null) {
         alert('WebSocket 已连接，无法编辑工具');
         return;
     }
-
     mcpEditingIndex = index;
     const errorContainer = document.getElementById('mcpErrorContainer');
     errorContainer.innerHTML = '';
-
     if (index !== null) {
         document.getElementById('mcpModalTitle').textContent = '编辑工具';
         const tool = mcpTools[index];
         document.getElementById('mcpToolName').value = tool.name;
         document.getElementById('mcpToolDescription').value = tool.description;
         document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
-
         mcpProperties = [];
         const schema = tool.inputSchema;
         if (schema.properties) {
@@ -275,9 +366,8 @@ function openMcpModal(index = null) {
         document.getElementById('mcpToolForm').reset();
         mcpProperties = [];
     }
-
     renderMcpProperties();
-    document.getElementById('mcpToolModal').style.display = 'block';
+    document.getElementById('mcpToolModal').style.display = 'flex';
 }
 
 /**
@@ -298,21 +388,15 @@ function handleMcpSubmit(e) {
     e.preventDefault();
     const errorContainer = document.getElementById('mcpErrorContainer');
     errorContainer.innerHTML = '';
-
     const name = document.getElementById('mcpToolName').value.trim();
     const description = document.getElementById('mcpToolDescription').value.trim();
     const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
-
     // 检查名称重复
-    const isDuplicate = mcpTools.some((tool, index) =>
+    const isDuplicate = mcpTools.some((tool, index) => tool.name === name && index !== mcpEditingIndex);
-        tool.name === name && index !== mcpEditingIndex
-    );
-
     if (isDuplicate) {
         showMcpError('工具名称已存在，请使用不同的名称');
         return;
     }
-
     // 解析模拟返回结果
     let mockResponse = null;
     if (mockResponseText) {
@@ -323,21 +407,13 @@ function handleMcpSubmit(e) {
             return;
         }
     }
-
     // 构建 inputSchema
-    const inputSchema = {
+    const inputSchema = { type: "object", properties: {}, required: [] };
-        type: "object",
-        properties: {},
-        required: []
-    };
-
     mcpProperties.forEach(prop => {
         const propSchema = { type: prop.type };
-
         if (prop.description) {
             propSchema.description = prop.description;
         }
-
         if ((prop.type === 'integer' || prop.type === 'number')) {
             if (prop.minimum !== undefined && prop.minimum !== '') {
                 propSchema.minimum = prop.minimum;
@@ -346,20 +422,15 @@ function handleMcpSubmit(e) {
                 propSchema.maximum = prop.maximum;
             }
         }
-
         inputSchema.properties[prop.name] = propSchema;
-
         if (prop.required) {
             inputSchema.required.push(prop.name);
         }
     });
-
     if (inputSchema.required.length === 0) {
         delete inputSchema.required;
     }
-
     const tool = { name, description, inputSchema, mockResponse };
-
     if (mcpEditingIndex !== null) {
         mcpTools[mcpEditingIndex] = tool;
         log(`已更新工具: ${name}`, 'success');
@@ -367,7 +438,6 @@ function handleMcpSubmit(e) {
         mcpTools.push(tool);
         log(`已添加工具: ${name}`, 'success');
     }
-
     saveMcpTools();
     renderMcpTools();
     closeMcpModal();
@@ -417,32 +487,36 @@ function saveMcpTools() {
  * 获取工具列表
  */
 export function getMcpTools() {
-    return mcpTools.map(tool => ({
+    return mcpTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
-        name: tool.name,
-        description: tool.description,
-        inputSchema: tool.inputSchema
-    }));
 }
 
 /**
  * 执行工具调用
  */
-export function executeMcpTool(toolName, toolArgs) {
+export async function executeMcpTool(toolName, toolArgs) {
     const tool = mcpTools.find(t => t.name === toolName);
-
     if (!tool) {
         log(`未找到工具: ${toolName}`, 'error');
-        return {
+        return { success: false, error: `未知工具: ${toolName}` };
-            success: false,
+    }
-            error: `未知工具: ${toolName}`
+
-        };
+    // 处理拍照工具
+    if (toolName === 'self_camera_take_photo') {
+        if (typeof window.takePhoto === 'function') {
+            const question = toolArgs && toolArgs.question ? toolArgs.question : '描述一下看到的物品';
+            log(`正在执行拍照: ${question}`, 'info');
+            const result = await window.takePhoto(question);
+            return result;
+        } else {
+            log('拍照功能不可用', 'warning');
+            return { success: false, error: '摄像头未启动或不支持拍照功能' };
+        }
     }
 
     // 如果有模拟返回结果，使用它
     if (tool.mockResponse) {
         // 替换模板变量
         let responseStr = JSON.stringify(tool.mockResponse);
-
         // 替换 ${paramName} 格式的变量
         if (toolArgs) {
             Object.keys(toolArgs).forEach(key => {
@@ -450,7 +524,6 @@ export function executeMcpTool(toolName, toolArgs) {
                 responseStr = responseStr.replace(regex, toolArgs[key]);
             });
         }
-
         try {
             const response = JSON.parse(responseStr);
             log(`工具 ${toolName} 执行成功，返回模拟结果: ${responseStr}`, 'success');
@@ -460,21 +533,10 @@ export function executeMcpTool(toolName, toolArgs) {
             return tool.mockResponse;
         }
     }
-
     // 没有模拟返回结果，返回默认成功消息
     log(`工具 ${toolName} 执行成功，返回默认结果`, 'success');
-    return {
+    return { success: true, message: `工具 ${toolName} 执行成功`, tool: toolName, arguments: toolArgs };
-        success: true,
-        message: `工具 ${toolName} 执行成功`,
-        tool: toolName,
-        arguments: toolArgs
-    };
 }
 
 // 暴露全局方法供 HTML 内联事件调用
-window.mcpModule = {
+window.mcpModule = { addMcpProperty, editMcpProperty, deleteMcpProperty, editMcpTool, deleteMcpTool };
-    updateMcpProperty,
-    deleteMcpProperty,
-    editMcpTool,
-    deleteMcpTool
-};

main/xiaozhi-server/test/js/core/network/ota-connector.js

@@ -1,4 +1,4 @@
-import { log } from '../../utils/logger.js';
+import { log } from '../../utils/logger.js?v=0127';
 
 // WebSocket 连接
 export async function webSocketConnect(otaUrl, config) {

main/xiaozhi-server/test/js/core/network/websocket.js

@@ -1,11 +1,11 @@
 // WebSocket消息处理模块
-import { log } from '../../utils/logger.js';
+import { getConfig, saveConnectionUrls } from '../../config/manager.js?v=0127';
-import { webSocketConnect } from './ota-connector.js';
+import { uiController } from '../../ui/controller.js?v=0127';
-import { getConfig, saveConnectionUrls } from '../../config/manager.js';
+import { log } from '../../utils/logger.js?v=0127';
-import { getAudioPlayer } from '../audio/player.js';
+import { getAudioPlayer } from '../audio/player.js?v=0127';
-import { getAudioRecorder } from '../audio/recorder.js';
+import { getAudioRecorder } from '../audio/recorder.js?v=0127';
-import { getMcpTools, executeMcpTool, setWebSocket as setMcpWebSocket } from '../mcp/tools.js';
+import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js?v=0127';
-import { uiController } from '../../ui/controller.js'
+import { webSocketConnect } from './ota-connector.js?v=0127';
 
 // WebSocket处理器类
 export class WebSocketHandler {
@@ -74,6 +74,9 @@ export class WebSocketHandler {
     handleTextMessage(message) {
         if (message.type === 'hello') {
             log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success');
+            window.cameraAvailable = true;
+            log('连接成功，摄像头已可用', 'success');
+            uiController.updateDialButton(true);
             uiController.startAIChatSession();
         } else if (message.type === 'tts') {
             this.handleTTSMessage(message);
@@ -81,6 +84,23 @@ export class WebSocketHandler {
             log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
         } else if (message.type === 'stt') {
             log(`识别结果: ${message.text}`, 'info');
+            // 检查是否需要绑定设备
+            if (message.text && (message.text.includes('绑定') || message.text.includes('bind'))) {
+                log('收到设备绑定提示，更新摄像头状态', 'warning');
+                window.cameraAvailable = false;
+                // 关闭摄像头
+                if (typeof window.stopCamera === 'function') {
+                    window.stopCamera();
+                }
+                // 更新摄像头按钮状态
+                const cameraBtn = document.getElementById('cameraBtn');
+                if (cameraBtn) {
+                    cameraBtn.classList.remove('camera-active');
+                    cameraBtn.querySelector('.btn-text').textContent = '摄像头';
+                    cameraBtn.disabled = true;
+                    cameraBtn.title = '请先绑定验证码';
+                }
+            }
             // 使用新的聊天消息回调显示STT消息
             if (this.onChatMessage && message.text) {
                 this.onChatMessage(message.text, true);
@@ -101,10 +121,10 @@ export class WebSocketHandler {
                 }
 
                 // 触发Live2D情绪动作
-            if (message.emotion) {
+                if (message.emotion) {
-                console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
+                    console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
-                this.triggerLive2DEmotionAction(message.emotion);
+                    this.triggerLive2DEmotionAction(message.emotion);
-            }
+                }
             }
 
             // 只有当文本不仅仅是表情时，才添加到对话中
@@ -249,7 +269,56 @@ export class WebSocketHandler {
 
             log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info');
 
-            const result = executeMcpTool(toolName, toolArgs);
+            executeMcpTool(toolName, toolArgs).then(result => {
+                const replyMessage = JSON.stringify({
+                    "session_id": message.session_id || "",
+                    "type": "mcp",
+                    "payload": {
+                        "jsonrpc": "2.0",
+                        "id": payload.id,
+                        "result": {
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": JSON.stringify(result)
+                                }
+                            ],
+                            "isError": false
+                        }
+                    }
+                });
+
+                log(`客户端上报: ${replyMessage}`, 'info');
+                this.websocket.send(replyMessage);
+            }).catch(error => {
+                log(`工具执行失败: ${error.message}`, 'error');
+                const errorReply = JSON.stringify({
+                    "session_id": message.session_id || "",
+                    "type": "mcp",
+                    "payload": {
+                        "jsonrpc": "2.0",
+                        "id": payload.id,
+                        "error": {
+                            "code": -32603,
+                            "message": error.message
+                        }
+                    }
+                });
+                this.websocket.send(errorReply);
+            });
+        } else if (payload.method === 'initialize') {
+            log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
+            // 保存视觉分析接口地址
+            const visionUrl = document.getElementById('visionUrl');
+            const visionConfig = payload?.params?.capabilities?.vision;
+            if (visionConfig && typeof visionConfig === 'object' && visionConfig.url && visionConfig.token) {
+                const visionConfigStr = JSON.stringify(visionConfig);
+                localStorage.setItem('xz_tester_vision', visionConfigStr);
+                if (visionUrl) visionUrl.value = visionConfig.url;
+            } else {
+                localStorage.removeItem('xz_tester_vision');
+                if (visionUrl) visionUrl.value = '';
+            }
 
             const replyMessage = JSON.stringify({
                 "session_id": message.session_id || "",
@@ -258,21 +327,19 @@ export class WebSocketHandler {
                     "jsonrpc": "2.0",
                     "id": payload.id,
                     "result": {
-                        "content": [
+                        "protocolVersion": "2024-11-05",
-                            {
+                        "capabilities": {
-                                "type": "text",
+                            "tools": {}
-                                "text": JSON.stringify(result)
+                        },
-                            }
+                        "serverInfo": {
-                        ],
+                            "name": "xiaozhi-web-test",
-                        "isError": false
+                            "version": "2.1.0"
+                        }
                     }
                 }
             });
-
+            log(`回复初始化响应`, 'info');
-            log(`客户端上报: ${replyMessage}`, 'info');
             this.websocket.send(replyMessage);
-        } else if (payload.method === 'initialize') {
-            log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
         } else {
             log(`未知的MCP方法: ${payload.method}`, 'warning');
         }
@@ -284,7 +351,6 @@ export class WebSocketHandler {
             let arrayBuffer;
             if (data instanceof ArrayBuffer) {
                 arrayBuffer = data;
-                log(`收到ArrayBuffer音频数据，大小: ${data.byteLength}字节`, 'debug');
             } else if (data instanceof Blob) {
                 arrayBuffer = await data.arrayBuffer();
                 log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`, 'debug');
@@ -368,11 +434,22 @@ export class WebSocketHandler {
 
             const audioRecorder = getAudioRecorder();
             audioRecorder.stop();
+
+            // 关闭摄像头
+            if (typeof window.stopCamera === 'function') {
+                window.stopCamera();
+            }
+
+            // 隐藏摄像头显示区域
+            const cameraContainer = document.getElementById('cameraContainer');
+            if (cameraContainer) {
+                cameraContainer.classList.remove('active');
+            }
         };
 
         this.websocket.onerror = (error) => {
             log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
-
+            uiController.addChatMessage(`⚠️ WebSocket错误: ${error.message || '未知错误'}`, false);
             if (this.onConnectionStateChange) {
                 this.onConnectionStateChange(false);
             }
@@ -401,6 +478,17 @@ export class WebSocketHandler {
         this.websocket.close();
         const audioRecorder = getAudioRecorder();
         audioRecorder.stop();
+
+        // 关闭摄像头
+        if (typeof window.stopCamera === 'function') {
+            window.stopCamera();
+        }
+
+        // 隐藏摄像头显示区域
+        const cameraContainer = document.getElementById('cameraContainer');
+        if (cameraContainer) {
+            cameraContainer.classList.remove('active');
+        }
     }
 
     // 发送文本消息

main/xiaozhi-server/test/js/live2d/live2d.js

@@ -12,7 +12,42 @@ class Live2DManager {
         this.audioContext = null;
         this.analyser = null;
         this.dataArray = null;
-        this.lastEmotionActionTime = null; // 上次情绪触发动作的时间
+        this.lastEmotionActionTime = null;
+        this.currentModelName = null;
+
+        // 模型特定配置
+        this.modelConfig = {
+            'hiyori_pro_zh': {
+                mouthParam: 'ParamMouthOpenY',
+                mouthAmplitude: 1.0,
+                mouthThresholds: { low: 0.3, high: 0.7 },
+                motionMap: {
+                    'FlickUp': 'FlickUp',
+                    'FlickDown': 'FlickDown',
+                    'Tap': 'Tap',
+                    'Tap@Body': 'Tap@Body',
+                    'Flick': 'Flick',
+                    'Flick@Body': 'Flick@Body'
+                }
+            },
+            'natori_pro_zh': {
+                mouthParam: 'ParamMouthOpenY',
+                mouthAmplitude: 1.0,
+                mouthThresholds: { low: 0.1, high: 0.4 },
+                mouthFormParam: 'ParamMouthForm',
+                mouthFormAmplitude: 1.0,
+                mouthForm2Param: 'ParamMouthForm2',
+                mouthForm2Amplitude: 0.8,
+                motionMap: {
+                    'FlickUp': 'FlickUp',
+                    'FlickDown': 'Flick@Body',
+                    'Tap': 'Tap',
+                    'Tap@Body': 'Tap@Head',
+                    'Flick': 'Tap',
+                    'Flick@Body': 'Flick@Body'
+                }
+            }
+        };
 
         // 情绪到动作的映射
         this.emotionToActionMap = {
@@ -67,10 +102,33 @@ class Live2DManager {
             const currentPath = window.location.pathname;
             const lastSlashIndex = currentPath.lastIndexOf('/');
             const basePath = currentPath.substring(0, lastSlashIndex + 1);
-            const modelPath = basePath + 'hiyori_pro_zh/runtime/hiyori_pro_t11.model3.json';
+
+            // 从 localStorage 读取上次选择的模型，如果没有则使用默认
+            const savedModelName = localStorage.getItem('live2dModel') || 'hiyori_pro_zh';
+            const modelFileMap = {
+                'hiyori_pro_zh': 'hiyori_pro_t11.model3.json',
+                'natori_pro_zh': 'natori_pro_t06.model3.json'
+            };
+            const modelFileName = modelFileMap[savedModelName] || 'hiyori_pro_t11.model3.json';
+            const modelPath = basePath + 'resources/' + savedModelName + '/runtime/' + modelFileName;
+
             this.live2dModel = await PIXI.live2d.Live2DModel.from(modelPath);
             this.live2dApp.stage.addChild(this.live2dModel);
 
+            // 保存当前模型名称
+            this.currentModelName = savedModelName;
+
+            // 更新下拉框显示
+            const modelSelect = document.getElementById('live2dModelSelect');
+            if (modelSelect) {
+                modelSelect.value = savedModelName;
+            }
+
+            // 设置模型特定的嘴部参数名
+            if (this.modelConfig[savedModelName]) {
+                this.mouthParam = this.modelConfig[savedModelName].mouthParam || 'ParamMouthOpenY';
+            }
+
             // 设置模型属性
             this.live2dModel.scale.set(0.33);
             this.live2dModel.x = (window.innerWidth - this.live2dModel.width) * 0.5;
@@ -364,33 +422,88 @@ class Live2DManager {
         if (internal && internal.coreModel) {
             const coreModel = internal.coreModel;
 
-            // 获取音频分贝值
+            let mouthOpenY = 0;
-            let mouthValue = 0;
+            let mouthForm = 0;
+            let mouthForm2 = 0;
             let average = 0;
+
             if (this.analyser && this.dataArray) {
                 this.analyser.getByteFrequencyData(this.dataArray);
                 average = this.dataArray.reduce((a, b) => a + b) / this.dataArray.length;
 
-                // 优化音量映射函数，使中等音量范围变化更明显
-                // 使用S形曲线函数，在中等音量范围有更好的响应
                 const normalizedVolume = average / 255;
 
-                // S形曲线：在0.3-0.7范围内有最大的斜率（变化最明显）
+                // 获取模型特定的阈值
-                if (normalizedVolume < 0.3) {
+                let lowThreshold = 0.3;
-                    // 低音量：缓慢增长
+                let highThreshold = 0.7;
-                    mouthValue = Math.pow(normalizedVolume / 0.3, 1.5) * 0.3;
+                if (this.currentModelName && this.modelConfig[this.currentModelName]) {
-                } else if (normalizedVolume < 0.7) {
+                    lowThreshold = this.modelConfig[this.currentModelName].mouthThresholds?.low || 0.3;
-                    // 中等音量：线性增长，变化最明显
+                    highThreshold = this.modelConfig[this.currentModelName].mouthThresholds?.high || 0.7;
-                    mouthValue = 0.3 + (normalizedVolume - 0.3) / 0.4 * 0.5;
-                } else {
-                    // 高音量：缓慢接近最大值
-                    mouthValue = 0.8 + Math.pow((normalizedVolume - 0.7) / 0.3, 1.2) * 0.2;
                 }
 
-                // 确保嘴部参数在0-1范围内
+                // 使用模型特定的阈值进行映射
-                mouthValue = Math.min(Math.max(mouthValue, 0), 1);
+                let minOpenY = 0.1;
+                if (this.currentModelName && this.modelConfig[this.currentModelName]) {
+                    minOpenY = this.modelConfig[this.currentModelName].mouthMinOpenY || 0.1;
+                }
+
+                if (normalizedVolume < lowThreshold) {
+                    mouthOpenY = minOpenY + Math.pow(normalizedVolume / lowThreshold, 1.5) * (0.4 - minOpenY);
+                } else if (normalizedVolume < highThreshold) {
+                    mouthOpenY = 0.4 + (normalizedVolume - lowThreshold) / (highThreshold - lowThreshold) * 0.4;
+                } else {
+                    mouthOpenY = 0.8 + Math.pow((normalizedVolume - highThreshold) / (1 - highThreshold), 1.2) * 0.2;
+                }
+
+                // 应用模型特定的嘴部开合幅度
+                let amplitudeMultiplier = 1.0;
+                let maxOpenY = 2.5;
+                if (this.currentModelName && this.modelConfig[this.currentModelName]) {
+                    amplitudeMultiplier = this.modelConfig[this.currentModelName].mouthAmplitude;
+                    maxOpenY = this.modelConfig[this.currentModelName].maxOpenY || 2.5;
+                }
+                mouthOpenY = mouthOpenY * amplitudeMultiplier;
+                mouthOpenY = Math.min(Math.max(mouthOpenY, 0), maxOpenY);
+
+                // 计算嘴型参数（仅对支持嘴型变化的模型）
+                if (this.currentModelName && this.modelConfig[this.currentModelName]?.mouthFormParam) {
+                    const config = this.modelConfig[this.currentModelName];
+                    const formAmplitude = config.mouthFormAmplitude || 0.5;
+                    const form2Amplitude = config.mouthForm2Amplitude || 0;
+
+                    // 嘴型随音量变化：
+                    // 低音量：嘴型偏"一"字（负值）
+                    // 高音量：嘴型偏"o"字（正值）
+                    // 音量=0时：嘴型=0（自然状态）
+                    mouthForm = (normalizedVolume - 0.5) * 2 * formAmplitude;
+                    mouthForm = Math.max(-formAmplitude, Math.min(formAmplitude, mouthForm));
+
+                    // 第二嘴型参数（natori特有）
+                    if (config.mouthForm2Param) {
+                        mouthForm2 = (normalizedVolume - 0.3) * 2 * form2Amplitude;
+                        mouthForm2 = Math.max(-form2Amplitude, Math.min(form2Amplitude, mouthForm2));
+                    }
+                }
+
+                // 调试日志：输出嘴部参数
+                console.log(`[Live2D] 模型: ${this.currentModelName || 'unknown'}, 音量: ${average?.toFixed(0)}, OpenY: ${mouthOpenY.toFixed(3)}, Form: ${mouthForm.toFixed(3)}, Form2: ${mouthForm2.toFixed(3)}`);
             }
-            coreModel.setParameterValueById(this.mouthParam, mouthValue);
+
+            // 设置嘴部开合参数
+            coreModel.setParameterValueById(this.mouthParam, mouthOpenY);
+
+            // 设置嘴型参数（仅对支持嘴型变化的模型）
+            if (this.currentModelName && this.modelConfig[this.currentModelName]?.mouthFormParam) {
+                const config = this.modelConfig[this.currentModelName];
+                const formParam = config.mouthFormParam;
+                coreModel.setParameterValueById(formParam, mouthForm);
+
+                // 设置第二嘴型参数（natori特有）
+                if (config.mouthForm2Param) {
+                    coreModel.setParameterValueById(config.mouthForm2Param, mouthForm2);
+                }
+            }
+
             coreModel.update();
         }
         this.mouthAnimationId = requestAnimationFrame(() => this.animateMouth());
@@ -473,12 +586,129 @@ class Live2DManager {
     motion(name) {
         try {
             if (!this.live2dModel) return;
-            this.live2dModel.motion(name);
+
+            // 根据当前模型获取对应的动作名称
+            let actualMotionName = name;
+            if (this.currentModelName && this.modelConfig[this.currentModelName]) {
+                const motionMap = this.modelConfig[this.currentModelName].motionMap;
+                actualMotionName = motionMap[name] || name;
+            }
+
+            this.live2dModel.motion(actualMotionName);
         } catch (error) {
             console.error('触发动作失败:', error);
         }
     }
 
+    /**
+     * 设置模型交互事件
+     */
+    setupModelInteractions() {
+        if (!this.live2dModel) return;
+
+        this.live2dModel.interactive = true;
+
+        this.live2dModel.on('doublehit', (args) => {
+            const area = Array.isArray(args) ? args[0] : args;
+
+            if (area === 'Body') {
+                this.motion('Flick@Body');
+            } else if (area === 'Head' || area === 'Face') {
+                this.motion('Flick');
+            }
+
+            const app = window.chatApp;
+            const payload = JSON.stringify({ type: 'live2d', event: 'doublehit', area });
+            if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
+                app.dataChannel.send(payload);
+            }
+        });
+
+        this.live2dModel.on('singlehit', (args) => {
+            const area = Array.isArray(args) ? args[0] : args;
+
+            if (area === 'Body') {
+                this.motion('Tap@Body');
+            } else if (area === 'Head' || area === 'Face') {
+                this.motion('Tap');
+            }
+
+            const app = window.chatApp;
+            const payload = JSON.stringify({ type: 'live2d', event: 'singlehit', area });
+            if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
+                app.dataChannel.send(payload);
+            }
+        });
+
+        this.live2dModel.on('swipe', (args) => {
+            const area = Array.isArray(args) ? args[0] : args;
+            const dir = Array.isArray(args) ? args[1] : undefined;
+
+            if (area === 'Body') {
+                if (dir === 'up') {
+                    this.motion('FlickUp');
+                } else if (dir === 'down') {
+                    this.motion('FlickDown');
+                }
+            }
+
+            const app = window.chatApp;
+            const payload = JSON.stringify({ type: 'live2d', event: 'swipe', area, dir });
+            if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
+                app.dataChannel.send(payload);
+            }
+        });
+
+        this.live2dModel.on('pointerdown', (event) => {
+            try {
+                const global = event.data.global;
+                const bounds = this.live2dModel.getBounds();
+                if (!bounds || !bounds.contains(global.x, global.y)) return;
+
+                const relX = (global.x - bounds.x) / (bounds.width || 1);
+                const relY = (global.y - bounds.y) / (bounds.height || 1);
+                let area = '';
+
+                if (relX >= 0.4 && relX <= 0.6) {
+                    if (relY <= 0.15) {
+                        area = 'Head';
+                    } else if (relY >= 0.7) {
+                        area = 'Body';
+                    }
+                }
+
+                if (!area) return;
+
+                const now = Date.now();
+                const dt = now - (this._lastClickTime || 0);
+                const dx = global.x - (this._lastClickPos?.x || 0);
+                const dy = global.y - (this._lastClickPos?.y || 0);
+                const dist = Math.hypot(dx, dy);
+
+                if (this._lastClickTime && dt <= this._doubleClickMs && dist <= this._doubleClickDist) {
+                    if (this._singleClickTimer) {
+                        clearTimeout(this._singleClickTimer);
+                        this._singleClickTimer = null;
+                    }
+
+                    this.live2dModel.emit('doublehit', area);
+                    this._lastClickTime = null;
+                    this._lastClickPos = null;
+                } else {
+                    this._lastClickTime = now;
+                    this._lastClickPos = { x: global.x, y: global.y };
+
+                    this._singleClickTimer = setTimeout(() => {
+                        this._singleClickTimer = null;
+                        this.live2dModel.emit('singlehit', area);
+                    }, this._doubleClickMs);
+                }
+            } catch (e) {
+                console.warn('pointerdown 处理出错:', e);
+            }
+        });
+    }
+
     /**
      * 清理资源
      */
@@ -500,6 +730,94 @@ class Live2DManager {
         }
         this.live2dModel = null;
     }
+
+    /**
+     * 切换 Live2D 模型
+     * @param {string} modelName - 模型目录名称，如 'hiyori_pro_zh'、'natori_pro_zh'
+     * @returns {Promise<boolean>} - 切换是否成功
+     */
+    async switchModel(modelName) {
+        try {
+            // 获取模型文件名映射
+            const modelFileMap = {
+                'hiyori_pro_zh': 'hiyori_pro_t11.model3.json',
+                'natori_pro_zh': 'natori_pro_t06.model3.json',
+                'chitose': 'chitose.model3.json',
+                'haru_greeter_pro_jp': 'haru_greeter_t05.model3.json'
+            };
+
+            const modelFileName = modelFileMap[modelName];
+            if (!modelFileName) {
+                console.error('未知的模型名称:', modelName);
+                return false;
+            }
+
+            // 获取基础路径
+            const currentPath = window.location.pathname;
+            const lastSlashIndex = currentPath.lastIndexOf('/');
+            const basePath = currentPath.substring(0, lastSlashIndex + 1);
+            const modelPath = basePath + 'resources/' + modelName + '/runtime/' + modelFileName;
+
+            // 如果已存在模型，先移除
+            if (this.live2dModel) {
+                this.live2dApp.stage.removeChild(this.live2dModel);
+                this.live2dModel.destroy();
+                this.live2dModel = null;
+            }
+
+            // 显示加载状态
+            const app = window.chatApp;
+            if (app) {
+                app.setModelLoadingStatus(true);
+            }
+
+            // 加载新模型
+            this.live2dModel = await PIXI.live2d.Live2DModel.from(modelPath);
+            this.live2dApp.stage.addChild(this.live2dModel);
+
+            // 设置模型属性
+            this.live2dModel.scale.set(0.33);
+            this.live2dModel.x = (window.innerWidth - this.live2dModel.width) * 0.5;
+            this.live2dModel.y = -50;
+
+            // 重新绑定交互事件
+            this.setupModelInteractions();
+
+            // 隐藏加载状态
+            if (app) {
+                app.setModelLoadingStatus(false);
+            }
+
+            // 保存当前模型名称
+            this.currentModelName = modelName;
+
+            // 设置模型特定的嘴部参数名
+            if (this.modelConfig[modelName]) {
+                this.mouthParam = this.modelConfig[modelName].mouthParam || 'ParamMouthOpenY';
+            }
+
+            // 保存到 localStorage
+            localStorage.setItem('live2dModel', modelName);
+
+            // 更新下拉框显示
+            const modelSelect = document.getElementById('live2dModelSelect');
+            if (modelSelect) {
+                modelSelect.value = modelName;
+            }
+
+            console.log('模型切换成功:', modelName);
+            return true;
+        } catch (error) {
+            console.error('切换模型失败:', error);
+            const app = window.chatApp;
+            if (app) {
+                app.setModelLoadingStatus(false);
+            }
+            return false;
+        }
+    }
+
+
 }
 
 // 导出全局实例

main/xiaozhi-server/test/js/ui/controller.js

@@ -1,31 +1,33 @@
-// UI控制模块
+// UI controller module
-import { loadConfig, saveConfig } from '../config/manager.js';
+import { loadConfig, saveConfig } from '../config/manager.js?v=0127';
-import { getAudioRecorder } from '../core/audio/recorder.js';
+import { getAudioPlayer } from '../core/audio/player.js?v=0127';
-import { getWebSocketHandler } from '../core/network/websocket.js';
+import { getAudioRecorder } from '../core/audio/recorder.js?v=0127';
-import { getAudioPlayer } from '../core/audio/player.js';
+import { getWebSocketHandler } from '../core/network/websocket.js?v=0127';
 
-// UI控制器类
+// UI controller class
 class UIController {
     constructor() {
         this.isEditing = false;
         this.visualizerCanvas = null;
         this.visualizerContext = null;
         this.audioStatsTimer = null;
-        this.currentBackgroundIndex = 0;
+        this.currentBackgroundIndex = localStorage.getItem('backgroundIndex') ? parseInt(localStorage.getItem('backgroundIndex')) : 0;
         this.backgroundImages = ['1.png', '2.png', '3.png'];
+        this.dialBtnDisabled = false;
 
-        // 绑定方法
+        // Bind methods
         this.init = this.init.bind(this);
         this.initEventListeners = this.initEventListeners.bind(this);
         this.updateDialButton = this.updateDialButton.bind(this);
         this.addChatMessage = this.addChatMessage.bind(this);
         this.switchBackground = this.switchBackground.bind(this);
+        this.switchLive2DModel = this.switchLive2DModel.bind(this);
         this.showModal = this.showModal.bind(this);
         this.hideModal = this.hideModal.bind(this);
         this.switchTab = this.switchTab.bind(this);
     }
 
-    // 初始化
+    // Initialize
     init() {
         console.log('UIController init started');
 
@@ -35,7 +37,7 @@ class UIController {
             this.initVisualizer();
         }
 
-        // 检查连接按钮在初始化时是否存在
+        // Check if connect button exists during initialization
         const connectBtn = document.getElementById('connectBtn');
         console.log('connectBtn during init:', connectBtn);
 
@@ -43,20 +45,26 @@ class UIController {
         this.startAudioStatsMonitor();
         loadConfig();
 
-        // 设置录音器回调
+        // Register recording callback
         const audioRecorder = getAudioRecorder();
         audioRecorder.onRecordingStart = (seconds) => {
             this.updateRecordButtonState(true, seconds);
         };
 
-        // 初始化状态显示
+        // Initialize status display
         this.updateConnectionUI(false);
+        // Apply saved background
+        const backgroundContainer = document.querySelector('.background-container');
+        if (backgroundContainer) {
+            backgroundContainer.style.backgroundImage = `url('./images/${this.backgroundImages[this.currentBackgroundIndex]}')`;
+        }
+
         this.updateDialButton(false);
 
         console.log('UIController init completed');
     }
 
-    // 初始化可视化器
+    // Initialize visualizer
     initVisualizer() {
         if (this.visualizerCanvas) {
             this.visualizerCanvas.width = this.visualizerCanvas.clientWidth;
@@ -66,9 +74,9 @@ class UIController {
         }
     }
 
-    // 初始化事件监听器
+    // Initialize event listeners
     initEventListeners() {
-        // 设置按钮
+        // Settings button
         const settingsBtn = document.getElementById('settingsBtn');
         if (settingsBtn) {
             settingsBtn.addEventListener('click', () => {
@@ -76,64 +84,133 @@ class UIController {
             });
         }
 
-        // 背景切换按钮
+        // Background switch button
         const backgroundBtn = document.getElementById('backgroundBtn');
         if (backgroundBtn) {
             backgroundBtn.addEventListener('click', this.switchBackground);
         }
 
-        // 拨号按钮
+        // Model select change event
+        const modelSelect = document.getElementById('live2dModelSelect');
+        if (modelSelect) {
+            modelSelect.addEventListener('change', () => {
+                this.switchLive2DModel();
+            });
+        }
+
+        // Dial button
         const dialBtn = document.getElementById('dialBtn');
         if (dialBtn) {
             dialBtn.addEventListener('click', () => {
+                dialBtn.disabled = true;
+                this.dialBtnDisabled = true;
+                setTimeout(() => {
+                    dialBtn.disabled = false;
+                    this.dialBtnDisabled = false;
+                }, 3000);
+
                 const wsHandler = getWebSocketHandler();
                 const isConnected = wsHandler.isConnected();
 
                 if (isConnected) {
                     wsHandler.disconnect();
                     this.updateDialButton(false);
-                    this.addChatMessage('已断开连接，期待下次再见~😉', false);
+                    this.addChatMessage('Disconnected, see you next time~😊', false);
                 } else {
-                    // 检查OTA地址是否已填写
+                    // Check if OTA URL is filled
                     const otaUrlInput = document.getElementById('otaUrl');
                     if (!otaUrlInput || !otaUrlInput.value.trim()) {
-                        // 如果OTA地址未填写，显示设置弹窗并切换到设备配置页
+                        // If OTA URL is not filled, show settings modal and switch to device tab
                         this.showModal('settingsModal');
                         this.switchTab('device');
-                        this.addChatMessage('请先填写OTA服务器地址', false);
+                        this.addChatMessage('Please fill in OTA server URL', false);
                         return;
                     }
 
-                    // 执行连接操作
+                    // Start connection process
                     this.handleConnect();
                 }
             });
         }
 
-        // 录音按钮
+        // Camera button
-        const recordBtn = document.getElementById('recordBtn');
+        const cameraBtn = document.getElementById('cameraBtn');
-        if (recordBtn) {
+        let cameraTimer = null;
-            recordBtn.addEventListener('click', () => {
+        if (cameraBtn) {
-                const audioRecorder = getAudioRecorder();
+            cameraBtn.addEventListener('click', () => {
-                if (audioRecorder.isRecording) {
+                if (cameraTimer) {
-                    audioRecorder.stop();
+                    clearTimeout(cameraTimer);
-                    // 停止录音时移除录音样式
+                    cameraTimer = null;
-                    recordBtn.classList.remove('recording');
-                    recordBtn.querySelector('.btn-text').textContent = '录音';
-                } else {
-                    // 先更新按钮状态为录音中
-                    recordBtn.classList.add('recording');
-                    recordBtn.querySelector('.btn-text').textContent = '录音中';
-
-                    // 延迟开始录音，确保按钮状态已更新
-                    setTimeout(() => {
-                        audioRecorder.start();
-                    }, 100);
                 }
+                cameraTimer = setTimeout(() => {
+                    const cameraContainer = document.getElementById('cameraContainer');
+                    if (!cameraContainer) {
+                        log('摄像头容器不存在', 'warning');
+                        return;
+                    }
+
+                    const isActive = cameraContainer.classList.contains('active');
+                    if (isActive) {
+                        // 关闭摄像头
+                        if (typeof window.stopCamera === 'function') {
+                            window.stopCamera();
+                        }
+                        cameraContainer.classList.remove('active');
+                        cameraBtn.classList.remove('camera-active');
+                        cameraBtn.querySelector('.btn-text').textContent = '摄像头';
+                        log('摄像头已关闭', 'info');
+                    } else {
+                        // 打开摄像头
+                        if (typeof window.startCamera === 'function') {
+                            window.startCamera().then(success => {
+                                if (success) {
+                                    cameraBtn.classList.add('camera-active');
+                                    cameraBtn.querySelector('.btn-text').textContent = '关闭';
+                                } else {
+                                    this.addChatMessage('⚠️ 摄像头启动失败，请检查浏览器权限', false);
+                                }
+                            }).catch(error => {
+                                log(`启动摄像头异常: ${error.message}`, 'error');
+                            });
+                        } else {
+                            log('startCamera函数未定义', 'warning');
+                        }
+                    }
+                }, 300);
             });
         }
 
-        // 消息输入框事件
+        // Record button
+        const recordBtn = document.getElementById('recordBtn');
+        if (recordBtn) {
+            let recordTimer = null;
+            recordBtn.addEventListener('click', () => {
+                if (recordTimer) {
+                    clearTimeout(recordTimer);
+                    recordTimer = null;
+                }
+                recordTimer = setTimeout(() => {
+                    const audioRecorder = getAudioRecorder();
+                    if (audioRecorder.isRecording) {
+                        audioRecorder.stop();
+                        // Restore record button to normal state
+                        recordBtn.classList.remove('recording');
+                        recordBtn.querySelector('.btn-text').textContent = '录音';
+                    } else {
+                        // Update button state to recording
+                        recordBtn.classList.add('recording');
+                        recordBtn.querySelector('.btn-text').textContent = '录音中';
+
+                        // Start recording, update button state after delay
+                        setTimeout(() => {
+                            audioRecorder.start();
+                        }, 100);
+                    }
+                }, 300);
+            });
+        }
+
+        // Chat input event listener
         const chatIpt = document.getElementById('chatIpt');
         if (chatIpt) {
             const wsHandler = getWebSocketHandler();
@@ -148,7 +225,7 @@ class UIController {
             });
         }
 
-        // 关闭按钮
+        // Close button
         const closeButtons = document.querySelectorAll('.close-btn');
         closeButtons.forEach(btn => {
             btn.addEventListener('click', (e) => {
@@ -163,7 +240,7 @@ class UIController {
             });
         });
 
-        // 设置标签页切换
+        // Settings tab switch
         const tabBtns = document.querySelectorAll('.tab-btn');
         tabBtns.forEach(btn => {
             btn.addEventListener('click', (e) => {
@@ -171,7 +248,7 @@ class UIController {
             });
         });
 
-        // 点击模态框外部关闭
+        // Click modal background to close
         const modals = document.querySelectorAll('.modal');
         modals.forEach(modal => {
             modal.addEventListener('click', (e) => {
@@ -184,7 +261,7 @@ class UIController {
             });
         });
 
-        // 添加MCP工具按钮
+        // Add MCP tool button
         const addMCPToolBtn = document.getElementById('addMCPToolBtn');
         if (addMCPToolBtn) {
             addMCPToolBtn.addEventListener('click', (e) => {
@@ -193,10 +270,10 @@ class UIController {
             });
         }
 
-        // 连接按钮和取消按钮已被移除，功能已集成到拨号按钮中
+        // Connect button and send button are not removed, can be added to dial button later
     }
 
-    // 更新连接状态UI
+    // Update connection status UI
     updateConnectionUI(isConnected) {
         const connectionStatus = document.getElementById('connectionStatus');
         const statusDot = document.querySelector('.status-dot');
@@ -216,48 +293,81 @@ class UIController {
         }
     }
 
-    // 更新拨号按钮状态
+    // Update dial button state
     updateDialButton(isConnected) {
         const dialBtn = document.getElementById('dialBtn');
         const recordBtn = document.getElementById('recordBtn');
+        const cameraBtn = document.getElementById('cameraBtn');
 
         if (dialBtn) {
             if (isConnected) {
                 dialBtn.classList.add('dial-active');
                 dialBtn.querySelector('.btn-text').textContent = '挂断';
-                // 更新拨号按钮图标为挂断图标
+                // Update dial button icon to hang up icon
                 dialBtn.querySelector('svg').innerHTML = `
                     <path d="M12,9C10.4,9 9,10.4 9,12C9,13.6 10.4,15 12,15C13.6,15 15,13.6 15,12C15,10.4 13.6,9 12,9M12,17C9.2,17 7,14.8 7,12C7,9.2 9.2,7 12,7C14.8,7 17,9.2 17,12C17,14.8 14.8,17 12,17M12,4.5C7,4.5 2.7,7.6 1,12C2.7,16.4 7,19.5 12,19.5C17,19.5 21.3,16.4 23,12C21.3,7.6 17,4.5 12,4.5Z"/>
                 `;
             } else {
                 dialBtn.classList.remove('dial-active');
                 dialBtn.querySelector('.btn-text').textContent = '拨号';
-                // 恢复拨号按钮图标
+                // Restore dial button icon
                 dialBtn.querySelector('svg').innerHTML = `
                     <path d="M6.62,10.79C8.06,13.62 10.38,15.94 13.21,17.38L15.41,15.18C15.69,14.9 16.08,14.82 16.43,14.93C17.55,15.3 18.75,15.5 20,15.5A1,1 0 0,1 21,16.5V20A1,1 0 0,1 20,21A17,17 0 0,1 3,4A1,1 0 0,1 4,3H7.5A1,1 0 0,1 8.5,4C8.5,5.25 8.7,6.45 9.07,7.57C9.18,7.92 9.1,8.31 8.82,8.59L6.62,10.79Z"/>
                 `;
             }
         }
 
-        // 更新录音按钮状态
+        // Update camera button state - reset to default when disconnected
+        if (cameraBtn && !isConnected) {
+            const cameraContainer = document.getElementById('cameraContainer');
+            if (cameraContainer && cameraContainer.classList.contains('active')) {
+                cameraContainer.classList.remove('active');
+            }
+            cameraBtn.classList.remove('camera-active');
+            cameraBtn.querySelector('.btn-text').textContent = '摄像头';
+            cameraBtn.disabled = true;
+            cameraBtn.title = '请先连接服务器';
+            // 关闭摄像头
+            if (typeof window.stopCamera === 'function') {
+                window.stopCamera();
+            }
+        }
+
+        // Update camera button state - enable when connected and camera is available
+        if (cameraBtn && isConnected) {
+            if (window.cameraAvailable) {
+                cameraBtn.disabled = false;
+                cameraBtn.title = '打开/关闭摄像头';
+            } else {
+                cameraBtn.disabled = true;
+                cameraBtn.title = '请先绑定验证码';
+            }
+        }
+
+        // Update record button state
         if (recordBtn) {
-            if (isConnected) {
+            const microphoneAvailable = window.microphoneAvailable !== false;
+            if (isConnected && microphoneAvailable) {
                 recordBtn.disabled = false;
                 recordBtn.title = '开始录音';
-                // 确保录音按钮恢复到正常状态
+                // Restore record button to normal state
                 recordBtn.querySelector('.btn-text').textContent = '录音';
                 recordBtn.classList.remove('recording');
             } else {
                 recordBtn.disabled = true;
-                recordBtn.title = '请先连接服务器';
+                if (!microphoneAvailable) {
-                // 确保录音按钮恢复到正常状态
+                    recordBtn.title = window.isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
+                } else {
+                    recordBtn.title = '请先连接服务器';
+                }
+                // Restore record button to normal state
                 recordBtn.querySelector('.btn-text').textContent = '录音';
                 recordBtn.classList.remove('recording');
             }
         }
     }
 
-    // 更新录音按钮状态
+    // Update record button state
     updateRecordButtonState(isRecording, seconds = 0) {
         const recordBtn = document.getElementById('recordBtn');
         if (recordBtn) {
@@ -268,11 +378,37 @@ class UIController {
                 recordBtn.querySelector('.btn-text').textContent = '录音';
                 recordBtn.classList.remove('recording');
             }
-            recordBtn.disabled = false;
+            // Only enable button when microphone is available
+            recordBtn.disabled = window.microphoneAvailable === false;
         }
     }
 
-    // 添加聊天消息
+    /**
+     * Update microphone availability state
+     * @param {boolean} isAvailable - Whether microphone is available
+     * @param {boolean} isHttpNonLocalhost - Whether it is HTTP non-localhost access
+     */
+    updateMicrophoneAvailability(isAvailable, isHttpNonLocalhost) {
+        const recordBtn = document.getElementById('recordBtn');
+        if (!recordBtn) return;
+        if (!isAvailable) {
+            // Disable record button
+            recordBtn.disabled = true;
+            // Update button text and title
+            recordBtn.querySelector('.btn-text').textContent = '录音';
+            recordBtn.title = isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
+
+        } else {
+            // If connected, enable record button
+            const wsHandler = getWebSocketHandler();
+            if (wsHandler && wsHandler.isConnected()) {
+                recordBtn.disabled = false;
+                recordBtn.title = '开始录音';
+            }
+        }
+    }
+
+    // Add chat message
     addChatMessage(content, isUser = false) {
         const chatStream = document.getElementById('chatStream');
         if (!chatStream) return;
@@ -282,20 +418,50 @@ class UIController {
         messageDiv.innerHTML = `<div class="message-bubble">${content}</div>`;
         chatStream.appendChild(messageDiv);
 
-        // 自动滚动到底部
+        // Scroll to bottom
         chatStream.scrollTop = chatStream.scrollHeight;
     }
 
-    // 切换背景
+    // Switch background
     switchBackground() {
         this.currentBackgroundIndex = (this.currentBackgroundIndex + 1) % this.backgroundImages.length;
         const backgroundContainer = document.querySelector('.background-container');
         if (backgroundContainer) {
             backgroundContainer.style.backgroundImage = `url('./images/${this.backgroundImages[this.currentBackgroundIndex]}')`;
         }
+        localStorage.setItem('backgroundIndex', this.currentBackgroundIndex);
     }
 
-    // 显示模态框
+    // Switch Live2D model
+    switchLive2DModel() {
+        const modelSelect = document.getElementById('live2dModelSelect');
+        if (!modelSelect) {
+            console.error('模型选择下拉框不存在');
+            return;
+        }
+
+        const selectedModel = modelSelect.value;
+        const app = window.chatApp;
+
+        if (app && app.live2dManager) {
+            app.live2dManager.switchModel(selectedModel)
+                .then(success => {
+                    if (success) {
+                        this.addChatMessage(`已切换到模型: ${selectedModel}`, false);
+                    } else {
+                        this.addChatMessage('模型切换失败', false);
+                    }
+                })
+                .catch(error => {
+                    console.error('模型切换错误:', error);
+                    this.addChatMessage('模型切换出错', false);
+                });
+        } else {
+            this.addChatMessage('Live2D管理器未初始化', false);
+        }
+    }
+
+    // Show modal
     showModal(modalId) {
         const modal = document.getElementById(modalId);
         if (modal) {
@@ -303,7 +469,7 @@ class UIController {
         }
     }
 
-    // 隐藏模态框
+    // Hide modal
     hideModal(modalId) {
         const modal = document.getElementById(modalId);
         if (modal) {
@@ -311,16 +477,16 @@ class UIController {
         }
     }
 
-    // 切换标签页
+    // Switch tab
     switchTab(tabName) {
-        // 移除所有标签页的active类
+        // Remove active class from all tabs
         const tabBtns = document.querySelectorAll('.tab-btn');
         const tabContents = document.querySelectorAll('.tab-content');
 
         tabBtns.forEach(btn => btn.classList.remove('active'));
         tabContents.forEach(content => content.classList.remove('active'));
 
-        // 激活选中的标签页
+        // Activate selected tab
         const activeTabBtn = document.querySelector(`[data-tab="${tabName}"]`);
         const activeTabContent = document.getElementById(`${tabName}Tab`);
 
@@ -330,24 +496,50 @@ class UIController {
         }
     }
 
-    // 连接成功后开始对话
+    // Start AI chat session after connection
     startAIChatSession() {
-        this.addChatMessage('连接成功，开始聊天吧~🙂', false);
+        this.addChatMessage('连接成功，开始聊天吧~😊', false);
-        // 开启录音
+        // Check microphone availability and show error messages if needed
-        const recordBtn = document.getElementById('recordBtn');
+        if (!window.microphoneAvailable) {
-        if (recordBtn) {
+            if (window.isHttpNonLocalhost) {
-            recordBtn.click();
+                this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
+            } else {
+                this.addChatMessage('⚠️ 麦克风不可用，请检查权限设置，只能用文字交互', false);
+            }
+        }
+        // Start recording only if microphone is available
+        if (window.microphoneAvailable) {
+            const recordBtn = document.getElementById('recordBtn');
+            if (recordBtn) {
+                recordBtn.click();
+            }
+        }
+        // Start camera only if camera is available (bound with verification code)
+        if (window.cameraAvailable && typeof window.startCamera === 'function') {
+            window.startCamera().then(success => {
+                if (success) {
+                    const cameraBtn = document.getElementById('cameraBtn');
+                    if (cameraBtn) {
+                        cameraBtn.classList.add('camera-active');
+                        cameraBtn.querySelector('.btn-text').textContent = '关闭';
+                    }
+                } else {
+                    this.addChatMessage('⚠️ 摄像头启动失败，可能被浏览器拒绝', false);
+                }
+            }).catch(error => {
+                log(`启动摄像头异常: ${error.message}`, 'error');
+            });
         }
     }
 
-    // 处理连接按钮点击
+    // Handle connect button click
     async handleConnect() {
         console.log('handleConnect called');
 
-        // 确保切换到设备配置标签页
+        // Switch to device settings tab
         this.switchTab('device');
 
-        // 等待DOM更新
+        // Wait for DOM update
         await new Promise(resolve => setTimeout(resolve, 50));
 
         const otaUrlInput = document.getElementById('otaUrl');
@@ -362,7 +554,7 @@ class UIController {
         const otaUrl = otaUrlInput.value;
         console.log('otaUrl value:', otaUrl);
 
-        // 更新拨号按钮状态为连接中
+        // Update dial button state to connecting
         const dialBtn = document.getElementById('dialBtn');
         if (dialBtn) {
             dialBtn.classList.add('dial-active');
@@ -370,7 +562,7 @@ class UIController {
             dialBtn.disabled = true;
         }
 
-        // 显示连接中消息
+        // Show connecting message
         this.addChatMessage('正在连接服务器...', false);
 
         const chatIpt = document.getElementById('chatIpt');
@@ -380,50 +572,61 @@ class UIController {
 
         try {
 
-            // 获取WebSocket处理器
+            // Get WebSocket handler instance
             const wsHandler = getWebSocketHandler();
+
+            // Register connection state callback BEFORE connecting
+            wsHandler.onConnectionStateChange = (isConnected) => {
+                this.updateConnectionUI(isConnected);
+                this.updateDialButton(isConnected);
+            };
+
+            // Register chat message callback BEFORE connecting
+            wsHandler.onChatMessage = (text, isUser) => {
+                this.addChatMessage(text, isUser);
+            };
+
+            // Register record button state callback BEFORE connecting
+            wsHandler.onRecordButtonStateChange = (isRecording) => {
+                const recordBtn = document.getElementById('recordBtn');
+                if (recordBtn) {
+                    if (isRecording) {
+                        recordBtn.classList.add('recording');
+                        recordBtn.querySelector('.btn-text').textContent = '录音中';
+                    } else {
+                        recordBtn.classList.remove('recording');
+                        recordBtn.querySelector('.btn-text').textContent = '录音';
+                    }
+                }
+            };
+
             const isConnected = await wsHandler.connect();
 
             if (isConnected) {
+                // Check microphone availability (check again after connection)
+                const { checkMicrophoneAvailability } = await import('../core/audio/recorder.js?v=0127');
+                const micAvailable = await checkMicrophoneAvailability();
 
-                // 设置连接状态回调
+                if (!micAvailable) {
-                wsHandler.onConnectionStateChange = (isConnected) => {
+                    const isHttp = window.isHttpNonLocalhost;
-                    this.updateConnectionUI(isConnected);
+                    if (isHttp) {
-                    this.updateDialButton(isConnected);
+                        this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
-                };
-
-                // 设置聊天消息回调
-                wsHandler.onChatMessage = (text, isUser) => {
-                    this.addChatMessage(text, isUser);
-                };
-
-                // 设置录音按钮状态回调
-                wsHandler.onRecordButtonStateChange = (isRecording) => {
-                    const recordBtn = document.getElementById('recordBtn');
-                    if (recordBtn) {
-                        if (isRecording) {
-                            recordBtn.classList.add('recording');
-                            recordBtn.querySelector('.btn-text').textContent = '录音中';
-                        } else {
-                            recordBtn.classList.remove('recording');
-                            recordBtn.querySelector('.btn-text').textContent = '录音';
-                        }
                     }
-                };
+                    // Update global state
+                    window.microphoneAvailable = false;
+                }
 
-                // 连接成功
+                // Update dial button state
-                this.addChatMessage('OTA连接成功，正在建立WebSocket连接...', false);
-
-                // 更新拨号按钮状态
                 const dialBtn = document.getElementById('dialBtn');
                 if (dialBtn) {
-                    dialBtn.disabled = false;
+                    if (!this.dialBtnDisabled) {
+                        dialBtn.disabled = false;
+                    }
                     dialBtn.querySelector('.btn-text').textContent = '挂断';
                     dialBtn.classList.add('dial-active');
                 }
 
                 this.hideModal('settingsModal');
-
             } else {
                 throw new Error('OTA连接失败');
             }
@@ -434,17 +637,19 @@ class UIController {
                 name: error.name
             });
 
-            // 显示错误消息
+            // Show error message
             const errorMessage = error.message.includes('Cannot set properties of null')
-                ? '连接失败：请刷新页面重试'
+                ? '连接失败：请检查设备连接'
                 : `连接失败: ${error.message}`;
 
             this.addChatMessage(errorMessage, false);
 
-            // 恢复拨号按钮状态
+            // Restore dial button state
             const dialBtn = document.getElementById('dialBtn');
             if (dialBtn) {
-                dialBtn.disabled = false;
+                if (!this.dialBtnDisabled) {
+                    dialBtn.disabled = false;
+                }
                 dialBtn.querySelector('.btn-text').textContent = '拨号';
                 dialBtn.classList.remove('dial-active');
                 console.log('Dial button state restored successfully');
@@ -452,7 +657,7 @@ class UIController {
         }
     }
 
-    // 添加MCP工具
+    // Add MCP tool
     addMCPTool() {
         const mcpToolsList = document.getElementById('mcpToolsList');
         if (!mcpToolsList) return;
@@ -471,7 +676,7 @@ class UIController {
         mcpToolsList.appendChild(toolDiv);
     }
 
-    // 移除MCP工具
+    // Remove MCP tool
     removeMCPTool(toolId) {
         const toolElement = document.getElementById(toolId);
         if (toolElement) {
@@ -479,24 +684,24 @@ class UIController {
         }
     }
 
-    // 更新音频统计信息
+    // Update audio statistics display
     updateAudioStats() {
         const audioPlayer = getAudioPlayer();
         if (!audioPlayer) return;
 
         const stats = audioPlayer.getAudioStats();
-        // 这里可以添加音频统计的UI更新逻辑
+        // Here can add audio statistics UI update logic
     }
 
-    // 启动音频统计监控
+    // Start audio statistics monitor
     startAudioStatsMonitor() {
-        // 每100ms更新一次音频统计
+        // Update audio statistics every 100ms
         this.audioStatsTimer = setInterval(() => {
             this.updateAudioStats();
         }, 100);
     }
 
-    // 停止音频统计监控
+    // Stop audio statistics monitor
     stopAudioStatsMonitor() {
         if (this.audioStatsTimer) {
             clearInterval(this.audioStatsTimer);
@@ -504,7 +709,7 @@ class UIController {
         }
     }
 
-    // 绘制音频可视化效果
+    // Draw audio visualizer waveform
     drawVisualizer(dataArray) {
         if (!this.visualizerContext || !this.visualizerCanvas) return;
 
@@ -518,7 +723,7 @@ class UIController {
         for (let i = 0; i < dataArray.length; i++) {
             barHeight = dataArray[i] / 2;
 
-            // 创建渐变色：从紫色到蓝色到青色
+            // Create gradient color: from purple to blue to green
             const gradient = this.visualizerContext.createLinearGradient(0, 0, 0, this.visualizerCanvas.height);
             gradient.addColorStop(0, '#8e44ad');
             gradient.addColorStop(0.5, '#3498db');
@@ -530,21 +735,21 @@ class UIController {
         }
     }
 
-    // 更新会话状态UI
+    // Update session status UI
     updateSessionStatus(isSpeaking) {
-        // 这里可以添加会话状态的UI更新逻辑
+        // Here can add session status UI update logic
-        // 例如：更新Live2D角色的表情或状态指示器
+        // For example: update Live2D model's mouth movement status
     }
 
-    // 更新会话表情
+    // Update session emotion
     updateSessionEmotion(emoji) {
-        // 这里可以添加表情更新的逻辑
+        // Here can add emotion update logic
-        // 例如：在状态指示器中显示表情
+        // For example: display emoji in status indicator
     }
 }
 
-// 创建全局实例
+// Create singleton instance
 export const uiController = new UIController();
 
-// 导出类供其他模块使用
+// Export class for module usage
-export { UIController };
+export { UIController };