Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py_test_typing' into feature/python-typing

Browse Source
This commit is contained in:
Sakura-RanChen
2026-02-05 17:14:47 +08:00
committed by GitHub
parent 4b573fb4e2 8a84428724
commit 4f61d21d17
133 changed files with 23165 additions and 1723 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-base-image.yml
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
file: Dockerfile-server-base
push: true
tags: ghcr.io/${{ github.repository }}:server-base
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha,scope=server-base
cache-to: type=gha,mode=max,scope=server-base
build-args: |
.github/workflows/docker-image.yml
+2 -2
View File
@@ -66,7 +66,7 @@ jobs:
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:server_{1},ghcr.io/{0}:server_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:server_latest', github.repository) }}
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
@@ -81,7 +81,7 @@ jobs:
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:web_{1},ghcr.io/{0}:web_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:web_latest', github.repository) }}
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
Dockerfile-web
+9 -8
View File
@@ -1,5 +1,5 @@
# 第一阶段：构建Vue前端
FROM node:18 as web-builder
FROM node:18 AS web-builder
WORKDIR /app
COPY main/manager-web/package*.json ./
RUN npm install
@@ -7,7 +7,7 @@ COPY main/manager-web .
RUN npm run build
# 第二阶段：构建Java后端
FROM maven:3.9.4-eclipse-temurin-21 as api-builder
FROM maven:3.9.4-eclipse-temurin-21 AS api-builder
WORKDIR /app
COPY main/manager-api/pom.xml .
COPY main/manager-api/src ./src
@@ -18,18 +18,19 @@ FROM bellsoft/liberica-runtime-container:jre-21-glibc
# 安装Nginx和字体库
RUN apk update && \
apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
apk add --no-cache --no-scripts \
nginx \
bash \
fontconfig \
ttf-dejavu \
msttcorefonts-installer \
&& ACCEPT_EULA=Y apk add --no-cache msttcorefonts-installer \
&& fc-cache -f -v \
&& rm -rf /var/cache/apk/*
&& rm -rf /var/cache/apk/* \
&& mkdir -p /run/nginx /var/log/nginx /var/tmp/nginx /etc/nginx/conf.d
# 复制项目自带的中文字体
COPY main/manager-web/public/generator/static/fonts/*.ttf /usr/share/fonts/
# 更新字体缓存
RUN (printf 'YES\n' | update-ms-fonts || true) && fc-cache -f -v
RUN fc-cache -f -v
# 配置Nginx
COPY docs/docker/nginx.conf /etc/nginx/nginx.conf
main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java
+1 -1
View File
@@ -304,7 +304,7 @@ public interface Constant {
/**
* 版本号
*/
public static final String VERSION = "0.8.11";
public static final String VERSION = "0.9.1";
/**
* 无效固件URL
main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java
+12 -5
View File
@@ -1,6 +1,12 @@
package xiaozhi.modules.agent.service.impl;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -81,9 +87,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
if (agent.getMemModelId() != null && agent.getMemModelId().equals(Constant.MEMORY_NO_MEM)) {
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.IGNORE.getCode());
if (agent.getChatHistoryConf() == null) {
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
}
}
if (agent.getChatHistoryConf() == null) {
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
}
// 查询上下文源配置
@@ -132,7 +138,8 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
if (StringUtils.isNotBlank(keyword)) {
if ("mac".equals(searchType)) {
// 按MAC地址搜索：先搜索设备，再获取对应的智能体
List<DeviceEntity> devices = Optional.ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
List<DeviceEntity> devices = Optional
.ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
// 获取设备对应的智能体ID列表
List<String> agentIds = devices.stream()
.map(DeviceEntity::getAgentId)
main/manager-api/src/main/java/xiaozhi/modules/model/controller/ModelController.java
+4
View File
@@ -129,6 +129,10 @@ public class ModelController {
if (entity == null) {
return new Result<Void>().error("模型配置不存在");
}
// 不能关闭默认模型
if (status == 0 && entity.getIsDefault() > 0) {
return new Result<Void>().error("默认模型配置不允许关闭");
}
// 不更新ConfigJson字段
entity.setConfigJson(null);
entity.setIsEnabled(status);
main/manager-api/src/main/resources/db/changelog/202601141645.sql
+31
View File
@@ -0,0 +1,31 @@
-- 批量清理 ai_model_provider 中的 sample_rate 字段定义
UPDATE `ai_model_provider` ap
JOIN (
SELECT
id,
JSON_ARRAYAGG(
JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t)
) AS new_fields
FROM `ai_model_provider`,
JSON_TABLE(`fields`, '$[*]' COLUMNS (
k VARCHAR(50) PATH '$.key',
l VARCHAR(100) PATH '$.label',
t VARCHAR(20) PATH '$.type'
)) AS jt
WHERE `model_type` = 'TTS'
AND jt.k != 'sample_rate'
GROUP BY id
) filtered ON ap.id = filtered.id
SET ap.fields = filtered.new_fields;
-- 清理 config_json 顶层的 sample_rate
UPDATE `ai_model_config`
SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate')
WHERE `model_type` = 'TTS'
AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL;
-- 清理Minimax流式TTS的sample_rate参数（位于audio_setting内部）
UPDATE `ai_model_config` SET
`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate'))
WHERE `id` = 'TTS_MinimaxStreamTTS'
AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL;
main/manager-api/src/main/resources/db/changelog/202601231530.sql
+87
View File
@@ -0,0 +1,87 @@
-- 更新HuoshanDoubleStreamTTS供应器配置，将分散的参数改为JSON字典配置
-- 将 speech_rate, loudness_rate, pitch, emotion, emotion_scale 等参数整合为 audio_params, additions, mix_speaker 三个JSON字典
UPDATE `ai_model_provider`
SET `fields` = '[
{"key": "ws_url", "type": "string", "label": "WebSocket地址"},
{"key": "appid", "type": "string", "label": "应用ID"},
{"key": "access_token", "type": "string", "label": "访问令牌"},
{"key": "resource_id", "type": "string", "label": "资源ID"},
{"key": "speaker", "type": "string", "label": "默认音色"},
{"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true},
{"key": "audio_params", "type": "dict", "label": "音频输出配置"},
{"key": "additions", "type": "dict", "label": "高级文本处理配置"},
{"key": "mix_speaker", "type": "dict", "label": "混音控制配置"}
]'
WHERE `id` = 'SYSTEM_TTS_HSDSTTS';
-- 更新现有配置，将旧的分散参数迁移到新的JSON字典结构
UPDATE `ai_model_config`
SET `config_json` = JSON_SET(
`config_json`,
'$.audio_params', JSON_OBJECT(
'speech_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.speech_rate')), ''), '0') AS SIGNED),
'loudness_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.loudness_rate')), ''), '0') AS SIGNED)
),
'$.additions', JSON_OBJECT(
'aigc_metadata', JSON_OBJECT(),
'cache_config', JSON_OBJECT(),
'post_process', JSON_OBJECT(
'pitch', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.pitch')), ''), '0') AS SIGNED)
)
),
'$.mix_speaker', JSON_OBJECT()
)
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
-- 删除旧的分散参数字段
UPDATE `ai_model_config`
SET `config_json` = JSON_REMOVE(
`config_json`,
'$.speech_rate',
'$.loudness_rate',
'$.pitch',
'$.emotion',
'$.emotion_scale'
)
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
-- 更新文档链接和备注说明
UPDATE `ai_model_config` SET
`doc_link` = 'https://www.volcengine.com/docs/6561/1329505',
`remark` = '火山引擎双向流式TTS配置说明：
1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号
2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型，购买音色
3. 在页面底部获取appid和access_token
4. 资源ID固定为：volc.service_type.10029（大模型语音合成及混音）
5. 链接复用：开启WebSocket连接复用，默认true减少链接损耗（注意：复用后设备处于聆听状态时空闲链接会占并发数）
详细参数文档：https://www.volcengine.com/docs/6561/1329505
【audio_params】音频输出配置 - 用户可自定义添加火山引擎支持的任何音频参数
- speech_rate: 语速(-50~100)，默认0
- loudness_rate: 音量(-50~100)，默认0
- emotion: 情感类型（仅部分音色支持），可选值：neutral、happy、sad、angry、fearful、disgusted、surprised
- emotion_scale: 情感强度(1~5)，默认4
示例：{"speech_rate": 10, "loudness_rate": 5, "emotion": "happy", "emotion_scale": 4}
【additions】高级文本处理配置 - 用户可自定义添加火山引擎支持的任何高级参数
- post_process.pitch: 音高(-12~12)，默认0
- aigc_metadata: AIGC元数据配置
- cache_config: 缓存配置
示例：{"post_process": {"pitch": 2}, "aigc_metadata": {}, "cache_config": {}}
【mix_speaker】混音控制配置 - 多音色混合（仅 TTS 1.0）
示例：
{"speakers": [
{"source_speaker": "zh_male_bvlazysheep","mix_factor": 0.3},
{"source_speaker": "BV120_streaming","mix_factor": 0.3},
{"source_speaker": "zh_male_ahu_conversation_wvae_bigtts","mix_factor": 0.4}
]}
注意：
- 多情感音色参数（emotion、emotion_scale）仅部分音色支持
- 相关音色列表：https://www.volcengine.com/docs/6561/1257544
- 用户可根据火山引擎API文档自行添加更多参数
- 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts
'
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
main/manager-api/src/main/resources/db/changelog/202601261730.sql
+14
View File
@@ -0,0 +1,14 @@
-- 更新小智参数中的默认采样率从 16000 改为 24000
UPDATE `sys_params`
SET `param_value` = '{
"type": "hello",
"version": 1,
"transport": "websocket",
"audio_params": {
"format": "opus",
"sample_rate": 24000,
"channels": 1,
"frame_duration": 60
}
}'
WHERE `id` = 309 AND `param_code` = 'xiaozhi';
main/manager-api/src/main/resources/db/changelog/202602021555.sql
+1
View File
@@ -0,0 +1 @@
INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (502, 'system_error_response', '主人，小智现在有点忙，我们稍后再试吧。', 'string', 1, '系统错误时的回复');
main/manager-api/src/main/resources/db/changelog/202602051125.sql
+2
View File
@@ -0,0 +1,2 @@
-- 删除无用业务表ai_voiceprint
DROP TABLE IF EXISTS ai_voiceprint;
main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
+35
View File
@@ -487,3 +487,38 @@ databaseChangeLog:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601051433.sql
- changeSet:
id: 202601141645
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601141645.sql
- changeSet:
id: 202601231530
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601231530.sql
- changeSet:
id: 202601261730
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601261730.sql
- changeSet:
id: 202602021555
author: shengzhou1216
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202602021555.sql
- changeSet:
id: 202602051125
author: DaGou12138
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202602051125.sql
main/manager-mobile/src/pages/settings/index.vue
+1 -1
View File
@@ -235,7 +235,7 @@ function showAbout() {
title: t('settings.aboutApp', { appName: import.meta.env.VITE_APP_TITLE }),
content: t('settings.aboutContent', {
appName: import.meta.env.VITE_APP_TITLE,
version: '0.8.11'
version: '0.9.1'
}),
showCancel: false,
confirmText: t('common.confirm'),
main/manager-web/public/generator/assets/index-B8r0c7xg.js
+81
View File
File diff suppressed because one or more lines are too long
main/manager-web/public/generator/assets/index-NXxBVrod.css → main/manager-web/public/generator/assets/index-CrIJdTCK.css
+1 -1
View File
File diff suppressed because one or more lines are too long
main/manager-web/public/generator/assets/index-FKVSBRAB.js
-81
View File
File diff suppressed because one or more lines are too long
main/manager-web/public/generator/favicon.ico
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

main/manager-web/public/generator/index.html
+3 -2
View File
@@ -4,9 +4,10 @@
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="./favicon.ico">
<title>Xiaozhi AI Customization</title>
<script type="module" crossorigin src="./assets/index-FKVSBRAB.js"></script>
<link rel="stylesheet" crossorigin href="./assets/index-NXxBVrod.css">
<script type="module" crossorigin src="./assets/index-B8r0c7xg.js"></script>
<link rel="stylesheet" crossorigin href="./assets/index-CrIJdTCK.css">
</head>
<body>
<div id="app"></div>
main/manager-web/src/i18n/de.js
+1
View File
@@ -836,6 +836,7 @@ export default {
'modelConfig.unknown': 'Unbekannt',
'modelConfig.isEnabled': 'Aktiviert',
'modelConfig.isDefault': 'Standard',
'modelConfig.defaultModelCannotDisable': 'Standardmodellkonfiguration kann nicht deaktiviert werden',
'modelConfig.action': 'Aktion',
'modelConfig.voiceManagement': 'Stimmverwaltung',
'modelConfig.edit': 'Bearbeiten',
main/manager-web/src/i18n/en.js
+1
View File
@@ -836,6 +836,7 @@ export default {
'modelConfig.unknown': 'Unknown',
'modelConfig.isEnabled': 'Enabled',
'modelConfig.isDefault': 'Default',
'modelConfig.defaultModelCannotDisable': 'Default model configuration cannot be disabled',
'modelConfig.action': 'Action',
'modelConfig.voiceManagement': 'Voice Management',
'modelConfig.edit': 'Edit',
main/manager-web/src/i18n/vi.js
+1
View File
@@ -836,6 +836,7 @@ export default {
'modelConfig.unknown': 'Không xác định',
'modelConfig.isEnabled': 'Đã bật',
'modelConfig.isDefault': 'Mặc định',
'modelConfig.defaultModelCannotDisable': 'Cấu hình mô hình mặc định không thể tắt',
'modelConfig.action': 'Hành động',
'modelConfig.voiceManagement': 'Quản lý giọng nói',
'modelConfig.edit': 'Chỉnh sửa',
main/manager-web/src/i18n/zh_CN.js
+1
View File
@@ -836,6 +836,7 @@ export default {
'modelConfig.unknown': '未知',
'modelConfig.isEnabled': '是否启用',
'modelConfig.isDefault': '是否默认',
'modelConfig.defaultModelCannotDisable': '默认模型配置不允许关闭',
'modelConfig.action': '操作',
'modelConfig.voiceManagement': '音色管理',
'modelConfig.edit': '修改',
main/manager-web/src/i18n/zh_TW.js
+1
View File
@@ -836,6 +836,7 @@ export default {
'modelConfig.unknown': '未知',
'modelConfig.isEnabled': '是否啟用',
'modelConfig.isDefault': '是否默認',
'modelConfig.defaultModelCannotDisable': '預設模型配置不允許關閉',
'modelConfig.action': '操作',
'modelConfig.voiceManagement': '音色管理',
'modelConfig.edit': '修改',
main/manager-web/src/views/DeviceManagement.vue
+3 -3
View File
@@ -64,7 +64,7 @@
<el-button size="mini" type="text" @click="handleUnbind(scope.row.device_id)">
{{ $t('device.unbind') }}
</el-button>
<el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor">
<el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor(scope.row)">
{{ $t('device.deviceThemeGeneration') }}
</el-button>
</template>
@@ -337,10 +337,10 @@ export default {
});
});
},
handleGenertor() {
handleGenertor(row) {
const pathname = window.location.pathname;
const basePath = pathname.split('/').slice(0, -1).join('/');
const url = `${window.location.origin}${basePath}/generator/`;
const url = `${window.location.origin}${basePath}/generator/?deviceId=${row.device_id}`;
sessionStorage.setItem('devicePath', window.location.href);
window.location.href = url;
},
main/manager-web/src/views/ModelConfig.vue
+16
View File
@@ -97,7 +97,23 @@
</el-table-column>
<el-table-column :label="$t('modelConfig.isEnabled')" align="center">
<template slot-scope="scope">
<el-tooltip
v-if="scope.row.isDefault === 1 && scope.row.isEnabled === 1"
:content="$t('modelConfig.defaultModelCannotDisable')"
placement="top"
effect="light"
>
<el-switch
v-model="scope.row.isEnabled"
class="custom-switch"
:active-value="1"
:inactive-value="0"
disabled
@change="handleStatusChange(scope.row)"
/>
</el-tooltip>
<el-switch
v-else
v-model="scope.row.isEnabled"
class="custom-switch"
:active-value="1"
main/xiaozhi-server/config.yaml
+30 -14
View File
@@ -89,7 +89,8 @@ xiaozhi:
transport: websocket
audio_params:
format: opus
sample_rate: 16000
# Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000]
sample_rate: 24000
channels: 1
frame_duration: 60
@@ -202,6 +203,9 @@ prompt: |
# 默认系统提示词模板文件
prompt_template: agent-base-prompt.txt
# 系统错误时的回复
system_error_response: "主人，小智现在有点忙，我们稍后再试吧。"
# 结束语prompt
end_prompt:
enable: true # 是否开启结束语
@@ -714,13 +718,31 @@ TTS:
speaker: zh_female_wanwanxiaohe_moon_bigtts
# 开启WebSocket连接复用，默认复用（注意：复用后设备处于聆听状态时空闲链接会占并发数）
enable_ws_reuse: True
speech_rate: 0
loudness_rate: 0
pitch: 0
# 多情感音色参数，注意：当前仅部分音色支持设置情感。
# 相关音色列表：https://www.volcengine.com/docs/6561/1257544
emotion: "neutral" # 情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised
emotion_scale: 4 # 情感强度，可选值为：1~5，默认值为4
# 相关参数文档：https://www.volcengine.com/docs/6561/1329505
# 音频输出配置（audio_params）- 用户可自定义添加火山引擎支持的任何音频参数
audio_params:
speech_rate: 0 # 语速(-50~100)
loudness_rate: 0 # 音量(-50~100)
# 多情感音色参数，注意：当前仅部分音色支持设置情感。
# 相关音色列表：https://www.volcengine.com/docs/6561/1257544
# emotion: "neutral" # 情感类型（仅部分音色支持）：neutral、happy、sad、angry、fearful、disgusted、surprised
# emotion_scale: 4 # 情感强度(1~5)
# 高级文本处理配置（additions）- 用户可自定义添加火山引擎支持的任何高级参数
additions:
post_process:
pitch: 0 # 音高(-12~12)
# aigc_metadata: {} # AIGC元数据配置
# cache_config: {} # 缓存配置
# 混音控制配置（mix_speaker）- 多音色混合（仅 TTS 1.0）
# 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts
# mix_speaker:
# speakers:
# - source_speaker: zh_male_bvlazysheep
# mix_factor: 0.3
# - source_speaker: BV120_streaming
# mix_factor: 0.3
# - source_speaker: zh_male_ahu_conversation_wvae_bigtts
# mix_factor: 0.4
CosyVoiceSiliconflow:
type: siliconflow
# 硅基流动TTS
@@ -837,7 +859,6 @@ TTS:
# - "处理/(chu3)(li3)"
# - "危险/dangerous"
# audio_setting:
# sample_rate: 24000
# bitrate: 128000
# format: "mp3"
# channel: 1
@@ -865,7 +886,6 @@ TTS:
# 以下可不用设置，使用默认设置
# format: wav
# sample_rate: 16000
# volume: 50
# speech_rate: 0
# pitch_rate: 0
@@ -889,7 +909,6 @@ TTS:
host: nls-gateway-cn-beijing.aliyuncs.com
# 以下可不用设置，使用默认设置
# format: pcm # 音频格式：pcm、wav、mp3
# sample_rate: 16000 # 采样率：8000、16000、24000
# volume: 50 # 音量：0-100
# speech_rate: 0 # 语速：-500到500
# pitch_rate: 0 # 语调：-500到500
@@ -1004,7 +1023,6 @@ TTS:
protocol: websocket # protocol choices = ['websocket', 'http']
url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址，指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming，http默认http://127.0.0.1:8090/paddlespeech/tts]
spk_id: 0 # 发音人 ID，0 通常表示默认的发音人
sample_rate: 24000 # 采样率 [websocket默认24000，http默认0 自动选择]
speed: 1.0 # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢
volume: 1.0 # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小
save_path: # 保存路径
@@ -1028,7 +1046,6 @@ TTS:
output_dir: tmp/
# 以下可不用设置，使用默认设置
# format: pcm # 音频格式：pcm、wav、mp3、opus
# sample_rate: 24000 # 采样率：16000, 24000, 48000
# volume: 50 # 音量：0-100
# rate: 1 # 语速：0.5~2
# pitch: 1 # 语调：0.5~2
@@ -1050,7 +1067,6 @@ TTS:
# stop_split: 0 # 关闭服务端拆句 不关闭：0，关闭：1
# remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0
# format: raw # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
# sample_rate: 24000 # 采样率：16000, 8000, 24000
# volume: 50 # 音量：0-100
# speed: 50 # 语速：0-100
# pitch: 50 # 语调：0-100
main/xiaozhi-server/config/logger.py
+1 -1
View File
@@ -5,7 +5,7 @@ from config.config_loader import load_config
from config.settings import check_config_file
from datetime import datetime
SERVER_VERSION = "0.8.11"
SERVER_VERSION = "0.9.1"
_logger_initialized = False
main/xiaozhi-server/core/api/vision_handler.py
+8
View File
@@ -29,7 +29,15 @@ class VisionHandler(BaseHandler):
def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]:
"""验证认证token"""
# 测试模式：允许特定测试令牌或跳过验证
auth_header = request.headers.get("Authorization", "")
client_id = request.headers.get("Client-Id", "")
# 允许测试客户端跳过认证
if client_id == "web_test_client":
device_id = request.headers.get("Device-Id", "test_device")
return True, device_id
if not auth_header.startswith("Bearer "):
return False, None
main/xiaozhi-server/core/connection.py
+78 -42
View File
@@ -40,8 +40,10 @@ from config.logger import setup_logging, build_module_string, create_connection_
from config.manage_api_client import DeviceNotFoundException, DeviceBindException
from core.utils.prompt_manager import PromptManager
from core.utils.voiceprint_provider import VoiceprintProvider
from core.utils.util import get_system_error_response
from core.utils import textUtils
TAG = __name__
auto_import_modules("plugins_func.functions")
@@ -85,6 +87,7 @@ class ConnectionHandler:
self.max_output_size = 0
self.chat_history_conf = 0
self.audio_format = "opus"
self.sample_rate = 24000 # 默认采样率，从客户端 hello 消息中动态更新
# 客户端状态相关
self.client_abort = False
@@ -134,7 +137,6 @@ class ConnectionHandler:
self.current_language_tag = None # 存储当前ASR识别的语言标签
# llm相关变量
self.llm_finish_task = True
self.dialogue = Dialogue()
# tts相关变量
@@ -206,6 +208,10 @@ class ConnectionHandler:
self.welcome_msg = self.config["xiaozhi"]
self.welcome_msg["session_id"] = self.session_id
# 从配置中读取采样率
self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
# 在后台初始化配置和组件（完全不阻塞主循环）
asyncio.create_task(self._background_initialize())
@@ -794,7 +800,6 @@ class ConnectionHandler:
# 为最顶层时新建会话ID和发送FIRST请求
if depth == 0:
self.llm_finish_task = False
self.sentence_id = str(uuid.uuid4().hex)
self.dialogue.put(Message(role="user", content=query))
self.tts.tts_text_queue.put(
@@ -869,46 +874,66 @@ class ConnectionHandler:
content_arguments = ""
self.client_abort = False
emotion_flag = True
for response in llm_responses:
if self.client_abort:
break
if self.intent_type == "function_call" and functions is not None:
content, tools_call = response
if "content" in response:
content = response["content"]
tools_call = None
if content is not None and len(content) > 0:
content_arguments += content
try:
for response in llm_responses:
if self.client_abort:
break
if self.intent_type == "function_call" and functions is not None:
content, tools_call = response
if "content" in response:
content = response["content"]
tools_call = None
if content is not None and len(content) > 0:
content_arguments += content
if not tool_call_flag and content_arguments.startswith("<tool_call>"):
# print("content_arguments", content_arguments)
tool_call_flag = True
if not tool_call_flag and content_arguments.startswith("<tool_call>"):
# print("content_arguments", content_arguments)
tool_call_flag = True
if tools_call is not None and len(tools_call) > 0:
tool_call_flag = True
self._merge_tool_calls(tool_calls_list, tools_call)
else:
content = response
if tools_call is not None and len(tools_call) > 0:
tool_call_flag = True
self._merge_tool_calls(tool_calls_list, tools_call)
else:
content = response
# 在llm回复中获取情绪表情，一轮对话只在开头获取一次
if emotion_flag and content is not None and content.strip():
asyncio.run_coroutine_threadsafe(
textUtils.get_emotion(self, content),
self.loop,
)
emotion_flag = False
if content is not None and len(content) > 0:
if not tool_call_flag:
response_message.append(content)
self.tts.tts_text_queue.put(
TTSMessageDTO(
sentence_id=self.sentence_id,
sentence_type=SentenceType.MIDDLE,
content_type=ContentType.TEXT,
content_detail=content,
)
# 在llm回复中获取情绪表情，一轮对话只在开头获取一次
if emotion_flag and content is not None and content.strip():
asyncio.run_coroutine_threadsafe(
textUtils.get_emotion(self, content),
self.loop,
)
emotion_flag = False
if content is not None and len(content) > 0:
if not tool_call_flag:
response_message.append(content)
self.tts.tts_text_queue.put(
TTSMessageDTO(
sentence_id=self.sentence_id,
sentence_type=SentenceType.MIDDLE,
content_type=ContentType.TEXT,
content_detail=content,
)
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"LLM stream processing error: {e}")
self.tts.tts_text_queue.put(
TTSMessageDTO(
sentence_id=self.sentence_id,
sentence_type=SentenceType.MIDDLE,
content_type=ContentType.TEXT,
content_detail=get_system_error_response(self.config),
)
)
if depth == 0:
self.tts.tts_text_queue.put(
TTSMessageDTO(
sentence_id=self.sentence_id,
sentence_type=SentenceType.LAST,
content_type=ContentType.ACTION,
)
)
return
# 处理function call
if tool_call_flag:
bHasError = False
@@ -989,7 +1014,6 @@ class ConnectionHandler:
content_type=ContentType.ACTION,
)
)
self.llm_finish_task = True
# 使用lambda延迟计算，只有在DEBUG级别时才执行get_llm_dialogue()
self.logger.bind(tag=TAG).debug(
lambda: json.dumps(
@@ -1161,6 +1185,8 @@ class ConnectionHandler:
if self.tts:
await self.tts.close()
if self.asr:
await self.asr.close()
# 最后关闭线程池（避免阻塞）
if self.executor:
@@ -1209,11 +1235,21 @@ class ConnectionHandler:
f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
)
def reset_vad_states(self):
self.client_audio_buffer = bytearray()
def reset_audio_states(self):
"""
重置所有音频相关状态(VAD + ASR)
"""
# Reset VAD states
self.client_audio_buffer.clear()
self.client_have_voice = False
self.client_voice_stop = False
self.logger.bind(tag=TAG).debug("VAD states reset.")
self.client_voice_window.clear()
self.last_is_voice = False
# Clear ASR buffers
self.asr_audio.clear()
self.logger.bind(tag=TAG).debug("All audio states reset.")
def chat_and_close(self, text):
"""Chat with the user and then close the connection"""
main/xiaozhi-server/core/handle/helloHandle.py
+2 -1
View File
@@ -143,7 +143,8 @@ async def wakeupWordsResponse(conn: "ConnectionHandler"):
# 获取当前音色
voice = getattr(conn.tts, "voice", "default")
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
# 使用链接的sample_rate
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
file_path = wakeup_words_config.generate_file_path(voice)
with open(file_path, "wb") as f:
f.write(wav_bytes)
main/xiaozhi-server/core/handle/receiveAudioHandle.py
-1
View File
@@ -21,7 +21,6 @@ async def handleAudioMessage(conn: "ConnectionHandler", audio):
if hasattr(conn, "just_woken_up") and conn.just_woken_up:
have_voice = False
# 设置一个短暂延迟后恢复VAD检测
conn.asr_audio.clear()
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
return
main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py
+4 -6
View File
@@ -29,10 +29,9 @@ class ListenTextMessageHandler(TextMessageHandler):
f"客户端拾音模式：{conn.client_listen_mode}"
)
if msg_json["state"] == "start":
conn.client_have_voice = True
conn.client_voice_stop = False
# 设备从播放模式切回录音模式,清除所有音频状态和缓冲区
conn.reset_audio_states()
elif msg_json["state"] == "stop":
conn.client_have_voice = True
conn.client_voice_stop = True
if conn.asr.interface_type == InterfaceType.STREAM:
# 流式模式下，发送结束请求
@@ -41,14 +40,13 @@ class ListenTextMessageHandler(TextMessageHandler):
# 非流式模式：直接触发ASR识别
if len(conn.asr_audio) > 0:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.reset_audio_states()
if len(asr_audio_task) > 0:
await conn.asr.handle_voice_stop(conn, asr_audio_task)
elif msg_json["state"] == "detect":
conn.client_have_voice = False
conn.asr_audio.clear()
conn.reset_audio_states()
if "text" in msg_json:
conn.last_activity_time = time.time() * 1000
original_text = msg_json["text"] # 保留原始文本
main/xiaozhi-server/core/providers/asr/aliyun.py
+7 -19
View File
@@ -213,36 +213,24 @@ class ASRProvider(ASRProviderBase):
return None
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if self._is_token_expired():
logger.warning("Token已过期，正在自动刷新...")
self._refresh_token()
file_path = None
try:
# 解码Opus为PCM
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 发送请求并获取文本
text = await self._send_request(combined_pcm_data)
text = await self._send_request(artifacts.pcm_bytes)
if text:
return text, file_path
return text, artifacts.file_path
return "", file_path
return "", artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
return "", None
main/xiaozhi-server/core/providers/asr/aliyun_stream.py
+12 -29
View File
@@ -129,17 +129,9 @@ class ASRProvider(ASRProviderBase):
async def open_audio_channels(self, conn):
await super().open_audio_channels(conn)
async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
# 初始化音频缓存
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
# 存储音频数据
if audio:
conn.asr_audio_for_voiceprint.append(audio)
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
async def receive_audio(self, conn, audio, audio_have_voice):
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 只在有声音且没有连接时建立连接（排除正在停止的情况）
if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -156,7 +148,7 @@ class ASRProvider(ASRProviderBase):
await self.asr_ws.send(pcm_frame)
except Exception as e:
logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}")
await self._cleanup(conn)
await self._cleanup()
async def _start_recognition(self, conn: "ConnectionHandler"):
"""开始识别会话"""
@@ -208,8 +200,10 @@ class ASRProvider(ASRProviderBase):
"""转发识别结果"""
try:
while not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = conn.asr_audio
try:
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
response = await self.asr_ws.recv()
result = json.loads(response)
header = result.get("header", {})
@@ -261,19 +255,12 @@ class ASRProvider(ASRProviderBase):
# 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
if conn.client_voice_stop:
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
break
else:
# 自动模式下直接覆盖
self.text = text
conn.reset_vad_states()
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
break
@@ -293,11 +280,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接的音频缓存
await self._cleanup()
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
conn.reset_audio_states()
async def _send_stop_request(self):
"""发送停止识别请求（不关闭连接）"""
@@ -345,7 +328,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
"""获取识别结果"""
result = self.text
self.text = ""
@@ -353,7 +336,7 @@ class ASRProvider(ASRProviderBase):
async def close(self):
"""关闭资源"""
await self._cleanup(None)
await self._cleanup()
if hasattr(self, 'decoder') and self.decoder is not None:
try:
del self.decoder
main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py
+9 -26
View File
@@ -55,17 +55,9 @@ class ASRProvider(ASRProviderBase):
async def open_audio_channels(self, conn):
await super().open_audio_channels(conn)
async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
# 初始化音频缓存
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
# 存储音频数据
if audio:
conn.asr_audio_for_voiceprint.append(audio)
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
async def receive_audio(self, conn, audio, audio_have_voice):
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 只在有声音且没有连接时建立连接
if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -170,6 +162,8 @@ class ASRProvider(ASRProviderBase):
"""转发识别结果"""
try:
while not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = conn.asr_audio
try:
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
result = json.loads(response)
@@ -218,19 +212,12 @@ class ASRProvider(ASRProviderBase):
# 手动模式下,只有在收到stop信号后才触发处理
if conn.client_voice_stop:
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
break
else:
# 自动模式下直接覆盖
self.text = text
conn.reset_vad_states()
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
break
@@ -261,11 +248,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接的音频缓存
await self._cleanup()
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
conn.reset_audio_states()
async def _send_stop_request(self):
"""发送停止请求(用于手动模式停止录音)"""
@@ -329,7 +312,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
"""获取识别结果"""
result = self.text
self.text = ""
main/xiaozhi-server/core/providers/asr/baidu.py
+8 -19
View File
@@ -30,37 +30,26 @@ class ASRProvider(ASRProviderBase):
os.makedirs(self.output_dir, exist_ok=True)
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if not opus_data:
logger.bind(tag=TAG).warning("音频数据为空！")
return None, None
file_path = None
try:
# 检查配置是否已设置
if not self.app_id or not self.api_key or not self.secret_key:
logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
return None, file_path
return None, None
# 将Opus音频数据解码为PCM
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
start_time = time.time()
# 识别本地文件
result = self.client.asr(
combined_pcm_data,
artifacts.pcm_bytes,
"pcm",
16000,
{
@@ -73,13 +62,13 @@ class ASRProvider(ASRProviderBase):
f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
)
result = result["result"][0]
return result, file_path
return result, artifacts.file_path
else:
raise Exception(
f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
)
return None, file_path
return None, artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
return None, file_path
return None, None
main/xiaozhi-server/core/providers/asr/base.py
+148 -33
View File
@@ -5,20 +5,25 @@ import uuid
import json
import time
import queue
import shutil
import asyncio
import tempfile
import traceback
import threading
import opuslib_next
from abc import ABC, abstractmethod
from config.logger import setup_logging
from typing import Optional, Tuple, List, TYPE_CHECKING
if TYPE_CHECKING:
from core.connection import ConnectionHandler
from core.providers.asr.dto.dto import InterfaceType
from core.handle.receiveAudioHandle import startToChat
from core.handle.reportHandle import enqueue_asr_report
from core.utils.util import remove_punctuation_and_length
from core.handle.receiveAudioHandle import handleAudioMessage
from typing import Optional, Tuple, List, NamedTuple, TYPE_CHECKING
if TYPE_CHECKING:
from core.connection import ConnectionHandler
TAG = __name__
logger = setup_logging()
@@ -60,18 +65,17 @@ class ASRProviderBase(ABC):
conn.asr_audio.append(audio)
else:
# 自动/实时模式：使用VAD检测
have_voice = audio_have_voice
conn.asr_audio.append(audio)
if not have_voice and not conn.client_have_voice:
# 如果没有语音，且之前也没有声音，缓存部分音频
if not audio_have_voice and not conn.client_have_voice:
conn.asr_audio = conn.asr_audio[-10:]
return
# 自动模式下通过VAD检测到语音停止时触发识别
if conn.client_voice_stop:
if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.reset_audio_states()
if len(asr_audio_task) > 15:
await self.handle_voice_stop(conn, asr_audio_task)
@@ -96,10 +100,14 @@ class ASRProviderBase(ABC):
wav_data = self._pcm_to_wav(combined_pcm_data)
# 定义ASR任务
asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
asr_task = self.speech_to_text_wrapper(
asr_audio_task, conn.session_id, conn.audio_format
)
if conn.voiceprint_provider and wav_data:
voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
voiceprint_task = conn.voiceprint_provider.identify_speaker(
wav_data, conn.session_id
)
# 并发等待两个结果
asr_result, voiceprint_result = await asyncio.gather(
asr_task, voiceprint_task, return_exceptions=True
@@ -162,20 +170,20 @@ class ASRProviderBase(ABC):
if text_len > 0:
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
audio_snapshot = asr_audio_task.copy()
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
except Exception as e:
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
import traceback
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
"""构建包含说话人信息的文本（仅用于纯文本ASR）"""
if speaker_name and speaker_name.strip():
return json.dumps({
"speaker": speaker_name,
"content": text
}, ensure_ascii=False)
return json.dumps(
{"speaker": speaker_name, "content": text}, ensure_ascii=False
)
else:
return text
@@ -184,23 +192,23 @@ class ASRProviderBase(ABC):
if len(pcm_data) == 0:
logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV")
return b""
# 确保数据长度是偶数（16位音频）
if len(pcm_data) % 2 != 0:
pcm_data = pcm_data[:-1]
# 创建WAV文件头
wav_buffer = io.BytesIO()
try:
with wave.open(wav_buffer, 'wb') as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
with wave.open(wav_buffer, "wb") as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
wav_file.setframerate(16000) # 16kHz采样率
wav_file.writeframes(pcm_data)
wav_buffer.seek(0)
wav_data = wav_buffer.read()
return wav_data
except Exception as e:
logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
@@ -209,6 +217,44 @@ class ASRProviderBase(ABC):
def stop_ws_connection(self):
pass
async def close(self):
pass
class AudioArtifacts(NamedTuple):
pcm_frames: List[bytes]
"""PCM音频帧列表"""
pcm_bytes: bytes
"""合并后的PCM音频字节数据"""
file_path: Optional[str]
"""WAV文件路径"""
temp_path: Optional[str]
"""临时WAV文件路径"""
def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]:
return self._current_artifacts
def requires_file(self) -> bool:
"""是否需要文件输入"""
return False
def prefers_temp_file(self) -> bool:
"""是否优先使用临时文件"""
return False
def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]:
try:
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
temp_path = temp_file.name
with wave.open(temp_path, "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(16000)
wav_file.writeframes(pcm_bytes)
return temp_path
except Exception as e:
logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}")
return None
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
@@ -223,11 +269,80 @@ class ASRProviderBase(ABC):
return file_path
@abstractmethod
async def speech_to_text(
async def speech_to_text_wrapper(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
temp_path = None
try:
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
free_space = shutil.disk_usage(self.output_dir).free
if free_space < len(combined_pcm_data) * 2:
raise OSError("磁盘空间不足")
if self.requires_file() and self.prefers_temp_file():
temp_path = self.build_temp_file(combined_pcm_data)
if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or (
self.requires_file() and not self.prefers_temp_file()
):
file_path = self.save_audio_to_file(pcm_data, session_id)
if len(combined_pcm_data) == 0:
artifacts = None
else:
artifacts = ASRProviderBase.AudioArtifacts(
pcm_frames=pcm_data,
pcm_bytes=combined_pcm_data,
file_path=file_path,
temp_path=temp_path,
)
text, _ = await self.speech_to_text(
opus_data, session_id, audio_format, artifacts
)
return text, file_path
except OSError as e:
logger.bind(tag=TAG).error(f"文件操作错误: {e}")
return None, None
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
return None, None
finally:
try:
if temp_path and os.path.exists(temp_path):
os.unlink(temp_path)
if (
hasattr(self, "delete_audio_file")
and self.delete_audio_file
and file_path
and os.path.exists(file_path)
):
os.remove(file_path)
except Exception as e:
logger.bind(tag=TAG).error(f"文件清理失败: {e}")
@abstractmethod
async def speech_to_text(
self,
opus_data: List[bytes],
session_id: str,
audio_format="opus",
artifacts: Optional[AudioArtifacts] = None,
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本
:param opus_data: 输入的Opus音频数据
:param session_id: 会话ID
:param audio_format: 音频格式，默认"opus"
:param artifacts: 音频工件，包含PCM数据、文件路径等
:return: 识别结果文本和文件路径（如果有）
"""
pass
@staticmethod
@@ -238,23 +353,23 @@ class ASRProviderBase(ABC):
decoder = opuslib_next.Decoder(16000, 1)
pcm_data = []
buffer_size = 960 # 每次处理960个采样点 (60ms at 16kHz)
for i, opus_packet in enumerate(opus_data):
try:
if not opus_packet or len(opus_packet) == 0:
continue
pcm_frame = decoder.decode(opus_packet, buffer_size)
if pcm_frame and len(pcm_frame) > 0:
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).warning(f"Opus解码错误，跳过数据包 {i}: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"音频处理错误，数据包 {i}: {e}")
return pcm_data
except Exception as e:
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
return []
main/xiaozhi-server/core/providers/asr/doubao.py
+7 -18
View File
@@ -232,24 +232,13 @@ class ASRProvider(ASRProviderBase):
yield data[offset:data_len], True
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
try:
# 合并所有opus数据包
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 直接使用PCM数据
# 计算分段大小 (单声道, 16bit, 16kHz采样率)
@@ -258,14 +247,14 @@ class ASRProvider(ASRProviderBase):
# 语音识别
start_time = time.time()
text = await self._send_request(combined_pcm_data, segment_size)
text = await self._send_request(artifacts.pcm_bytes, segment_size)
if text:
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, file_path
return "", file_path
return text, artifacts.file_path
return "", artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
return "", None
main/xiaozhi-server/core/providers/asr/doubao_stream.py
+12 -57
View File
@@ -66,22 +66,9 @@ class ASRProvider(ASRProviderBase):
await super().open_audio_channels(conn)
async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 存储音频数据
if not hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
conn.asr_audio_for_voiceprint.append(audio)
# 当没有音频数据时处理完整语音片段
if (
conn.client_listen_mode != "manual"
and not audio
and len(conn.asr_audio_for_voiceprint) > 0
):
await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
conn.asr_audio_for_voiceprint = []
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 如果本次有声音，且之前没有建立连接
if audio_have_voice and self.asr_ws is None and not self.is_processing:
try:
@@ -174,7 +161,7 @@ class ASRProvider(ASRProviderBase):
try:
while self.asr_ws and not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = getattr(conn, "asr_audio_for_voiceprint", [])
audio_data = conn.asr_audio
try:
response = await self.asr_ws.recv()
result = self.parse_response(response)
@@ -200,7 +187,6 @@ class ASRProvider(ASRProviderBase):
):
logger.bind(tag=TAG).error(f"识别文本：空")
self.text = ""
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(conn, audio_data)
break
@@ -211,18 +197,9 @@ class ASRProvider(ASRProviderBase):
if self.enable_multilingual:
continue
if (
conn.client_listen_mode == "manual"
and conn.client_voice_stop
and len(audio_data) > 0
):
logger.bind(tag=TAG).debug(
"消息结束收到停止信号，触发处理"
)
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 15:
logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
for utterance in utterances:
@@ -240,24 +217,13 @@ class ASRProvider(ASRProviderBase):
self.text = current_text
# 在接收消息中途时收到停止信号
if (
conn.client_voice_stop
and len(audio_data) > 0
):
logger.bind(tag=TAG).debug(
"消息中途收到停止信号，触发处理"
)
await self.handle_voice_stop(
conn, audio_data
)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
if conn.client_voice_stop and len(audio_data) > 0:
logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
await self.handle_voice_stop(conn, audio_data)
break
else:
# 自动模式下直接覆盖
self.text = current_text
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(
conn, audio_data
@@ -288,11 +254,8 @@ class ASRProvider(ASRProviderBase):
await self.asr_ws.close()
self.asr_ws = None
self.is_processing = False
if conn:
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
# 重置所有音频相关状态
conn.reset_audio_states()
def stop_ws_connection(self):
if self.asr_ws:
@@ -436,7 +399,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(f"原始响应数据: {res.hex()}")
raise
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
result = self.text
self.text = "" # 清空text
return result, None
@@ -463,11 +426,3 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("Doubao decoder resources released")
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, "_connections"):
for conn in self._connections.values():
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
main/xiaozhi-server/core/providers/asr/fun_local.py
+8 -37
View File
@@ -64,39 +64,21 @@ class ASRProvider(ASRProviderBase):
)
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
retry_count = 0
while retry_count < MAX_RETRIES:
try:
# 合并所有opus数据包
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 检查磁盘空间
if not self.delete_audio_file:
free_space = shutil.disk_usage(self.output_dir).free
if free_space < len(combined_pcm_data) * 2: # 预留2倍空间
raise OSError("磁盘空间不足")
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 语音识别 - 使用线程池避免阻塞事件循环
start_time = time.time()
result = await asyncio.to_thread(
self.model.generate,
input=combined_pcm_data,
input=artifacts.pcm_bytes,
cache={},
language="auto",
use_itn=True,
@@ -107,7 +89,7 @@ class ASRProvider(ASRProviderBase):
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
)
return text, file_path
return text, artifacts.file_path
except OSError as e:
retry_count += 1
@@ -115,7 +97,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(
f"语音识别失败（已重试{retry_count}次）: {e}", exc_info=True
)
return "", file_path
return "", None
logger.bind(tag=TAG).warning(
f"语音识别失败，正在重试（{retry_count}/{MAX_RETRIES}）: {e}"
)
@@ -123,15 +105,4 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(
f"文件删除失败: {file_path} | 错误: {e}"
)
return "", None
main/xiaozhi-server/core/providers/asr/fun_server.py
+8 -17
View File
@@ -101,7 +101,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""
Convert speech data to text using FunASR.
@@ -109,18 +109,9 @@ class ASRProvider(ASRProviderBase):
:param session_id: Unique session identifier.
:return: Tuple containing recognized text and optional timestamp.
"""
file_path = None
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
async with websockets.connect(
self.uri,
@@ -132,7 +123,7 @@ class ASRProvider(ASRProviderBase):
try:
# Use asyncio to handle WebSocket communication
send_task = asyncio.create_task(
self._send_data(ws, combined_pcm_data, session_id)
self._send_data(ws, artifacts.pcm_bytes, session_id)
)
receive_task = asyncio.create_task(self._receive_responses(ws))
@@ -161,14 +152,14 @@ class ASRProvider(ASRProviderBase):
result = lang_tag_filter(result)
return (
result,
file_path,
artifacts.file_path,
) # Return the recognized text and timestamp (if any)
except websockets.exceptions.ConnectionClosed as e:
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
return "", file_path
return "", artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(
f"Error during speech-to-text conversion: {e}", exc_info=True
)
return "", file_path
return "", artifacts.file_path
main/xiaozhi-server/core/providers/asr/openai.py
+8 -20
View File
@@ -21,20 +21,16 @@ class ASRProvider(ASRProviderBase):
os.makedirs(self.output_dir, exist_ok=True)
async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]:
def requires_file(self) -> bool:
return True
async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None) -> Tuple[Optional[str], Optional[str]]:
file_path = None
try:
start_time = time.time()
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
file_path = self.save_audio_to_file(pcm_data, session_id)
logger.bind(tag=TAG).debug(
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
)
if artifacts is None:
return "", None
file_path = artifacts.file_path
logger.bind(tag=TAG).info(f"file path: {file_path}")
headers = {
"Authorization": f"Bearer {self.api_key}",
@@ -71,12 +67,4 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
return "", None
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py
+10 -50
View File
@@ -1,5 +1,4 @@
import os
import tempfile
from typing import Optional, Tuple, List
import dashscope
from config.logger import setup_logging
@@ -35,56 +34,25 @@ class ASRProvider(ASRProviderBase):
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def _prepare_audio_file(self, pcm_data: bytes) -> str:
"""将PCM数据转换为WAV文件并返回文件路径"""
try:
import wave
# 创建临时WAV文件
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
temp_path = temp_file.name
# 写入WAV格式
with wave.open(temp_path, 'wb') as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
wav_file.setframerate(16000) # 16kHz采样率
wav_file.writeframes(pcm_data)
return temp_path
except Exception as e:
logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
return None
def prefers_temp_file(self) -> bool:
return True
def requires_file(self) -> bool:
return True
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
temp_file_path = None
file_path = None
try:
# 解码音频数据
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
if len(combined_pcm_data) == 0:
logger.bind(tag=tag).warning("音频数据为空")
if artifacts is None:
return "", None
# 准备音频文件
temp_file_path = self._prepare_audio_file(combined_pcm_data)
temp_file_path = artifacts.temp_path
file_path = artifacts.file_path
if not temp_file_path:
return "", None
# 保存音频文件（如果需要）
if not self.delete_audio_file:
file_path = self.save_audio_to_file(pcm_data, session_id)
return "", file_path
# 构造请求消息
messages = [
{
@@ -141,11 +109,3 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=tag).error(f"语音识别失败: {e}")
return "", file_path
finally:
# 清理临时文件
if temp_file_path and os.path.exists(temp_file_path):
try:
os.unlink(temp_file_path)
except Exception as e:
logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")
main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py
+7 -20
View File
@@ -120,24 +120,19 @@ class ASRProvider(ASRProviderBase):
samples_float32 = samples_float32 / 32768
return samples_float32, f.getframerate()
def requires_file(self) -> bool:
return True
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
try:
# 保存音频文件
start_time = time.time()
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
file_path = self.save_audio_to_file(pcm_data, session_id)
logger.bind(tag=TAG).debug(
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
)
if artifacts is None:
return "", None
file_path = artifacts.file_path
# 语音识别
start_time = time.time()
s = self.model.create_stream()
samples, sample_rate = self.read_wave(file_path)
@@ -153,11 +148,3 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
main/xiaozhi-server/core/providers/asr/tencent.py
+7 -18
View File
@@ -32,35 +32,24 @@ class ASRProvider(ASRProviderBase):
os.makedirs(self.output_dir, exist_ok=True)
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if not opus_data:
logger.bind(tag=TAG).warning("音频数据为空！")
return None, None
file_path = None
try:
# 检查配置是否已设置
if not self.secret_id or not self.secret_key:
logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
return None, file_path
return None, None
# 将Opus音频数据解码为PCM
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 将音频数据转换为Base64编码
base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8")
base64_audio = base64.b64encode(artifacts.pcm_bytes).decode("utf-8")
# 构建请求体
request_body = self._build_request_body(base64_audio)
@@ -77,11 +66,11 @@ class ASRProvider(ASRProviderBase):
f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
)
return result, file_path
return result, artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
return None, file_path
return None, None
def _build_request_body(self, base64_audio: str) -> str:
"""构建请求体"""
main/xiaozhi-server/core/providers/asr/vosk.py
+7 -30
View File
@@ -44,36 +44,21 @@ class ASRProvider(ASRProviderBase):
raise
async def speech_to_text(
self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
try:
# 检查模型是否加载成功
if not self.model:
logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别")
return "", None
# 解码音频（如果原始格式是Opus）
if audio_format == "pcm":
pcm_data = audio_data
else:
pcm_data = self.decode_opus(audio_data)
if not pcm_data:
logger.bind(tag=TAG).warning("解码后的PCM数据为空，无法进行识别")
if artifacts is None:
return "", None
# 合并PCM数据
combined_pcm_data = b"".join(pcm_data)
if len(combined_pcm_data) == 0:
if not artifacts.pcm_bytes:
logger.bind(tag=TAG).warning("合并后的PCM数据为空")
return "", None
# 判断是否保存为WAV文件
if not self.delete_audio_file:
file_path = self.save_audio_to_file(pcm_data, session_id)
start_time = time.time()
@@ -81,8 +66,8 @@ class ASRProvider(ASRProviderBase):
chunk_size = 2000
text_result = ""
for i in range(0, len(combined_pcm_data), chunk_size):
chunk = combined_pcm_data[i:i+chunk_size]
for i in range(0, len(artifacts.pcm_bytes), chunk_size):
chunk = artifacts.pcm_bytes[i:i+chunk_size]
if self.recognizer.AcceptWaveform(chunk):
result = json.loads(self.recognizer.Result())
text = result.get('text', '')
@@ -99,16 +84,8 @@ class ASRProvider(ASRProviderBase):
f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
)
return text_result.strip(), file_path
return text_result.strip(), artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
return "", None
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
main/xiaozhi-server/core/providers/asr/xunfei_stream.py
+4 -28
View File
@@ -104,11 +104,6 @@ class ASRProvider(ASRProviderBase):
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 存储音频数据用于声纹识别
if not hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
conn.asr_audio_for_voiceprint.append(audio)
# 如果本次有声音，且之前没有建立连接
if audio_have_voice and self.asr_ws is None and not self.is_processing:
try:
@@ -234,14 +229,8 @@ class ASRProvider(ASRProviderBase):
self.text += w
if status == 2:
if conn.client_listen_mode == "manual":
audio_data = getattr(conn, "asr_audio_for_voiceprint", [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, conn.asr_audio)
break
except asyncio.TimeoutError:
@@ -265,13 +254,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接资源
await self._cleanup()
# 清理连接的音频缓存
if conn:
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
conn.reset_audio_states()
async def handle_voice_stop(
self, conn: "ConnectionHandler", asr_audio_task: List[bytes]
@@ -339,7 +322,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
"""获取识别结果"""
result = self.text
self.text = ""
@@ -368,10 +351,3 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, "_connections"):
for conn in self._connections.values():
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+20 -9
View File
@@ -5,11 +5,14 @@ if TYPE_CHECKING:
from ..base import IntentProviderBase
from plugins_func.functions.play_music import initialize_music_handler
from config.logger import setup_logging
from core.utils.util import get_system_error_response
import re
import json
import hashlib
import time
TAG = __name__
logger = setup_logging()
@@ -118,12 +121,16 @@ class IntentProvider(IntentProviderBase):
return prompt
def replyResult(self, text: str, original_text: str):
llm_result = self.llm.response_no_stream(
system_prompt=text,
user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+ original_text,
)
return llm_result
try:
llm_result = self.llm.response_no_stream(
system_prompt=text,
user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+ original_text,
)
return llm_result
except Exception as e:
logger.bind(tag=TAG).error(f"Error in generating reply result: {e}")
return get_system_error_response(self.config)
async def detect_intent(
self, conn: "ConnectionHandler", dialogue_history: List[Dict], text: str
@@ -199,9 +206,13 @@ class IntentProvider(IntentProviderBase):
llm_start_time = time.time()
logger.bind(tag=TAG).debug(f"开始LLM意图识别调用, 模型: {model_info}")
intent = self.llm.response_no_stream(
system_prompt=prompt_music, user_prompt=user_prompt
)
try:
intent = self.llm.response_no_stream(
system_prompt=prompt_music, user_prompt=user_prompt
)
except Exception as e:
logger.bind(tag=TAG).error(f"Error in intent detection LLM call: {e}")
return '{"function_call": {"name": "continue_chat"}}'
# 记录LLM调用完成时间
llm_time = time.time() - llm_start_time
main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py
+68 -73
View File
@@ -21,83 +21,78 @@ class LLMProvider(LLMProviderBase):
check_model_key("AliBLLLM", self.api_key)
def response(self, session_id, dialogue):
try:
# 处理dialogue
if self.is_No_prompt:
dialogue.pop(0)
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】处理后的dialogue: {dialogue}"
)
# 构造调用参数
call_params = {
"api_key": self.api_key,
"app_id": self.app_id,
"session_id": session_id,
"messages": dialogue,
# 开启SDK原生流式
"stream": True,
}
if self.memory_id != False:
# 百练memory需要prompt参数
prompt = dialogue[-1].get("content")
call_params["memory_id"] = self.memory_id
call_params["prompt"] = prompt
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】处理后的prompt: {prompt}"
)
# 可选地设置自定义API基地址（若配置为兼容模式URL则忽略）
if self.base_url and ("/api/" in self.base_url):
dashscope.base_http_api_url = self.base_url
responses = Application.call(**call_params)
# 流式处理（SDK在stream=True时返回可迭代对象；否则返回单次响应对象）
# 处理dialogue
if self.is_No_prompt:
dialogue.pop(0)
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】构造参数: {dict(call_params, api_key='***')}"
f"【阿里百练API服务】处理后的dialogue: {dialogue}"
)
last_text = ""
try:
for resp in responses:
if resp.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={resp.status_code}, message={resp.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
continue
current_text = getattr(getattr(resp, "output", None), "text", None)
if current_text is None:
continue
# SDK流式为增量覆盖，计算差量输出
if len(current_text) >= len(last_text):
delta = current_text[len(last_text):]
else:
# 避免偶发回退
delta = current_text
if delta:
yield delta
last_text = current_text
except TypeError:
# 非流式回落（一次性返回）
if responses.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={responses.status_code}, message={responses.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
yield "【阿里百练API服务响应异常】"
else:
full_text = getattr(getattr(responses, "output", None), "text", "")
logger.bind(tag=TAG).info(
f"【阿里百练API服务】完整响应长度: {len(full_text)}"
)
for i in range(0, len(full_text), self.streaming_chunk_size):
chunk = full_text[i:i + self.streaming_chunk_size]
if chunk:
yield chunk
# 构造调用参数
call_params = {
"api_key": self.api_key,
"app_id": self.app_id,
"session_id": session_id,
"messages": dialogue,
# 开启SDK原生流式
"stream": True,
}
if self.memory_id != False:
# 百练memory需要prompt参数
prompt = dialogue[-1].get("content")
call_params["memory_id"] = self.memory_id
call_params["prompt"] = prompt
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】处理后的prompt: {prompt}"
)
except Exception as e:
logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
yield "【LLM服务响应异常】"
# 可选地设置自定义API基地址（若配置为兼容模式URL则忽略）
if self.base_url and ("/api/" in self.base_url):
dashscope.base_http_api_url = self.base_url
responses = Application.call(**call_params)
# 流式处理（SDK在stream=True时返回可迭代对象；否则返回单次响应对象）
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】构造参数: {dict(call_params, api_key='***')}"
)
last_text = ""
try:
for resp in responses:
if resp.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={resp.status_code}, message={resp.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
continue
current_text = getattr(getattr(resp, "output", None), "text", None)
if current_text is None:
continue
# SDK流式为增量覆盖，计算差量输出
if len(current_text) >= len(last_text):
delta = current_text[len(last_text):]
else:
# 避免偶发回退
delta = current_text
if delta:
yield delta
last_text = current_text
except TypeError:
# 非流式回落（一次性返回）
if responses.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={responses.status_code}, message={responses.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
yield "【阿里百练API服务响应异常】"
else:
full_text = getattr(getattr(responses, "output", None), "text", "")
logger.bind(tag=TAG).info(
f"【阿里百练API服务】完整响应长度: {len(full_text)}"
)
for i in range(0, len(full_text), self.streaming_chunk_size):
chunk = full_text[i:i + self.streaming_chunk_size]
if chunk:
yield chunk
def response_with_functions(self, session_id, dialogue, functions=None):
# 阿里百练当前未支持原生的 function call。为保持兼容，这里回退到普通文本流式输出。
main/xiaozhi-server/core/providers/llm/base.py
+9 -14
View File
@@ -11,20 +11,15 @@ class LLMProviderBase(ABC):
pass
def response_no_stream(self, system_prompt, user_prompt, **kwargs):
try:
# 构造对话格式
dialogue = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
]
result = ""
for part in self.response("", dialogue, **kwargs):
result += part
return result
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
return "【LLM服务响应异常】"
# 构造对话格式
dialogue = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
]
result = ""
for part in self.response("", dialogue, **kwargs):
result += part
return result
def response_with_functions(self, session_id, dialogue, functions=None):
"""
main/xiaozhi-server/core/providers/llm/dify/dify.py
+61 -66
View File
@@ -20,76 +20,71 @@ class LLMProvider(LLMProviderBase):
logger.bind(tag=TAG).error(model_key_msg)
def response(self, session_id, dialogue, **kwargs):
try:
# 取最后一条用户消息
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
conversation_id = self.session_conversation_map.get(session_id)
# 取最后一条用户消息
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
conversation_id = self.session_conversation_map.get(session_id)
# 发起流式请求
# 发起流式请求
if self.mode == "chat-messages":
request_json = {
"query": last_msg["content"],
"response_mode": "streaming",
"user": session_id,
"inputs": {},
"conversation_id": conversation_id,
}
elif self.mode == "workflows/run":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id,
}
elif self.mode == "completion-messages":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id,
}
with requests.post(
f"{self.base_url}/{self.mode}",
headers={"Authorization": f"Bearer {self.api_key}"},
json=request_json,
stream=True,
) as r:
if self.mode == "chat-messages":
request_json = {
"query": last_msg["content"],
"response_mode": "streaming",
"user": session_id,
"inputs": {},
"conversation_id": conversation_id,
}
for line in r.iter_lines():
if line.startswith(b"data: "):
event = json.loads(line[6:])
# 如果没有找到conversation_id，则获取此次conversation_id
if not conversation_id:
conversation_id = event.get("conversation_id")
self.session_conversation_map[session_id] = (
conversation_id # 更新映射
)
# 过滤 message_replace 事件，此事件会全量推一次
if event.get("event") != "message_replace" and event.get(
"answer"
):
yield event["answer"]
elif self.mode == "workflows/run":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id,
}
for line in r.iter_lines():
if line.startswith(b"data: "):
event = json.loads(line[6:])
if event.get("event") == "workflow_finished":
if event["data"]["status"] == "succeeded":
yield event["data"]["outputs"]["answer"]
else:
yield "【服务响应异常】"
elif self.mode == "completion-messages":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id,
}
with requests.post(
f"{self.base_url}/{self.mode}",
headers={"Authorization": f"Bearer {self.api_key}"},
json=request_json,
stream=True,
) as r:
if self.mode == "chat-messages":
for line in r.iter_lines():
if line.startswith(b"data: "):
event = json.loads(line[6:])
# 如果没有找到conversation_id，则获取此次conversation_id
if not conversation_id:
conversation_id = event.get("conversation_id")
self.session_conversation_map[session_id] = (
conversation_id # 更新映射
)
# 过滤 message_replace 事件，此事件会全量推一次
if event.get("event") != "message_replace" and event.get(
"answer"
):
yield event["answer"]
elif self.mode == "workflows/run":
for line in r.iter_lines():
if line.startswith(b"data: "):
event = json.loads(line[6:])
if event.get("event") == "workflow_finished":
if event["data"]["status"] == "succeeded":
yield event["data"]["outputs"]["answer"]
else:
yield "【服务响应异常】"
elif self.mode == "completion-messages":
for line in r.iter_lines():
if line.startswith(b"data: "):
event = json.loads(line[6:])
# 过滤 message_replace 事件，此事件会全量推一次
if event.get("event") != "message_replace" and event.get(
"answer"
):
yield event["answer"]
except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
yield "【服务响应异常】"
for line in r.iter_lines():
if line.startswith(b"data: "):
event = json.loads(line[6:])
# 过滤 message_replace 事件，此事件会全量推一次
if event.get("event") != "message_replace" and event.get(
"answer"
):
yield event["answer"]
def response_with_functions(self, session_id, dialogue, functions=None):
if len(dialogue) == 2 and functions is not None and len(functions) > 0:
main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py
+39 -44
View File
@@ -19,53 +19,48 @@ class LLMProvider(LLMProviderBase):
logger.bind(tag=TAG).error(model_key_msg)
def response(self, session_id, dialogue, **kwargs):
try:
# 取最后一条用户消息
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
# 取最后一条用户消息
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
# 发起流式请求
with requests.post(
f"{self.base_url}/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"stream": True,
"chatId": session_id,
"detail": self.detail,
"variables": self.variables,
"messages": [{"role": "user", "content": last_msg["content"]}],
},
stream=True,
) as r:
for line in r.iter_lines():
if line:
try:
if line.startswith(b"data: "):
if line[6:].decode("utf-8") == "[DONE]":
break
# 发起流式请求
with requests.post(
f"{self.base_url}/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"stream": True,
"chatId": session_id,
"detail": self.detail,
"variables": self.variables,
"messages": [{"role": "user", "content": last_msg["content"]}],
},
stream=True,
) as r:
for line in r.iter_lines():
if line:
try:
if line.startswith(b"data: "):
if line[6:].decode("utf-8") == "[DONE]":
break
data = json.loads(line[6:])
if "choices" in data and len(data["choices"]) > 0:
delta = data["choices"][0].get("delta", {})
if (
delta
and "content" in delta
and delta["content"] is not None
):
content = delta["content"]
if "<think>" in content:
continue
if "</think>" in content:
continue
yield content
data = json.loads(line[6:])
if "choices" in data and len(data["choices"]) > 0:
delta = data["choices"][0].get("delta", {})
if (
delta
and "content" in delta
and delta["content"] is not None
):
content = delta["content"]
if "<think>" in content:
continue
if "</think>" in content:
continue
yield content
except json.JSONDecodeError as e:
continue
except Exception as e:
continue
except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
yield "【服务响应异常】"
except json.JSONDecodeError as e:
continue
except Exception as e:
continue
def response_with_functions(self, session_id, dialogue, functions=None):
logger.bind(tag=TAG).error(
main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py
+37 -43
View File
@@ -15,55 +15,49 @@ class LLMProvider(LLMProviderBase):
self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
def response(self, session_id, dialogue, **kwargs):
try:
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
# 提取最后一个 role 为 'user' 的 content
input_text = None
if isinstance(dialogue, list): # 确保 dialogue 是一个列表
# 逆序遍历，找到最后一个 role 为 'user' 的消息
for message in reversed(dialogue):
if message.get("role") == "user": # 找到 role 为 'user' 的消息
input_text = message.get("content", "")
break # 找到后立即退出循环
# 提取最后一个 role 为 'user' 的 content
input_text = None
if isinstance(dialogue, list): # 确保 dialogue 是一个列表
# 逆序遍历，找到最后一个 role 为 'user' 的消息
for message in reversed(dialogue):
if message.get("role") == "user": # 找到 role 为 'user' 的消息
input_text = message.get("content", "")
break # 找到后立即退出循环
# 构造请求数据
payload = {
"text": input_text,
"agent_id": self.agent_id,
"conversation_id": session_id, # 使用 session_id 作为 conversation_id
}
# 设置请求头
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
# 构造请求数据
payload = {
"text": input_text,
"agent_id": self.agent_id,
"conversation_id": session_id, # 使用 session_id 作为 conversation_id
}
# 设置请求头
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
# 发起 POST 请求
response = requests.post(self.api_url, json=payload, headers=headers)
# 发起 POST 请求
response = requests.post(self.api_url, json=payload, headers=headers)
# 检查请求是否成功
response.raise_for_status()
# 检查请求是否成功
response.raise_for_status()
# 解析返回数据
data = response.json()
speech = (
data.get("response", {})
.get("speech", {})
.get("plain", {})
.get("speech", "")
)
# 解析返回数据
data = response.json()
speech = (
data.get("response", {})
.get("speech", {})
.get("plain", {})
.get("speech", "")
)
# 返回生成的内容
if speech:
yield speech
else:
logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
except RequestException as e:
logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
# 返回生成的内容
if speech:
yield speech
else:
logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
def response_with_functions(self, session_id, dialogue, functions=None):
logger.bind(tag=TAG).error(
main/xiaozhi-server/core/providers/llm/ollama/ollama.py
+118 -128
View File
@@ -25,151 +25,141 @@ class LLMProvider(LLMProviderBase):
self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
def response(self, session_id, dialogue, **kwargs):
try:
# 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
if self.is_qwen3:
# 复制对话列表，避免修改原始对话
dialogue_copy = dialogue.copy()
# 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
if self.is_qwen3:
# 复制对话列表，避免修改原始对话
dialogue_copy = dialogue.copy()
# 找到最后一条用户消息
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = (
"/no_think " + dialogue_copy[i]["content"]
)
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
# 使用修改后的对话
dialogue = dialogue_copy
responses = self.client.chat.completions.create(
model=self.model_name, messages=dialogue, stream=True
)
is_active = True
# 用于处理跨chunk的标签
buffer = ""
for chunk in responses:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
# 找到最后一条用户消息
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = (
"/no_think " + dialogue_copy[i]["content"]
)
content = delta.content if hasattr(delta, "content") else ""
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
if content:
# 将内容添加到缓冲区
buffer += content
# 使用修改后的对话
dialogue = dialogue_copy
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
responses = self.client.chat.completions.create(
model=self.model_name, messages=dialogue, stream=True
)
is_active = True
# 用于处理跨chunk的标签
buffer = ""
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
for chunk in responses:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
if content:
# 将内容添加到缓冲区
buffer += content
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
yield buffer
buffer = "" # 清空缓冲区
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
yield "【Ollama服务响应异常】"
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
yield buffer
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
def response_with_functions(self, session_id, dialogue, functions=None):
try:
# 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
if self.is_qwen3:
# 复制对话列表，避免修改原始对话
dialogue_copy = dialogue.copy()
# 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
if self.is_qwen3:
# 复制对话列表，避免修改原始对话
dialogue_copy = dialogue.copy()
# 找到最后一条用户消息
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = (
"/no_think " + dialogue_copy[i]["content"]
)
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
# 使用修改后的对话
dialogue = dialogue_copy
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True,
tools=functions,
)
is_active = True
buffer = ""
for chunk in stream:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else None
tool_calls = (
delta.tool_calls if hasattr(delta, "tool_calls") else None
# 找到最后一条用户消息
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = (
"/no_think " + dialogue_copy[i]["content"]
)
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
# 如果是工具调用，直接传递
if tool_calls:
yield None, tool_calls
continue
# 使用修改后的对话
dialogue = dialogue_copy
# 处理文本内容
if content:
# 将内容添加到缓冲区
buffer += content
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True,
tools=functions,
)
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
is_active = True
buffer = ""
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
for chunk in stream:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else None
tool_calls = (
delta.tool_calls if hasattr(delta, "tool_calls") else None
)
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
yield buffer, None
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
# 如果是工具调用，直接传递
if tool_calls:
yield None, tool_calls
continue
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
yield f"【Ollama服务响应异常: {str(e)}】", None
# 处理文本内容
if content:
# 将内容添加到缓冲区
buffer += content
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
yield buffer, None
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
continue
main/xiaozhi-server/core/providers/llm/openai/openai.py
+63 -72
View File
@@ -56,87 +56,78 @@ class LLMProvider(LLMProviderBase):
return dialogue
def response(self, session_id, dialogue, **kwargs):
try:
dialogue = self.normalize_dialogue(dialogue)
dialogue = self.normalize_dialogue(dialogue)
request_params = {
"model": self.model_name,
"messages": dialogue,
"stream": True,
}
request_params = {
"model": self.model_name,
"messages": dialogue,
"stream": True,
}
# 添加可选参数,只有当参数不为None时才添加
optional_params = {
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
}
# 添加可选参数,只有当参数不为None时才添加
optional_params = {
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
}
for key, value in optional_params.items():
if value is not None:
request_params[key] = value
for key, value in optional_params.items():
if value is not None:
request_params[key] = value
responses = self.client.chat.completions.create(**request_params)
responses = self.client.chat.completions.create(**request_params)
is_active = True
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
content = getattr(delta, "content", "") if delta else ""
except IndexError:
content = ""
if content:
if "<think>" in content:
is_active = False
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split("</think>")[-1]
if is_active:
yield content
except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
is_active = True
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
content = getattr(delta, "content", "") if delta else ""
except IndexError:
content = ""
if content:
if "<think>" in content:
is_active = False
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split("</think>")[-1]
if is_active:
yield content
def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
try:
dialogue = self.normalize_dialogue(dialogue)
dialogue = self.normalize_dialogue(dialogue)
request_params = {
"model": self.model_name,
"messages": dialogue,
"stream": True,
"tools": functions,
}
request_params = {
"model": self.model_name,
"messages": dialogue,
"stream": True,
"tools": functions,
}
optional_params = {
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
}
optional_params = {
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
}
for key, value in optional_params.items():
if value is not None:
request_params[key] = value
for key, value in optional_params.items():
if value is not None:
request_params[key] = value
stream = self.client.chat.completions.create(**request_params)
stream = self.client.chat.completions.create(**request_params)
for chunk in stream:
if getattr(chunk, "choices", None):
delta = chunk.choices[0].delta
content = getattr(delta, "content", "")
tool_calls = getattr(delta, "tool_calls", None)
yield content, tool_calls
elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
usage_info = getattr(chunk, "usage", None)
logger.bind(tag=TAG).info(
f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，"
f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
)
except Exception as e:
logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
yield f"【OpenAI服务响应异常: {e}】", None
for chunk in stream:
if getattr(chunk, "choices", None):
delta = chunk.choices[0].delta
content = getattr(delta, "content", "")
tool_calls = getattr(delta, "tool_calls", None)
yield content, tool_calls
elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
usage_info = getattr(chunk, "usage", None)
logger.bind(tag=TAG).info(
f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，"
f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
)
main/xiaozhi-server/core/providers/llm/xinference/xinference.py
+45 -58
View File
@@ -31,68 +31,55 @@ class LLMProvider(LLMProviderBase):
raise
def response(self, session_id, dialogue, **kwargs):
try:
logger.bind(tag=TAG).debug(
f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
)
responses = self.client.chat.completions.create(
model=self.model_name, messages=dialogue, stream=True
)
is_active = True
for chunk in responses:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
if content:
if "<think>" in content:
is_active = False
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split("</think>")[-1]
if is_active:
yield content
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Xinference response generation: {e}")
yield "【Xinference服务响应异常】"
logger.bind(tag=TAG).debug(
f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
)
responses = self.client.chat.completions.create(
model=self.model_name, messages=dialogue, stream=True
)
is_active = True
for chunk in responses:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
if content:
if "<think>" in content:
is_active = False
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split("</think>")[-1]
if is_active:
yield content
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
def response_with_functions(self, session_id, dialogue, functions=None):
try:
logger.bind(tag=TAG).debug(
f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
)
if functions:
logger.bind(tag=TAG).debug(
f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
)
if functions:
logger.bind(tag=TAG).debug(
f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}"
)
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True,
tools=functions,
f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}"
)
for chunk in stream:
delta = chunk.choices[0].delta
content = delta.content
tool_calls = delta.tool_calls
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True,
tools=functions,
)
if content:
yield content, tool_calls
elif tool_calls:
yield None, tool_calls
for chunk in stream:
delta = chunk.choices[0].delta
content = delta.content
tool_calls = delta.tool_calls
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Xinference function call: {e}")
yield {
"type": "content",
"content": f"【Xinference服务响应异常: {str(e)}】",
}
if content:
yield content, tool_calls
elif tool_calls:
yield None, tool_calls
main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py
+8 -8
View File
@@ -162,19 +162,19 @@ class MemoryProvider(MemoryProviderBase):
msgStr += f"当前时间：{time_str}"
if self.save_to_file:
result = self.llm.response_no_stream(
short_term_memory_prompt,
msgStr,
max_tokens=2000,
temperature=0.2,
)
json_str = extract_json_data(result)
try:
result = self.llm.response_no_stream(
short_term_memory_prompt,
msgStr,
max_tokens=2000,
temperature=0.2,
)
json_str = extract_json_data(result)
json.loads(json_str) # 检查json格式是否正确
self.short_memory = json_str
self.save_memory_to_file()
except Exception as e:
print("Error:", e)
logger.bind(tag=TAG).error(f"Error in saving memory: {e}")
else:
# 当save_to_file为False时，调用Java端的聊天记录总结接口
summary_id = session_id if session_id else self.role_id
main/xiaozhi-server/core/providers/tts/alibl_stream.py
+2 -9
View File
@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
# 音频参数配置
self.format = config.get("format", "pcm")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
volume = config.get("volume", "50")
self.volume = int(volume) if volume else 50
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
"X-DashScope-DataInspection": "enable",
}
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
async def _ensure_connection(self):
"""确保WebSocket连接可用，支持60秒内连接复用"""
try:
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
main/xiaozhi-server/core/providers/tts/aliyun.py
+1 -3
View File
@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
self.appkey = config.get("appkey")
self.format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
"token": self.token,
"text": text,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"voice": self.voice,
"volume": self.volume,
"speech_rate": self.speech_rate,
main/xiaozhi-server/core/providers/tts/aliyun_stream.py
+2 -11
View File
@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
self.format = config.get("format", "pcm")
self.audio_file_type = config.get("format", "pcm")
# 采样率配置
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
# 音色配置 - CosyVoice大模型音色
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
# 专属tts设置
self.task_id = uuid.uuid4().hex
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# Token管理
if self.access_key_id and self.access_key_secret:
self._refresh_token()
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
main/xiaozhi-server/core/providers/tts/base.py
+16 -5
View File
@@ -1,17 +1,18 @@
import os
import re
import time
import uuid
import queue
import asyncio
import threading
import traceback
from core.utils import p3
from datetime import datetime
from core.utils import textUtils
from typing import Callable, Any
from abc import ABC, abstractmethod
from config.logger import setup_logging
from core.utils import opus_encoder_utils
from core.utils.tts import MarkdownCleaner
from core.utils.output_counter import add_device_output
from core.handle.reportHandle import enqueue_tts_report
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
file_type=self.audio_file_type,
is_opus=True,
callback=opus_handler,
sample_rate=self.conn.sample_rate,
opus_encoder=self.opus_encoder,
)
break
else:
@@ -138,7 +141,7 @@ class TTSProviderBase(ABC):
logger.bind(tag=TAG).error(
f"语音生成失败: {text}，请检查网络或服务是否正常"
)
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
self._process_audio_file_stream(tmp_file, callback=opus_handler)
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
audio_bytes,
file_type=self.audio_file_type,
is_opus=True,
callback=lambda data: audio_datas.append(data)
callback=lambda data: audio_datas.append(data),
sample_rate=self.conn.sample_rate,
)
return audio_datas
else:
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为PCM编码"""
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为Opus编码"""
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
def tts_one_sentence(
self,
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
async def open_audio_channels(self, conn):
self.conn = conn
# 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理）
if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
)
# tts 消化线程
self.tts_priority_thread = threading.Thread(
target=self.tts_text_priority_thread, daemon=True
main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py
+37 -36
View File
@@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("speaker")
speech_rate = config.get("speech_rate", "0")
loudness_rate = config.get("loudness_rate", "0")
pitch = config.get("pitch", "0")
self.speech_rate = int(speech_rate) if speech_rate else 0
self.loudness_rate = int(loudness_rate) if loudness_rate else 0
self.pitch = int(pitch) if pitch else 0
# 多情感音色参数
self.emotion = config.get("emotion", "neutral")
emotion_scale = config.get("emotion_scale", "4")
self.emotion_scale = int(emotion_scale) if emotion_scale else 4
# 默认 audio_params 配置
default_audio_params = {
"speech_rate": 0,
"loudness_rate": 0
}
# 默认 additions 配置
default_additions = {
"aigc_metadata": {},
"cache_config": {},
"post_process": {
"pitch": 0
}
}
# 默认 mix_speaker 配置
default_mix_speaker = {}
# 合并用户配置
self.audio_params = {**default_audio_params, **config.get("audio_params", {})}
self.additions = {**default_additions, **config.get("additions", {})}
self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})}
self.ws_url = config.get("ws_url")
self.authorization = config.get("authorization")
@@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase):
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
self.tts_text = ""
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
model_key_msg = check_model_key("TTS", self.access_token)
if model_key_msg:
logger.bind(tag=TAG).error(model_key_msg)
@@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase):
async def open_audio_channels(self, conn):
try:
await super().open_audio_channels(conn)
# 更新 audio_params 中的采样率为实际的 conn.sample_rate
self.audio_params["sample_rate"] = conn.sample_rate
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}")
self.ws = None
@@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase):
text="",
speaker="",
audio_format="pcm",
audio_sample_rate=16000,
):
audio_params = {
"format": audio_format,
"sample_rate": audio_sample_rate,
"speech_rate": self.speech_rate,
"loudness_rate": self.loudness_rate
# 构建 req_params
req_params = {
"text": text,
"speaker": speaker,
"audio_params": {**self.audio_params, "format": audio_format},
"additions": json.dumps(self.additions)
}
# 如果是多情感音色,添加情感参数
if '_emo_' in self.voice:
if self.emotion:
audio_params["emotion"] = self.emotion
audio_params["emotion_scale"] = self.emotion_scale
# 如果有 mix_speaker 配置，添加到 req_params
if self.mix_speaker:
req_params["mix_speaker"] = self.mix_speaker
return str.encode(
json.dumps(
@@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase):
"user": {"uid": uid},
"event": event,
"namespace": "BidirectionalTTS",
"req_params": {
"text": text,
"speaker": speaker,
"audio_params": audio_params,
"additions": json.dumps({
"post_process": {
"pitch": self.pitch
}
})
},
"req_params": req_params
}
)
)
main/xiaozhi-server/core/providers/tts/index_stream.py
+14
View File
@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def audio_to_pcm_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为PCM编码，使用24kHz采样率"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为Opus编码，使用24kHz采样率和自己的编码器"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
async def close(self):
"""资源清理"""
await super().close()
main/xiaozhi-server/core/providers/tts/linkerai.py
+3 -8
View File
@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
self.audio_format = "pcm"
self.before_stop_play_files = []
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_durition": 60,
"stream": "true",
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": "pcm",
"instruct_text": "请生成一段自然流畅的语音",
}
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
"Content-Type": "application/json",
}
# 一帧 PCM 所需字节数：60 ms &times; 16 kHz &times; 1 ch &times; 2 B = 1 920
# 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B
frame_bytes = int(
self.opus_encoder.sample_rate
* self.opus_encoder.channels # 1
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_duration": 60,
"stream": False,
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": self.audio_format,
"instruct_text": "请生成一段自然流畅的语音",
}
main/xiaozhi-server/core/providers/tts/minimax_httpstream.py
+20 -4
View File
@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
}
self.audio_file_type = defult_audio_setting.get("format", "pcm")
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=24000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
async def open_audio_channels(self, conn):
"""初始化音频通道,并根据conn.sample_rate更新配置"""
# 调用父类方法
await super().open_audio_channels(conn)
# 更新audio_setting中的采样率为实际的conn.sample_rate
self.audio_setting["sample_rate"] = conn.sample_rate
def tts_text_priority_thread(self):
"""流式文本处理线程"""
while not self.conn.stop_event.is_set():
@@ -212,6 +216,18 @@ class TTSProvider(TTSProviderBase):
try:
data = json.loads(json_str)
# 检查业务层错误
base_resp = data.get("base_resp", {})
status_code = base_resp.get("status_code", 0)
if status_code != 0:
status_msg = base_resp.get("status_msg", "未知错误")
logger.bind(tag=TAG).error(
f"TTS请求失败, 错误码:{status_code}, 错误消息:{status_msg}"
)
self.tts_audio_queue.put((SentenceType.LAST, [], None))
return
status = data.get("data", {}).get("status", 1)
audio_hex = data.get("data", {}).get("audio")
main/xiaozhi-server/core/providers/tts/paddle_speech.py
+1 -4
View File
@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
self.spk_id = int(config.get("private_voice"))
else:
self.spk_id = int(config.get("spk_id", "0"))
sample_rate = config.get("sample_rate", 24000)
self.sample_rate = float(sample_rate) if sample_rate else 24000
speed = config.get("speed", 1.0)
self.speed = float(speed) if speed else 1.0
main/xiaozhi-server/core/providers/tts/siliconflow.py
-1
View File
@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("voice")
self.response_format = config.get("response_format", "mp3")
self.audio_file_type = config.get("response_format", "mp3")
self.sample_rate = config.get("sample_rate")
self.speed = float(config.get("speed", 1.0))
self.gain = config.get("gain")
main/xiaozhi-server/core/providers/tts/xunfei_stream.py
+1 -9
View File
@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
# 音频编码配置
self.format = config.get("format", "raw")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
# 口语化配置
self.oral_level = config.get("oral_level", "mid")
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
# 序列号管理
self.text_seq = 0
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
# 验证必需参数
if not all([self.app_id, self.api_key, self.api_secret]):
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"rhy": 0,
"audio": {
"encoding": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"channels": 1,
"bit_depth": 16,
"frame_size": 0
main/xiaozhi-server/core/utils/util.py
+44 -16
View File
@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
def audio_to_data_stream(
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
) -> None:
# 获取文件后缀名
file_type = os.path.splitext(audio_file_path)[1]
@@ -238,12 +238,12 @@ def audio_to_data_stream(
audio_file_path, format=file_type, parameters=["-nostdin"]
)
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
# 转换为单声道/指定采样率/16位小端编码（确保与编码器匹配）
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
# 获取原始PCM数据（16位小端）
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
async def audio_to_data(
@@ -325,7 +325,7 @@ async def audio_to_data(
def audio_bytes_to_data_stream(
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
) -> None:
"""
直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
audio = AudioSegment.from_file(
BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
)
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
# 初始化Opus编码器
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
"""
将PCM数据流式编码为Opus或直接输出PCM
Args:
raw_data: PCM原始数据
is_opus: 是否编码为Opus
callback: 回调函数
sample_rate: 采样率
opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
"""
using_temp_encoder = False
if is_opus and opus_encoder is None:
encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
using_temp_encoder = True
# 编码参数
frame_duration = 60 # 60ms per frame
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
frame_size = int(sample_rate * frame_duration / 1000) # samples/frame
# 按帧处理所有音频数据（包括最后一帧可能补零）
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
chunk += b"\x00" * (frame_size * 2 - len(chunk))
if is_opus:
# 转换为numpy数组处理
np_frame = np.frombuffer(chunk, dtype=np.int16)
# 编码Opus数据
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
if using_temp_encoder:
# 使用临时编码器（仅用于独立音频场景）
np_frame = np.frombuffer(chunk, dtype=np.int16)
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
else:
# 使用外部编码器（TTS流式场景,保持状态连续）
is_last = (i + frame_size * 2 >= len(raw_data))
opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
else:
# PCM模式,直接输出
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
callback(frame_data)
@@ -573,3 +590,14 @@ def validate_mcp_endpoint(mcp_endpoint: str) -> bool:
return False
return True
def get_system_error_response(config: dict) -> str:
"""获取系统错误时的回复
Args:
config: 配置字典
Returns:
str: 系统错误时的回复
"""
return config.get("system_error_response", "主人，小智现在有点忙，我们稍后再试吧。")
main/xiaozhi-server/test/css/test_page.css
+320 -27
View File
@@ -108,26 +108,68 @@ body {
animation: shadow-float 1.26s ease-in-out infinite;
}
.loading-char-float .main-char span:nth-child(1) { animation-delay: 0s; }
.loading-char-float .main-char span:nth-child(2) { animation-delay: 0.11s; }
.loading-char-float .main-char span:nth-child(3) { animation-delay: 0.22s; }
.loading-char-float .main-char span:nth-child(4) { animation-delay: 0.33s; }
.loading-char-float .main-char span:nth-child(5) { animation-delay: 0.44s; }
.loading-char-float .main-char span:nth-child(6) { animation-delay: 0.55s; }
.loading-char-float .main-char span:nth-child(1) {
animation-delay: 0s;
}
.loading-char-float .main-char span:nth-child(2) {
animation-delay: 0.11s;
}
.loading-char-float .main-char span:nth-child(3) {
animation-delay: 0.22s;
}
.loading-char-float .main-char span:nth-child(4) {
animation-delay: 0.33s;
}
.loading-char-float .main-char span:nth-child(5) {
animation-delay: 0.44s;
}
.loading-char-float .main-char span:nth-child(6) {
animation-delay: 0.55s;
}
@keyframes scan {
0% { background-position: 0 0; }
100% { background-position: 0 100px; }
0% {
background-position: 0 0;
}
100% {
background-position: 0 100px;
}
}
@keyframes char-float {
0%, 100% { transform: translateY(0); opacity: 1; text-shadow: 0 0 5px #5865f2, 0 0 10px rgba(88, 101, 242, 0.8); }
50% { transform: translateY(-5px); opacity: 0.9; text-shadow: 0 0 10px #5865f2, 0 0 20px rgba(88, 101, 242, 0.8); }
0%,
100% {
transform: translateY(0);
opacity: 1;
text-shadow: 0 0 5px #5865f2, 0 0 10px rgba(88, 101, 242, 0.8);
}
50% {
transform: translateY(-5px);
opacity: 0.9;
text-shadow: 0 0 10px #5865f2, 0 0 20px rgba(88, 101, 242, 0.8);
}
}
@keyframes shadow-float {
0%, 100% { transform: translateY(3px); opacity: 0.3; }
50% { transform: translateY(8px); opacity: 0.15; }
0%,
100% {
transform: translateY(3px);
opacity: 0.3;
}
50% {
transform: translateY(8px);
opacity: 0.15;
}
}
/* ==================== Live2D显示区域 ==================== */
@@ -402,6 +444,15 @@ body {
background: rgba(237, 66, 69, 1);
}
/* 摄像头按钮特殊样式 */
#cameraBtn.camera-active {
background: rgba(237, 66, 69, 0.9);
}
#cameraBtn.camera-active:hover {
background: rgba(237, 66, 69, 1);
}
/* 录音按钮脉冲动画 */
@keyframes pulse {
0% {
@@ -761,6 +812,72 @@ body {
font-size: 14px;
}
.model-select {
width: 100%;
padding: 10px 40px 10px 14px;
border: 1px solid #40444b;
border-radius: 8px;
background: #40444b;
color: #ffffff;
font-size: 14px;
font-weight: 500;
cursor: pointer;
appearance: none;
-webkit-appearance: none;
-moz-appearance: none;
background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='%23ffffff'%3E%3Cpath d='M7 10l5 5 5-5z'/%3E%3C/svg%3E");
background-repeat: no-repeat;
background-position: right 12px center;
background-size: 16px 16px;
}
.model-select:hover {
border-color: #5865f2;
}
.model-select:focus {
outline: none;
border-color: #5865f2;
}
.model-select option {
background: #2f3136;
color: #ffffff;
padding: 12px;
font-size: 14px;
}
.background-btn {
width: 100%;
padding: 10px 14px;
border: 1px solid #40444b;
border-radius: 8px;
background: #40444b;
color: #ffffff;
font-size: 14px;
font-weight: 500;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
gap: 8px;
transition: all 0.3s ease;
}
.background-btn:hover {
background: #4f545c;
}
.background-btn:active {
background: #40444b;
}
.background-btn .btn-icon {
width: 20px;
height: 20px;
fill: #ffffff;
}
/* ==================== MCP工具样式 ==================== */
.mcp-tools-container {
background: #2f3136;
@@ -911,35 +1028,174 @@ body {
margin-bottom: 12px;
}
.property-item {
.properties-container #addMcpPropertyBtn {
display: block;
margin: 0 auto;
}
.properties-container #addMcpPropertyBtn:hover {
background: #4752c4;
}
.mcp-checkbox-label {
display: flex;
gap: 8px;
margin-bottom: 8px;
align-items: center;
gap: 8px;
margin-top: 8px;
color: #b9bbbe;
font-size: 13px;
cursor: pointer;
justify-content: flex-start;
}
.property-item input {
flex: 1;
padding: 6px 8px;
border: 1px solid #40444b;
border-radius: 4px;
background: #40444b;
.mcp-checkbox-label input[type="checkbox"] {
width: 16px;
height: 16px;
cursor: pointer;
}
.mcp-property-name {
color: white;
font-size: 12px;
font-size: 14px;
font-weight: 500;
}
.remove-property {
background: #ed4245;
.mcp-property-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 8px;
}
.mcp-properties-list {
display: flex;
flex-wrap: wrap;
gap: 12px;
min-height: 60px;
padding: 8px;
}
.input-group-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 8px;
}
.input-group-header label {
margin-bottom: 0;
}
.input-group-header .properties-btn-primary {
margin: 0;
}
.mcp-empty-state {
text-align: center;
padding: 20px;
color: #999;
font-size: 14px;
width: 100%;
display: none;
}
.properties-btn-primary {
background: #2196f3;
color: white;
border: none;
padding: 4px 8px;
padding: 6px 12px;
border-radius: 4px;
cursor: pointer;
font-size: 12px;
font-weight: 500;
transition: background-color 0.2s;
display: block;
margin: 0 auto;
}
.remove-property:hover {
background: #c03537;
.properties-btn-primary:hover {
background: #2196f3;
}
.mcp-property-card {
background: #36393f;
border: 1px solid #40444b;
border-radius: 8px;
padding: 12px;
width: 100%;
cursor: pointer;
transition: all 0.2s ease;
}
.mcp-property-card:hover {
background: #40444b;
border-color: #5865f2;
}
.mcp-property-card:active {
transform: none;
}
.mcp-property-row-label {
display: flex;
align-items: flex-start;
margin-bottom: 6px;
gap: 8px;
}
.mcp-property-label {
color: #b9bbbe;
font-size: 13px;
width: 60px;
flex-shrink: 0;
text-align: left;
}
.mcp-property-value {
color: white;
font-size: 13px;
flex: 1;
word-break: break-all;
}
.mcp-property-required-badge {
color: #f44336;
font-size: 12px;
}
.mcp-property-row-action {
display: flex;
justify-content: flex-end;
margin-top: 6px;
padding-top: 6px;
border-top: 1px solid #40444b;
}
.mcp-property-delete-btn {
background: #f44336;
color: white;
border: none;
padding: 4px 12px;
border-radius: 4px;
cursor: pointer;
font-size: 11px;
transition: background-color 0.2s;
}
.mcp-property-delete-btn:hover {
background: #d32f2f;
}
.mcp-property-card-optional {
background: #4f545c;
color: #b9bbbe;
font-size: 11px;
padding: 2px 8px;
border-radius: 4px;
}
.property-modal {
max-width: 450px;
}
/* ==================== 音频可视化器样式 ==================== */
@@ -1125,4 +1381,41 @@ body {
.modal-body::-webkit-scrollbar-thumb:hover,
.mcp-tools-list::-webkit-scrollbar-thumb:hover {
background: #4f545c;
}
/* ==================== 摄像头显示区域样式 ==================== */
.camera-container {
position: fixed;
top: 20px;
left: 20px;
width: 240px;
height: 180px;
background: rgba(0, 0, 0, 0.5);
backdrop-filter: blur(10px);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.2);
z-index: 1000;
display: none;
overflow: hidden;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
cursor: move;
}
.camera-container.active {
display: block;
}
.camera-container.dragging {
cursor: grabbing;
box-shadow: 0 8px 30px rgba(0, 0, 0, 0.4);
border-color: rgba(88, 101, 242, 0.5);
}
#cameraVideo {
width: 100%;
height: 100%;
object-fit: cover;
background: #1a1a1a;
pointer-events: none;
transform: scaleX(-1);
}
main/xiaozhi-server/test/favicon.ico
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

main/xiaozhi-server/test/images/1.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 142 KiB

After

Width:  |  Height:  |  Size: 136 KiB

main/xiaozhi-server/test/js/app.js
+250 -21
View File
@@ -1,9 +1,22 @@
// 主应用入口
import { log } from './utils/logger.js';
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js';
import { uiController } from './ui/controller.js';
import { getAudioPlayer } from './core/audio/player.js';
import { initMcpTools } from './core/mcp/tools.js';
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js?v=0127';
import { getAudioPlayer } from './core/audio/player.js?v=0127';
import { checkMicrophoneAvailability, isHttpNonLocalhost } from './core/audio/recorder.js?v=0127';
import { initMcpTools } from './core/mcp/tools.js?v=0127';
import { uiController } from './ui/controller.js?v=0127';
import { log } from './utils/logger.js?v=0127';
// 辅助函数：将Base64数据转换为Blob
function dataURItoBlob(dataURI) {
const byteString = atob(dataURI.split(',')[1]);
const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
const ab = new ArrayBuffer(byteString.length);
const ia = new Uint8Array(ab);
for (let i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
return new Blob([ab], { type: mimeString });
}
// 应用类
class App {
@@ -11,35 +24,34 @@ class App {
this.uiController = null;
this.audioPlayer = null;
this.live2dManager = null;
this.cameraStream = null;
}
// 初始化应用
async init() {
log('正在初始化应用...', 'info');
// 初始化UI控制器
this.uiController = uiController;
this.uiController.init();
// 检查Opus库
checkOpusLoaded();
// 初始化Opus编码器
initOpusEncoder();
// 初始化音频播放器
this.audioPlayer = getAudioPlayer();
await this.audioPlayer.start();
// 初始化MCP工具
initMcpTools();
// 检查麦克风可用性
await this.checkMicrophoneAvailability();
// 检查摄像头可用性
this.checkCameraAvailability();
// 初始化Live2D
await this.initLive2D();
// 初始化摄像头
this.initCamera();
// 关闭加载loading
this.setModelLoadingStatus(false);
log('应用初始化完成', 'success');
}
@@ -50,21 +62,17 @@ class App {
if (typeof window.Live2DManager === 'undefined') {
throw new Error('Live2DManager未加载，请检查脚本引入顺序');
}
this.live2dManager = new window.Live2DManager();
await this.live2dManager.initializeLive2D();
// 更新UI状态
const live2dStatus = document.getElementById('live2dStatus');
if (live2dStatus) {
live2dStatus.textContent = '● 已加载';
live2dStatus.className = 'status loaded';
}
log('Live2D初始化完成', 'success');
} catch (error) {
log(`Live2D初始化失败: ${error.message}`, 'error');
// 更新UI状态
const live2dStatus = document.getElementById('live2dStatus');
if (live2dStatus) {
@@ -81,18 +89,239 @@ class App {
modelLoading.style.display = isLoading ? 'flex' : 'none';
}
}
/**
* 检查麦克风可用性
* 在应用初始化时调用，检查麦克风是否可用并更新UI状态
*/
async checkMicrophoneAvailability() {
try {
const isAvailable = await checkMicrophoneAvailability();
const isHttp = isHttpNonLocalhost();
// 保存可用性状态到全局变量
window.microphoneAvailable = isAvailable;
window.isHttpNonLocalhost = isHttp;
// 更新UI
if (this.uiController) {
this.uiController.updateMicrophoneAvailability(isAvailable, isHttp);
}
log(`麦克风可用性检查完成: ${isAvailable ? '可用' : '不可用'}`, isAvailable ? 'success' : 'warning');
} catch (error) {
log(`检查麦克风可用性失败: ${error.message}`, 'error');
// 默认设置为不可用
window.microphoneAvailable = false;
window.isHttpNonLocalhost = isHttpNonLocalhost();
if (this.uiController) {
this.uiController.updateMicrophoneAvailability(false, window.isHttpNonLocalhost);
}
}
}
// 检查摄像头可用性
checkCameraAvailability() {
window.cameraAvailable = true;
log('摄像头可用性检查完成: 默认已绑定验证码', 'success');
}
// 初始化摄像头
async initCamera() {
const cameraContainer = document.getElementById('cameraContainer');
const cameraVideo = document.getElementById('cameraVideo');
if (!cameraContainer || !cameraVideo) {
log('摄像头元素未找到，跳过初始化', 'warning');
return Promise.resolve(false);
}
let isDragging = false;
let currentX, currentY, initialX, initialY;
let xOffset = 0, yOffset = 0;
cameraContainer.addEventListener('mousedown', dragStart);
document.addEventListener('mousemove', drag);
document.addEventListener('mouseup', dragEnd);
cameraContainer.addEventListener('touchstart', dragStart, { passive: false });
document.addEventListener('touchmove', drag, { passive: false });
document.addEventListener('touchend', dragEnd);
function dragStart(e) {
if (e.type === 'touchstart') {
initialX = e.touches[0].clientX - xOffset;
initialY = e.touches[0].clientY - yOffset;
} else {
initialX = e.clientX - xOffset;
initialY = e.clientY - yOffset;
}
isDragging = true;
cameraContainer.classList.add('dragging');
}
function drag(e) {
if (isDragging) {
e.preventDefault();
if (e.type === 'touchmove') {
currentX = e.touches[0].clientX - initialX;
currentY = e.touches[0].clientY - initialY;
} else {
currentX = e.clientX - initialX;
currentY = e.clientY - initialY;
}
xOffset = currentX;
yOffset = currentY;
cameraContainer.style.transform = `translate3d(${currentX}px, ${currentY}px, 0)`;
}
}
function dragEnd() {
initialX = currentX;
initialY = currentY;
isDragging = false;
cameraContainer.classList.remove('dragging');
}
return new Promise((resolve) => {
window.startCamera = async () => {
try {
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
log('浏览器不支持摄像头API', 'warning');
return false;
}
log('正在请求摄像头权限...', 'info');
this.cameraStream = await navigator.mediaDevices.getUserMedia({
video: { width: 320, height: 240, facingMode: 'user' },
audio: false
});
cameraVideo.srcObject = this.cameraStream;
cameraContainer.classList.add('active');
log('摄像头已启动', 'success');
return true;
} catch (error) {
log(`启动摄像头失败: ${error.name} - ${error.message}`, 'error');
if (error.name === 'NotAllowedError') {
log('摄像头权限被拒绝，请检查浏览器设置', 'warning');
} else if (error.name === 'NotFoundError') {
log('未找到摄像头设备', 'warning');
} else if (error.name === 'NotReadableError') {
log('摄像头已被其他程序占用', 'warning');
}
return false;
}
};
window.stopCamera = () => {
if (this.cameraStream) {
this.cameraStream.getTracks().forEach(track => track.stop());
this.cameraStream = null;
cameraVideo.srcObject = null;
log('摄像头已关闭', 'info');
}
};
window.takePhoto = (question = '描述一下看到的物品') => {
return new Promise(async (resolve) => {
const canvas = document.createElement('canvas');
const video = cameraVideo;
if (!video || video.readyState !== video.HAVE_ENOUGH_DATA) {
log('无法拍照：摄像头未就绪', 'warning');
resolve({
success: false,
error: '摄像头未就绪，请确保已连接且摄像头已启动'
});
return;
}
canvas.width = video.videoWidth || 320;
canvas.height = video.videoHeight || 240;
const ctx = canvas.getContext('2d');
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
const photoData = canvas.toDataURL('image/jpeg', 0.8);
log(`拍照成功，图像数据长度: ${photoData.length}`, 'success');
try {
const xz_tester_vision = localStorage.getItem('xz_tester_vision');
if (xz_tester_vision) {
let visionInfo = null;
try {
visionInfo = JSON.parse(xz_tester_vision);
} catch (err) {
throw new Error(`视觉配置解析失败`);
}
const { url, token } = visionInfo || {};
if (!url || !token) {
throw new Error('视觉分析失败：配置缺少接口地址(url)或令牌(token)');
}
log(`正在发送图片到视觉分析接口: ${url}`, 'info');
const deviceId = document.getElementById('deviceMac')?.value || '';
const clientId = document.getElementById('clientId')?.value || 'web_test_client';
const formData = new FormData();
formData.append('question', question);
formData.append('image', dataURItoBlob(photoData), 'photo.jpg');
const response = await fetch(url, {
method: 'POST',
body: formData,
headers: {
'Device-Id': deviceId,
'Client-Id': clientId,
'Authorization': `Bearer ${token}`
}
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const analysisResult = await response.json();
log(`视觉分析完成: ${JSON.stringify(analysisResult).substring(0, 200)}...`, 'success');
resolve({
success: true,
message: question,
photo_data: photoData,
photo_width: canvas.width,
photo_height: canvas.height,
vision_analysis: analysisResult
});
} else {
log('未配置视觉分析服务', 'warning');
}
} catch (error) {
log(`视觉分析失败: ${error.message}`, 'error');
resolve({
success: true,
message: question,
photo_data: photoData,
photo_width: canvas.width,
photo_height: canvas.height,
vision_analysis: {
success: false,
error: error.message,
fallback: '无法连接到视觉分析服务'
}
});
}
});
};
log('摄像头初始化完成', 'success');
resolve(true);
});
}
}
// 创建并启动应用
const app = new App();
// 将应用实例暴露到全局，供其他模块访问
window.chatApp = app;
document.addEventListener('DOMContentLoaded', () => {
// 初始化应用
app.init();
});
export default app;
main/xiaozhi-server/test/js/config/default-mcp-tools.json
+18
View File
@@ -1,4 +1,22 @@
[
{
"name": "self_camera_take_photo",
"description": "Take a photo using the device's camera. This tool captures the current camera frame and returns the image data. You MUST call this tool when user asks to 'take a photo', 'what do you see', 'describe what you see', or similar requests. After getting the photo data, describe what you see in the image. Parameter 'question' is optional, defaults to 'describe objects seen'.",
"inputSchema": {
"type": "object",
"properties": {
"question": {
"type": "string",
"description": "Question to guide the photo analysis, e.g., 'describe the objects seen'. Can be empty."
}
}
},
"mockResponse": {
"success": true,
"photo_data": "base64_image_data",
"response": "Please describe what you see based on the returned photo_data"
}
},
{
"name": "self.get_device_status",
"description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
main/xiaozhi-server/test/js/core/audio/opus-codec.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// 检查Opus库是否已加载
main/xiaozhi-server/test/js/core/audio/player.js
+3 -3
View File
@@ -1,7 +1,7 @@
// 音频播放模块
import { log } from '../../utils/logger.js';
import BlockingQueue from '../../utils/blocking-queue.js';
import { createStreamingContext } from './stream-context.js';
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
import { createStreamingContext } from './stream-context.js?v=0127';
// 音频播放器类
export class AudioPlayer {
main/xiaozhi-server/test/js/core/audio/recorder.js
+90 -123
View File
@@ -1,9 +1,9 @@
// 音频录制模块
import { log } from '../../utils/logger.js';
import { initOpusEncoder } from './opus-codec.js';
import { getAudioPlayer } from './player.js';
// Audio recording module
import { log } from '../../utils/logger.js?v=0127';
import { initOpusEncoder } from './opus-codec.js?v=0127';
import { getAudioPlayer } from './player.js?v=0127';
// 音频录制器类
// Audio recorder class
export class AudioRecorder {
constructor() {
this.isRecording = false;
@@ -19,25 +19,23 @@ export class AudioRecorder {
this.visualizationRequest = null;
this.recordingTimer = null;
this.websocket = null;
// 回调函数
// Callback functions
this.onRecordingStart = null;
this.onRecordingStop = null;
this.onVisualizerUpdate = null;
}
// 设置WebSocket实例
// Set WebSocket instance
setWebSocket(ws) {
this.websocket = ws;
}
// 获取AudioContext实例
// Get AudioContext instance
getAudioContext() {
const audioPlayer = getAudioPlayer();
return audioPlayer.getAudioContext();
return getAudioPlayer().getAudioContext();
}
// 初始化编码器
// Initialize encoder
initEncoder() {
if (!this.opusEncoder) {
this.opusEncoder = initOpusEncoder();
@@ -45,7 +43,7 @@ export class AudioRecorder {
return this.opusEncoder;
}
// PCM处理器代码
// PCM processor code
getAudioProcessorCode() {
return `
class AudioRecorderProcessor extends AudioWorkletProcessor {
@@ -56,166 +54,132 @@ export class AudioRecorder {
this.buffer = new Int16Array(this.frameSize);
this.bufferIndex = 0;
this.isRecording = false;
this.port.onmessage = (event) => {
if (event.data.command === 'start') {
this.isRecording = true;
this.port.postMessage({ type: 'status', status: 'started' });
} else if (event.data.command === 'stop') {
this.isRecording = false;
if (this.bufferIndex > 0) {
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
this.port.postMessage({
type: 'buffer',
buffer: finalBuffer
});
this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
this.bufferIndex = 0;
}
this.port.postMessage({ type: 'status', status: 'stopped' });
}
};
}
process(inputs, outputs, parameters) {
if (!this.isRecording) return true;
const input = inputs[0][0];
if (!input) return true;
for (let i = 0; i < input.length; i++) {
if (this.bufferIndex >= this.frameSize) {
this.port.postMessage({
type: 'buffer',
buffer: this.buffer.slice(0)
});
this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
this.bufferIndex = 0;
}
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
return true;
}
}
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
`;
}
// 创建音频处理器
// Create audio processor
async createAudioProcessor() {
this.audioContext = this.getAudioContext();
try {
if (this.audioContext.audioWorklet) {
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
await this.audioContext.audioWorklet.addModule(url);
URL.revokeObjectURL(url);
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
audioProcessor.port.onmessage = (event) => {
if (event.data.type === 'buffer') {
this.processPCMBuffer(event.data.buffer);
}
};
log('使用AudioWorklet处理音频', 'success');
const silent = this.audioContext.createGain();
silent.gain.value = 0;
audioProcessor.connect(silent);
silent.connect(this.audioContext.destination);
return { node: audioProcessor, type: 'worklet' };
} else {
log('AudioWorklet不可用，使用ScriptProcessorNode作为回退方案', 'warning');
log('AudioWorklet不可用，使用ScriptProcessorNode作为后备方案', 'warning');
return this.createScriptProcessor();
}
} catch (error) {
log(`创建音频处理器失败: ${error.message}，尝试回退方案`, 'error');
log(`创建音频处理器失败: ${error.message}，尝试后备方案`, 'error');
return this.createScriptProcessor();
}
}
// 创建ScriptProcessor作为回退
// Create ScriptProcessor as fallback
createScriptProcessor() {
try {
const frameSize = 4096;
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
scriptProcessor.onaudioprocess = (event) => {
if (!this.isRecording) return;
const input = event.inputBuffer.getChannelData(0);
const buffer = new Int16Array(input.length);
for (let i = 0; i < input.length; i++) {
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
this.processPCMBuffer(buffer);
};
const silent = this.audioContext.createGain();
silent.gain.value = 0;
scriptProcessor.connect(silent);
silent.connect(this.audioContext.destination);
log('使用ScriptProcessorNode作为回退方案成功', 'warning');
log('使用ScriptProcessorNode作为后备方案成功', 'warning');
return { node: scriptProcessor, type: 'processor' };
} catch (fallbackError) {
log(`回退方案也失败: ${fallbackError.message}`, 'error');
log(`后备方案也失败: ${fallbackError.message}`, 'error');
return null;
}
}
// 处理PCM缓冲数据
// Process PCM buffer data
processPCMBuffer(buffer) {
if (!this.isRecording) return;
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
newBuffer.set(this.pcmDataBuffer);
newBuffer.set(buffer, this.pcmDataBuffer.length);
this.pcmDataBuffer = newBuffer;
const samplesPerFrame = 960;
while (this.pcmDataBuffer.length >= samplesPerFrame) {
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
this.encodeAndSendOpus(frameData);
}
}
// 编码并发送Opus数据
// Encode and send Opus data
encodeAndSendOpus(pcmData = null) {
if (!this.opusEncoder) {
log('Opus编码器未初始化', 'error');
return;
}
try {
if (pcmData) {
const opusData = this.opusEncoder.encode(pcmData);
if (opusData && opusData.length > 0) {
this.audioBuffers.push(opusData.buffer);
this.totalAudioSize += opusData.length;
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
try {
this.websocket.send(opusData.buffer);
log(`发送Opus帧，大小：${opusData.length}字节`, 'debug');
} catch (error) {
log(`WebSocket发送错误: ${error.message}`, 'error');
}
}
} else {
log('Opus编码失败，无有效数据返回', 'error');
log('Opus编码失败，未返回有效数据', 'error');
}
} else {
if (this.pcmDataBuffer.length > 0) {
@@ -235,96 +199,67 @@ export class AudioRecorder {
}
}
// 开始录音
// Start recording
async start() {
if (this.isRecording) return false;
try {
// 检查是否有WebSocketHandler实例
const { getWebSocketHandler } = await import('../network/websocket.js');
// Check if WebSocketHandler instance exists
const { getWebSocketHandler } = await import('../network/websocket.js?v=0127');
const wsHandler = getWebSocketHandler();
// 如果机器正在说话，发送打断消息
// If machine is speaking, send abort message
if (wsHandler && wsHandler.isRemoteSpeaking && wsHandler.currentSessionId) {
const abortMessage = {
session_id: wsHandler.currentSessionId,
type: 'abort',
reason: 'wake_word_detected'
};
const abortMessage = { session_id: wsHandler.currentSessionId, type: 'abort', reason: 'wake_word_detected' };
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
this.websocket.send(JSON.stringify(abortMessage));
log('发送打断消息', 'info');
log('已发送中止消息', 'info');
}
}
if (!this.initEncoder()) {
log('无法启动录音: Opus编码器初始化失败', 'error');
log('无法开始录音: Opus编码器初始化失败', 'error');
return false;
}
log('请至少录制1-2秒钟的音频，确保采集到足够数据', 'info');
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
sampleRate: 16000,
channelCount: 1
}
});
log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
this.audioContext = this.getAudioContext();
if (this.audioContext.state === 'suspended') {
await this.audioContext.resume();
}
const processorResult = await this.createAudioProcessor();
if (!processorResult) {
log('无法创建音频处理器', 'error');
return false;
}
this.audioProcessor = processorResult.node;
this.audioProcessorType = processorResult.type;
this.audioSource = this.audioContext.createMediaStreamSource(stream);
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 2048;
this.audioSource.connect(this.analyser);
this.audioSource.connect(this.audioProcessor);
this.pcmDataBuffer = new Int16Array();
this.audioBuffers = [];
this.totalAudioSize = 0;
this.isRecording = true;
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
this.audioProcessor.port.postMessage({ command: 'start' });
}
// 发送监听开始消息
// Send listening start message
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
log(`发送录音开始消息`, 'info');
log(`已发送录音开始消息`, 'info');
} else {
log('WebSocket未连接，无法发送开始消息', 'error');
return false;
}
// 开始可视化
// Start visualization
if (this.onVisualizerUpdate) {
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
this.startVisualization(dataArray);
}
// 立即通知录音开始，更新按钮状态
// Immediately notify recording start, update button state
if (this.onRecordingStart) {
this.onRecordingStart(0);
}
// 启动录音计时器
// Start recording timer
let recordingSeconds = 0;
this.recordingTimer = setInterval(() => {
recordingSeconds += 0.1;
@@ -332,8 +267,7 @@ export class AudioRecorder {
this.onRecordingStart(recordingSeconds);
}
}, 100);
log('开始PCM直接录音', 'success');
log('已开始PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音启动错误: ${error.message}`, 'error');
@@ -342,15 +276,12 @@ export class AudioRecorder {
}
}
// 开始可视化
// Start visualization
startVisualization(dataArray) {
const draw = () => {
this.visualizationRequest = requestAnimationFrame(() => draw());
if (!this.isRecording) return;
this.analyser.getByteFrequencyData(dataArray);
if (this.onVisualizerUpdate) {
this.onVisualizerUpdate(dataArray);
}
@@ -358,52 +289,42 @@ export class AudioRecorder {
draw();
}
// 停止录音
// Stop recording
stop() {
if (!this.isRecording) return false;
try {
this.isRecording = false;
if (this.audioProcessor) {
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
this.audioProcessor.port.postMessage({ command: 'stop' });
}
this.audioProcessor.disconnect();
this.audioProcessor = null;
}
if (this.audioSource) {
this.audioSource.disconnect();
this.audioSource = null;
}
if (this.visualizationRequest) {
cancelAnimationFrame(this.visualizationRequest);
this.visualizationRequest = null;
}
if (this.recordingTimer) {
clearInterval(this.recordingTimer);
this.recordingTimer = null;
}
// 编码并发送剩余的数据
// Encode and send remaining data
this.encodeAndSendOpus();
// 发送结束信号
// Send end signal
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
const emptyOpusFrame = new Uint8Array(0);
this.websocket.send(emptyOpusFrame);
log('已发送录音停止信号', 'info');
}
if (this.onRecordingStop) {
this.onRecordingStop();
}
log('停止PCM直接录音', 'success');
log('已停止PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音停止错误: ${error.message}`, 'error');
@@ -411,13 +332,13 @@ export class AudioRecorder {
}
}
// 获取分析器
// Get analyser
getAnalyser() {
return this.analyser;
}
}
// 创建单例
// Create singleton instance
let audioRecorderInstance = null;
export function getAudioRecorder() {
@@ -426,3 +347,49 @@ export function getAudioRecorder() {
}
return audioRecorderInstance;
}
/**
* Check if microphone is available
* @returns {Promise<boolean>} Returns true if available, false if not available
*/
export async function checkMicrophoneAvailability() {
// Check if browser supports getUserMedia API
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
log('浏览器不支持getUserMedia API', 'warning');
return false;
}
try {
// Try to access microphone
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
// Immediately stop all tracks to release microphone
stream.getTracks().forEach(track => track.stop());
log('麦克风可用性检查成功', 'success');
return true;
} catch (error) {
log(`麦克风不可用: ${error.message}`, 'warning');
return false;
}
}
/**
* Check if it is HTTP non-localhost access
* @returns {boolean} Returns true if it is HTTP non-localhost access
*/
export function isHttpNonLocalhost() {
const protocol = window.location.protocol;
const hostname = window.location.hostname;
// Check if it is HTTP protocol
if (protocol !== 'http:') {
return false;
}
// localhost and 127.0.0.1 can use microphone
if (hostname === 'localhost' || hostname === '127.0.0.1') {
return false;
}
// Private IP addresses can also use microphone (browser allows)
if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
return false;
}
// Other HTTP access is considered non-localhost
return true;
}
main/xiaozhi-server/test/js/core/audio/stream-context.js
+2 -2
View File
@@ -1,5 +1,5 @@
import BlockingQueue from '../../utils/blocking-queue.js';
import { log } from '../../utils/logger.js';
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
// 音频流播放上下文类
export class StreamingContext {
main/xiaozhi-server/test/js/core/mcp/tools.js
+201 -139
View File
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// ==========================================
// MCP 工具管理逻辑
@@ -24,11 +24,19 @@ export function setWebSocket(ws) {
export async function initMcpTools() {
// 加载默认工具数据
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
const savedTools = localStorage.getItem('mcpTools');
if (savedTools) {
try {
mcpTools = JSON.parse(savedTools);
const parsedTools = JSON.parse(savedTools);
// 合并默认工具和用户保存的工具，保留用户自定义的工具
const defaultToolNames = new Set(defaultMcpTools.map(t => t.name));
// 添加默认工具中不存在的新工具
parsedTools.forEach(tool => {
if (!defaultToolNames.has(tool.name)) {
defaultMcpTools.push(tool);
}
});
mcpTools = defaultMcpTools;
} catch (e) {
log('加载MCP工具失败，使用默认工具', 'warning');
mcpTools = [...defaultMcpTools];
@@ -36,9 +44,11 @@ export async function initMcpTools() {
} else {
mcpTools = [...defaultMcpTools];
}
renderMcpTools();
setupMcpEventListeners();
// Only setup event listeners if DOM elements exist
if (document.getElementById('toggleMcpTools')) {
setupMcpEventListeners();
}
}
/**
@@ -47,21 +57,20 @@ export async function initMcpTools() {
function renderMcpTools() {
const container = document.getElementById('mcpToolsContainer');
const countSpan = document.getElementById('mcpToolsCount');
if (!container) {
return; // Container not found, skip rendering
}
if (countSpan) {
countSpan.textContent = `${mcpTools.length} 个工具`;
}
if (mcpTools.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
return;
}
container.innerHTML = mcpTools.map((tool, index) => {
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
return `
<div class="mcp-tool-card">
<div class="mcp-tool-header">
@@ -96,100 +105,167 @@ function renderMcpTools() {
*/
function renderMcpProperties() {
const container = document.getElementById('mcpPropertiesContainer');
const emptyState = document.getElementById('mcpEmptyState');
if (!container) {
return; // Container not found, skip rendering
}
if (mcpProperties.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 20px; color: #999; font-size: 14px;">暂无参数，点击下方按钮添加参数</div>';
if (emptyState) {
emptyState.style.display = 'block';
}
container.innerHTML = '';
return;
}
if (emptyState) {
emptyState.style.display = 'none';
}
container.innerHTML = mcpProperties.map((prop, index) => `
<div class="mcp-property-item">
<div class="mcp-property-header">
<span class="mcp-property-name">${prop.name}</span>
<button type="button" onclick="window.mcpModule.deleteMcpProperty(${index})"
style="padding: 3px 8px; border: none; border-radius: 3px; background-color: #f44336; color: white; cursor: pointer; font-size: 11px;">
删除
</button>
<div class="mcp-property-card" onclick="window.mcpModule.editMcpProperty(${index})">
<div class="mcp-property-row-label">
<span class="mcp-property-label">参数名称</span>
<span class="mcp-property-value">${prop.name}${prop.required ? ' <span class="mcp-property-required-badge">[必填]</span>' : ''}</span>
</div>
<div class="mcp-property-row">
<div>
<label class="mcp-small-label">参数名称 *</label>
<input type="text" class="mcp-small-input" value="${prop.name}"
onchange="window.mcpModule.updateMcpProperty(${index}, 'name', this.value)" required>
</div>
<div>
<label class="mcp-small-label">数据类型 *</label>
<select class="mcp-small-input" onchange="window.mcpModule.updateMcpProperty(${index}, 'type', this.value)">
<option value="string" ${prop.type === 'string' ? 'selected' : ''}>字符串</option>
<option value="integer" ${prop.type === 'integer' ? 'selected' : ''}>整数</option>
<option value="number" ${prop.type === 'number' ? 'selected' : ''}>数字</option>
<option value="boolean" ${prop.type === 'boolean' ? 'selected' : ''}>布尔值</option>
<option value="array" ${prop.type === 'array' ? 'selected' : ''}>数组</option>
<option value="object" ${prop.type === 'object' ? 'selected' : ''}>对象</option>
</select>
</div>
<div class="mcp-property-row-label">
<span class="mcp-property-label">数据类型</span>
<span class="mcp-property-value">${getTypeLabel(prop.type)}</span>
</div>
${(prop.type === 'integer' || prop.type === 'number') ? `
<div class="mcp-property-row">
<div>
<label class="mcp-small-label">最小值</label>
<input type="number" class="mcp-small-input" value="${prop.minimum !== undefined ? prop.minimum : ''}"
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'minimum', this.value ? parseFloat(this.value) : undefined)">
</div>
<div>
<label class="mcp-small-label">最大值</label>
<input type="number" class="mcp-small-input" value="${prop.maximum !== undefined ? prop.maximum : ''}"
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'maximum', this.value ? parseFloat(this.value) : undefined)">
</div>
<div class="mcp-property-row-label">
<span class="mcp-property-label">描述</span>
<span class="mcp-property-value">${prop.description || '-'}</span>
</div>
` : ''}
<div class="mcp-property-row-full">
<label class="mcp-small-label">参数描述</label>
<input type="text" class="mcp-small-input" value="${prop.description || ''}"
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'description', this.value)">
<div class="mcp-property-row-action">
<button class="mcp-property-delete-btn" onclick="event.stopPropagation(); window.mcpModule.deleteMcpProperty(${index})">删除</button>
</div>
<label class="mcp-checkbox-label">
<input type="checkbox" ${prop.required ? 'checked' : ''}
onchange="window.mcpModule.updateMcpProperty(${index}, 'required', this.checked)">
必填参数
</label>
</div>
`).join('');
}
/**
* 添加参数
* 获取数据类型标签
*/
function addMcpProperty() {
mcpProperties.push({
name: `param_${mcpProperties.length + 1}`,
type: 'string',
required: false,
description: ''
});
renderMcpProperties();
function getTypeLabel(type) {
const typeMap = {
'string': '字符串',
'integer': '整数',
'number': '数字',
'boolean': '布尔值',
'array': '数组',
'object': '对象'
};
return typeMap[type] || type;
}
/**
* 更新参数
* 添加参数 - 打开参数编辑模态框
*/
function updateMcpProperty(index, field, value) {
if (field === 'name') {
const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === value);
if (isDuplicate) {
alert('参数名称已存在，请使用不同的名称');
renderMcpProperties();
return;
function addMcpProperty() {
openPropertyModal();
}
/**
* 编辑参数 - 打开参数编辑模态框
*/
function editMcpProperty(index) {
openPropertyModal(index);
}
/**
* 打开参数编辑模态框
*/
function openPropertyModal(index = null) {
const form = document.getElementById('mcpPropertyForm');
const title = document.getElementById('mcpPropertyModalTitle');
document.getElementById('mcpPropertyIndex').value = index !== null ? index : -1;
if (index !== null) {
const prop = mcpProperties[index];
title.textContent = '编辑参数';
document.getElementById('mcpPropertyName').value = prop.name;
document.getElementById('mcpPropertyType').value = prop.type || 'string';
document.getElementById('mcpPropertyMinimum').value = prop.minimum !== undefined ? prop.minimum : '';
document.getElementById('mcpPropertyMaximum').value = prop.maximum !== undefined ? prop.maximum : '';
document.getElementById('mcpPropertyDescription').value = prop.description || '';
document.getElementById('mcpPropertyRequired').checked = prop.required || false;
} else {
title.textContent = '添加参数';
form.reset();
document.getElementById('mcpPropertyName').value = `param_${mcpProperties.length + 1}`;
document.getElementById('mcpPropertyType').value = 'string';
document.getElementById('mcpPropertyMinimum').value = '';
document.getElementById('mcpPropertyMaximum').value = '';
document.getElementById('mcpPropertyDescription').value = '';
document.getElementById('mcpPropertyRequired').checked = false;
}
updatePropertyRangeVisibility();
document.getElementById('mcpPropertyModal').style.display = 'flex';
}
/**
* 关闭参数编辑模态框
*/
function closePropertyModal() {
document.getElementById('mcpPropertyModal').style.display = 'none';
}
/**
* 更新数值范围输入框的可见性
*/
function updatePropertyRangeVisibility() {
const type = document.getElementById('mcpPropertyType').value;
const rangeGroup = document.getElementById('mcpPropertyRangeGroup');
if (type === 'integer' || type === 'number') {
rangeGroup.style.display = 'block';
} else {
rangeGroup.style.display = 'none';
}
}
/**
* 处理参数表单提交
*/
function handlePropertySubmit(e) {
e.preventDefault();
const index = parseInt(document.getElementById('mcpPropertyIndex').value);
const name = document.getElementById('mcpPropertyName').value.trim();
const type = document.getElementById('mcpPropertyType').value;
const minimum = document.getElementById('mcpPropertyMinimum').value;
const maximum = document.getElementById('mcpPropertyMaximum').value;
const description = document.getElementById('mcpPropertyDescription').value.trim();
const required = document.getElementById('mcpPropertyRequired').checked;
// 检查名称重复
const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === name);
if (isDuplicate) {
alert('参数名称已存在，请使用不同的名称');
return;
}
const propData = {
name,
type,
description,
required
};
// 数值类型添加范围限制
if (type === 'integer' || type === 'number') {
if (minimum !== '') {
propData.minimum = parseFloat(minimum);
}
if (maximum !== '') {
propData.maximum = parseFloat(maximum);
}
}
mcpProperties[index][field] = value;
if (field === 'type' && value !== 'integer' && value !== 'number') {
delete mcpProperties[index].minimum;
delete mcpProperties[index].maximum;
renderMcpProperties();
if (index >= 0) {
mcpProperties[index] = propData;
} else {
mcpProperties.push(propData);
}
renderMcpProperties();
closePropertyModal();
}
/**
@@ -213,25 +289,43 @@ function setupMcpEventListeners() {
const form = document.getElementById('mcpToolForm');
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
// 参数编辑模态框相关元素
const propertyModal = document.getElementById('mcpPropertyModal');
const closePropertyBtn = document.getElementById('closeMcpPropertyModalBtn');
const cancelPropertyBtn = document.getElementById('cancelMcpPropertyBtn');
const propertyForm = document.getElementById('mcpPropertyForm');
const propertyTypeSelect = document.getElementById('mcpPropertyType');
// Return early if required elements don't exist (e.g., in test environment)
if (!toggleBtn || !panel || !addBtn || !modal || !closeBtn || !cancelBtn || !form || !addPropertyBtn) {
return;
}
toggleBtn.addEventListener('click', () => {
const isExpanded = panel.classList.contains('expanded');
panel.classList.toggle('expanded');
toggleBtn.textContent = isExpanded ? '展开' : '收起';
toggleBtn.textContent = isExpanded ? '收起' : '展开';
});
// 确保面板默认展开
panel.classList.add('expanded');
addBtn.addEventListener('click', () => openMcpModal());
closeBtn.addEventListener('click', closeMcpModal);
cancelBtn.addEventListener('click', closeMcpModal);
addPropertyBtn.addEventListener('click', addMcpProperty);
modal.addEventListener('click', (e) => {
if (e.target === modal) closeMcpModal();
});
form.addEventListener('submit', handleMcpSubmit);
// 参数编辑模态框事件
if (propertyModal && closePropertyBtn && cancelPropertyBtn && propertyForm && propertyTypeSelect) {
closePropertyBtn.addEventListener('click', closePropertyModal);
cancelPropertyBtn.addEventListener('click', closePropertyModal);
propertyModal.addEventListener('click', (e) => {
if (e.target === propertyModal) closePropertyModal();
});
propertyForm.addEventListener('submit', handlePropertySubmit);
propertyTypeSelect.addEventListener('change', updatePropertyRangeVisibility);
}
}
/**
@@ -243,18 +337,15 @@ function openMcpModal(index = null) {
alert('WebSocket 已连接，无法编辑工具');
return;
}
mcpEditingIndex = index;
const errorContainer = document.getElementById('mcpErrorContainer');
errorContainer.innerHTML = '';
if (index !== null) {
document.getElementById('mcpModalTitle').textContent = '编辑工具';
const tool = mcpTools[index];
document.getElementById('mcpToolName').value = tool.name;
document.getElementById('mcpToolDescription').value = tool.description;
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
mcpProperties = [];
const schema = tool.inputSchema;
if (schema.properties) {
@@ -275,9 +366,8 @@ function openMcpModal(index = null) {
document.getElementById('mcpToolForm').reset();
mcpProperties = [];
}
renderMcpProperties();
document.getElementById('mcpToolModal').style.display = 'block';
document.getElementById('mcpToolModal').style.display = 'flex';
}
/**
@@ -298,21 +388,15 @@ function handleMcpSubmit(e) {
e.preventDefault();
const errorContainer = document.getElementById('mcpErrorContainer');
errorContainer.innerHTML = '';
const name = document.getElementById('mcpToolName').value.trim();
const description = document.getElementById('mcpToolDescription').value.trim();
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
// 检查名称重复
const isDuplicate = mcpTools.some((tool, index) =>
tool.name === name && index !== mcpEditingIndex
);
const isDuplicate = mcpTools.some((tool, index) => tool.name === name && index !== mcpEditingIndex);
if (isDuplicate) {
showMcpError('工具名称已存在，请使用不同的名称');
return;
}
// 解析模拟返回结果
let mockResponse = null;
if (mockResponseText) {
@@ -323,21 +407,13 @@ function handleMcpSubmit(e) {
return;
}
}
// 构建 inputSchema
const inputSchema = {
type: "object",
properties: {},
required: []
};
const inputSchema = { type: "object", properties: {}, required: [] };
mcpProperties.forEach(prop => {
const propSchema = { type: prop.type };
if (prop.description) {
propSchema.description = prop.description;
}
if ((prop.type === 'integer' || prop.type === 'number')) {
if (prop.minimum !== undefined && prop.minimum !== '') {
propSchema.minimum = prop.minimum;
@@ -346,20 +422,15 @@ function handleMcpSubmit(e) {
propSchema.maximum = prop.maximum;
}
}
inputSchema.properties[prop.name] = propSchema;
if (prop.required) {
inputSchema.required.push(prop.name);
}
});
if (inputSchema.required.length === 0) {
delete inputSchema.required;
}
const tool = { name, description, inputSchema, mockResponse };
if (mcpEditingIndex !== null) {
mcpTools[mcpEditingIndex] = tool;
log(`已更新工具: ${name}`, 'success');
@@ -367,7 +438,6 @@ function handleMcpSubmit(e) {
mcpTools.push(tool);
log(`已添加工具: ${name}`, 'success');
}
saveMcpTools();
renderMcpTools();
closeMcpModal();
@@ -417,32 +487,36 @@ function saveMcpTools() {
* 获取工具列表
*/
export function getMcpTools() {
return mcpTools.map(tool => ({
name: tool.name,
description: tool.description,
inputSchema: tool.inputSchema
}));
return mcpTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
}
/**
* 执行工具调用
*/
export function executeMcpTool(toolName, toolArgs) {
export async function executeMcpTool(toolName, toolArgs) {
const tool = mcpTools.find(t => t.name === toolName);
if (!tool) {
log(`未找到工具: ${toolName}`, 'error');
return {
success: false,
error: `未知工具: ${toolName}`
};
return { success: false, error: `未知工具: ${toolName}` };
}
// 处理拍照工具
if (toolName === 'self_camera_take_photo') {
if (typeof window.takePhoto === 'function') {
const question = toolArgs && toolArgs.question ? toolArgs.question : '描述一下看到的物品';
log(`正在执行拍照: ${question}`, 'info');
const result = await window.takePhoto(question);
return result;
} else {
log('拍照功能不可用', 'warning');
return { success: false, error: '摄像头未启动或不支持拍照功能' };
}
}
// 如果有模拟返回结果，使用它
if (tool.mockResponse) {
// 替换模板变量
let responseStr = JSON.stringify(tool.mockResponse);
// 替换 ${paramName} 格式的变量
if (toolArgs) {
Object.keys(toolArgs).forEach(key => {
@@ -450,7 +524,6 @@ export function executeMcpTool(toolName, toolArgs) {
responseStr = responseStr.replace(regex, toolArgs[key]);
});
}
try {
const response = JSON.parse(responseStr);
log(`工具 ${toolName} 执行成功，返回模拟结果: ${responseStr}`, 'success');
@@ -460,21 +533,10 @@ export function executeMcpTool(toolName, toolArgs) {
return tool.mockResponse;
}
}
// 没有模拟返回结果，返回默认成功消息
log(`工具 ${toolName} 执行成功，返回默认结果`, 'success');
return {
success: true,
message: `工具 ${toolName} 执行成功`,
tool: toolName,
arguments: toolArgs
};
return { success: true, message: `工具 ${toolName} 执行成功`, tool: toolName, arguments: toolArgs };
}
// 暴露全局方法供 HTML 内联事件调用
window.mcpModule = {
updateMcpProperty,
deleteMcpProperty,
editMcpTool,
deleteMcpTool
};
window.mcpModule = { addMcpProperty, editMcpProperty, deleteMcpProperty, editMcpTool, deleteMcpTool };
main/xiaozhi-server/test/js/core/network/ota-connector.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// WebSocket 连接
export async function webSocketConnect(otaUrl, config) {
main/xiaozhi-server/test/js/core/network/websocket.js
+113 -25
View File
@@ -1,11 +1,11 @@
// WebSocket消息处理模块
import { log } from '../../utils/logger.js';
import { webSocketConnect } from './ota-connector.js';
import { getConfig, saveConnectionUrls } from '../../config/manager.js';
import { getAudioPlayer } from '../audio/player.js';
import { getAudioRecorder } from '../audio/recorder.js';
import { getMcpTools, executeMcpTool, setWebSocket as setMcpWebSocket } from '../mcp/tools.js';
import { uiController } from '../../ui/controller.js'
import { getConfig, saveConnectionUrls } from '../../config/manager.js?v=0127';
import { uiController } from '../../ui/controller.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
import { getAudioPlayer } from '../audio/player.js?v=0127';
import { getAudioRecorder } from '../audio/recorder.js?v=0127';
import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js?v=0127';
import { webSocketConnect } from './ota-connector.js?v=0127';
// WebSocket处理器类
export class WebSocketHandler {
@@ -74,6 +74,9 @@ export class WebSocketHandler {
handleTextMessage(message) {
if (message.type === 'hello') {
log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success');
window.cameraAvailable = true;
log('连接成功，摄像头已可用', 'success');
uiController.updateDialButton(true);
uiController.startAIChatSession();
} else if (message.type === 'tts') {
this.handleTTSMessage(message);
@@ -81,6 +84,23 @@ export class WebSocketHandler {
log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
} else if (message.type === 'stt') {
log(`识别结果: ${message.text}`, 'info');
// 检查是否需要绑定设备
if (message.text && (message.text.includes('绑定') || message.text.includes('bind'))) {
log('收到设备绑定提示，更新摄像头状态', 'warning');
window.cameraAvailable = false;
// 关闭摄像头
if (typeof window.stopCamera === 'function') {
window.stopCamera();
}
// 更新摄像头按钮状态
const cameraBtn = document.getElementById('cameraBtn');
if (cameraBtn) {
cameraBtn.classList.remove('camera-active');
cameraBtn.querySelector('.btn-text').textContent = '摄像头';
cameraBtn.disabled = true;
cameraBtn.title = '请先绑定验证码';
}
}
// 使用新的聊天消息回调显示STT消息
if (this.onChatMessage && message.text) {
this.onChatMessage(message.text, true);
@@ -101,10 +121,10 @@ export class WebSocketHandler {
}
// 触发Live2D情绪动作
if (message.emotion) {
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
this.triggerLive2DEmotionAction(message.emotion);
}
if (message.emotion) {
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
this.triggerLive2DEmotionAction(message.emotion);
}
}
// 只有当文本不仅仅是表情时，才添加到对话中
@@ -249,7 +269,56 @@ export class WebSocketHandler {
log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info');
const result = executeMcpTool(toolName, toolArgs);
executeMcpTool(toolName, toolArgs).then(result => {
const replyMessage = JSON.stringify({
"session_id": message.session_id || "",
"type": "mcp",
"payload": {
"jsonrpc": "2.0",
"id": payload.id,
"result": {
"content": [
{
"type": "text",
"text": JSON.stringify(result)
}
],
"isError": false
}
}
});
log(`客户端上报: ${replyMessage}`, 'info');
this.websocket.send(replyMessage);
}).catch(error => {
log(`工具执行失败: ${error.message}`, 'error');
const errorReply = JSON.stringify({
"session_id": message.session_id || "",
"type": "mcp",
"payload": {
"jsonrpc": "2.0",
"id": payload.id,
"error": {
"code": -32603,
"message": error.message
}
}
});
this.websocket.send(errorReply);
});
} else if (payload.method === 'initialize') {
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
// 保存视觉分析接口地址
const visionUrl = document.getElementById('visionUrl');
const visionConfig = payload?.params?.capabilities?.vision;
if (visionConfig && typeof visionConfig === 'object' && visionConfig.url && visionConfig.token) {
const visionConfigStr = JSON.stringify(visionConfig);
localStorage.setItem('xz_tester_vision', visionConfigStr);
if (visionUrl) visionUrl.value = visionConfig.url;
} else {
localStorage.removeItem('xz_tester_vision');
if (visionUrl) visionUrl.value = '';
}
const replyMessage = JSON.stringify({
"session_id": message.session_id || "",
@@ -258,21 +327,19 @@ export class WebSocketHandler {
"jsonrpc": "2.0",
"id": payload.id,
"result": {
"content": [
{
"type": "text",
"text": JSON.stringify(result)
}
],
"isError": false
"protocolVersion": "2024-11-05",
"capabilities": {
"tools": {}
},
"serverInfo": {
"name": "xiaozhi-web-test",
"version": "2.1.0"
}
}
}
});
log(`客户端上报: ${replyMessage}`, 'info');
log(`回复初始化响应`, 'info');
this.websocket.send(replyMessage);
} else if (payload.method === 'initialize') {
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
} else {
log(`未知的MCP方法: ${payload.method}`, 'warning');
}
@@ -284,7 +351,6 @@ export class WebSocketHandler {
let arrayBuffer;
if (data instanceof ArrayBuffer) {
arrayBuffer = data;
log(`收到ArrayBuffer音频数据，大小: ${data.byteLength}字节`, 'debug');
} else if (data instanceof Blob) {
arrayBuffer = await data.arrayBuffer();
log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`, 'debug');
@@ -368,11 +434,22 @@ export class WebSocketHandler {
const audioRecorder = getAudioRecorder();
audioRecorder.stop();
// 关闭摄像头
if (typeof window.stopCamera === 'function') {
window.stopCamera();
}
// 隐藏摄像头显示区域
const cameraContainer = document.getElementById('cameraContainer');
if (cameraContainer) {
cameraContainer.classList.remove('active');
}
};
this.websocket.onerror = (error) => {
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
uiController.addChatMessage(`⚠️ WebSocket错误: ${error.message || '未知错误'}`, false);
if (this.onConnectionStateChange) {
this.onConnectionStateChange(false);
}
@@ -401,6 +478,17 @@ export class WebSocketHandler {
this.websocket.close();
const audioRecorder = getAudioRecorder();
audioRecorder.stop();
// 关闭摄像头
if (typeof window.stopCamera === 'function') {
window.stopCamera();
}
// 隐藏摄像头显示区域
const cameraContainer = document.getElementById('cameraContainer');
if (cameraContainer) {
cameraContainer.classList.remove('active');
}
}
// 发送文本消息
main/xiaozhi-server/test/js/live2d/live2d.js
+338 -20
View File
@@ -12,7 +12,42 @@ class Live2DManager {
this.audioContext = null;
this.analyser = null;
this.dataArray = null;
this.lastEmotionActionTime = null; // 上次情绪触发动作的时间
this.lastEmotionActionTime = null;
this.currentModelName = null;
// 模型特定配置
this.modelConfig = {
'hiyori_pro_zh': {
mouthParam: 'ParamMouthOpenY',
mouthAmplitude: 1.0,
mouthThresholds: { low: 0.3, high: 0.7 },
motionMap: {
'FlickUp': 'FlickUp',
'FlickDown': 'FlickDown',
'Tap': 'Tap',
'Tap@Body': 'Tap@Body',
'Flick': 'Flick',
'Flick@Body': 'Flick@Body'
}
},
'natori_pro_zh': {
mouthParam: 'ParamMouthOpenY',
mouthAmplitude: 1.0,
mouthThresholds: { low: 0.1, high: 0.4 },
mouthFormParam: 'ParamMouthForm',
mouthFormAmplitude: 1.0,
mouthForm2Param: 'ParamMouthForm2',
mouthForm2Amplitude: 0.8,
motionMap: {
'FlickUp': 'FlickUp',
'FlickDown': 'Flick@Body',
'Tap': 'Tap',
'Tap@Body': 'Tap@Head',
'Flick': 'Tap',
'Flick@Body': 'Flick@Body'
}
}
};
// 情绪到动作的映射
this.emotionToActionMap = {
@@ -67,10 +102,33 @@ class Live2DManager {
const currentPath = window.location.pathname;
const lastSlashIndex = currentPath.lastIndexOf('/');
const basePath = currentPath.substring(0, lastSlashIndex + 1);
const modelPath = basePath + 'hiyori_pro_zh/runtime/hiyori_pro_t11.model3.json';
// 从 localStorage 读取上次选择的模型，如果没有则使用默认
const savedModelName = localStorage.getItem('live2dModel') || 'hiyori_pro_zh';
const modelFileMap = {
'hiyori_pro_zh': 'hiyori_pro_t11.model3.json',
'natori_pro_zh': 'natori_pro_t06.model3.json'
};
const modelFileName = modelFileMap[savedModelName] || 'hiyori_pro_t11.model3.json';
const modelPath = basePath + 'resources/' + savedModelName + '/runtime/' + modelFileName;
this.live2dModel = await PIXI.live2d.Live2DModel.from(modelPath);
this.live2dApp.stage.addChild(this.live2dModel);
// 保存当前模型名称
this.currentModelName = savedModelName;
// 更新下拉框显示
const modelSelect = document.getElementById('live2dModelSelect');
if (modelSelect) {
modelSelect.value = savedModelName;
}
// 设置模型特定的嘴部参数名
if (this.modelConfig[savedModelName]) {
this.mouthParam = this.modelConfig[savedModelName].mouthParam || 'ParamMouthOpenY';
}
// 设置模型属性
this.live2dModel.scale.set(0.33);
this.live2dModel.x = (window.innerWidth - this.live2dModel.width) * 0.5;
@@ -364,33 +422,88 @@ class Live2DManager {
if (internal && internal.coreModel) {
const coreModel = internal.coreModel;
// 获取音频分贝值
let mouthValue = 0;
let mouthOpenY = 0;
let mouthForm = 0;
let mouthForm2 = 0;
let average = 0;
if (this.analyser && this.dataArray) {
this.analyser.getByteFrequencyData(this.dataArray);
average = this.dataArray.reduce((a, b) => a + b) / this.dataArray.length;
// 优化音量映射函数，使中等音量范围变化更明显
// 使用S形曲线函数，在中等音量范围有更好的响应
const normalizedVolume = average / 255;
// S形曲线：在0.3-0.7范围内有最大的斜率（变化最明显）
if (normalizedVolume < 0.3) {
// 低音量：缓慢增长
mouthValue = Math.pow(normalizedVolume / 0.3, 1.5) * 0.3;
} else if (normalizedVolume < 0.7) {
// 中等音量：线性增长，变化最明显
mouthValue = 0.3 + (normalizedVolume - 0.3) / 0.4 * 0.5;
} else {
// 高音量：缓慢接近最大值
mouthValue = 0.8 + Math.pow((normalizedVolume - 0.7) / 0.3, 1.2) * 0.2;
// 获取模型特定的阈值
let lowThreshold = 0.3;
let highThreshold = 0.7;
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
lowThreshold = this.modelConfig[this.currentModelName].mouthThresholds?.low || 0.3;
highThreshold = this.modelConfig[this.currentModelName].mouthThresholds?.high || 0.7;
}
// 确保嘴部参数在0-1范围内
mouthValue = Math.min(Math.max(mouthValue, 0), 1);
// 使用模型特定的阈值进行映射
let minOpenY = 0.1;
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
minOpenY = this.modelConfig[this.currentModelName].mouthMinOpenY || 0.1;
}
if (normalizedVolume < lowThreshold) {
mouthOpenY = minOpenY + Math.pow(normalizedVolume / lowThreshold, 1.5) * (0.4 - minOpenY);
} else if (normalizedVolume < highThreshold) {
mouthOpenY = 0.4 + (normalizedVolume - lowThreshold) / (highThreshold - lowThreshold) * 0.4;
} else {
mouthOpenY = 0.8 + Math.pow((normalizedVolume - highThreshold) / (1 - highThreshold), 1.2) * 0.2;
}
// 应用模型特定的嘴部开合幅度
let amplitudeMultiplier = 1.0;
let maxOpenY = 2.5;
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
amplitudeMultiplier = this.modelConfig[this.currentModelName].mouthAmplitude;
maxOpenY = this.modelConfig[this.currentModelName].maxOpenY || 2.5;
}
mouthOpenY = mouthOpenY * amplitudeMultiplier;
mouthOpenY = Math.min(Math.max(mouthOpenY, 0), maxOpenY);
// 计算嘴型参数（仅对支持嘴型变化的模型）
if (this.currentModelName && this.modelConfig[this.currentModelName]?.mouthFormParam) {
const config = this.modelConfig[this.currentModelName];
const formAmplitude = config.mouthFormAmplitude || 0.5;
const form2Amplitude = config.mouthForm2Amplitude || 0;
// 嘴型随音量变化：
// 低音量：嘴型偏"一"字（负值）
// 高音量：嘴型偏"o"字（正值）
// 音量=0时：嘴型=0（自然状态）
mouthForm = (normalizedVolume - 0.5) * 2 * formAmplitude;
mouthForm = Math.max(-formAmplitude, Math.min(formAmplitude, mouthForm));
// 第二嘴型参数（natori特有）
if (config.mouthForm2Param) {
mouthForm2 = (normalizedVolume - 0.3) * 2 * form2Amplitude;
mouthForm2 = Math.max(-form2Amplitude, Math.min(form2Amplitude, mouthForm2));
}
}
// 调试日志：输出嘴部参数
console.log(`[Live2D] 模型: ${this.currentModelName || 'unknown'}, 音量: ${average?.toFixed(0)}, OpenY: ${mouthOpenY.toFixed(3)}, Form: ${mouthForm.toFixed(3)}, Form2: ${mouthForm2.toFixed(3)}`);
}
coreModel.setParameterValueById(this.mouthParam, mouthValue);
// 设置嘴部开合参数
coreModel.setParameterValueById(this.mouthParam, mouthOpenY);
// 设置嘴型参数（仅对支持嘴型变化的模型）
if (this.currentModelName && this.modelConfig[this.currentModelName]?.mouthFormParam) {
const config = this.modelConfig[this.currentModelName];
const formParam = config.mouthFormParam;
coreModel.setParameterValueById(formParam, mouthForm);
// 设置第二嘴型参数（natori特有）
if (config.mouthForm2Param) {
coreModel.setParameterValueById(config.mouthForm2Param, mouthForm2);
}
}
coreModel.update();
}
this.mouthAnimationId = requestAnimationFrame(() => this.animateMouth());
@@ -473,12 +586,129 @@ class Live2DManager {
motion(name) {
try {
if (!this.live2dModel) return;
this.live2dModel.motion(name);
// 根据当前模型获取对应的动作名称
let actualMotionName = name;
if (this.currentModelName && this.modelConfig[this.currentModelName]) {
const motionMap = this.modelConfig[this.currentModelName].motionMap;
actualMotionName = motionMap[name] || name;
}
this.live2dModel.motion(actualMotionName);
} catch (error) {
console.error('触发动作失败:', error);
}
}
/**
* 设置模型交互事件
*/
setupModelInteractions() {
if (!this.live2dModel) return;
this.live2dModel.interactive = true;
this.live2dModel.on('doublehit', (args) => {
const area = Array.isArray(args) ? args[0] : args;
if (area === 'Body') {
this.motion('Flick@Body');
} else if (area === 'Head' || area === 'Face') {
this.motion('Flick');
}
const app = window.chatApp;
const payload = JSON.stringify({ type: 'live2d', event: 'doublehit', area });
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
app.dataChannel.send(payload);
}
});
this.live2dModel.on('singlehit', (args) => {
const area = Array.isArray(args) ? args[0] : args;
if (area === 'Body') {
this.motion('Tap@Body');
} else if (area === 'Head' || area === 'Face') {
this.motion('Tap');
}
const app = window.chatApp;
const payload = JSON.stringify({ type: 'live2d', event: 'singlehit', area });
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
app.dataChannel.send(payload);
}
});
this.live2dModel.on('swipe', (args) => {
const area = Array.isArray(args) ? args[0] : args;
const dir = Array.isArray(args) ? args[1] : undefined;
if (area === 'Body') {
if (dir === 'up') {
this.motion('FlickUp');
} else if (dir === 'down') {
this.motion('FlickDown');
}
}
const app = window.chatApp;
const payload = JSON.stringify({ type: 'live2d', event: 'swipe', area, dir });
if (app && app.dataChannel && app.dataChannel.readyState === 'open') {
app.dataChannel.send(payload);
}
});
this.live2dModel.on('pointerdown', (event) => {
try {
const global = event.data.global;
const bounds = this.live2dModel.getBounds();
if (!bounds || !bounds.contains(global.x, global.y)) return;
const relX = (global.x - bounds.x) / (bounds.width || 1);
const relY = (global.y - bounds.y) / (bounds.height || 1);
let area = '';
if (relX >= 0.4 && relX <= 0.6) {
if (relY <= 0.15) {
area = 'Head';
} else if (relY >= 0.7) {
area = 'Body';
}
}
if (!area) return;
const now = Date.now();
const dt = now - (this._lastClickTime || 0);
const dx = global.x - (this._lastClickPos?.x || 0);
const dy = global.y - (this._lastClickPos?.y || 0);
const dist = Math.hypot(dx, dy);
if (this._lastClickTime && dt <= this._doubleClickMs && dist <= this._doubleClickDist) {
if (this._singleClickTimer) {
clearTimeout(this._singleClickTimer);
this._singleClickTimer = null;
}
this.live2dModel.emit('doublehit', area);
this._lastClickTime = null;
this._lastClickPos = null;
} else {
this._lastClickTime = now;
this._lastClickPos = { x: global.x, y: global.y };
this._singleClickTimer = setTimeout(() => {
this._singleClickTimer = null;
this.live2dModel.emit('singlehit', area);
}, this._doubleClickMs);
}
} catch (e) {
console.warn('pointerdown 处理出错:', e);
}
});
}
/**
* 清理资源
*/
@@ -500,6 +730,94 @@ class Live2DManager {
}
this.live2dModel = null;
}
/**
* 切换 Live2D 模型
* @param {string} modelName - 模型目录名称，如 'hiyori_pro_zh'、'natori_pro_zh'
* @returns {Promise<boolean>} - 切换是否成功
*/
async switchModel(modelName) {
try {
// 获取模型文件名映射
const modelFileMap = {
'hiyori_pro_zh': 'hiyori_pro_t11.model3.json',
'natori_pro_zh': 'natori_pro_t06.model3.json',
'chitose': 'chitose.model3.json',
'haru_greeter_pro_jp': 'haru_greeter_t05.model3.json'
};
const modelFileName = modelFileMap[modelName];
if (!modelFileName) {
console.error('未知的模型名称:', modelName);
return false;
}
// 获取基础路径
const currentPath = window.location.pathname;
const lastSlashIndex = currentPath.lastIndexOf('/');
const basePath = currentPath.substring(0, lastSlashIndex + 1);
const modelPath = basePath + 'resources/' + modelName + '/runtime/' + modelFileName;
// 如果已存在模型，先移除
if (this.live2dModel) {
this.live2dApp.stage.removeChild(this.live2dModel);
this.live2dModel.destroy();
this.live2dModel = null;
}
// 显示加载状态
const app = window.chatApp;
if (app) {
app.setModelLoadingStatus(true);
}
// 加载新模型
this.live2dModel = await PIXI.live2d.Live2DModel.from(modelPath);
this.live2dApp.stage.addChild(this.live2dModel);
// 设置模型属性
this.live2dModel.scale.set(0.33);
this.live2dModel.x = (window.innerWidth - this.live2dModel.width) * 0.5;
this.live2dModel.y = -50;
// 重新绑定交互事件
this.setupModelInteractions();
// 隐藏加载状态
if (app) {
app.setModelLoadingStatus(false);
}
// 保存当前模型名称
this.currentModelName = modelName;
// 设置模型特定的嘴部参数名
if (this.modelConfig[modelName]) {
this.mouthParam = this.modelConfig[modelName].mouthParam || 'ParamMouthOpenY';
}
// 保存到 localStorage
localStorage.setItem('live2dModel', modelName);
// 更新下拉框显示
const modelSelect = document.getElementById('live2dModelSelect');
if (modelSelect) {
modelSelect.value = modelName;
}
console.log('模型切换成功:', modelName);
return true;
} catch (error) {
console.error('切换模型失败:', error);
const app = window.chatApp;
if (app) {
app.setModelLoadingStatus(false);
}
return false;
}
}
}
// 导出全局实例
main/xiaozhi-server/test/js/ui/controller.js
+334 -129
View File
@@ -1,31 +1,33 @@
// UI控制模块
import { loadConfig, saveConfig } from '../config/manager.js';
import { getAudioRecorder } from '../core/audio/recorder.js';
import { getWebSocketHandler } from '../core/network/websocket.js';
import { getAudioPlayer } from '../core/audio/player.js';
// UI controller module
import { loadConfig, saveConfig } from '../config/manager.js?v=0127';
import { getAudioPlayer } from '../core/audio/player.js?v=0127';
import { getAudioRecorder } from '../core/audio/recorder.js?v=0127';
import { getWebSocketHandler } from '../core/network/websocket.js?v=0127';
// UI控制器类
// UI controller class
class UIController {
constructor() {
this.isEditing = false;
this.visualizerCanvas = null;
this.visualizerContext = null;
this.audioStatsTimer = null;
this.currentBackgroundIndex = 0;
this.currentBackgroundIndex = localStorage.getItem('backgroundIndex') ? parseInt(localStorage.getItem('backgroundIndex')) : 0;
this.backgroundImages = ['1.png', '2.png', '3.png'];
this.dialBtnDisabled = false;
// 绑定方法
// Bind methods
this.init = this.init.bind(this);
this.initEventListeners = this.initEventListeners.bind(this);
this.updateDialButton = this.updateDialButton.bind(this);
this.addChatMessage = this.addChatMessage.bind(this);
this.switchBackground = this.switchBackground.bind(this);
this.switchLive2DModel = this.switchLive2DModel.bind(this);
this.showModal = this.showModal.bind(this);
this.hideModal = this.hideModal.bind(this);
this.switchTab = this.switchTab.bind(this);
}
// 初始化
// Initialize
init() {
console.log('UIController init started');
@@ -35,7 +37,7 @@ class UIController {
this.initVisualizer();
}
// 检查连接按钮在初始化时是否存在
// Check if connect button exists during initialization
const connectBtn = document.getElementById('connectBtn');
console.log('connectBtn during init:', connectBtn);
@@ -43,20 +45,26 @@ class UIController {
this.startAudioStatsMonitor();
loadConfig();
// 设置录音器回调
// Register recording callback
const audioRecorder = getAudioRecorder();
audioRecorder.onRecordingStart = (seconds) => {
this.updateRecordButtonState(true, seconds);
};
// 初始化状态显示
// Initialize status display
this.updateConnectionUI(false);
// Apply saved background
const backgroundContainer = document.querySelector('.background-container');
if (backgroundContainer) {
backgroundContainer.style.backgroundImage = `url('./images/${this.backgroundImages[this.currentBackgroundIndex]}')`;
}
this.updateDialButton(false);
console.log('UIController init completed');
}
// 初始化可视化器
// Initialize visualizer
initVisualizer() {
if (this.visualizerCanvas) {
this.visualizerCanvas.width = this.visualizerCanvas.clientWidth;
@@ -66,9 +74,9 @@ class UIController {
}
}
// 初始化事件监听器
// Initialize event listeners
initEventListeners() {
// 设置按钮
// Settings button
const settingsBtn = document.getElementById('settingsBtn');
if (settingsBtn) {
settingsBtn.addEventListener('click', () => {
@@ -76,64 +84,133 @@ class UIController {
});
}
// 背景切换按钮
// Background switch button
const backgroundBtn = document.getElementById('backgroundBtn');
if (backgroundBtn) {
backgroundBtn.addEventListener('click', this.switchBackground);
}
// 拨号按钮
// Model select change event
const modelSelect = document.getElementById('live2dModelSelect');
if (modelSelect) {
modelSelect.addEventListener('change', () => {
this.switchLive2DModel();
});
}
// Dial button
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.addEventListener('click', () => {
dialBtn.disabled = true;
this.dialBtnDisabled = true;
setTimeout(() => {
dialBtn.disabled = false;
this.dialBtnDisabled = false;
}, 3000);
const wsHandler = getWebSocketHandler();
const isConnected = wsHandler.isConnected();
if (isConnected) {
wsHandler.disconnect();
this.updateDialButton(false);
this.addChatMessage('已断开连接，期待下次再见~😉', false);
this.addChatMessage('Disconnected, see you next time~😊', false);
} else {
// 检查OTA地址是否已填写
// Check if OTA URL is filled
const otaUrlInput = document.getElementById('otaUrl');
if (!otaUrlInput || !otaUrlInput.value.trim()) {
// 如果OTA地址未填写，显示设置弹窗并切换到设备配置页
// If OTA URL is not filled, show settings modal and switch to device tab
this.showModal('settingsModal');
this.switchTab('device');
this.addChatMessage('请先填写OTA服务器地址', false);
this.addChatMessage('Please fill in OTA server URL', false);
return;
}
// 执行连接操作
// Start connection process
this.handleConnect();
}
});
}
// 录音按钮
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.addEventListener('click', () => {
const audioRecorder = getAudioRecorder();
if (audioRecorder.isRecording) {
audioRecorder.stop();
// 停止录音时移除录音样式
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
} else {
// 先更新按钮状态为录音中
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
// 延迟开始录音，确保按钮状态已更新
setTimeout(() => {
audioRecorder.start();
}, 100);
// Camera button
const cameraBtn = document.getElementById('cameraBtn');
let cameraTimer = null;
if (cameraBtn) {
cameraBtn.addEventListener('click', () => {
if (cameraTimer) {
clearTimeout(cameraTimer);
cameraTimer = null;
}
cameraTimer = setTimeout(() => {
const cameraContainer = document.getElementById('cameraContainer');
if (!cameraContainer) {
log('摄像头容器不存在', 'warning');
return;
}
const isActive = cameraContainer.classList.contains('active');
if (isActive) {
// 关闭摄像头
if (typeof window.stopCamera === 'function') {
window.stopCamera();
}
cameraContainer.classList.remove('active');
cameraBtn.classList.remove('camera-active');
cameraBtn.querySelector('.btn-text').textContent = '摄像头';
log('摄像头已关闭', 'info');
} else {
// 打开摄像头
if (typeof window.startCamera === 'function') {
window.startCamera().then(success => {
if (success) {
cameraBtn.classList.add('camera-active');
cameraBtn.querySelector('.btn-text').textContent = '关闭';
} else {
this.addChatMessage('⚠️ 摄像头启动失败，请检查浏览器权限', false);
}
}).catch(error => {
log(`启动摄像头异常: ${error.message}`, 'error');
});
} else {
log('startCamera函数未定义', 'warning');
}
}
}, 300);
});
}
// 消息输入框事件
// Record button
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
let recordTimer = null;
recordBtn.addEventListener('click', () => {
if (recordTimer) {
clearTimeout(recordTimer);
recordTimer = null;
}
recordTimer = setTimeout(() => {
const audioRecorder = getAudioRecorder();
if (audioRecorder.isRecording) {
audioRecorder.stop();
// Restore record button to normal state
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
} else {
// Update button state to recording
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
// Start recording, update button state after delay
setTimeout(() => {
audioRecorder.start();
}, 100);
}
}, 300);
});
}
// Chat input event listener
const chatIpt = document.getElementById('chatIpt');
if (chatIpt) {
const wsHandler = getWebSocketHandler();
@@ -148,7 +225,7 @@ class UIController {
});
}
// 关闭按钮
// Close button
const closeButtons = document.querySelectorAll('.close-btn');
closeButtons.forEach(btn => {
btn.addEventListener('click', (e) => {
@@ -163,7 +240,7 @@ class UIController {
});
});
// 设置标签页切换
// Settings tab switch
const tabBtns = document.querySelectorAll('.tab-btn');
tabBtns.forEach(btn => {
btn.addEventListener('click', (e) => {
@@ -171,7 +248,7 @@ class UIController {
});
});
// 点击模态框外部关闭
// Click modal background to close
const modals = document.querySelectorAll('.modal');
modals.forEach(modal => {
modal.addEventListener('click', (e) => {
@@ -184,7 +261,7 @@ class UIController {
});
});
// 添加MCP工具按钮
// Add MCP tool button
const addMCPToolBtn = document.getElementById('addMCPToolBtn');
if (addMCPToolBtn) {
addMCPToolBtn.addEventListener('click', (e) => {
@@ -193,10 +270,10 @@ class UIController {
});
}
// 连接按钮和取消按钮已被移除，功能已集成到拨号按钮中
// Connect button and send button are not removed, can be added to dial button later
}
// 更新连接状态UI
// Update connection status UI
updateConnectionUI(isConnected) {
const connectionStatus = document.getElementById('connectionStatus');
const statusDot = document.querySelector('.status-dot');
@@ -216,48 +293,81 @@ class UIController {
}
}
// 更新拨号按钮状态
// Update dial button state
updateDialButton(isConnected) {
const dialBtn = document.getElementById('dialBtn');
const recordBtn = document.getElementById('recordBtn');
const cameraBtn = document.getElementById('cameraBtn');
if (dialBtn) {
if (isConnected) {
dialBtn.classList.add('dial-active');
dialBtn.querySelector('.btn-text').textContent = '挂断';
// 更新拨号按钮图标为挂断图标
// Update dial button icon to hang up icon
dialBtn.querySelector('svg').innerHTML = `
<path d="M12,9C10.4,9 9,10.4 9,12C9,13.6 10.4,15 12,15C13.6,15 15,13.6 15,12C15,10.4 13.6,9 12,9M12,17C9.2,17 7,14.8 7,12C7,9.2 9.2,7 12,7C14.8,7 17,9.2 17,12C17,14.8 14.8,17 12,17M12,4.5C7,4.5 2.7,7.6 1,12C2.7,16.4 7,19.5 12,19.5C17,19.5 21.3,16.4 23,12C21.3,7.6 17,4.5 12,4.5Z"/>
`;
} else {
dialBtn.classList.remove('dial-active');
dialBtn.querySelector('.btn-text').textContent = '拨号';
// 恢复拨号按钮图标
// Restore dial button icon
dialBtn.querySelector('svg').innerHTML = `
<path d="M6.62,10.79C8.06,13.62 10.38,15.94 13.21,17.38L15.41,15.18C15.69,14.9 16.08,14.82 16.43,14.93C17.55,15.3 18.75,15.5 20,15.5A1,1 0 0,1 21,16.5V20A1,1 0 0,1 20,21A17,17 0 0,1 3,4A1,1 0 0,1 4,3H7.5A1,1 0 0,1 8.5,4C8.5,5.25 8.7,6.45 9.07,7.57C9.18,7.92 9.1,8.31 8.82,8.59L6.62,10.79Z"/>
`;
}
}
// 更新录音按钮状态
// Update camera button state - reset to default when disconnected
if (cameraBtn && !isConnected) {
const cameraContainer = document.getElementById('cameraContainer');
if (cameraContainer && cameraContainer.classList.contains('active')) {
cameraContainer.classList.remove('active');
}
cameraBtn.classList.remove('camera-active');
cameraBtn.querySelector('.btn-text').textContent = '摄像头';
cameraBtn.disabled = true;
cameraBtn.title = '请先连接服务器';
// 关闭摄像头
if (typeof window.stopCamera === 'function') {
window.stopCamera();
}
}
// Update camera button state - enable when connected and camera is available
if (cameraBtn && isConnected) {
if (window.cameraAvailable) {
cameraBtn.disabled = false;
cameraBtn.title = '打开/关闭摄像头';
} else {
cameraBtn.disabled = true;
cameraBtn.title = '请先绑定验证码';
}
}
// Update record button state
if (recordBtn) {
if (isConnected) {
const microphoneAvailable = window.microphoneAvailable !== false;
if (isConnected && microphoneAvailable) {
recordBtn.disabled = false;
recordBtn.title = '开始录音';
// 确保录音按钮恢复到正常状态
// Restore record button to normal state
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
} else {
recordBtn.disabled = true;
recordBtn.title = '请先连接服务器';
// 确保录音按钮恢复到正常状态
if (!microphoneAvailable) {
recordBtn.title = window.isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
} else {
recordBtn.title = '请先连接服务器';
}
// Restore record button to normal state
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
}
}
}
// 更新录音按钮状态
// Update record button state
updateRecordButtonState(isRecording, seconds = 0) {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
@@ -268,11 +378,37 @@ class UIController {
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
}
recordBtn.disabled = false;
// Only enable button when microphone is available
recordBtn.disabled = window.microphoneAvailable === false;
}
}
// 添加聊天消息
/**
* Update microphone availability state
* @param {boolean} isAvailable - Whether microphone is available
* @param {boolean} isHttpNonLocalhost - Whether it is HTTP non-localhost access
*/
updateMicrophoneAvailability(isAvailable, isHttpNonLocalhost) {
const recordBtn = document.getElementById('recordBtn');
if (!recordBtn) return;
if (!isAvailable) {
// Disable record button
recordBtn.disabled = true;
// Update button text and title
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.title = isHttpNonLocalhost ? '当前由于是http访问，无法录音，只能用文字交互' : '麦克风不可用';
} else {
// If connected, enable record button
const wsHandler = getWebSocketHandler();
if (wsHandler && wsHandler.isConnected()) {
recordBtn.disabled = false;
recordBtn.title = '开始录音';
}
}
}
// Add chat message
addChatMessage(content, isUser = false) {
const chatStream = document.getElementById('chatStream');
if (!chatStream) return;
@@ -282,20 +418,50 @@ class UIController {
messageDiv.innerHTML = `<div class="message-bubble">${content}</div>`;
chatStream.appendChild(messageDiv);
// 自动滚动到底部
// Scroll to bottom
chatStream.scrollTop = chatStream.scrollHeight;
}
// 切换背景
// Switch background
switchBackground() {
this.currentBackgroundIndex = (this.currentBackgroundIndex + 1) % this.backgroundImages.length;
const backgroundContainer = document.querySelector('.background-container');
if (backgroundContainer) {
backgroundContainer.style.backgroundImage = `url('./images/${this.backgroundImages[this.currentBackgroundIndex]}')`;
}
localStorage.setItem('backgroundIndex', this.currentBackgroundIndex);
}
// 显示模态框
// Switch Live2D model
switchLive2DModel() {
const modelSelect = document.getElementById('live2dModelSelect');
if (!modelSelect) {
console.error('模型选择下拉框不存在');
return;
}
const selectedModel = modelSelect.value;
const app = window.chatApp;
if (app && app.live2dManager) {
app.live2dManager.switchModel(selectedModel)
.then(success => {
if (success) {
this.addChatMessage(`已切换到模型: ${selectedModel}`, false);
} else {
this.addChatMessage('模型切换失败', false);
}
})
.catch(error => {
console.error('模型切换错误:', error);
this.addChatMessage('模型切换出错', false);
});
} else {
this.addChatMessage('Live2D管理器未初始化', false);
}
}
// Show modal
showModal(modalId) {
const modal = document.getElementById(modalId);
if (modal) {
@@ -303,7 +469,7 @@ class UIController {
}
}
// 隐藏模态框
// Hide modal
hideModal(modalId) {
const modal = document.getElementById(modalId);
if (modal) {
@@ -311,16 +477,16 @@ class UIController {
}
}
// 切换标签页
// Switch tab
switchTab(tabName) {
// 移除所有标签页的active类
// Remove active class from all tabs
const tabBtns = document.querySelectorAll('.tab-btn');
const tabContents = document.querySelectorAll('.tab-content');
tabBtns.forEach(btn => btn.classList.remove('active'));
tabContents.forEach(content => content.classList.remove('active'));
// 激活选中的标签页
// Activate selected tab
const activeTabBtn = document.querySelector(`[data-tab="${tabName}"]`);
const activeTabContent = document.getElementById(`${tabName}Tab`);
@@ -330,24 +496,50 @@ class UIController {
}
}
// 连接成功后开始对话
// Start AI chat session after connection
startAIChatSession() {
this.addChatMessage('连接成功，开始聊天吧~🙂', false);
// 开启录音
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.click();
this.addChatMessage('连接成功，开始聊天吧~😊', false);
// Check microphone availability and show error messages if needed
if (!window.microphoneAvailable) {
if (window.isHttpNonLocalhost) {
this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
} else {
this.addChatMessage('⚠️ 麦克风不可用，请检查权限设置，只能用文字交互', false);
}
}
// Start recording only if microphone is available
if (window.microphoneAvailable) {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.click();
}
}
// Start camera only if camera is available (bound with verification code)
if (window.cameraAvailable && typeof window.startCamera === 'function') {
window.startCamera().then(success => {
if (success) {
const cameraBtn = document.getElementById('cameraBtn');
if (cameraBtn) {
cameraBtn.classList.add('camera-active');
cameraBtn.querySelector('.btn-text').textContent = '关闭';
}
} else {
this.addChatMessage('⚠️ 摄像头启动失败，可能被浏览器拒绝', false);
}
}).catch(error => {
log(`启动摄像头异常: ${error.message}`, 'error');
});
}
}
// 处理连接按钮点击
// Handle connect button click
async handleConnect() {
console.log('handleConnect called');
// 确保切换到设备配置标签页
// Switch to device settings tab
this.switchTab('device');
// 等待DOM更新
// Wait for DOM update
await new Promise(resolve => setTimeout(resolve, 50));
const otaUrlInput = document.getElementById('otaUrl');
@@ -362,7 +554,7 @@ class UIController {
const otaUrl = otaUrlInput.value;
console.log('otaUrl value:', otaUrl);
// 更新拨号按钮状态为连接中
// Update dial button state to connecting
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.classList.add('dial-active');
@@ -370,7 +562,7 @@ class UIController {
dialBtn.disabled = true;
}
// 显示连接中消息
// Show connecting message
this.addChatMessage('正在连接服务器...', false);
const chatIpt = document.getElementById('chatIpt');
@@ -380,50 +572,61 @@ class UIController {
try {
// 获取WebSocket处理器
// Get WebSocket handler instance
const wsHandler = getWebSocketHandler();
// Register connection state callback BEFORE connecting
wsHandler.onConnectionStateChange = (isConnected) => {
this.updateConnectionUI(isConnected);
this.updateDialButton(isConnected);
};
// Register chat message callback BEFORE connecting
wsHandler.onChatMessage = (text, isUser) => {
this.addChatMessage(text, isUser);
};
// Register record button state callback BEFORE connecting
wsHandler.onRecordButtonStateChange = (isRecording) => {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
if (isRecording) {
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
} else {
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
}
}
};
const isConnected = await wsHandler.connect();
if (isConnected) {
// Check microphone availability (check again after connection)
const { checkMicrophoneAvailability } = await import('../core/audio/recorder.js?v=0127');
const micAvailable = await checkMicrophoneAvailability();
// 设置连接状态回调
wsHandler.onConnectionStateChange = (isConnected) => {
this.updateConnectionUI(isConnected);
this.updateDialButton(isConnected);
};
// 设置聊天消息回调
wsHandler.onChatMessage = (text, isUser) => {
this.addChatMessage(text, isUser);
};
// 设置录音按钮状态回调
wsHandler.onRecordButtonStateChange = (isRecording) => {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
if (isRecording) {
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
} else {
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
}
if (!micAvailable) {
const isHttp = window.isHttpNonLocalhost;
if (isHttp) {
this.addChatMessage('⚠️ 当前由于是http访问，无法录音，只能用文字交互', false);
}
};
// Update global state
window.microphoneAvailable = false;
}
// 连接成功
this.addChatMessage('OTA连接成功，正在建立WebSocket连接...', false);
// 更新拨号按钮状态
// Update dial button state
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.disabled = false;
if (!this.dialBtnDisabled) {
dialBtn.disabled = false;
}
dialBtn.querySelector('.btn-text').textContent = '挂断';
dialBtn.classList.add('dial-active');
}
this.hideModal('settingsModal');
} else {
throw new Error('OTA连接失败');
}
@@ -434,17 +637,19 @@ class UIController {
name: error.name
});
// 显示错误消息
// Show error message
const errorMessage = error.message.includes('Cannot set properties of null')
? '连接失败：请刷新页面重试'
? '连接失败：请检查设备连接'
: `连接失败: ${error.message}`;
this.addChatMessage(errorMessage, false);
// 恢复拨号按钮状态
// Restore dial button state
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.disabled = false;
if (!this.dialBtnDisabled) {
dialBtn.disabled = false;
}
dialBtn.querySelector('.btn-text').textContent = '拨号';
dialBtn.classList.remove('dial-active');
console.log('Dial button state restored successfully');
@@ -452,7 +657,7 @@ class UIController {
}
}
// 添加MCP工具
// Add MCP tool
addMCPTool() {
const mcpToolsList = document.getElementById('mcpToolsList');
if (!mcpToolsList) return;
@@ -471,7 +676,7 @@ class UIController {
mcpToolsList.appendChild(toolDiv);
}
// 移除MCP工具
// Remove MCP tool
removeMCPTool(toolId) {
const toolElement = document.getElementById(toolId);
if (toolElement) {
@@ -479,24 +684,24 @@ class UIController {
}
}
// 更新音频统计信息
// Update audio statistics display
updateAudioStats() {
const audioPlayer = getAudioPlayer();
if (!audioPlayer) return;
const stats = audioPlayer.getAudioStats();
// 这里可以添加音频统计的UI更新逻辑
// Here can add audio statistics UI update logic
}
// 启动音频统计监控
// Start audio statistics monitor
startAudioStatsMonitor() {
// 每100ms更新一次音频统计
// Update audio statistics every 100ms
this.audioStatsTimer = setInterval(() => {
this.updateAudioStats();
}, 100);
}
// 停止音频统计监控
// Stop audio statistics monitor
stopAudioStatsMonitor() {
if (this.audioStatsTimer) {
clearInterval(this.audioStatsTimer);
@@ -504,7 +709,7 @@ class UIController {
}
}
// 绘制音频可视化效果
// Draw audio visualizer waveform
drawVisualizer(dataArray) {
if (!this.visualizerContext || !this.visualizerCanvas) return;
@@ -518,7 +723,7 @@ class UIController {
for (let i = 0; i < dataArray.length; i++) {
barHeight = dataArray[i] / 2;
// 创建渐变色：从紫色到蓝色到青色
// Create gradient color: from purple to blue to green
const gradient = this.visualizerContext.createLinearGradient(0, 0, 0, this.visualizerCanvas.height);
gradient.addColorStop(0, '#8e44ad');
gradient.addColorStop(0.5, '#3498db');
@@ -530,21 +735,21 @@ class UIController {
}
}
// 更新会话状态UI
// Update session status UI
updateSessionStatus(isSpeaking) {
// 这里可以添加会话状态的UI更新逻辑
// 例如：更新Live2D角色的表情或状态指示器
// Here can add session status UI update logic
// For example: update Live2D model's mouth movement status
}
// 更新会话表情
// Update session emotion
updateSessionEmotion(emoji) {
// 这里可以添加表情更新的逻辑
// 例如：在状态指示器中显示表情
// Here can add emotion update logic
// For example: display emoji in status indicator
}
}
// 创建全局实例
// Create singleton instance
export const uiController = new UIController();
// 导出类供其他模块使用
export { UIController };
// Export class for module usage
export { UIController };
main/xiaozhi-server/test/hiyori_pro_zh/ReadMe.txt → main/xiaozhi-server/test/resources/hiyori_pro_zh/ReadMe.txt
View File
main/xiaozhi-server/test/hiyori_pro_zh/hiyori_pro_t04.can3 → main/xiaozhi-server/test/resources/hiyori_pro_zh/hiyori_pro_t04.can3
View File
main/xiaozhi-server/test/hiyori_pro_zh/hiyori_pro_t11.cmo3 → main/xiaozhi-server/test/resources/hiyori_pro_zh/hiyori_pro_t11.cmo3
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/hiyori_pro_t11.2048/texture_00.png → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/hiyori_pro_t11.2048/texture_00.png
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.7 MiB

After

Width:  |  Height:  |  Size: 1.7 MiB

main/xiaozhi-server/test/hiyori_pro_zh/runtime/hiyori_pro_t11.2048/texture_01.png → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/hiyori_pro_t11.2048/texture_01.png
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.4 MiB

After

Width:  |  Height:  |  Size: 2.4 MiB

main/xiaozhi-server/test/hiyori_pro_zh/runtime/hiyori_pro_t11.cdi3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/hiyori_pro_t11.cdi3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/hiyori_pro_t11.moc3 → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/hiyori_pro_t11.moc3
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/hiyori_pro_t11.model3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/hiyori_pro_t11.model3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/hiyori_pro_t11.physics3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/hiyori_pro_t11.physics3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/hiyori_pro_t11.pose3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/hiyori_pro_t11.pose3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/motion/hiyori_m01.motion3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/motion/hiyori_m01.motion3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/motion/hiyori_m02.motion3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/motion/hiyori_m02.motion3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/motion/hiyori_m03.motion3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/motion/hiyori_m03.motion3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/motion/hiyori_m04.motion3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/motion/hiyori_m04.motion3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/motion/hiyori_m05.motion3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/motion/hiyori_m05.motion3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/motion/hiyori_m06.motion3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/motion/hiyori_m06.motion3.json
View File
main/xiaozhi-server/test/hiyori_pro_zh/runtime/motion/hiyori_m07.motion3.json → main/xiaozhi-server/test/resources/hiyori_pro_zh/runtime/motion/hiyori_m07.motion3.json
View File

Some files were not shown because too many files have changed in this diff Show More