diff --git a/docs/fish-speech-integration.md b/docs/fish-speech-integration.md new file mode 100644 index 00000000..b916e619 --- /dev/null +++ b/docs/fish-speech-integration.md @@ -0,0 +1,72 @@ +登录AutoDL，租赁镜像 +选择镜像: +``` +PyTorch / 2.1.0 / 3.10(ubuntu22.04) / cuda 12.1 +``` + +机器开机后，设置学术加速 +``` +source /etc/network_turbo +``` + +进入工作目录 +``` +cd autodl-tmp/ +``` + +拉取项目 +``` +git clone https://gitclone.com/github.com/fishaudio/fish-speech.git ; cd fish-speech +``` + +安装依赖 +``` +pip install -e. +``` + +如果报错，安装portaudio +``` +apt-get install portaudio19-dev -y +``` + +安装后执行 +``` +pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cu121 +``` + +下载模型 +``` +cd tools +python download_models.py +``` + +下载完模型后运行接口 +``` +python -m tools.api_server --listen 0.0.0.0:6006 +``` + +然后用浏览器去到aotodl实例页面 +``` +https://autodl.com/console/instance/list +``` + +如下图点击你刚才机器的`自定义服务`按钮，开启端口转发服务 +![自定义服务](images/fishspeech/autodl-01.png) + +端口转发服务设置完成后，你本地电脑打开网址`http://localhost:6006/`，就可以访问fish-speech的接口了 +![服务预览](images/fishspeech/autodl-02.png) + + +如果你是单模块部署，核心配置如下 +``` +selected_module: + TTS: FishSpeech +TTS: + FishSpeech: + reference_audio: ["config/assets/wakeup_words.wav",] + reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",] + api_key: "123" + api_url: "http://127.0.0.1:6006/v1/tts" +``` + +然后重启服务 \ No newline at end of file diff --git a/docs/images/fishspeech/autodl-01.png b/docs/images/fishspeech/autodl-01.png new file mode 100644 index 00000000..695b94b8 Binary files /dev/null and b/docs/images/fishspeech/autodl-01.png differ diff --git a/docs/images/fishspeech/autodl-02.png b/docs/images/fishspeech/autodl-02.png new file mode 100644 index 00000000..e3f72afe Binary files /dev/null and b/docs/images/fishspeech/autodl-02.png differ diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 78154b5c..52e840f4 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -1,5 +1,7 @@ package xiaozhi.common.constant; +import lombok.Getter; + /** * 常量 * Copyright (c) 人人开源 All rights reserved. @@ -109,6 +111,11 @@ public interface Constant { */ String FILE_EXTENSION_SEG = "."; + /** + * 无记忆 + */ + String MEMORY_NO_MEM = "Memory_nomem"; + enum SysBaseParam { /** * 系统全称 @@ -174,8 +181,23 @@ public interface Constant { } } + @Getter + enum ChatHistoryConfEnum { + IGNORE(0, "不记录"), + RECORD_TEXT(1, "记录文本"), + RECORD_TEXT_AUDIO(2, "文本音频都记录"); + + private final int code; + private final String name; + + ChatHistoryConfEnum(int code, String name) { + this.code = code; + this.name = name; + } + } + /** * 版本号 */ - public static final String VERSION = "0.4.2"; + public static final String VERSION = "0.4.3"; } \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java index 9368cf6b..597cf985 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java @@ -109,6 +109,7 @@ public class AgentController { entity.setMemModelId(template.getMemModelId()); entity.setIntentModelId(template.getIntentModelId()); entity.setSystemPrompt(template.getSystemPrompt()); + entity.setChatHistoryConf(template.getChatHistoryConf()); entity.setLangCode(template.getLangCode()); entity.setLanguage(template.getLanguage()); } @@ -166,6 +167,9 @@ public class AgentController { if (dto.getSystemPrompt() != null) { existingEntity.setSystemPrompt(dto.getSystemPrompt()); } + if (dto.getChatHistoryConf() != null) { + existingEntity.setChatHistoryConf(dto.getChatHistoryConf()); + } if (dto.getLangCode() != null) { existingEntity.setLangCode(dto.getLangCode()); } @@ -183,6 +187,15 @@ public class AgentController { agentService.updateById(existingEntity); + // 更新记忆策略 + if (existingEntity.getMemModelId() == null || existingEntity.getMemModelId().equals(Constant.MEMORY_NO_MEM)) { + // 删除所有记录 + agentChatHistoryService.deleteByAgentId(existingEntity.getId(), true, true); + } else if (existingEntity.getChatHistoryConf() != null && existingEntity.getChatHistoryConf() == 1) { + // 删除音频数据 + agentChatHistoryService.deleteByAgentId(existingEntity.getId(), true, false); + } + return new Result<>(); } @@ -193,7 +206,7 @@ public class AgentController { // 先删除关联的设备 deviceService.deleteByAgentId(id); // 删除关联的聊天记录 - agentChatHistoryService.deleteByAgentId(id); + agentChatHistoryService.deleteByAgentId(id, true, true); // 再删除智能体 agentService.deleteById(id); return new Result<>(); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AiAgentChatHistoryDao.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AiAgentChatHistoryDao.java index 43a5f1ce..b75312d2 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AiAgentChatHistoryDao.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AiAgentChatHistoryDao.java @@ -28,4 +28,11 @@ public interface AiAgentChatHistoryDao extends BaseMapper imp @Override public AgentEntity getAgentById(String id) { - return agentDao.selectById(id); + AgentEntity agent = agentDao.selectById(id); + if (agent != null && agent.getMemModelId() != null && agent.getMemModelId().equals(Constant.MEMORY_NO_MEM)) { + agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.IGNORE.getCode()); + } else if (agent != null && agent.getMemModelId() != null + && !agent.getMemModelId().equals(Constant.MEMORY_NO_MEM) + && agent.getChatHistoryConf() == null) { + agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode()); + } + return agent; } @Override @@ -93,6 +102,9 @@ public class AgentServiceImpl extends BaseServiceImpl imp // 获取 LLM 模型名称 dto.setLlmModelName(modelConfigService.getModelNameById(agent.getLlmModelId())); + // 获取记忆模型名称 + dto.setMemModelId(agent.getMemModelId()); + // 获取 TTS 音色名称 dto.setTtsVoiceName(timbreModelService.getTimbreNameById(agent.getTtsVoiceId())); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index ee42c934..6ce70ba7 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -9,6 +9,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import lombok.AllArgsConstructor; +import xiaozhi.common.constant.Constant; import xiaozhi.common.exception.ErrorCode; import xiaozhi.common.exception.RenException; import xiaozhi.common.redis.RedisKeys; @@ -108,6 +109,17 @@ public class ConfigServiceImpl implements ConfigService { // 获取单台设备每天最多输出字数 String deviceMaxOutputSize = sysParamsService.getValue("device_max_output_size", true); result.put("device_max_output_size", deviceMaxOutputSize); + + // 获取聊天记录配置 + Integer chatHistoryConf = agent.getChatHistoryConf(); + if (agent.getMemModelId() != null && agent.getMemModelId().equals(Constant.MEMORY_NO_MEM)) { + chatHistoryConf = Constant.ChatHistoryConfEnum.IGNORE.getCode(); + } else if (agent.getMemModelId() != null + && !agent.getMemModelId().equals(Constant.MEMORY_NO_MEM) + && agent.getChatHistoryConf() == null) { + chatHistoryConf = Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode(); + } + result.put("chat_history_conf", chatHistoryConf); // 如果客户端已实例化模型，则不返回 String alreadySelectedVadModelId = (String) selectedModule.get("VAD"); if (alreadySelectedVadModelId != null && alreadySelectedVadModelId.equals(agent.getVadModelId())) { diff --git a/main/manager-api/src/main/resources/db/changelog/202505091555.sql b/main/manager-api/src/main/resources/db/changelog/202505091555.sql new file mode 100644 index 00000000..b7ad91d4 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202505091555.sql @@ -0,0 +1,25 @@ +-- 更新模型供应器表 +UPDATE `ai_model_provider` SET fields = '[{"key": "host", "type": "string", "label": "服务地址"}, {"key": "port", "type": "number", "label": "端口号"}, {"key": "type", "type": "string", "label": "服务类型"}, {"key": "is_ssl", "type": "boolean", "label": "是否使用SSL"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]' WHERE id = 'SYSTEM_ASR_FunASRServer'; + +-- 更新模型配置表 +UPDATE `ai_model_config` SET +config_json = '{"host": "127.0.0.1", "port": 10096, "type": "fun_server", "is_ssl": true, "api_key": "none", "output_dir": "tmp/"}', +`doc_link` = 'https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md', +`remark` = '独立部署FunASR，使用FunASR的API服务，只需要五句话 +第一句：mkdir -p ./funasr-runtime-resources/models +第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12 +上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime +不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & +上一句话执行后会进入到容器，继续第五句：tail -f log.txt +第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 +以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer'; + +-- FishSpeech配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md', +`remark` = 'FishSpeech配置说明： +1. 需要本地部署FishSpeech服务 +2. 支持自定义音色 +3. 本地推理，无需网络连接 +4. 输出文件保存在tmp/目录 +5. 可参照教程https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md' WHERE `id` = 'TTS_FishSpeech'; diff --git a/main/manager-api/src/main/resources/db/changelog/202505111914.sql b/main/manager-api/src/main/resources/db/changelog/202505111914.sql new file mode 100644 index 00000000..c67f3477 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202505111914.sql @@ -0,0 +1,6 @@ +-- 添加聊天记录配置字段 +ALTER TABLE `ai_agent` +ADD COLUMN `chat_history_conf` tinyint NOT NULL DEFAULT 0 COMMENT '聊天记录配置（0不记录 1仅记录文本 2记录文本和语音）' AFTER `system_prompt`; + +ALTER TABLE `ai_agent_template` +ADD COLUMN `chat_history_conf` tinyint NOT NULL DEFAULT 0 COMMENT '聊天记录配置（0不记录 1仅记录文本 2记录文本和语音）' AFTER `system_prompt`; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 53f4609d..2dfd9831 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -113,4 +113,18 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505091409.sql \ No newline at end of file + path: classpath:db/changelog/202505091409.sql + - changeSet: + id: 202505091555 + author: whosmyqueen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202505091555.sql + - changeSet: + id: 202505111914 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202505111914.sql diff --git a/main/manager-api/src/main/resources/mapper/agent/AiAgentChatHistoryDao.xml b/main/manager-api/src/main/resources/mapper/agent/AiAgentChatHistoryDao.xml index 24a669c1..3e80f7fd 100644 --- a/main/manager-api/src/main/resources/mapper/agent/AiAgentChatHistoryDao.xml +++ b/main/manager-api/src/main/resources/mapper/agent/AiAgentChatHistoryDao.xml @@ -28,11 +28,17 @@ SELECT audio_id FROM ai_agent_chat_history WHERE agent_id = #{agentId} - ); + ) + + + UPDATE ai_agent_chat_history + SET audio_id = NULL + WHERE agent_id = #{agentId} + DELETE FROM ai_agent_chat_history - WHERE agent_id = #{agentId}; + WHERE agent_id = #{agentId} diff --git a/main/manager-web/src/apis/module/agent.js b/main/manager-web/src/apis/module/agent.js index 8aa89072..756e1bef 100644 --- a/main/manager-web/src/apis/module/agent.js +++ b/main/manager-web/src/apis/module/agent.js @@ -50,9 +50,9 @@ export default { }).send(); }, // 获取智能体配置 - getDeviceConfig(deviceId, callback) { + getDeviceConfig(agentId, callback) { RequestService.sendRequest() - .url(`${getServiceUrl()}/agent/${deviceId}`) + .url(`${getServiceUrl()}/agent/${agentId}`) .method('GET') .success((res) => { RequestService.clearRequestTime(); @@ -61,7 +61,7 @@ export default { .fail((err) => { console.error('获取配置失败:', err); RequestService.reAjaxFun(() => { - this.getDeviceConfig(deviceId, callback); + this.getDeviceConfig(agentId, callback); }); }).send(); }, diff --git a/main/manager-web/src/assets/login/login-person.png b/main/manager-web/src/assets/login/login-person.png index 27477f2f..081eff3e 100644 Binary files a/main/manager-web/src/assets/login/login-person.png and b/main/manager-web/src/assets/login/login-person.png differ diff --git a/main/manager-web/src/assets/model/model.png b/main/manager-web/src/assets/model/model.png index f08ecb86..21132b76 100644 Binary files a/main/manager-web/src/assets/model/model.png and b/main/manager-web/src/assets/model/model.png differ diff --git a/main/manager-web/src/components/DeviceItem.vue b/main/manager-web/src/components/DeviceItem.vue index 60f54b9d..0ecbcefa 100644 --- a/main/manager-web/src/components/DeviceItem.vue +++ b/main/manager-web/src/components/DeviceItem.vue @@ -26,8 +26,12 @@
设备管理({{ device.deviceCount }})
-
- 聊天记录 +
+ + 聊天记录 + + 聊天记录
@@ -77,6 +81,9 @@ export default { this.$router.push({ path: '/device-management', query: { agentId: this.device.agentId } }); }, handleChatHistory() { + if (this.device.memModelId === 'Memory_nomem') { + return + } this.$emit('chat-history', { agentId: this.device.agentId, agentName: this.device.agentName }) } } @@ -120,6 +127,12 @@ export default { color: #979db1; font-weight: 400; } + +.disabled-btn { + background: #e6e6e6; + color: #999; + cursor: not-allowed; +} \ No newline at end of file diff --git a/main/manager-web/src/components/FunctionDialog.vue b/main/manager-web/src/components/FunctionDialog.vue new file mode 100644 index 00000000..b4fec417 --- /dev/null +++ b/main/manager-web/src/components/FunctionDialog.vue @@ -0,0 +1,410 @@ + + + + + \ No newline at end of file diff --git a/main/manager-web/src/views/roleConfig.vue b/main/manager-web/src/views/roleConfig.vue index 8b9504da..00148b7d 100644 --- a/main/manager-web/src/views/roleConfig.vue +++ b/main/manager-web/src/views/roleConfig.vue @@ -60,12 +60,44 @@
- - - +
+ + + +
+ +
+
功能名称: {{ func.name }}
+
+ 参数配置: +
+ {{ key }}: {{ value }} +
+
+
无参数配置
+
+
+ {{ func.name.charAt(0) }} +
+ + + 编辑功能 + +
+
+ + 上报文字 + 上报文字+语音 + +
+
- + @@ -75,27 +107,31 @@
- + +