diff --git a/docs/fish-speech-integration.md b/docs/fish-speech-integration.md new file mode 100644 index 00000000..b916e619 --- /dev/null +++ b/docs/fish-speech-integration.md @@ -0,0 +1,72 @@ +登录AutoDL，租赁镜像 +选择镜像: +``` +PyTorch / 2.1.0 / 3.10(ubuntu22.04) / cuda 12.1 +``` + +机器开机后，设置学术加速 +``` +source /etc/network_turbo +``` + +进入工作目录 +``` +cd autodl-tmp/ +``` + +拉取项目 +``` +git clone https://gitclone.com/github.com/fishaudio/fish-speech.git ; cd fish-speech +``` + +安装依赖 +``` +pip install -e. +``` + +如果报错，安装portaudio +``` +apt-get install portaudio19-dev -y +``` + +安装后执行 +``` +pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cu121 +``` + +下载模型 +``` +cd tools +python download_models.py +``` + +下载完模型后运行接口 +``` +python -m tools.api_server --listen 0.0.0.0:6006 +``` + +然后用浏览器去到aotodl实例页面 +``` +https://autodl.com/console/instance/list +``` + +如下图点击你刚才机器的`自定义服务`按钮，开启端口转发服务 +![自定义服务](images/fishspeech/autodl-01.png) + +端口转发服务设置完成后，你本地电脑打开网址`http://localhost:6006/`，就可以访问fish-speech的接口了 +![服务预览](images/fishspeech/autodl-02.png) + + +如果你是单模块部署，核心配置如下 +``` +selected_module: + TTS: FishSpeech +TTS: + FishSpeech: + reference_audio: ["config/assets/wakeup_words.wav",] + reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",] + api_key: "123" + api_url: "http://127.0.0.1:6006/v1/tts" +``` + +然后重启服务 \ No newline at end of file diff --git a/docs/images/fishspeech/autodl-01.png b/docs/images/fishspeech/autodl-01.png new file mode 100644 index 00000000..695b94b8 Binary files /dev/null and b/docs/images/fishspeech/autodl-01.png differ diff --git a/docs/images/fishspeech/autodl-02.png b/docs/images/fishspeech/autodl-02.png new file mode 100644 index 00000000..e3f72afe Binary files /dev/null and b/docs/images/fishspeech/autodl-02.png differ diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java index 0d1ec7b2..075db842 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java @@ -27,6 +27,9 @@ public class AgentDTO { @Schema(description = "大语言模型名称", example = "llm_model_01") private String llmModelName; + @Schema(description = "记忆模型ID", example = "mem_model_01") + private String memModelId; + @Schema(description = "角色设定参数", example = "你是一个专业的客服助手，负责回答用户问题并提供帮助") private String systemPrompt; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java index 067a8f1e..37bb7d93 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java @@ -102,6 +102,9 @@ public class AgentServiceImpl extends BaseServiceImpl imp // 获取 LLM 模型名称 dto.setLlmModelName(modelConfigService.getModelNameById(agent.getLlmModelId())); + // 获取记忆模型名称 + dto.setMemModelId(agent.getMemModelId()); + // 获取 TTS 音色名称 dto.setTtsVoiceName(timbreModelService.getTimbreNameById(agent.getTtsVoiceId())); diff --git a/main/manager-api/src/main/resources/db/changelog/202505091555.sql b/main/manager-api/src/main/resources/db/changelog/202505091555.sql new file mode 100644 index 00000000..b7ad91d4 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202505091555.sql @@ -0,0 +1,25 @@ +-- 更新模型供应器表 +UPDATE `ai_model_provider` SET fields = '[{"key": "host", "type": "string", "label": "服务地址"}, {"key": "port", "type": "number", "label": "端口号"}, {"key": "type", "type": "string", "label": "服务类型"}, {"key": "is_ssl", "type": "boolean", "label": "是否使用SSL"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]' WHERE id = 'SYSTEM_ASR_FunASRServer'; + +-- 更新模型配置表 +UPDATE `ai_model_config` SET +config_json = '{"host": "127.0.0.1", "port": 10096, "type": "fun_server", "is_ssl": true, "api_key": "none", "output_dir": "tmp/"}', +`doc_link` = 'https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md', +`remark` = '独立部署FunASR，使用FunASR的API服务，只需要五句话 +第一句：mkdir -p ./funasr-runtime-resources/models +第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12 +上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime +不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & +上一句话执行后会进入到容器，继续第五句：tail -f log.txt +第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 +以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer'; + +-- FishSpeech配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md', +`remark` = 'FishSpeech配置说明： +1. 需要本地部署FishSpeech服务 +2. 支持自定义音色 +3. 本地推理，无需网络连接 +4. 输出文件保存在tmp/目录 +5. 可参照教程https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md' WHERE `id` = 'TTS_FishSpeech'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 459a056b..43ce1034 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -114,6 +114,13 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202505091409.sql + - changeSet: + id: 202505091555 + author: whosmyqueen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202505091555.sql - changeSet: id: 202505111914 author: hrz @@ -127,4 +134,4 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505122348.sql \ No newline at end of file + path: classpath:db/changelog/202505122348.sql diff --git a/main/manager-web/src/components/DeviceItem.vue b/main/manager-web/src/components/DeviceItem.vue index 60f54b9d..0ecbcefa 100644 --- a/main/manager-web/src/components/DeviceItem.vue +++ b/main/manager-web/src/components/DeviceItem.vue @@ -26,8 +26,12 @@
设备管理({{ device.deviceCount }})
-
- 聊天记录 +
+ + 聊天记录 + + 聊天记录
@@ -77,6 +81,9 @@ export default { this.$router.push({ path: '/device-management', query: { agentId: this.device.agentId } }); }, handleChatHistory() { + if (this.device.memModelId === 'Memory_nomem') { + return + } this.$emit('chat-history', { agentId: this.device.agentId, agentName: this.device.agentName }) } } @@ -120,6 +127,12 @@ export default { color: #979db1; font-weight: 400; } + +.disabled-btn { + background: #e6e6e6; + color: #999; + cursor: not-allowed; +}