From be6fba596319cf727a776872f603a504584f6c54 Mon Sep 17 00:00:00 2001 From: shane0411 Date: Mon, 9 Jun 2025 15:20:27 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0tts=E6=94=AF=E6=8F=B4=E9=9D=A2=E6=9D=BF=E7=9A=84"?= =?UTF-8?q?=E8=A7=92=E8=89=B2=E9=9F=B3=E8=89=B2"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/service/impl/ConfigServiceImpl.java | 13 ++++++++++--- .../xiaozhi-server/core/providers/tts/fishspeech.py | 7 +++++++ .../core/providers/tts/gpt_sovits_v2.py | 8 ++++++++ .../core/providers/tts/gpt_sovits_v3.py | 8 ++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index b9a7918c..a5256a66 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -66,6 +66,7 @@ public class ConfigServiceImpl implements ConfigService { null, null, null, + null, agent.getVadModelId(), agent.getAsrModelId(), null, @@ -102,9 +103,11 @@ public class ConfigServiceImpl implements ConfigService { } // 获取音色信息 String voice = null; + String voiceRemark = null; TimbreDetailsVO timbre = timbreService.get(agent.getTtsVoiceId()); if (timbre != null) { voice = timbre.getTtsVoice(); + voiceRemark = timbre.getRemark(); } // 构建返回数据 Map result = new HashMap<>(); @@ -138,6 +141,7 @@ public class ConfigServiceImpl implements ConfigService { agent.getSystemPrompt(), agent.getSummaryMemory(), voice, + voiceRemark, agent.getVadModelId(), agent.getAsrModelId(), agent.getLlmModelId(), @@ -154,7 +158,7 @@ public class ConfigServiceImpl implements ConfigService { /** * 构建配置信息 * - * @param paramsList 系统参数列表 + * @param config 系统参数列表 * @return 配置信息 */ private Object buildConfig(Map config) { @@ -227,6 +231,7 @@ public class ConfigServiceImpl implements ConfigService { * * @param prompt 提示词 * @param voice 音色 + * @param voiceRemark 備註 * @param vadModelId VAD模型ID * @param asrModelId ASR模型ID * @param llmModelId LLM模型ID @@ -240,6 +245,7 @@ public class ConfigServiceImpl implements ConfigService { String prompt, String summaryMemory, String voice, + String voiceRemark, String vadModelId, String asrModelId, String llmModelId, @@ -265,8 +271,9 @@ public class ConfigServiceImpl implements ConfigService { if (model.getConfigJson() != null) { typeConfig.put(model.getId(), model.getConfigJson()); // 如果是TTS类型，添加private_voice属性 - if ("TTS".equals(modelTypes[i]) && voice != null) { - ((Map) model.getConfigJson()).put("private_voice", voice); + if ("TTS".equals(modelTypes[i])){ + if (voice != null) ((Map) model.getConfigJson()).put("private_voice", voice); + if (voiceRemark != null) ((Map) model.getConfigJson()).put("voice_remark", voiceRemark); } // 如果是Intent类型，且type=intent_llm，则给他添加附加模型 if ("Intent".equals(modelTypes[i])) { diff --git a/main/xiaozhi-server/core/providers/tts/fishspeech.py b/main/xiaozhi-server/core/providers/tts/fishspeech.py index 3bcb1229..8e8782df 100644 --- a/main/xiaozhi-server/core/providers/tts/fishspeech.py +++ b/main/xiaozhi-server/core/providers/tts/fishspeech.py @@ -129,6 +129,13 @@ class TTSProvider(TTSProviderBase): self.use_memory_cache = config.get("use_memory_cache", "on") self.seed = int(config.get("seed")) if config.get("seed") else None self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts") + self.get_voice_data(config) + + def get_voice_data(self, config: dict): + if not config.get('private_voice', '') and not config.get('voice_remark', ''): + return + self.reference_audio = config.get('private_voice') + self.reference_text = config.get('voice_remark') async def text_to_speak(self, text, output_file): # Prepare reference data diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py index 7f43ad7d..a22be6d4 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py @@ -67,6 +67,14 @@ class TTSProvider(TTSProviderBase): ) self.audio_file_type = config.get("format", "wav") + self.get_voice_data(config) + + def get_voice_data(self, config: dict): + if not config.get('private_voice', '') and not config.get('voice_remark', ''): + return + self.ref_audio_path = config.get('private_voice') + self.prompt_text = config.get('voice_remark') + async def text_to_speak(self, text, output_file): request_json = { "text": text, diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py index ae41fc4e..577a2794 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py @@ -34,6 +34,14 @@ class TTSProvider(TTSProviderBase): self.if_sr = str(config.get("if_sr", False)).lower() in ("true", "1", "yes") self.audio_file_type = config.get("format", "wav") + self.get_voice_data(config) + + def get_voice_data(self, config: dict): + if not config.get('private_voice', '') and not config.get('voice_remark', ''): + return + self.refer_wav_path = config.get('private_voice') + self.prompt_text = config.get('voice_remark') + async def text_to_speak(self, text, output_file): request_params = { "refer_wav_path": self.refer_wav_path, From 2eb5539c9b5c790d8ea98daf4fda74cbabdf107c Mon Sep 17 00:00:00 2001 From: shane0411 Date: Mon, 9 Jun 2025 19:18:58 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E`=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E8=B7=AF=E5=BE=84`=E8=88=87`=E9=9F=B3=E9=A2=91?= =?UTF-8?q?=E6=96=87=E6=9C=AC`=E6=AC=84=E4=BD=8D=EF=BC=8C=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E7=B5=A6=E6=9C=AC=E5=9C=B0tts=E9=81=B8=E5=8F=96?= =?UTF-8?q?=E9=9F=B3=E8=89=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/xinnan-tech/xiaozhi-esp32-server/issues/1503#issuecomment-2954952028 --- .../service/impl/ConfigServiceImpl.java | 37 +++++++++++-------- .../modules/timbre/dto/TimbreDataDTO.java | 6 +++ .../modules/timbre/entity/TimbreEntity.java | 6 +++ .../modules/timbre/vo/TimbreDetailsVO.java | 6 +++ .../resources/db/changelog/202506091720.sql | 3 ++ .../db/changelog/db.changelog-master.yaml | 9 ++++- main/manager-web/src/apis/module/timbre.js | 4 ++ main/manager-web/src/components/TtsModel.vue | 18 +++++++++ .../core/providers/tts/fishspeech.py | 15 +++----- .../core/providers/tts/gpt_sovits_v2.py | 12 +----- .../core/providers/tts/gpt_sovits_v3.py | 12 +----- 11 files changed, 83 insertions(+), 45 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202506091720.sql diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index a5256a66..bf143802 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -67,6 +67,7 @@ public class ConfigServiceImpl implements ConfigService { null, null, null, + null, agent.getVadModelId(), agent.getAsrModelId(), null, @@ -103,11 +104,13 @@ public class ConfigServiceImpl implements ConfigService { } // 获取音色信息 String voice = null; - String voiceRemark = null; + String referenceAudio = null; + String referenceText = null; TimbreDetailsVO timbre = timbreService.get(agent.getTtsVoiceId()); if (timbre != null) { voice = timbre.getTtsVoice(); - voiceRemark = timbre.getRemark(); + referenceAudio = timbre.getReferenceAudio(); + referenceText = timbre.getReferenceText(); } // 构建返回数据 Map result = new HashMap<>(); @@ -141,7 +144,8 @@ public class ConfigServiceImpl implements ConfigService { agent.getSystemPrompt(), agent.getSummaryMemory(), voice, - voiceRemark, + referenceAudio, + referenceText, agent.getVadModelId(), agent.getAsrModelId(), agent.getLlmModelId(), @@ -229,23 +233,25 @@ public class ConfigServiceImpl implements ConfigService { /** * 构建模块配置 * - * @param prompt 提示词 - * @param voice 音色 - * @param voiceRemark 備註 - * @param vadModelId VAD模型ID - * @param asrModelId ASR模型ID - * @param llmModelId LLM模型ID - * @param ttsModelId TTS模型ID - * @param memModelId 记忆模型ID - * @param intentModelId 意图模型ID - * @param result 结果Map + * @param prompt 提示词 + * @param voice 音色 + * @param referenceAudio 参考音频路径 + * @param referenceText 参考文本 + * @param vadModelId VAD模型ID + * @param asrModelId ASR模型ID + * @param llmModelId LLM模型ID + * @param ttsModelId TTS模型ID + * @param memModelId 记忆模型ID + * @param intentModelId 意图模型ID + * @param result 结果Map */ private void buildModuleConfig( String assistantName, String prompt, String summaryMemory, String voice, - String voiceRemark, + String referenceAudio, + String referenceText, String vadModelId, String asrModelId, String llmModelId, @@ -273,7 +279,8 @@ public class ConfigServiceImpl implements ConfigService { // 如果是TTS类型，添加private_voice属性 if ("TTS".equals(modelTypes[i])){ if (voice != null) ((Map) model.getConfigJson()).put("private_voice", voice); - if (voiceRemark != null) ((Map) model.getConfigJson()).put("voice_remark", voiceRemark); + if (referenceAudio != null) ((Map) model.getConfigJson()).put("ref_audio", referenceAudio); + if (referenceText != null) ((Map) model.getConfigJson()).put("ref_text", referenceText); } // 如果是Intent类型，且type=intent_llm，则给他添加附加模型 if ("Intent".equals(modelTypes[i])) { diff --git a/main/manager-api/src/main/java/xiaozhi/modules/timbre/dto/TimbreDataDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/timbre/dto/TimbreDataDTO.java index 030e2b4a..202c4eaf 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/timbre/dto/TimbreDataDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/timbre/dto/TimbreDataDTO.java @@ -26,6 +26,12 @@ public class TimbreDataDTO { @Schema(description = "备注") private String remark; + @Schema(description = "参考音频路径") + private String referenceAudio; + + @Schema(description = "參考文本") + private String referenceText; + @Schema(description = "排序") @Min(value = 0, message = "{sort.number}") private long sort; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/timbre/entity/TimbreEntity.java b/main/manager-api/src/main/java/xiaozhi/modules/timbre/entity/TimbreEntity.java index 3d6e5ba1..508c3f32 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/timbre/entity/TimbreEntity.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/timbre/entity/TimbreEntity.java @@ -34,6 +34,12 @@ public class TimbreEntity { @Schema(description = "备注") private String remark; + @Schema(description = "参考音频路径") + private String referenceAudio; + + @Schema(description = "參考文本") + private String referenceText; + @Schema(description = "排序") private long sort; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/timbre/vo/TimbreDetailsVO.java b/main/manager-api/src/main/java/xiaozhi/modules/timbre/vo/TimbreDetailsVO.java index 3f0ace40..2cae1a53 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/timbre/vo/TimbreDetailsVO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/timbre/vo/TimbreDetailsVO.java @@ -25,6 +25,12 @@ public class TimbreDetailsVO implements Serializable { @Schema(description = "备注") private String remark; + @Schema(description = "参考音频路径") + private String referenceAudio; + + @Schema(description = "參考文本") + private String referenceText; + @Schema(description = "排序") private long sort; diff --git a/main/manager-api/src/main/resources/db/changelog/202506091720.sql b/main/manager-api/src/main/resources/db/changelog/202506091720.sql new file mode 100644 index 00000000..97b457ee --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202506091720.sql @@ -0,0 +1,3 @@ +ALTER TABLE `ai_tts_voice` +ADD COLUMN `reference_audio` VARCHAR(500) DEFAULT NULL COMMENT '参考音频路径' AFTER `remark`, +ADD COLUMN `reference_text` VARCHAR(500) DEFAULT NULL COMMENT '参考文本' AFTER `reference_audio`; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index f33d28a2..e7757d3d 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -204,4 +204,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506080955.sql \ No newline at end of file + path: classpath:db/changelog/202506080955.sql + - changeSet: + id: 202506091720 + author: shane0411 + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202506091720.sql diff --git a/main/manager-web/src/apis/module/timbre.js b/main/manager-web/src/apis/module/timbre.js index 7aa6a711..3a149b1d 100644 --- a/main/manager-web/src/apis/module/timbre.js +++ b/main/manager-web/src/apis/module/timbre.js @@ -34,6 +34,8 @@ export default { languages: params.languageType, name: params.voiceName, remark: params.remark, + referenceAudio: params.referenceAudio, + referenceText: params.referenceText, sort: params.sort, ttsModelId: params.ttsModelId, ttsVoice: params.voiceCode, @@ -75,6 +77,8 @@ export default { languages: params.languageType, name: params.voiceName, remark: params.remark, + referenceAudio: params.referenceAudio, + referenceText: params.referenceText, ttsModelId: params.ttsModelId, ttsVoice: params.voiceCode, voiceDemo: params.voiceDemo || '' diff --git a/main/manager-web/src/components/TtsModel.vue b/main/manager-web/src/components/TtsModel.vue index 87356523..5f7b631a 100644 --- a/main/manager-web/src/components/TtsModel.vue +++ b/main/manager-web/src/components/TtsModel.vue @@ -74,6 +74,18 @@ {{ scope.row.remark }} + + + + + +