Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 23:23:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2353 from xinnan-tech/voice-clone

Browse Source 
uptate:音色复刻功能完善
This commit is contained in:
欣南科技
2025-10-19 13:11:23 +08:00
committed by GitHub
parent d3371c2875 48e2369964
commit 6ad9ae8d1b
30 changed files with 666 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java
+37
View File
@@ -146,6 +146,11 @@ public interface Constant {
*/
String MEMORY_NO_MEM = "Memory_nomem";
/**
* 火山引擎双声道语音克隆
*/
String VOICE_CLONE_HUOSHAN_DOUBLE_STREAM = "huoshan_double_stream";
enum SysBaseParam {
/**
* ICP备案号
@@ -171,6 +176,38 @@ public interface Constant {
}
}
/**
* 训练状态
*/
enum TrainStatus {
/**
* 未训练
*/
NOT_TRAINED(0),
/**
* 训练中
*/
TRAINING(1),
/**
* 已训练
*/
TRAINED(2),
/**
* 训练失败
*/
TRAIN_FAILED(3);
private final int code;
TrainStatus(int code) {
this.code = code;
}
public int getCode() {
return code;
}
}
/**
* 系统短信
*/
main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java
+20 -10
View File
@@ -55,7 +55,7 @@ public interface ErrorCode {
int OTA_DEVICE_NOT_FOUND = 10041;
int OTA_DEVICE_NEED_BIND = 10042;
// 新增错误编码
int DELETE_DATA_FAILED = 10043;
int USER_NOT_LOGIN = 10044;
@@ -66,7 +66,7 @@ public interface ErrorCode {
int INVALID_LLM_TYPE = 10049;
int TOKEN_GENERATE_ERROR = 10050;
int RESOURCE_NOT_FOUND = 10051;
// 新增错误编码
int DEFAULT_AGENT_NOT_FOUND = 10052;
int AGENT_NOT_FOUND = 10053;
@@ -89,7 +89,7 @@ public interface ErrorCode {
int LLM_NOT_EXIST = 10092; // 设置的LLM不存在
int MODEL_REFERENCED_BY_AGENT = 10093; // 该模型配置已被智能体引用，无法删除
int LLM_REFERENCED_BY_INTENT = 10094; // 该LLM模型已被意图识别配置引用，无法删除
// 登录相关错误码
int ADD_DATA_FAILED = 10065; // 新增数据失败
int UPDATE_DATA_FAILED = 10066; // 修改数据失败
@@ -102,17 +102,17 @@ public interface ErrorCode {
int RETRIEVE_PASSWORD_DISABLED = 10073; // 未开启找回密码功能
int PHONE_FORMAT_ERROR = 10074; // 手机号码格式不正确
int SMS_CODE_ERROR = 10075; // 手机验证码错误
// 字典类型相关错误码
int DICT_TYPE_NOT_EXIST = 10076; // 字典类型不存在
int DICT_TYPE_DUPLICATE = 10077; // 字典类型编码重复
// 资源处理相关错误码
int RESOURCE_READ_ERROR = 10078; // 读取资源失败
// 智能体相关错误码
int LLM_INTENT_PARAMS_MISMATCH = 10079; // LLM大模型和Intent意图识别，选择参数不匹配
// 声纹相关错误码
int VOICEPRINT_ALREADY_REGISTERED = 10080; // 此声音声纹已经注册
int VOICEPRINT_DELETE_ERROR = 10081; // 删除声纹出现错误
@@ -126,12 +126,12 @@ public interface ErrorCode {
int VOICEPRINT_UNREGISTER_REQUEST_ERROR = 10089; // 声纹注销请求失败
int VOICEPRINT_UNREGISTER_PROCESS_ERROR = 10090; // 声纹注销处理失败
int VOICEPRINT_IDENTIFY_REQUEST_ERROR = 10091; // 声纹识别请求失败
// 服务端管理相关错误码
int INVALID_SERVER_ACTION = 10095; // 无效服务端操作
int SERVER_WEBSOCKET_NOT_CONFIGURED = 10096; // 未配置服务端WebSocket地址
int TARGET_WEBSOCKET_NOT_EXIST = 10097; // 目标WebSocket地址不存在
// 参数验证相关错误码
int WEBSOCKET_URLS_EMPTY = 10098; // WebSocket地址列表不能为空
int WEBSOCKET_URL_LOCALHOST = 10099; // WebSocket地址不能使用localhost或127.0.0.1
@@ -175,7 +175,7 @@ public interface ErrorCode {
int DOWNLOAD_LINK_INVALID = 10137; // 下载链接无效
int CHAT_ROLE_USER = 10138; // 用户角色
int CHAT_ROLE_AGENT = 10139; // 智能体角色
// 声音克隆相关错误码
int VOICE_CLONE_AUDIO_EMPTY = 10140; // 音频文件不能为空
int VOICE_CLONE_NOT_AUDIO_FILE = 10141; // 只支持音频文件
@@ -188,4 +188,14 @@ public interface ErrorCode {
int VOICE_RESOURCE_ACCOUNT_EMPTY = 10148; // 归属账号不能为空
int VOICE_RESOURCE_DELETE_ID_EMPTY = 10149; // 删除的音色资源ID不能为空
int VOICE_RESOURCE_NO_PERMISSION = 10150; // 您没有权限操作该记录
int VOICE_CLONE_AUDIO_NOT_UPLOADED = 10151; // 请先上传音频文件
int VOICE_CLONE_MODEL_CONFIG_NOT_FOUND = 10152; // 模型配置未找到
int VOICE_CLONE_MODEL_TYPE_NOT_FOUND = 10153; // 模型类型未找到
int VOICE_CLONE_TRAINING_FAILED = 10154; // 训练失败
int VOICE_CLONE_HUOSHAN_CONFIG_MISSING = 10155; // 火山引擎缺少配置
int VOICE_CLONE_RESPONSE_FORMAT_ERROR = 10156; // 响应格式错误
int VOICE_CLONE_REQUEST_FAILED = 10157; // 请求失败
int VOICE_CLONE_PREFIX = 10158; // 复刻音色前缀
int VOICE_ID_ALREADY_EXISTS = 10159; // 音色ID已存在
int VOICE_CLONE_HUOSHAN_VOICE_ID_ERROR = 10160; // 火山引擎音色ID格式错误
}
main/manager-api/src/main/java/xiaozhi/common/utils/MessageUtils.java
+4 -4
View File
@@ -11,15 +11,15 @@ import org.springframework.context.i18n.LocaleContextHolder;
public class MessageUtils {
private static MessageSource messageSource;
static {
messageSource = (MessageSource) SpringContextUtils.getBean("messageSource");
}
public static String getMessage(int code) {
return getMessage(code, new String[0]);
}
public static String getMessage(int code, String... params) {
if (messageSource == null) {
// 延迟初始化，确保Spring上下文已完全初始化
messageSource = (MessageSource) SpringContextUtils.getBean("messageSource");
}
return messageSource.getMessage(code + "", params, LocaleContextHolder.getLocale());
}
}
main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java
+17
View File
@@ -42,6 +42,7 @@ import xiaozhi.modules.agent.service.AgentTemplateService;
import xiaozhi.modules.agent.vo.AgentInfoVO;
import xiaozhi.modules.device.service.DeviceService;
import xiaozhi.modules.model.dto.ModelProviderDTO;
import xiaozhi.modules.model.dto.VoiceDTO;
import xiaozhi.modules.model.entity.ModelConfigEntity;
import xiaozhi.modules.model.service.ModelConfigService;
import xiaozhi.modules.model.service.ModelProviderService;
@@ -373,6 +374,22 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
entity.setLlmModelId(template.getLlmModelId());
entity.setVllmModelId(template.getVllmModelId());
entity.setTtsModelId(template.getTtsModelId());
if (template.getTtsVoiceId() == null && template.getTtsModelId() != null) {
ModelConfigEntity ttsModel = modelConfigService.selectById(template.getTtsModelId());
if (ttsModel != null && ttsModel.getConfigJson() != null) {
Map<String, Object> config = ttsModel.getConfigJson();
String voice = (String) config.get("voice");
if (StringUtils.isBlank(voice)) {
voice = (String) config.get("speaker");
}
VoiceDTO timbre = timbreModelService.getByVoiceCode(template.getTtsModelId(), voice);
if (timbre != null) {
template.setTtsVoiceId(timbre.getId());
}
}
}
entity.setTtsVoiceId(template.getTtsVoiceId());
entity.setMemModelId(template.getMemModelId());
entity.setIntentModelId(template.getIntentModelId());
main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java
+17
View File
@@ -38,6 +38,8 @@ import xiaozhi.modules.sys.dto.SysParamsDTO;
import xiaozhi.modules.sys.service.SysParamsService;
import xiaozhi.modules.timbre.service.TimbreService;
import xiaozhi.modules.timbre.vo.TimbreDetailsVO;
import xiaozhi.modules.voiceclone.entity.VoiceCloneEntity;
import xiaozhi.modules.voiceclone.service.VoiceCloneService;
@Service
@AllArgsConstructor
@@ -51,6 +53,7 @@ public class ConfigServiceImpl implements ConfigService {
private final TimbreService timbreService;
private final AgentPluginMappingService agentPluginMappingService;
private final AgentMcpAccessPointService agentMcpAccessPointService;
private final VoiceCloneService cloneVoiceService;
private final AgentVoicePrintDao agentVoicePrintDao;
@Override
@@ -124,6 +127,11 @@ public class ConfigServiceImpl implements ConfigService {
voice = timbre.getTtsVoice();
referenceAudio = timbre.getReferenceAudio();
referenceText = timbre.getReferenceText();
} else {
VoiceCloneEntity voice_print = cloneVoiceService.selectById(agent.getTtsVoiceId());
if (voice_print != null) {
voice = voice_print.getVoiceId();
}
}
// 构建返回数据
Map<String, Object> result = new HashMap<>();
@@ -392,6 +400,15 @@ public class ConfigServiceImpl implements ConfigService {
((Map<String, Object>) model.getConfigJson()).put("ref_audio", referenceAudio);
if (referenceText != null)
((Map<String, Object>) model.getConfigJson()).put("ref_text", referenceText);
// 火山引擎声音克隆需要替换resource_id
Map<String, Object> map = (Map<String, Object>) model.getConfigJson();
if (Constant.VOICE_CLONE_HUOSHAN_DOUBLE_STREAM.equals(map.get("type"))) {
// 如果voice是”S_“开头的，使用seed-icl-1.0
if (voice != null && voice.startsWith("S_")) {
map.put("resource_id", "seed-icl-1.0");
}
}
}
// 如果是Intent类型，且type=intent_llm，则给他添加附加模型
if ("Intent".equals(modelTypes[i])) {
main/manager-api/src/main/java/xiaozhi/modules/timbre/service/TimbreService.java
+9
View File
@@ -64,4 +64,13 @@ public interface TimbreService extends BaseService<TimbreEntity> {
* @return 音色名称
*/
String getTimbreNameById(String id);
/**
* 根据音色编码获取音色信息
*
* @param ttsModelId 音色模型ID
* @param voiceCode 音色编码
* @return 音色信息
*/
VoiceDTO getByVoiceCode(String ttsModelId, String voiceCode);
}
main/manager-api/src/main/java/xiaozhi/modules/timbre/service/impl/TimbreServiceImpl.java
+52 -3
View File
@@ -1,9 +1,11 @@
package xiaozhi.modules.timbre.service.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@@ -15,18 +17,23 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import cn.hutool.core.collection.CollectionUtil;
import lombok.AllArgsConstructor;
import xiaozhi.common.constant.Constant;
import xiaozhi.common.exception.ErrorCode;
import xiaozhi.common.page.PageData;
import xiaozhi.common.redis.RedisKeys;
import xiaozhi.common.redis.RedisUtils;
import xiaozhi.common.service.impl.BaseServiceImpl;
import xiaozhi.common.utils.ConvertUtils;
import xiaozhi.common.utils.MessageUtils;
import xiaozhi.modules.model.dto.VoiceDTO;
import xiaozhi.modules.security.user.SecurityUser;
import xiaozhi.modules.timbre.dao.TimbreDao;
import xiaozhi.modules.timbre.dto.TimbreDataDTO;
import xiaozhi.modules.timbre.dto.TimbrePageDTO;
import xiaozhi.modules.timbre.entity.TimbreEntity;
import xiaozhi.modules.timbre.service.TimbreService;
import xiaozhi.modules.timbre.vo.TimbreDetailsVO;
import xiaozhi.modules.voiceclone.dao.VoiceCloneDao;
import xiaozhi.modules.voiceclone.entity.VoiceCloneEntity;
/**
* 音色的业务层的实现
@@ -39,6 +46,7 @@ import xiaozhi.modules.timbre.vo.TimbreDetailsVO;
public class TimbreServiceImpl extends BaseServiceImpl<TimbreDao, TimbreEntity> implements TimbreService {
private final TimbreDao timbreDao;
private final VoiceCloneDao voiceCloneDao;
private final RedisUtils redisUtils;
@Override
@@ -121,11 +129,30 @@ public class TimbreServiceImpl extends BaseServiceImpl<TimbreDao, TimbreEntity>
queryWrapper.like("name", voiceName);
}
List<TimbreEntity> timbreEntities = timbreDao.selectList(queryWrapper);
if (CollectionUtil.isEmpty(timbreEntities)) {
return null;
if (timbreEntities == null) {
timbreEntities = new ArrayList<>();
}
List<VoiceDTO> voiceDTOs = timbreEntities.stream()
.map(entity -> new VoiceDTO(entity.getId(), entity.getName()))
.collect(Collectors.toList());
// 获取当前登录用户ID
Long currentUserId = SecurityUser.getUser().getId();
if (currentUserId != null) {
// 查询用户的所有克隆音色记录
List<VoiceDTO> cloneEntities = voiceCloneDao.getTrainSuccess(ttsModelId, currentUserId);
for (VoiceDTO entity : cloneEntities) {
// 只添加训练成功的克隆音色，且模型ID匹配
VoiceDTO voiceDTO = new VoiceDTO();
voiceDTO.setId(entity.getId());
voiceDTO.setName(MessageUtils.getMessage(ErrorCode.VOICE_CLONE_PREFIX) + entity.getName());
redisUtils.set(RedisKeys.getTimbreNameById(voiceDTO.getId()), voiceDTO.getName(),
RedisUtils.NOT_EXPIRE);
voiceDTOs.add(0, voiceDTO);
}
}
return ConvertUtils.sourceToTarget(timbreEntities, VoiceDTO.class);
return CollectionUtil.isEmpty(voiceDTOs) ? null : voiceDTOs;
}
/**
@@ -154,8 +181,30 @@ public class TimbreServiceImpl extends BaseServiceImpl<TimbreDao, TimbreEntity>
redisUtils.set(RedisKeys.getTimbreNameById(id), name);
}
return name;
} else {
VoiceCloneEntity cloneEntity = voiceCloneDao.selectById(id);
if (cloneEntity != null) {
String name = MessageUtils.getMessage(ErrorCode.VOICE_CLONE_PREFIX) + cloneEntity.getName();
redisUtils.set(RedisKeys.getTimbreNameById(id), name);
return name;
}
}
return null;
}
@Override
public VoiceDTO getByVoiceCode(String ttsModelId, String voiceCode) {
if (StringUtils.isBlank(voiceCode)) {
return null;
}
QueryWrapper<TimbreEntity> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("tts_model_id", ttsModelId);
queryWrapper.eq("tts_voice", voiceCode);
List<TimbreEntity> list = timbreDao.selectList(queryWrapper);
if (list.isEmpty()) {
return null;
}
return new VoiceDTO(list.get(0).getId(), list.get(0).getName());
}
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/controller/VoiceCloneController.java
+3
View File
@@ -116,6 +116,7 @@ public class VoiceCloneController {
checkPermission(id);
voiceCloneService.updateName(id, name);
redisUtils.delete(RedisKeys.getTimbreNameById(id));
return new Result<String>();
} catch (Exception e) {
return new Result<String>().error(ErrorCode.UPDATE_DATA_FAILED, "更新失败: " + e.getMessage());
@@ -175,6 +176,8 @@ public class VoiceCloneController {
public Result<String> cloneAudio(@RequestBody Map<String, String> params) {
String cloneId = params.get("cloneId");
checkPermission(cloneId);
// 调用服务层进行语音克隆训练
voiceCloneService.cloneAudio(cloneId);
return new Result<String>();
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/controller/VoiceResourceController.java
+2
View File
@@ -80,6 +80,8 @@ public class VoiceResourceController {
try {
voiceCloneService.save(dto);
return new Result<Void>();
} catch (xiaozhi.common.exception.RenException e) {
return new Result<Void>().error(e.getCode(), e.getMsg());
} catch (RuntimeException e) {
return new Result<Void>().error(ErrorCode.ADD_DATA_FAILED, e.getMessage());
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/dao/VoiceCloneDao.java
+11
View File
@@ -1,9 +1,12 @@
package xiaozhi.modules.voiceclone.dao;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import xiaozhi.modules.model.dto.VoiceDTO;
import xiaozhi.modules.voiceclone.entity.VoiceCloneEntity;
/**
@@ -11,5 +14,13 @@ import xiaozhi.modules.voiceclone.entity.VoiceCloneEntity;
*/
@Mapper
public interface VoiceCloneDao extends BaseMapper<VoiceCloneEntity> {
/**
* 获取用户训练成功的音色列表
*
* @param modelId 模型ID
* @param userId 用户ID
* @return 训练成功的音色列表
*/
List<VoiceDTO> getTrainSuccess(String modelId, Long userId);
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/service/VoiceCloneService.java
+13 -3
View File
@@ -33,19 +33,22 @@ public interface VoiceCloneService extends BaseService<VoiceCloneEntity> {
/**
* 根据用户ID查询声音克隆列表
*
* @param userId 用户ID
* @return 声音克隆列表
*/
List<VoiceCloneEntity> getByUserId(Long userId);
/**
* 分页查询带模型名称和用户名称的声音克隆列表
*/
PageData<VoiceCloneResponseDTO> pageWithNames(Map<String, Object> params);
/**
* 根据ID查询带模型名称和用户名称的声音克隆信息
*/
VoiceCloneResponseDTO getByIdWithNames(String id);
/**
* 根据用户ID查询带模型名称的声音克隆列表
*/
@@ -65,4 +68,11 @@ public interface VoiceCloneService extends BaseService<VoiceCloneEntity> {
* 获取音频数据
*/
byte[] getVoiceData(String id);
/**
* 克隆音频，调用火山引擎进行语音复刻训练
*
* @param cloneId 语音克隆记录ID
*/
void cloneAudio(String cloneId);
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/service/impl/VoiceCloneServiceImpl.java
+166 -2
View File
@@ -1,7 +1,13 @@
package xiaozhi.modules.voiceclone.service.impl;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -12,14 +18,19 @@ import org.springframework.web.multipart.MultipartFile;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import xiaozhi.common.constant.Constant;
import xiaozhi.common.exception.ErrorCode;
import xiaozhi.common.exception.RenException;
import xiaozhi.common.page.PageData;
import xiaozhi.common.service.impl.BaseServiceImpl;
import xiaozhi.common.utils.ConvertUtils;
import xiaozhi.common.utils.DateUtils;
import xiaozhi.common.exception.ErrorCode;
import xiaozhi.common.exception.RenException;
import xiaozhi.modules.model.entity.ModelConfigEntity;
import xiaozhi.modules.model.service.ModelConfigService;
import xiaozhi.modules.sys.service.SysUserService;
import xiaozhi.modules.voiceclone.dao.VoiceCloneDao;
@@ -28,6 +39,7 @@ import xiaozhi.modules.voiceclone.dto.VoiceCloneResponseDTO;
import xiaozhi.modules.voiceclone.entity.VoiceCloneEntity;
import xiaozhi.modules.voiceclone.service.VoiceCloneService;
@Slf4j
@Service
@RequiredArgsConstructor
public class VoiceCloneServiceImpl extends BaseServiceImpl<VoiceCloneDao, VoiceCloneEntity>
@@ -35,6 +47,7 @@ public class VoiceCloneServiceImpl extends BaseServiceImpl<VoiceCloneDao, VoiceC
private final ModelConfigService modelConfigService;
private final SysUserService sysUserService;
private final ObjectMapper objectMapper;
@Override
public PageData<VoiceCloneEntity> page(Map<String, Object> params) {
@@ -61,6 +74,36 @@ public class VoiceCloneServiceImpl extends BaseServiceImpl<VoiceCloneDao, VoiceC
@Override
@Transactional(rollbackFor = Exception.class)
public void save(VoiceCloneDTO dto) {
ModelConfigEntity modelConfig = modelConfigService.getModelByIdFromCache(dto.getModelId());
if (modelConfig == null || modelConfig.getConfigJson() == null) {
throw new RenException(ErrorCode.VOICE_CLONE_MODEL_CONFIG_NOT_FOUND);
}
Map<String, Object> config = modelConfig.getConfigJson();
String type = (String) config.get("type");
if (StringUtils.isBlank(type)) {
throw new RenException(ErrorCode.VOICE_CLONE_MODEL_TYPE_NOT_FOUND);
}
// 检查Voice ID是否已经被使用
for (String voiceId : dto.getVoiceIds()) {
if (StringUtils.isBlank(voiceId)) {
continue;
}
if (Constant.VOICE_CLONE_HUOSHAN_DOUBLE_STREAM.equals(type)) {
if (voiceId.indexOf("S_") == -1) {
throw new RenException(ErrorCode.VOICE_CLONE_HUOSHAN_VOICE_ID_ERROR);
}
}
QueryWrapper<VoiceCloneEntity> wrapper = new QueryWrapper<>();
wrapper.eq("voice_id", voiceId);
wrapper.eq("model_id", dto.getModelId());
Long count = baseDao.selectCount(wrapper);
if (count > 0) {
throw new RenException(ErrorCode.VOICE_ID_ALREADY_EXISTS, "音色ID " + voiceId + " 已存在");
}
}
// 遍历选择的音色ID，为每个音色ID创建一条记录
int index = 0;
String namePrefix = DateUtils.format(new java.util.Date(), "MMddHHmm");
@@ -207,4 +250,125 @@ public class VoiceCloneServiceImpl extends BaseServiceImpl<VoiceCloneDao, VoiceC
}
return entity.getVoice();
}
@Override
// @Transactional(rollbackFor = Exception.class)
public void cloneAudio(String cloneId) {
VoiceCloneEntity entity = baseDao.selectById(cloneId);
if (entity == null) {
throw new RenException(ErrorCode.VOICE_CLONE_RECORD_NOT_EXIST);
}
if (entity.getVoice() == null || entity.getVoice().length == 0) {
throw new RenException(ErrorCode.VOICE_CLONE_AUDIO_NOT_UPLOADED);
}
try {
ModelConfigEntity modelConfig = modelConfigService.getModelByIdFromCache(entity.getModelId());
if (modelConfig == null || modelConfig.getConfigJson() == null) {
throw new RenException(ErrorCode.VOICE_CLONE_MODEL_CONFIG_NOT_FOUND);
}
Map<String, Object> config = modelConfig.getConfigJson();
String type = (String) config.get("type");
if (StringUtils.isBlank(type)) {
throw new RenException(ErrorCode.VOICE_CLONE_MODEL_TYPE_NOT_FOUND);
}
if (Constant.VOICE_CLONE_HUOSHAN_DOUBLE_STREAM.equals(type)) {
huoshanClone(config, entity);
}
} catch (RenException re) {
entity.setTrainStatus(3);
entity.setTrainError(re.getMsg());
baseDao.updateById(entity);
throw re;
} catch (Exception e) {
e.printStackTrace();
entity.setTrainStatus(3);
entity.setTrainError(e.getMessage());
baseDao.updateById(entity);
throw new RenException(ErrorCode.VOICE_CLONE_TRAINING_FAILED, e.getMessage());
}
}
/**
* 调用火山引擎进行语音复刻训练
*
* @param config 模型配置
* @param entity 语音克隆记录实体
* @throws Exception
*/
private void huoshanClone(Map<String, Object> config, VoiceCloneEntity entity) throws Exception {
String appid = (String) config.get("appid");
String accessToken = (String) config.get("access_token");
if (StringUtils.isAnyBlank(appid, accessToken)) {
throw new RenException(ErrorCode.VOICE_CLONE_HUOSHAN_CONFIG_MISSING);
}
String audioBase64 = Base64.getEncoder().encodeToString(entity.getVoice());
Map<String, Object> reqBody = new HashMap<>();
reqBody.put("appid", appid);
List<Map<String, String>> audios = new ArrayList<>();
Map<String, String> audioMap = new HashMap<>();
audioMap.put("audio_bytes", audioBase64);
audioMap.put("audio_format", "wav");
audios.add(audioMap);
reqBody.put("audios", audios);
reqBody.put("source", 2);
reqBody.put("language", 0);
reqBody.put("model_type", 1);
reqBody.put("speaker_id", entity.getVoiceId());
String apiUrl = "https://openspeech.bytedance.com/api/v1/mega_tts/audio/upload";
HttpClient client = HttpClient.newHttpClient();
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(apiUrl))
.header("Content-Type", "application/json")
.header("Authorization", "Bearer;" + accessToken)
.header("Resource-Id", "seed-icl-1.0")
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(reqBody)))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(">>> HTTP status = " + response.statusCode());
System.out.println(">>> response body = " + response.body());
Map<String, Object> rsp = objectMapper.readValue(response.body(),
new TypeReference<Map<String, Object>>() {
});
// 获取BaseResp对象
Map<String, Object> baseResp = objectMapper.convertValue(rsp.get("BaseResp"),
new TypeReference<Map<String, Object>>() {
});
if (baseResp != null) {
Integer statusCode = objectMapper.convertValue(baseResp.get("StatusCode"), Integer.class);
String statusMessage = objectMapper.convertValue(baseResp.getOrDefault("StatusMessage", ""),
String.class);
// 获取speaker_id
String speakerId = objectMapper.convertValue(rsp.get("speaker_id"), String.class);
// StatusCode == 0 表示成功
if (statusCode != null && statusCode == 0 && StringUtils.isNotBlank(speakerId)) {
entity.setTrainStatus(2);
entity.setVoiceId(speakerId);
entity.setTrainError("");
baseDao.updateById(entity);
} else {
// 失败时使用StatusMessage作为错误信息
String errorMsg = StringUtils.isNotBlank(statusMessage) ? statusMessage : "训练失败";
throw new RenException(errorMsg);
}
} else {
String errorMsg = objectMapper.convertValue(rsp.get("message"),
new TypeReference<String>() {
});
if (StringUtils.isNotBlank(errorMsg)) {
throw new RenException(errorMsg);
}
throw new RenException(ErrorCode.VOICE_CLONE_RESPONSE_FORMAT_ERROR);
}
}
}
main/manager-api/src/main/resources/db/changelog/202510071521.sql → main/manager-api/src/main/resources/db/changelog/202510071522.sql
+5 -1
View File
@@ -11,5 +11,9 @@ CREATE TABLE `ai_voice_clone` (
`train_error` VARCHAR(255) COMMENT '训练错误原因',
`creator` BIGINT COMMENT '创建者 ID',
`create_date` DATETIME COMMENT '创建时间',
PRIMARY KEY (`id`)
PRIMARY KEY (`id`),
INDEX idx_ai_voice_clone_user_id_model_id_train_status (model_id,user_id, train_status),
INDEX idx_ai_voice_clone_voice_id (voice_id),
INDEX idx_ai_voice_clone_user_id (user_id),
INDEX idx_ai_voice_clone_model_id_voice_id (model_id, voice_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='声音克隆表';
main/manager-api/src/main/resources/db/changelog/202510191042.sql
+2
View File
@@ -0,0 +1,2 @@
UPDATE `ai_model_config` SET model_code = 'XunfeiSparkLLM' WHERE id = 'LLM_XunfeiSparkLLM';
UPDATE `ai_model_config` SET model_code = 'XunfeiStreamASR' WHERE id = 'ASR_XunfeiStream';
main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
+9 -2
View File
@@ -389,9 +389,16 @@ databaseChangeLog:
encoding: utf8
path: classpath:db/changelog/202509220958.sql
- changeSet:
id: 202510071521
id: 202510071522
author: hrz
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202510071521.sql
path: classpath:db/changelog/202510071522.sql
- changeSet:
id: 202510191042
author: hrz
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202510191042.sql
main/manager-api/src/main/resources/i18n/messages.properties
+14 -2
View File
@@ -135,7 +135,9 @@
10124=\u60A8\u7684mqtt\u5BC6\u94A5\u957F\u5EA6\u4E0D\u5B89\u5168\uFF0Cmqtt\u5BC6\u94A5\u5FC5\u987B\u540C\u65F6\u5305\u542B\u5927\u5C0F\u5199\u5B57\u6BCD
10125=\u60A8\u7684mqtt\u5BC6\u94A5\u5305\u542B\u5F31\u5BC6\u7801
10128=\u5B57\u5178\u6807\u7B7E\u91CD\u590D
10129=modelType\u548CprovideCode\u4E0D\u80FD\u4E3A\u7A7A
10129=SM2\u5BC6\u94A5\u672A\u914D\u7F6E
10130=SM2\u89E3\u5BC6\u5931\u8D25
10131=modelType\u548CprovideCode\u4E0D\u80FD\u4E3A\u7A7A
10132=\u6CA1\u6709\u6743\u9650\u67E5\u770B\u8BE5\u667A\u80FD\u4F53\u7684\u804A\u5929\u8BB0\u5F55
10133=\u4F1A\u8BDDID\u4E0D\u80FD\u4E3A\u7A7A
10134=\u667A\u80FD\u4F53ID\u4E0D\u80FD\u4E3A\u7A7A
@@ -154,4 +156,14 @@
10147=\u97F3\u8272ID\u4E0D\u80FD\u4E3A\u7A7A
10148=\u5F52\u5C5E\u8D26\u53F7\u4E0D\u80FD\u4E3A\u7A7A
10149=\u5220\u9664\u7684\u97F3\u8272\u8D44\u6E90ID\u4E0D\u80FD\u4E3A\u7A7A
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10151=\u8BF7\u5148\u4E0A\u4F20\u97F3\u9891\u6587\u4EF6
10152=\u6A21\u578B\u914D\u7F6E\u672A\u627E\u5230
10153=\u6A21\u578B\u7C7B\u578B\u672A\u627E\u5230
10154=\u8BAD\u7EC3\u5931\u8D25
10155=\u706B\u5C71\u5F15\u64CE\u7F3A\u5C11\u914D\u7F6E
10156=\u54CD\u5E94\u683C\u5F0F\u9519\u8BEF\uFF0C\u7F3A\u5C11BaseResp\u5B57\u6BB5
10157=\u8BF7\u6C42\u5931\u8D25
10158=\u8BF7\u6C42\u5931\u8D25\uFF0C\u72B6\u6001\u7801{0}
10159=\u97F3\u8272ID\u5DF2\u5B58\u5728
10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u9519\u8BEF\uFF0C\u5FC5\u987B\u4EE5S_\u5F00\u5934
main/manager-api/src/main/resources/i18n/messages_en_US.properties
+11 -1
View File
@@ -156,4 +156,14 @@
10147=Voice ID cannot be empty
10148=Owner account cannot be empty
10149=Voice resource ID to delete cannot be empty
10150=You do not have permission to operate this record
10150=You do not have permission to operate this record
10151=Please upload audio file first
10152=Model configuration not found
10153=Model type not found
10154=Training failed: {0}
10155=Huoshan Engine configuration missing
10156=Response format error, missing BaseResp field
10157=Request failed
10158=Clone Voice:
10159=Voice ID already exists
10160=Huoshan Engine voice ID format error, must start with S_
main/manager-api/src/main/resources/i18n/messages_zh_CN.properties
+11 -1
View File
@@ -156,4 +156,14 @@
10147=\u97F3\u8272ID\u4E0D\u80FD\u4E3A\u7A7A
10148=\u5F52\u5C5E\u8D26\u53F7\u4E0D\u80FD\u4E3A\u7A7A
10149=\u5220\u9664\u7684\u97F3\u8272\u8D44\u6E90ID\u4E0D\u80FD\u4E3A\u7A7A
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10151=\u8BF7\u5148\u4E0A\u4F20\u97F3\u9891\u6587\u4EF6
10152=\u6A21\u578B\u914D\u7F6E\u672A\u627E\u5230
10153=\u6A21\u578B\u7C7B\u578B\u672A\u627E\u5230
10154=\u8BAD\u7EC3\u5931\u8D25
10155=\u706B\u5C71\u5F15\u64CE\u7F3A\u5C11\u914D\u7F6E
10156=\u54CD\u5E94\u683C\u5F0F\u9519\u8BEF\uFF0C\u7F3A\u5C11BaseResp\u5B57\u6BB5
10157=\u8BF7\u6C42\u5931\u8D25
10158=\u514B\u9686\u8272\u97F3:
10159=\u97F3\u8272ID\u5DF2\u5B58\u5728
10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u9519\u8BEF\uFF0C\u5FC5\u987B\u4EE5S_\u5F00\u5934
main/manager-api/src/main/resources/i18n/messages_zh_TW.properties
+11 -1
View File
@@ -156,4 +156,14 @@
10147=\u97F3\u8272ID\u4E0D\u80FD\u70BA\u7A7A
10148=\u6B78\u5C6C\u5E33\u865F\u4E0D\u80FD\u70BA\u7A7A
10149=\u522A\u9664\u7684\u97F3\u8272\u8CC7\u6E90ID\u4E0D\u80FD\u70BA\u7A7A
10150=\u60A8\u6C92\u6709\u6B0A\u9650\u64CD\u4F5C\u8A72\u8A18\u9304
10150=\u60A8\u6C92\u6709\u6B0A\u9650\u64CD\u4F5C\u8A72\u8A18\u9304
10151=\u8ACB\u5148\u4E0A\u50B3\u97F3\u983B\u6587\u4EF6
10152=\u6A21\u578B\u914D\u7F6E\u672A\u627E\u5230
10153=\u6A21\u578B\u985E\u578B\u672A\u627E\u5230
10154=\u6A21\u578B\u8AB2\u7DF4\u5931\u6557: {0}
10155=\u706B\u5C71\u5F15\u64CE\u7F3A\u5C11appid\u6216access_token
10156=\u97FF\u61C9\u683C\u5F0F\u932F\u8AA4\uFF0C\u7F3A\u5C11BaseResp\u5B57\u6BB5
10157=\u8ACB\u6C42\u5931\u6557
10158=\u514B\u9686\u8A9E\u97F3:
10159=\u97F3\u8272ID\u5DF2\u5B58\u5728
10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u932F\u8AA4\uFF0C\u5FC5\u9808\u4EE5S_\u958B\u982D
main/manager-api/src/main/resources/mapper/voiceclone/VoiceCloneDao.xml
+10
View File
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="xiaozhi.modules.voiceclone.dao.VoiceCloneDao">
<select id="getTrainSuccess" resultType="xiaozhi.modules.model.dto.VoiceDTO">
select id, name
from ai_voice_clone
where model_id = #{modelId} and user_id = #{userId} and train_status = 2
</select>
</mapper>