Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'voice-clone' of https://github.com/xinnan-tech/xiaozhi-esp32-server

Browse Source
This commit is contained in:
rainv123
2025-10-14 10:27:02 +08:00
parent 13ce67e4c7 bccda3e7b2
commit 1400e48fc7
14 changed files with 273 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java
+8
View File
@@ -188,4 +188,12 @@ public interface ErrorCode {
int VOICE_RESOURCE_ACCOUNT_EMPTY = 10148; // 归属账号不能为空
int VOICE_RESOURCE_DELETE_ID_EMPTY = 10149; // 删除的音色资源ID不能为空
int VOICE_RESOURCE_NO_PERMISSION = 10150; // 您没有权限操作该记录
int VOICE_CLONE_AUDIO_NOT_UPLOADED = 10151; // 请先上传音频文件
int VOICE_CLONE_MODEL_CONFIG_NOT_FOUND = 10152; // 模型配置未找到
int VOICE_CLONE_MODEL_TYPE_NOT_FOUND = 10153; // 模型类型未找到
int VOICE_CLONE_TRAINING_FAILED = 10154; // 训练失败
int VOICE_CLONE_HUOSHAN_CONFIG_MISSING = 10155; // 火山引擎缺少配置
int VOICE_CLONE_RESPONSE_FORMAT_ERROR = 10156; // 响应格式错误
int VOICE_CLONE_REQUEST_FAILED = 10157; // 请求失败
int VOICE_CLONE_PREFIX = 10158; // 复刻音色前缀
}
main/manager-api/src/main/java/xiaozhi/modules/timbre/service/TimbreService.java
+8
View File
@@ -64,4 +64,12 @@ public interface TimbreService extends BaseService<TimbreEntity> {
* @return 音色名称
*/
String getTimbreNameById(String id);
/**
* 检查音色编码（ttsVoice/speaker_id）是否已存在
*
* @param ttsVoice 音色编码
* @return 是否存在
*/
boolean existsByTtsVoice(String ttsVoice);
}
main/manager-api/src/main/java/xiaozhi/modules/timbre/service/impl/TimbreServiceImpl.java
+44 -1
View File
@@ -1,9 +1,12 @@
package xiaozhi.modules.timbre.service.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import xiaozhi.common.exception.ErrorCode;
import xiaozhi.common.utils.MessageUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@@ -120,12 +123,39 @@ public class TimbreServiceImpl extends BaseServiceImpl<TimbreDao, TimbreEntity>
if (StringUtils.isNotBlank(voiceName)) {
queryWrapper.like("name", voiceName);
}
List<TimbreEntity> timbreEntities = timbreDao.selectList(queryWrapper);
if (CollectionUtil.isEmpty(timbreEntities)) {
return null;
}
return ConvertUtils.sourceToTarget(timbreEntities, VoiceDTO.class);
// 自定义排序：音色编码以S_开头的排在最前面，其他按sort字段升序排序
timbreEntities.sort((t1, t2) -> {
// 先判断是否以S_开头
boolean isT1S_Start = StringUtils.isNotBlank(t1.getTtsVoice()) && t1.getTtsVoice().startsWith("S_");
boolean isT2S_Start = StringUtils.isNotBlank(t2.getTtsVoice()) && t2.getTtsVoice().startsWith("S_");
if (isT1S_Start && !isT2S_Start) return -1; // t1以S_开头，排在前面
if (!isT1S_Start && isT2S_Start) return 1; // t2以S_开头，排在前面
// 都以S_开头或都不以S_开头，则按sort字段升序排序
return Long.compare(t1.getSort(), t2.getSort());
});
// 排序完成后转换为DTO列表，并为复刻音色添加前缀
List<VoiceDTO> voiceDTOs = new ArrayList<>();
for (TimbreEntity entity : timbreEntities) {
VoiceDTO dto = new VoiceDTO();
dto.setId(entity.getId());
// 对于音色编码以S_开头的复刻音色，在名称前加上"克隆："前缀
String name = entity.getName();
if (StringUtils.isNotBlank(entity.getTtsVoice()) && entity.getTtsVoice().startsWith("S_") && !name.startsWith("克隆：")) {
name = MessageUtils.getMessage(ErrorCode.VOICE_CLONE_PREFIX) + name;
}
dto.setName(name);
voiceDTOs.add(dto);
}
return voiceDTOs;
}
/**
@@ -158,4 +188,17 @@ public class TimbreServiceImpl extends BaseServiceImpl<TimbreDao, TimbreEntity>
return null;
}
@Override
public boolean existsByTtsVoice(String ttsVoice) {
if (StringUtils.isBlank(ttsVoice)) {
return false;
}
QueryWrapper<TimbreEntity> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("tts_voice", ttsVoice);
// 检查数据库中是否存在相同的ttsVoice值
return timbreDao.exists(queryWrapper);
}
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/controller/VoiceCloneController.java
+2
View File
@@ -175,6 +175,8 @@ public class VoiceCloneController {
public Result<String> cloneAudio(@RequestBody Map<String, String> params) {
String cloneId = params.get("cloneId");
checkPermission(cloneId);
// 调用服务层进行语音克隆训练
voiceCloneService.cloneAudio(cloneId);
return new Result<String>();
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/service/VoiceCloneService.java
+10 -3
View File
@@ -35,17 +35,17 @@ public interface VoiceCloneService extends BaseService<VoiceCloneEntity> {
* 根据用户ID查询声音克隆列表
*/
List<VoiceCloneEntity> getByUserId(Long userId);
/**
* 分页查询带模型名称和用户名称的声音克隆列表
*/
PageData<VoiceCloneResponseDTO> pageWithNames(Map<String, Object> params);
/**
* 根据ID查询带模型名称和用户名称的声音克隆信息
*/
VoiceCloneResponseDTO getByIdWithNames(String id);
/**
* 根据用户ID查询带模型名称的声音克隆列表
*/
@@ -65,4 +65,11 @@ public interface VoiceCloneService extends BaseService<VoiceCloneEntity> {
* 获取音频数据
*/
byte[] getVoiceData(String id);
/**
* 克隆音频，调用火山引擎进行语音复刻训练
*
* @param cloneId 语音克隆记录ID
*/
void cloneAudio(String cloneId);
}
main/manager-api/src/main/java/xiaozhi/modules/voiceclone/service/impl/VoiceCloneServiceImpl.java
+153 -2
View File
@@ -1,7 +1,13 @@
package xiaozhi.modules.voiceclone.service.impl;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -12,14 +18,18 @@ import org.springframework.web.multipart.MultipartFile;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import xiaozhi.common.exception.ErrorCode;
import xiaozhi.common.exception.RenException;
import xiaozhi.common.page.PageData;
import xiaozhi.common.service.impl.BaseServiceImpl;
import xiaozhi.common.utils.ConvertUtils;
import xiaozhi.common.utils.DateUtils;
import xiaozhi.common.exception.ErrorCode;
import xiaozhi.common.exception.RenException;
import xiaozhi.modules.model.entity.ModelConfigEntity;
import xiaozhi.modules.model.service.ModelConfigService;
import xiaozhi.modules.sys.service.SysUserService;
import xiaozhi.modules.voiceclone.dao.VoiceCloneDao;
@@ -28,6 +38,7 @@ import xiaozhi.modules.voiceclone.dto.VoiceCloneResponseDTO;
import xiaozhi.modules.voiceclone.entity.VoiceCloneEntity;
import xiaozhi.modules.voiceclone.service.VoiceCloneService;
@Slf4j
@Service
@RequiredArgsConstructor
public class VoiceCloneServiceImpl extends BaseServiceImpl<VoiceCloneDao, VoiceCloneEntity>
@@ -35,6 +46,8 @@ public class VoiceCloneServiceImpl extends BaseServiceImpl<VoiceCloneDao, VoiceC
private final ModelConfigService modelConfigService;
private final SysUserService sysUserService;
private final ObjectMapper objectMapper;
private final xiaozhi.modules.timbre.service.TimbreService timbreService;
@Override
public PageData<VoiceCloneEntity> page(Map<String, Object> params) {
@@ -207,4 +220,142 @@ public class VoiceCloneServiceImpl extends BaseServiceImpl<VoiceCloneDao, VoiceC
}
return entity.getVoice();
}
@Override
@Transactional(rollbackFor = Exception.class)
public void cloneAudio(String cloneId) {
VoiceCloneEntity entity = baseDao.selectById(cloneId);
if (entity == null) {
throw new RenException(ErrorCode.VOICE_CLONE_RECORD_NOT_EXIST);
}
if (entity.getVoice() == null || entity.getVoice().length == 0) {
throw new RenException(ErrorCode.VOICE_CLONE_AUDIO_NOT_UPLOADED);
}
try {
ModelConfigEntity modelConfig = modelConfigService.getModelByIdFromCache(entity.getModelId());
if (modelConfig == null || modelConfig.getConfigJson() == null) {
throw new RenException(ErrorCode.VOICE_CLONE_MODEL_CONFIG_NOT_FOUND);
}
Map<String, Object> config = modelConfig.getConfigJson();
String type = (String) config.get("type");
if (StringUtils.isBlank(type)) {
throw new RenException(ErrorCode.VOICE_CLONE_MODEL_TYPE_NOT_FOUND);
}
if (type.equals("huoshan_double_stream")) {
huoshanClone(config, entity);
}
} catch (RenException re) {
throw re;
} catch (Exception e) {
e.printStackTrace();
entity.setTrainStatus(3);
entity.setTrainError(e.getMessage());
baseDao.updateById(entity);
throw new RenException(ErrorCode.VOICE_CLONE_TRAINING_FAILED, e.getMessage());
}
}
/**
* 调用火山引擎进行语音复刻训练
*
* @param config 模型配置
* @param entity 语音克隆记录实体
* @throws Exception
*/
private void huoshanClone(Map<String, Object> config, VoiceCloneEntity entity) throws Exception {
String appid = (String) config.get("appid");
String accessToken = (String) config.get("access_token");
if (StringUtils.isAnyBlank(appid, accessToken)) {
throw new RenException(ErrorCode.VOICE_CLONE_HUOSHAN_CONFIG_MISSING);
}
String audioBase64 = Base64.getEncoder().encodeToString(entity.getVoice());
Map<String, Object> reqBody = new HashMap<>();
reqBody.put("appid", appid);
List<Map<String, String>> audios = new ArrayList<>();
Map<String, String> audioMap = new HashMap<>();
audioMap.put("audio_bytes", audioBase64);
audioMap.put("audio_format", "wav");
audios.add(audioMap);
reqBody.put("audios", audios);
reqBody.put("source", 2);
reqBody.put("language", 0);
reqBody.put("model_type", 1);
reqBody.put("speaker_id", entity.getVoiceId());
String apiUrl = "https://openspeech.bytedance.com/api/v1/mega_tts/audio/upload";
HttpClient client = HttpClient.newHttpClient();
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(apiUrl))
.header("Content-Type", "application/json")
.header("Authorization", "Bearer;" + accessToken)
.header("Resource-Id", "seed-icl-1.0")
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(reqBody)))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(">>> HTTP status = " + response.statusCode());
System.out.println(">>> response body = " + response.body());
if (response.statusCode() == 200) {
Map<String, Object> rsp = objectMapper.readValue(response.body(),
new TypeReference<Map<String, Object>>() {
});
// 获取BaseResp对象
Map<String, Object> baseResp = objectMapper.convertValue(rsp.get("BaseResp"),
new TypeReference<Map<String, Object>>() {
});
if (baseResp != null) {
Integer statusCode = objectMapper.convertValue(baseResp.get("StatusCode"), Integer.class);
String statusMessage = objectMapper.convertValue(baseResp.getOrDefault("StatusMessage", ""),
String.class);
// 获取speaker_id
String speakerId = objectMapper.convertValue(rsp.get("speaker_id"), String.class);
// StatusCode == 0 表示成功
if (statusCode != null && statusCode == 0 && StringUtils.isNotBlank(speakerId)) {
entity.setTrainStatus(2);
entity.setVoiceId(speakerId);
entity.setTrainError("");
baseDao.updateById(entity);
try {
// 检查speaker_id是否已经被使用
if (timbreService.existsByTtsVoice(speakerId)) {
log.info("音色编码speaker_id[{}]已存在，不重复写入数据库", speakerId);
} else {
xiaozhi.modules.timbre.dto.TimbreDataDTO timbreDataDTO = new xiaozhi.modules.timbre.dto.TimbreDataDTO();
timbreDataDTO.setTtsModelId(entity.getModelId());
timbreDataDTO.setName(entity.getName());
timbreDataDTO.setTtsVoice(speakerId);
timbreDataDTO.setSort(1);
timbreDataDTO.setLanguages("zh-CN");
timbreDataDTO.setRemark("复刻音色");
// 保存到音色表
timbreService.save(timbreDataDTO);
log.info("成功将复刻音色添加到音色表，speaker_id: {}", speakerId);
}
} catch (Exception e) {
// 记录错误但不影响主流程
log.error("将复刻音色添加到音色表失败: " + e.getMessage(), e);
}
} else {
// 失败时使用StatusMessage作为错误信息
String errorMsg = StringUtils.isNotBlank(statusMessage) ? statusMessage : "训练失败";
throw new RenException(errorMsg);
}
} else {
throw new RenException(ErrorCode.VOICE_CLONE_RESPONSE_FORMAT_ERROR);
}
} else {
throw new RenException(ErrorCode.VOICE_CLONE_REQUEST_FAILED + "，状态码: " + response.statusCode());
}
}
}
main/manager-api/src/main/resources/i18n/messages.properties
+12 -2
View File
@@ -135,7 +135,9 @@
10124=\u60A8\u7684mqtt\u5BC6\u94A5\u957F\u5EA6\u4E0D\u5B89\u5168\uFF0Cmqtt\u5BC6\u94A5\u5FC5\u987B\u540C\u65F6\u5305\u542B\u5927\u5C0F\u5199\u5B57\u6BCD
10125=\u60A8\u7684mqtt\u5BC6\u94A5\u5305\u542B\u5F31\u5BC6\u7801
10128=\u5B57\u5178\u6807\u7B7E\u91CD\u590D
10129=modelType\u548CprovideCode\u4E0D\u80FD\u4E3A\u7A7A
10129=SM2\u5BC6\u94A5\u672A\u914D\u7F6E
10130=SM2\u89E3\u5BC6\u5931\u8D25
10131=modelType\u548CprovideCode\u4E0D\u80FD\u4E3A\u7A7A
10132=\u6CA1\u6709\u6743\u9650\u67E5\u770B\u8BE5\u667A\u80FD\u4F53\u7684\u804A\u5929\u8BB0\u5F55
10133=\u4F1A\u8BDDID\u4E0D\u80FD\u4E3A\u7A7A
10134=\u667A\u80FD\u4F53ID\u4E0D\u80FD\u4E3A\u7A7A
@@ -154,4 +156,12 @@
10147=\u97F3\u8272ID\u4E0D\u80FD\u4E3A\u7A7A
10148=\u5F52\u5C5E\u8D26\u53F7\u4E0D\u80FD\u4E3A\u7A7A
10149=\u5220\u9664\u7684\u97F3\u8272\u8D44\u6E90ID\u4E0D\u80FD\u4E3A\u7A7A
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10151=\u8BF7\u5148\u4E0A\u4F20\u97F3\u9891\u6587\u4EF6
10152=\u6A21\u578B\u914D\u7F6E\u672A\u627E\u5230
10153=\u6A21\u578B\u7C7B\u578B\u672A\u627E\u5230
10154=\u8BAD\u7EC3\u5931\u8D25
10155=\u706B\u5C71\u5F15\u64CE\u7F3A\u5C11\u914D\u7F6E
10156=\u54CD\u5E94\u683C\u5F0F\u9519\u8BEF\uFF0C\u7F3A\u5C11BaseResp\u5B57\u6BB5
10157=\u8BF7\u6C42\u5931\u8D25
10158=\u8BF7\u6C42\u5931\u8D25\uFF0C\u72B6\u6001\u7801
main/manager-api/src/main/resources/i18n/messages_en_US.properties
+9 -1
View File
@@ -156,4 +156,12 @@
10147=Voice ID cannot be empty
10148=Owner account cannot be empty
10149=Voice resource ID to delete cannot be empty
10150=You do not have permission to operate this record
10150=You do not have permission to operate this record
10151=Please upload audio file first
10152=Model configuration not found
10153=Model type not found
10154=Training failed: {0}
10155=Huoshan Engine configuration missing
10156=Response format error, missing BaseResp field
10157=Request failed
10158=Clone Voice:
main/manager-api/src/main/resources/i18n/messages_zh_CN.properties
+9 -1
View File
@@ -156,4 +156,12 @@
10147=\u97F3\u8272ID\u4E0D\u80FD\u4E3A\u7A7A
10148=\u5F52\u5C5E\u8D26\u53F7\u4E0D\u80FD\u4E3A\u7A7A
10149=\u5220\u9664\u7684\u97F3\u8272\u8D44\u6E90ID\u4E0D\u80FD\u4E3A\u7A7A
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10150=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55
10151=\u8BF7\u5148\u4E0A\u4F20\u97F3\u9891\u6587\u4EF6
10152=\u6A21\u578B\u914D\u7F6E\u672A\u627E\u5230
10153=\u6A21\u578B\u7C7B\u578B\u672A\u627E\u5230
10154=\u8BAD\u7EC3\u5931\u8D25
10155=\u706B\u5C71\u5F15\u64CE\u7F3A\u5C11\u914D\u7F6E
10156=\u54CD\u5E94\u683C\u5F0F\u9519\u8BEF\uFF0C\u7F3A\u5C11BaseResp\u5B57\u6BB5
10157=\u8BF7\u6C42\u5931\u8D25
10158=\u8BF7\u6C42\u5931\u8D25\uFF0C\u72B6\u6001\u7801
main/manager-api/src/main/resources/i18n/messages_zh_TW.properties
+9 -1
View File
@@ -156,4 +156,12 @@
10147=\u97F3\u8272ID\u4E0D\u80FD\u70BA\u7A7A
10148=\u6B78\u5C6C\u5E33\u865F\u4E0D\u80FD\u70BA\u7A7A
10149=\u522A\u9664\u7684\u97F3\u8272\u8CC7\u6E90ID\u4E0D\u80FD\u70BA\u7A7A
10150=\u60A8\u6C92\u6709\u6B0A\u9650\u64CD\u4F5C\u8A72\u8A18\u9304
10150=\u60A8\u6C92\u6709\u6B0A\u9650\u64CD\u4F5C\u8A72\u8A18\u9304
10151=\u8ACB\u5148\u4E0A\u50B3\u97F3\u983B\u6587\u4EF6
10152=\u6A21\u578B\u914D\u7F6E\u672A\u627E\u5230
10153=\u6A21\u578B\u985E\u578B\u672A\u627E\u5230
10154=\u6A21\u578B\u8AB2\u7DF4\u5931\u6557: {0}
10155=\u706B\u5C71\u5F15\u64CE\u7F3A\u5C11appid\u6216access_token
10156=\u97FF\u61C9\u683C\u5F0F\u932F\u8AA4\uFF0C\u7F3A\u5C11BaseResp\u5B57\u6BB5
10157=\u8ACB\u6C42\u5931\u6557
10158=\u590D\u8B58\u97F3\u8272\uFF1A\uFF0C\u72C0\u614B\u78BC
main/manager-web/src/i18n/en.js
+2
View File
@@ -1095,6 +1095,8 @@ export default {
'voiceClone.play': 'Play',
'voiceClone.pause': 'Pause',
'voiceClone.cancel': 'Cancel',
'voiceClone.warning': 'Warning',
'voiceClone.ok': 'OK',
'voiceClone.nextStep': 'Next',
'voiceClone.prevStep': 'Previous',
'voiceClone.upload': 'Upload Audio',
main/manager-web/src/i18n/zh_CN.js
+2
View File
@@ -1095,6 +1095,8 @@ export default {
'voiceClone.play': '播放',
'voiceClone.pause': '暂停',
'voiceClone.cancel': '取消',
'voiceClone.warning': '警告',
'voiceClone.ok': '确定',
'voiceClone.nextStep': '下一步',
'voiceClone.prevStep': '上一步',
'voiceClone.upload': '上传音频',
main/manager-web/src/i18n/zh_TW.js
+2
View File
@@ -1095,6 +1095,8 @@ export default {
'voiceClone.play': '播放',
'voiceClone.pause': '暫停',
'voiceClone.cancel': '取消',
'voiceClone.warning': '警告',
'voiceClone.ok': '確定',
'voiceClone.nextStep': '下一步',
'voiceClone.prevStep': '上一步',
'voiceClone.upload': '上傳音頻',
main/manager-web/src/views/VoiceResourceManagement.vue
+3 -3
View File
@@ -249,9 +249,9 @@ export default {
}
const itemCount = items.length;
this.$confirm(this.$t('voiceClone.confirmDelete', { count: itemCount }), 'Warning', {
confirmButtonText: 'OK',
cancelButtonText: 'Cancel',
this.$confirm(this.$t('voiceClone.confirmDelete', { count: itemCount }), this.$t('voiceClone.warning'), {
confirmButtonText: this.$t('voiceClone.ok'),
cancelButtonText: this.$t('voiceClone.cancel'),
type: 'warning',
distinguishCancelAndClose: true
}).then(() => {