From 4912f385c69f8e84bbd61b5bc0283d903cfbf57d Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 13 Apr 2025 19:01:19 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E6=99=BA=E8=83=BD=E4=BD=93=E9=9F=B3?= =?UTF-8?q?=E8=89=B2=E6=9C=AA=E9=9A=8Fmanager=E7=94=9F=E6=95=88bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix:连接manager后无法使用functioncallbug * fix:意图识别使用llm无法播放音乐bug * fix：manager第一图识别使用独立llm无法初始化llm的bug * fix:智能体音色未随manager生效bug * update:添加edgeTTS音色 --- .../resources/db/changelog/202504131543.sql | 18 ++++ .../db/changelog/db.changelog-master.yaml | 4 +- .../core/providers/tts/cozecn.py | 16 +++- .../core/providers/tts/doubao.py | 7 +- .../xiaozhi-server/core/providers/tts/edge.py | 16 ++-- .../core/providers/tts/minimax.py | 44 +++++---- .../core/providers/tts/openai.py | 5 +- .../core/providers/tts/siliconflow.py | 16 +++- .../core/providers/tts/tencent.py | 89 +++++++++++-------- .../core/providers/tts/ttson.py | 58 +++++++----- 10 files changed, 181 insertions(+), 92 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202504131543.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202504131543.sql b/main/manager-api/src/main/resources/db/changelog/202504131543.sql new file mode 100644 index 00000000..55c69912 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202504131543.sql @@ -0,0 +1,18 @@ +-- 对0.3.0版本之前的参数进行修改 +update `sys_params` set param_value = '.mp3;.wav;.p3' where param_code = 'plugins.play_music.music_ext'; +update `ai_model_config` set config_json = '{\"type\": \"intent_llm\", \"llm\": \"LLM_ChatGLMLLM\"}' where id = 'Intent_intent_llm'; + +-- 添加edge音色 +delete from `ai_tts_voice` where tts_model_id = 'TTS_EdgeTTS'; +INSERT INTO `ai_tts_voice` VALUES +('TTS_EdgeTTS0001', 'TTS_EdgeTTS', 'EdgeTTS女声-晓晓', 'zh-CN-XiaoxiaoNeural', '普通话', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0002', 'TTS_EdgeTTS', 'EdgeTTS男声-云扬', 'zh-CN-YunyangNeural', '普通话', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0003', 'TTS_EdgeTTS', 'EdgeTTS女声-晓伊', 'zh-CN-XiaoyiNeural', '普通话', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0004', 'TTS_EdgeTTS', 'EdgeTTS男声-云健', 'zh-CN-YunjianNeural', '普通话', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0005', 'TTS_EdgeTTS', 'EdgeTTS男声-云希', 'zh-CN-YunxiNeural', '普通话', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0006', 'TTS_EdgeTTS', 'EdgeTTS男声-云夏', 'zh-CN-YunxiaNeural', '普通话', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0007', 'TTS_EdgeTTS', 'EdgeTTS女声-辽宁小贝', 'zh-CN-liaoning-XiaobeiNeural', '辽宁', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0008', 'TTS_EdgeTTS', 'EdgeTTS女声-陕西小妮', 'zh-CN-shaanxi-XiaoniNeural', '陕西', NULL, NULL, 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0009', 'TTS_EdgeTTS', 'EdgeTTS女声-香港海佳', 'zh-HK-HiuGaaiNeural', '粤语', 'General', 'Friendly, Positive', 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0010', 'TTS_EdgeTTS', 'EdgeTTS女声-香港海曼', 'zh-HK-HiuMaanNeural', '粤语', 'General', 'Friendly, Positive', 1, NULL, NULL, NULL, NULL), +('TTS_EdgeTTS0011', 'TTS_EdgeTTS', 'EdgeTTS男声-香港万龙', 'zh-HK-WanLungNeural', '粤语', 'General', 'Friendly, Positive', 1, NULL, NULL, NULL, NULL); \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 41f65488..37797d3f 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -52,9 +52,9 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202504112058.sql - changeSet: - id: 202504131542 + id: 202504131543 author: John changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202504131542.sql \ No newline at end of file + path: classpath:db/changelog/202504131543.sql diff --git a/main/xiaozhi-server/core/providers/tts/cozecn.py b/main/xiaozhi-server/core/providers/tts/cozecn.py index 74de9ab3..1f9244f6 100644 --- a/main/xiaozhi-server/core/providers/tts/cozecn.py +++ b/main/xiaozhi-server/core/providers/tts/cozecn.py @@ -12,14 +12,20 @@ class TTSProvider(TTSProviderBase): super().__init__(config, delete_audio_file) self.model = config.get("model") self.access_token = config.get("access_token") - self.voice = config.get("voice") + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice") self.response_format = config.get("response_format") self.host = "api.coze.cn" self.api_url = f"https://{self.host}/v1/audio/speech" def generate_filename(self, extension=".wav"): - return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + return os.path.join( + self.output_file, + f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) async def text_to_speak(self, text, output_file): request_json = { @@ -30,9 +36,11 @@ class TTSProvider(TTSProviderBase): } headers = { "Authorization": f"Bearer {self.access_token}", - "Content-Type": "application/json" + "Content-Type": "application/json", } - response = requests.request("POST", self.api_url, json=request_json, headers=headers) + response = requests.request( + "POST", self.api_url, json=request_json, headers=headers + ) data = response.content file_to_save = open(output_file, "wb") file_to_save.write(data) diff --git a/main/xiaozhi-server/core/providers/tts/doubao.py b/main/xiaozhi-server/core/providers/tts/doubao.py index b40121e3..0051d53e 100644 --- a/main/xiaozhi-server/core/providers/tts/doubao.py +++ b/main/xiaozhi-server/core/providers/tts/doubao.py @@ -18,7 +18,12 @@ class TTSProvider(TTSProviderBase): self.appid = config.get("appid") self.access_token = config.get("access_token") self.cluster = config.get("cluster") - self.voice = config.get("voice") + + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice") + self.api_url = config.get("api_url") self.authorization = config.get("authorization") self.header = {"Authorization": f"{self.authorization}{self.access_token}"} diff --git a/main/xiaozhi-server/core/providers/tts/edge.py b/main/xiaozhi-server/core/providers/tts/edge.py index 289961cb..b86e4087 100644 --- a/main/xiaozhi-server/core/providers/tts/edge.py +++ b/main/xiaozhi-server/core/providers/tts/edge.py @@ -8,20 +8,26 @@ from core.providers.tts.base import TTSProviderBase class TTSProvider(TTSProviderBase): def __init__(self, config, delete_audio_file): super().__init__(config, delete_audio_file) - self.voice = config.get("voice") + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice") def generate_filename(self, extension=".mp3"): - return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + return os.path.join( + self.output_file, + f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) async def text_to_speak(self, text, output_file): communicate = edge_tts.Communicate(text, voice=self.voice) # 确保目录存在并创建空文件 os.makedirs(os.path.dirname(output_file), exist_ok=True) - with open(output_file, 'wb') as f: + with open(output_file, "wb") as f: pass - + # 流式写入音频数据 - with open(output_file, 'ab') as f: # 改为追加模式避免覆盖 + with open(output_file, "ab") as f: # 改为追加模式避免覆盖 async for chunk in communicate.stream(): if chunk["type"] == "audio": # 只处理音频数据块 f.write(chunk["data"]) diff --git a/main/xiaozhi-server/core/providers/tts/minimax.py b/main/xiaozhi-server/core/providers/tts/minimax.py index 1fe7be50..2b63ea7e 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax.py +++ b/main/xiaozhi-server/core/providers/tts/minimax.py @@ -12,28 +12,33 @@ class TTSProvider(TTSProviderBase): self.group_id = config.get("group_id") self.api_key = config.get("api_key") self.model = config.get("model") - self.voice_id = config.get("voice_id") + if config.get("private_voice"): + self.voice_id = config.get("private_voice") + else: + self.voice_id = config.get("voice_id") default_voice_setting = { "voice_id": "female-shaonv", "speed": 1, "vol": 1, "pitch": 0, - "emotion": "happy" - } - default_pronunciation_dict = { - "tone": [ - "处理/(chu3)(li3)", "危险/dangerous" - ] + "emotion": "happy", } + default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]} defult_audio_setting = { "sample_rate": 32000, "bitrate": 128000, "format": "mp3", - "channel": 1 + "channel": 1, + } + self.voice_setting = { + **default_voice_setting, + **config.get("voice_setting", {}), + } + self.pronunciation_dict = { + **default_pronunciation_dict, + **config.get("pronunciation_dict", {}), } - self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})} - self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})} self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})} self.timber_weights = config.get("timber_weights", []) @@ -44,11 +49,14 @@ class TTSProvider(TTSProviderBase): self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}" self.header = { "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}" + "Authorization": f"Bearer {self.api_key}", } def generate_filename(self, extension=".mp3"): - return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + return os.path.join( + self.output_file, + f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) async def text_to_speak(self, text, output_file): request_json = { @@ -65,13 +73,17 @@ class TTSProvider(TTSProviderBase): request_json["voice_setting"]["voice_id"] = "" try: - resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header) + resp = requests.post( + self.api_url, json.dumps(request_json), headers=self.header + ) # 检查返回请求数据的status_code是否为0 if resp.json()["base_resp"]["status_code"] == 0: - data = resp.json()['data']['audio'] + data = resp.json()["data"]["audio"] file_to_save = open(output_file, "wb") file_to_save.write(bytes.fromhex(data)) else: - raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}") + raise Exception( + f"{__name__} status_code: {resp.status_code} response: {resp.content}" + ) except Exception as e: - raise Exception(f"{__name__} error: {e}") \ No newline at end of file + raise Exception(f"{__name__} error: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/openai.py b/main/xiaozhi-server/core/providers/tts/openai.py index 8d967df6..18daf29b 100644 --- a/main/xiaozhi-server/core/providers/tts/openai.py +++ b/main/xiaozhi-server/core/providers/tts/openai.py @@ -16,7 +16,10 @@ class TTSProvider(TTSProviderBase): self.api_key = config.get("api_key") self.api_url = config.get("api_url", "https://api.openai.com/v1/audio/speech") self.model = config.get("model", "tts-1") - self.voice = config.get("voice", "alloy") + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice", "alloy") self.response_format = "wav" self.speed = config.get("speed", 1.0) self.output_file = config.get("output_dir", "tmp/") diff --git a/main/xiaozhi-server/core/providers/tts/siliconflow.py b/main/xiaozhi-server/core/providers/tts/siliconflow.py index 4c0cc356..76d131bb 100644 --- a/main/xiaozhi-server/core/providers/tts/siliconflow.py +++ b/main/xiaozhi-server/core/providers/tts/siliconflow.py @@ -10,7 +10,10 @@ class TTSProvider(TTSProviderBase): super().__init__(config, delete_audio_file) self.model = config.get("model") self.access_token = config.get("access_token") - self.voice = config.get("voice") + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice") self.response_format = config.get("response_format") self.sample_rate = config.get("sample_rate") self.speed = config.get("speed") @@ -20,7 +23,10 @@ class TTSProvider(TTSProviderBase): self.api_url = f"https://{self.host}/v1/audio/speech" def generate_filename(self, extension=".wav"): - return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + return os.path.join( + self.output_file, + f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) async def text_to_speak(self, text, output_file): request_json = { @@ -31,9 +37,11 @@ class TTSProvider(TTSProviderBase): } headers = { "Authorization": f"Bearer {self.access_token}", - "Content-Type": "application/json" + "Content-Type": "application/json", } - response = requests.request("POST", self.api_url, json=request_json, headers=headers) + response = requests.request( + "POST", self.api_url, json=request_json, headers=headers + ) data = response.content file_to_save = open(output_file, "wb") file_to_save.write(data) diff --git a/main/xiaozhi-server/core/providers/tts/tencent.py b/main/xiaozhi-server/core/providers/tts/tencent.py index ed4db622..c05261fd 100644 --- a/main/xiaozhi-server/core/providers/tts/tencent.py +++ b/main/xiaozhi-server/core/providers/tts/tencent.py @@ -16,7 +16,10 @@ class TTSProvider(TTSProviderBase): self.appid = config.get("appid") self.secret_id = config.get("secret_id") self.secret_key = config.get("secret_key") - self.voice = config.get("voice") + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice") self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点 self.region = config.get("region") self.output_file = config.get("output_dir") @@ -25,35 +28,36 @@ class TTSProvider(TTSProviderBase): """生成鉴权请求头""" # 获取当前UTC时间戳 timestamp = int(time.time()) - + # 使用UTC时间计算日期 - utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d') - + utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime( + "%Y-%m-%d" + ) + # 服务名称必须是 "tts" service = "tts" - + # 拼接凭证范围 credential_scope = f"{utc_date}/{service}/tc3_request" - + # 使用TC3-HMAC-SHA256签名方法 algorithm = "TC3-HMAC-SHA256" - + # 构建规范请求字符串 http_request_method = "POST" canonical_uri = "/" canonical_querystring = "" - + # 请求头必须包含host和content-type，且按字典序排列 canonical_headers = ( - f"content-type:application/json\n" - f"host:tts.tencentcloudapi.com\n" + f"content-type:application/json\n" f"host:tts.tencentcloudapi.com\n" ) signed_headers = "content-type;host" - + # 请求体哈希值 payload = json.dumps(request_body) - payload_hash = hashlib.sha256(payload.encode('utf-8')).hexdigest() - + payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + # 构建规范请求字符串 canonical_request = ( f"{http_request_method}\n" @@ -63,10 +67,12 @@ class TTSProvider(TTSProviderBase): f"{signed_headers}\n" f"{payload_hash}" ) - + # 计算规范请求的哈希值 - hashed_canonical_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest() - + hashed_canonical_request = hashlib.sha256( + canonical_request.encode("utf-8") + ).hexdigest() + # 构建待签名字符串 string_to_sign = ( f"{algorithm}\n" @@ -74,19 +80,19 @@ class TTSProvider(TTSProviderBase): f"{credential_scope}\n" f"{hashed_canonical_request}" ) - + # 计算签名密钥 - secret_date = self._hmac_sha256(f"TC3{self.secret_key}".encode('utf-8'), utc_date) + secret_date = self._hmac_sha256( + f"TC3{self.secret_key}".encode("utf-8"), utc_date + ) secret_service = self._hmac_sha256(secret_date, service) secret_signing = self._hmac_sha256(secret_service, "tc3_request") - + # 计算签名 signature = hmac.new( - secret_signing, - string_to_sign.encode('utf-8'), - hashlib.sha256 + secret_signing, string_to_sign.encode("utf-8"), hashlib.sha256 ).hexdigest() - + # 构建授权头 authorization = ( f"{algorithm} " @@ -94,7 +100,7 @@ class TTSProvider(TTSProviderBase): f"SignedHeaders={signed_headers}, " f"Signature={signature}" ) - + # 构建请求头 headers = { "Content-Type": "application/json", @@ -104,19 +110,22 @@ class TTSProvider(TTSProviderBase): "X-TC-Timestamp": str(timestamp), "X-TC-Version": "2019-08-23", "X-TC-Region": self.region, - "X-TC-Language": "zh-CN" + "X-TC-Language": "zh-CN", } - + return headers - + def _hmac_sha256(self, key, msg): """HMAC-SHA256加密""" if isinstance(msg, str): - msg = msg.encode('utf-8') + msg = msg.encode("utf-8") return hmac.new(key, msg, hashlib.sha256).digest() def generate_filename(self, extension=".wav"): - return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + return os.path.join( + self.output_file, + f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) async def text_to_speak(self, text, output_file): # 构建请求体 @@ -129,19 +138,23 @@ class TTSProvider(TTSProviderBase): try: # 获取请求头（每次请求都重新生成，以确保时间戳和签名是最新的） headers = self._get_auth_headers(request_json) - + # 发送请求 - resp = requests.post(self.api_url, json.dumps(request_json), headers=headers) - + resp = requests.post( + self.api_url, json.dumps(request_json), headers=headers + ) + # 检查响应 if resp.status_code == 200: response_data = resp.json() - + # 检查是否成功 if response_data.get("Response", {}).get("Error") is not None: error_info = response_data["Response"]["Error"] - raise Exception(f"API返回错误: {error_info['Code']}: {error_info['Message']}") - + raise Exception( + f"API返回错误: {error_info['Code']}: {error_info['Message']}" + ) + # 提取音频数据 audio_data = response_data["Response"].get("Audio") if audio_data: @@ -151,6 +164,8 @@ class TTSProvider(TTSProviderBase): else: raise Exception(f"{__name__}: 没有返回音频数据: {response_data}") else: - raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}") + raise Exception( + f"{__name__} status_code: {resp.status_code} response: {resp.content}" + ) except Exception as e: - raise Exception(f"{__name__} error: {e}") \ No newline at end of file + raise Exception(f"{__name__} error: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/ttson.py b/main/xiaozhi-server/core/providers/tts/ttson.py index e9fee109..ba0a1222 100644 --- a/main/xiaozhi-server/core/providers/tts/ttson.py +++ b/main/xiaozhi-server/core/providers/tts/ttson.py @@ -10,8 +10,14 @@ from core.providers.tts.base import TTSProviderBase class TTSProvider(TTSProviderBase): def __init__(self, config, delete_audio_file): super().__init__(config, delete_audio_file) - self.url = config.get("url", "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=") - self.voice_id = config.get("voice_id", 1695) + self.url = config.get( + "url", + "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=", + ) + if config.get("private_voice"): + self.voice_id = int(config.get("private_voice")) + else: + self.voice_id = int(config.get("voice_id", 1695)) self.token = config.get("token") self.to_lang = config.get("to_lang") self.volume_change_dB = config.get("volume_change_dB", 0) @@ -21,37 +27,45 @@ class TTSProvider(TTSProviderBase): self.pitch_factor = config.get("pitch_factor", 0) self.format = config.get("format", "mp3") self.emotion = config.get("emotion", 1) - self.header = { - "Content-Type": "application/json" - } + self.header = {"Content-Type": "application/json"} def generate_filename(self, extension=".mp3"): - return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + return os.path.join( + self.output_file, + f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) async def text_to_speak(self, text, output_file): - url = f'{self.url}{self.token}' + url = f"{self.url}{self.token}" result = "firefly" - payload = json.dumps({ - "to_lang": self.to_lang, - "text": text, - "emotion": self.emotion, - "format": self.format, - "volume_change_dB": self.volume_change_dB, - "voice_id": self.voice_id, - "pitch_factor": self.pitch_factor, - "speed_factor": self.speed_factor, - "token": self.token - }) + payload = json.dumps( + { + "to_lang": self.to_lang, + "text": text, + "emotion": self.emotion, + "format": self.format, + "volume_change_dB": self.volume_change_dB, + "voice_id": self.voice_id, + "pitch_factor": self.pitch_factor, + "speed_factor": self.speed_factor, + "token": self.token, + } + ) resp = requests.request("POST", url, data=payload) if resp.status_code != 200: return None resp_json = resp.json() try: - result = resp_json['url'] + ':' + str( - resp_json[ - 'port']) + '/flashsummary/retrieveFileData?stream=True&token=' + self.token + '&voice_audio_path=' + \ - resp_json['voice_path'] + result = ( + resp_json["url"] + + ":" + + str(resp_json["port"]) + + "/flashsummary/retrieveFileData?stream=True&token=" + + self.token + + "&voice_audio_path=" + + resp_json["voice_path"] + ) except Exception as e: print("error:", e)