diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index ea8627be..9df50884 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -6,6 +6,9 @@ import hashlib import base64 import requests from datetime import datetime + +from pydub import AudioSegment + from core.providers.tts.base import TTSProviderBase import http.client @@ -13,6 +16,10 @@ import urllib.parse import time import uuid from urllib import parse + +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType + + class AccessToken: @staticmethod def _encode_text(text): @@ -105,7 +112,7 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".wav"): return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): request_json = { "appkey": self.appkey, "token": self.token, @@ -119,14 +126,26 @@ class TTSProvider(TTSProviderBase): } print(self.api_url, json.dumps(request_json, ensure_ascii=False)) + tmp_file = self.generate_filename() try: resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header) # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的 if resp.headers['Content-Type'].startswith('audio/'): - with open(output_file, 'wb') as f: + with open(tmp_file, 'wb') as f: f.write(resp.content) - return output_file else: raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}") + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="wav") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, + tts_finish_text=text, sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass except Exception as e: raise Exception(f"{__name__} error: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/cozecn.py b/main/xiaozhi-server/core/providers/tts/cozecn.py index 74de9ab3..51ab4df9 100644 --- a/main/xiaozhi-server/core/providers/tts/cozecn.py +++ b/main/xiaozhi-server/core/providers/tts/cozecn.py @@ -4,7 +4,11 @@ import json import base64 import requests from datetime import datetime + +from pydub import AudioSegment + from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType class TTSProvider(TTSProviderBase): @@ -21,7 +25,7 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".wav"): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): request_json = { "model": self.model, "input": text, @@ -34,5 +38,18 @@ class TTSProvider(TTSProviderBase): } response = requests.request("POST", self.api_url, json=request_json, headers=headers) data = response.content - file_to_save = open(output_file, "wb") + tmp_file = self.generate_filename() + file_to_save = open(tmp_file, "wb") file_to_save.write(data) + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="wav") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass diff --git a/main/xiaozhi-server/core/providers/tts/custom.py b/main/xiaozhi-server/core/providers/tts/custom.py index 3417790f..695117ab 100644 --- a/main/xiaozhi-server/core/providers/tts/custom.py +++ b/main/xiaozhi-server/core/providers/tts/custom.py @@ -1,13 +1,17 @@ import os import uuid import requests +from pydub import AudioSegment + from config.logger import setup_logging from datetime import datetime from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType TAG = __name__ logger = setup_logging() + class TTSProvider(TTSProviderBase): def __init__(self, config, delete_audio_file): super().__init__(config, delete_audio_file) @@ -20,8 +24,9 @@ class TTSProvider(TTSProviderBase): def generate_filename(self): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): request_params = {} + tmp_file = self.generate_filename() for k, v in self.params.items(): if isinstance(v, str) and "{prompt_text}" in v: v = v.replace("{prompt_text}", text) @@ -29,7 +34,19 @@ class TTSProvider(TTSProviderBase): resp = requests.get(self.url, params=request_params, headers=self.headers) if resp.status_code == 200: - with open(output_file, "wb") as file: + with open(tmp_file, "wb") as file: file.write(resp.content) else: logger.bind(tag=TAG).error(f"Custom TTS请求失败: {resp.status_code} - {resp.text}") + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format=self.format) + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass diff --git a/main/xiaozhi-server/core/providers/tts/doubao.py b/main/xiaozhi-server/core/providers/tts/doubao.py index 55438a0c..dc55cf6a 100644 --- a/main/xiaozhi-server/core/providers/tts/doubao.py +++ b/main/xiaozhi-server/core/providers/tts/doubao.py @@ -4,6 +4,10 @@ import json import base64 import requests from datetime import datetime + +from pydub import AudioSegment + +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType from core.utils.util import check_model_key from core.providers.tts.base import TTSProviderBase @@ -23,7 +27,8 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".wav"): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): + tmp_file = self.generate_filename() request_json = { "app": { "appid": f"{self.appid}", @@ -54,9 +59,21 @@ class TTSProvider(TTSProviderBase): resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header) if "data" in resp.json(): data = resp.json()["data"] - file_to_save = open(output_file, "wb") + file_to_save = open(tmp_file, "wb") file_to_save.write(base64.b64decode(data)) else: raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}") except Exception as e: raise Exception(f"{__name__} error: {e}") + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="wav") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py index ae20beb6..014c2649 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py @@ -3,9 +3,12 @@ import uuid import json import base64 import requests +from pydub import AudioSegment + from config.logger import setup_logging from datetime import datetime from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType TAG = __name__ logger = setup_logging() @@ -36,7 +39,8 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".wav"): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): + tmp_file = self.generate_filename() request_json = { "text": text, "text_lang": self.text_lang, @@ -61,7 +65,19 @@ class TTSProvider(TTSProviderBase): resp = requests.post(self.url, json=request_json) if resp.status_code == 200: - with open(output_file, "wb") as file: + with open(tmp_file, "wb") as file: file.write(resp.content) else: logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}") + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="wav") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py index ebfb12f1..57b63ae1 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py @@ -1,9 +1,12 @@ import os import uuid import requests +from pydub import AudioSegment + from config.logger import setup_logging from datetime import datetime from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType TAG = __name__ logger = setup_logging() @@ -29,7 +32,8 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".wav"): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): + tmp_file = self.generate_filename() request_params = { "refer_wav_path": self.refer_wav_path, "prompt_text": self.prompt_text, @@ -48,7 +52,19 @@ class TTSProvider(TTSProviderBase): resp = requests.get(self.url, params=request_params) if resp.status_code == 200: - with open(output_file, "wb") as file: + with open(tmp_file, "wb") as file: file.write(resp.content) else: logger.bind(tag=TAG).error(f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}") + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="wav") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass diff --git a/main/xiaozhi-server/core/providers/tts/minimax.py b/main/xiaozhi-server/core/providers/tts/minimax.py index 1fe7be50..33f12075 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax.py +++ b/main/xiaozhi-server/core/providers/tts/minimax.py @@ -3,7 +3,11 @@ import uuid import json import requests from datetime import datetime + +from pydub import AudioSegment + from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType class TTSProvider(TTSProviderBase): @@ -50,7 +54,8 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".mp3"): return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): + tmp_file = self.generate_filename() request_json = { "model": self.model, "text": text, @@ -69,9 +74,21 @@ class TTSProvider(TTSProviderBase): # 检查返回请求数据的status_code是否为0 if resp.json()["base_resp"]["status_code"] == 0: data = resp.json()['data']['audio'] - file_to_save = open(output_file, "wb") + file_to_save = open(tmp_file, "wb") file_to_save.write(bytes.fromhex(data)) else: raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}") except Exception as e: - raise Exception(f"{__name__} error: {e}") \ No newline at end of file + raise Exception(f"{__name__} error: {e}") + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="mp3") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/openai.py b/main/xiaozhi-server/core/providers/tts/openai.py index 1849c128..6ea5ea5e 100644 --- a/main/xiaozhi-server/core/providers/tts/openai.py +++ b/main/xiaozhi-server/core/providers/tts/openai.py @@ -2,6 +2,10 @@ import os import uuid import requests from datetime import datetime + +from pydub import AudioSegment + +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType from core.utils.util import check_model_key from core.providers.tts.base import TTSProviderBase @@ -20,7 +24,8 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".wav"): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): + tmp_file = self.generate_filename() headers = { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json" @@ -34,7 +39,19 @@ class TTSProvider(TTSProviderBase): } response = requests.post(self.api_url, json=data, headers=headers) if response.status_code == 200: - with open(output_file, "wb") as audio_file: + with open(tmp_file, "wb") as audio_file: audio_file.write(response.content) else: raise Exception(f"OpenAI TTS请求失败: {response.status_code} - {response.text}") + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="wav") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass diff --git a/main/xiaozhi-server/core/providers/tts/siliconflow.py b/main/xiaozhi-server/core/providers/tts/siliconflow.py index 4c0cc356..6361c412 100644 --- a/main/xiaozhi-server/core/providers/tts/siliconflow.py +++ b/main/xiaozhi-server/core/providers/tts/siliconflow.py @@ -2,7 +2,11 @@ import os import uuid import requests from datetime import datetime + +from pydub import AudioSegment + from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType class TTSProvider(TTSProviderBase): @@ -22,7 +26,8 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".wav"): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): + tmp_file = self.generate_filename() request_json = { "model": self.model, "input": text, @@ -35,5 +40,17 @@ class TTSProvider(TTSProviderBase): } response = requests.request("POST", self.api_url, json=request_json, headers=headers) data = response.content - file_to_save = open(output_file, "wb") + file_to_save = open(tmp_file, "wb") file_to_save.write(data) + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="wav") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass diff --git a/main/xiaozhi-server/core/providers/tts/ttson.py b/main/xiaozhi-server/core/providers/tts/ttson.py index e9fee109..9fd7d2a1 100644 --- a/main/xiaozhi-server/core/providers/tts/ttson.py +++ b/main/xiaozhi-server/core/providers/tts/ttson.py @@ -4,7 +4,11 @@ import json import requests import shutil from datetime import datetime + +from pydub import AudioSegment + from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType class TTSProvider(TTSProviderBase): @@ -28,7 +32,8 @@ class TTSProvider(TTSProviderBase): def generate_filename(self, extension=".mp3"): return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") - async def text_to_speak(self, text, output_file): + async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False): + tmp_file = self.generate_filename() url = f'{self.url}{self.token}' result = "firefly" payload = json.dumps({ @@ -45,7 +50,7 @@ class TTSProvider(TTSProviderBase): resp = requests.request("POST", url, data=payload) if resp.status_code != 200: - return None + return resp_json = resp.json() try: result = resp_json['url'] + ':' + str( @@ -56,9 +61,21 @@ class TTSProvider(TTSProviderBase): print("error:", e) audio_content = requests.get(result) - with open(output_file, "wb") as f: + with open(tmp_file, "wb") as f: f.write(audio_content.content) - return True + # 使用 pydub 读取临时文件 + audio = AudioSegment.from_file(tmp_file, format="mp3") + audio = audio.set_channels(1).set_frame_rate(16000) + opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data) + yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text, + sentence_type=SentenceType.SENTENCE_START) + # 用完后删除临时文件 + try: + os.remove(tmp_file) + except FileNotFoundError: + # 若文件不存在，忽略该异常 + pass voice_path = resp_json.get("voice_path") - des_path = output_file + des_path = tmp_file shutil.move(voice_path, des_path) +