Merge pull request #612 from HiCoderMonkey/tts-response

fix：aliyun,cozecn,custom,doubao,gpt_sovits_v2,gpt_sovits_v3,minimax,o…

main/xiaozhi-server/core/providers/tts/aliyun.py

@@ -6,6 +6,9 @@ import hashlib
 import base64
 import requests
 from datetime import datetime
+
+from pydub import AudioSegment
+
 from core.providers.tts.base import TTSProviderBase
 
 import http.client
@@ -13,6 +16,10 @@ import urllib.parse
 import time
 import uuid
 from urllib import parse
+
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
+
+
 class AccessToken:
     @staticmethod
     def _encode_text(text):
@@ -105,7 +112,7 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
         request_json = {
             "appkey": self.appkey,
             "token": self.token,
@@ -119,14 +126,26 @@ class TTSProvider(TTSProviderBase):
         }
 
         print(self.api_url, json.dumps(request_json, ensure_ascii=False))
+        tmp_file = self.generate_filename()
         try:
             resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
             # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
             if resp.headers['Content-Type'].startswith('audio/'):
-                with open(output_file, 'wb') as f:
+                with open(tmp_file, 'wb') as f:
                     f.write(resp.content)
-                return output_file
             else:
                 raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+            # 使用 pydub 读取临时文件
+            audio = AudioSegment.from_file(tmp_file, format="wav")
+            audio = audio.set_channels(1).set_frame_rate(16000)
+            opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+            yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas,
+                                tts_finish_text=text, sentence_type=SentenceType.SENTENCE_START)
+            # 用完后删除临时文件
+            try:
+                os.remove(tmp_file)
+            except FileNotFoundError:
+                # 若文件不存在，忽略该异常
+                pass
         except Exception as e:
             raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/cozecn.py

@@ -4,7 +4,11 @@ import json
 import base64
 import requests
 from datetime import datetime
+
+from pydub import AudioSegment
+
 from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 
 
 class TTSProvider(TTSProviderBase):
@@ -21,7 +25,7 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
         request_json = {
             "model": self.model,
             "input": text,
@@ -34,5 +38,18 @@ class TTSProvider(TTSProviderBase):
         }
         response = requests.request("POST", self.api_url, json=request_json, headers=headers)
         data = response.content
-        file_to_save = open(output_file, "wb")
+        tmp_file = self.generate_filename()
+        file_to_save = open(tmp_file, "wb")
         file_to_save.write(data)
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/custom.py

@@ -1,13 +1,17 @@
 import os
 import uuid
 import requests
+from pydub import AudioSegment
+
 from config.logger import setup_logging
 from datetime import datetime
 from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 
 TAG = __name__
 logger = setup_logging()
 
+
 class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
         super().__init__(config, delete_audio_file)
@@ -20,8 +24,9 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
         request_params = {}
+        tmp_file = self.generate_filename()
         for k, v in self.params.items():
             if isinstance(v, str) and "{prompt_text}" in v:
                 v = v.replace("{prompt_text}", text)
@@ -29,7 +34,19 @@ class TTSProvider(TTSProviderBase):
 
         resp = requests.get(self.url, params=request_params, headers=self.headers)
         if resp.status_code == 200:
-            with open(output_file, "wb") as file:
+            with open(tmp_file, "wb") as file:
                 file.write(resp.content)
         else:
             logger.bind(tag=TAG).error(f"Custom TTS请求失败: {resp.status_code} - {resp.text}")
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format=self.format)
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -4,6 +4,10 @@ import json
 import base64
 import requests
 from datetime import datetime
+
+from pydub import AudioSegment
+
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 from core.utils.util import check_model_key
 from core.providers.tts.base import TTSProviderBase
 
@@ -23,7 +27,8 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        tmp_file = self.generate_filename()
         request_json = {
             "app": {
                 "appid": f"{self.appid}",
@@ -54,9 +59,21 @@ class TTSProvider(TTSProviderBase):
             resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
             if "data" in resp.json():
                 data = resp.json()["data"]
-                file_to_save = open(output_file, "wb")
+                file_to_save = open(tmp_file, "wb")
                 file_to_save.write(base64.b64decode(data))
             else:
                 raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
         except Exception as e:
             raise Exception(f"{__name__} error: {e}")
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py

@@ -3,9 +3,12 @@ import uuid
 import json
 import base64
 import requests
+from pydub import AudioSegment
+
 from config.logger import setup_logging
 from datetime import datetime
 from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 
 TAG = __name__
 logger = setup_logging()
@@ -36,7 +39,8 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        tmp_file = self.generate_filename()
         request_json = {
             "text": text,
             "text_lang": self.text_lang,
@@ -61,7 +65,19 @@ class TTSProvider(TTSProviderBase):
 
         resp = requests.post(self.url, json=request_json)
         if resp.status_code == 200:
-            with open(output_file, "wb") as file:
+            with open(tmp_file, "wb") as file:
                 file.write(resp.content)
         else:
             logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -1,9 +1,12 @@
 import os
 import uuid
 import requests
+from pydub import AudioSegment
+
 from config.logger import setup_logging
 from datetime import datetime
 from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 
 TAG = __name__
 logger = setup_logging()
@@ -29,7 +32,8 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        tmp_file = self.generate_filename()
         request_params = {
             "refer_wav_path": self.refer_wav_path,
             "prompt_text": self.prompt_text,
@@ -48,7 +52,19 @@ class TTSProvider(TTSProviderBase):
 
         resp = requests.get(self.url, params=request_params)
         if resp.status_code == 200:
-            with open(output_file, "wb") as file:
+            with open(tmp_file, "wb") as file:
                 file.write(resp.content)
         else:
             logger.bind(tag=TAG).error(f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}")
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/minimax.py

@@ -3,7 +3,11 @@ import uuid
 import json
 import requests
 from datetime import datetime
+
+from pydub import AudioSegment
+
 from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 
 
 class TTSProvider(TTSProviderBase):
@@ -50,7 +54,8 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".mp3"):
         return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        tmp_file = self.generate_filename()
         request_json = {
             "model": self.model,
             "text": text,
@@ -69,9 +74,21 @@ class TTSProvider(TTSProviderBase):
             # 检查返回请求数据的status_code是否为0
             if resp.json()["base_resp"]["status_code"] == 0:
                 data = resp.json()['data']['audio']
-                file_to_save = open(output_file, "wb")
+                file_to_save = open(tmp_file, "wb")
                 file_to_save.write(bytes.fromhex(data))
             else:
                 raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
         except Exception as e:
-            raise Exception(f"{__name__} error: {e}")
+            raise Exception(f"{__name__} error: {e}")
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format="mp3")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/openai.py

@@ -2,6 +2,10 @@ import os
 import uuid
 import requests
 from datetime import datetime
+
+from pydub import AudioSegment
+
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 from core.utils.util import check_model_key
 from core.providers.tts.base import TTSProviderBase
 
@@ -20,7 +24,8 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        tmp_file = self.generate_filename()
         headers = {
             "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json"
@@ -34,7 +39,19 @@ class TTSProvider(TTSProviderBase):
         }
         response = requests.post(self.api_url, json=data, headers=headers)
         if response.status_code == 200:
-            with open(output_file, "wb") as audio_file:
+            with open(tmp_file, "wb") as audio_file:
                 audio_file.write(response.content)
         else:
             raise Exception(f"OpenAI TTS请求失败: {response.status_code} - {response.text}")
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/siliconflow.py

@@ -2,7 +2,11 @@ import os
 import uuid
 import requests
 from datetime import datetime
+
+from pydub import AudioSegment
+
 from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 
 
 class TTSProvider(TTSProviderBase):
@@ -22,7 +26,8 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        tmp_file = self.generate_filename()
         request_json = {
             "model": self.model,
             "input": text,
@@ -35,5 +40,17 @@ class TTSProvider(TTSProviderBase):
         }
         response = requests.request("POST", self.api_url, json=request_json, headers=headers)
         data = response.content
-        file_to_save = open(output_file, "wb")
+        file_to_save = open(tmp_file, "wb")
         file_to_save.write(data)
+        # 使用 pydub 读取临时文件
+        audio = AudioSegment.from_file(tmp_file, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                            sentence_type=SentenceType.SENTENCE_START)
+        # 用完后删除临时文件
+        try:
+            os.remove(tmp_file)
+        except FileNotFoundError:
+            # 若文件不存在，忽略该异常
+            pass

main/xiaozhi-server/core/providers/tts/ttson.py

@@ -4,7 +4,11 @@ import json
 import requests
 import shutil
 from datetime import datetime
+
+from pydub import AudioSegment
+
 from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 
 
 class TTSProvider(TTSProviderBase):
@@ -28,7 +32,8 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".mp3"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        tmp_file = self.generate_filename()
         url = f'{self.url}{self.token}'
         result = "firefly"
         payload = json.dumps({
@@ -45,7 +50,7 @@ class TTSProvider(TTSProviderBase):
 
         resp = requests.request("POST", url, data=payload)
         if resp.status_code != 200:
-            return None
+            return
         resp_json = resp.json()
         try:
             result = resp_json['url'] + ':' + str(
@@ -56,9 +61,21 @@ class TTSProvider(TTSProviderBase):
             print("error:", e)
 
         audio_content = requests.get(result)
-        with open(output_file, "wb") as f:
+        with open(tmp_file, "wb") as f:
             f.write(audio_content.content)
-            return True
+            # 使用 pydub 读取临时文件
+            audio = AudioSegment.from_file(tmp_file, format="mp3")
+            audio = audio.set_channels(1).set_frame_rate(16000)
+            opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+            yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
+                                sentence_type=SentenceType.SENTENCE_START)
+            # 用完后删除临时文件
+            try:
+                os.remove(tmp_file)
+            except FileNotFoundError:
+                # 若文件不存在，忽略该异常
+                pass
         voice_path = resp_json.get("voice_path")
-        des_path = output_file
+        des_path = tmp_file
         shutil.move(voice_path, des_path)
+