merge main

main/xiaozhi-server/core/providers/tts/aliyun.py

@@ -0,0 +1,132 @@
+import os
+import uuid
+import json
+import hmac
+import hashlib
+import base64
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+import http.client
+import urllib.parse
+import time
+import uuid
+from urllib import parse
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {'AccessKeyId': access_key_id,
+                      'Action': 'CreateToken',
+                      'Format': 'JSON',
+                      'RegionId': 'cn-shanghai',
+                      'SignatureMethod': 'HMAC-SHA1',
+                      'SignatureNonce': str(uuid.uuid1()),
+                      'SignatureVersion': '1.0',
+                      'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                      'Version': '2019-02-28'}
+        # 构造规范化的请求字符串
+        query_string = AccessToken._encode_dict(parameters)
+        print('规范化的请求字符串: %s' % query_string)
+        # 构造待签名字符串
+        string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string)
+        print('待签名的字符串: %s' % string_to_sign)
+        # 计算签名
+        secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'),
+                                   bytes(string_to_sign, encoding='utf-8'),
+                                   hashlib.sha1).digest()
+        signature = base64.b64encode(secreted_string)
+        print('签名: %s' % signature)
+        # 进行URL编码
+        signature = AccessToken._encode_text(signature)
+        print('URL编码后的签名: %s' % signature)
+        # 调用服务
+        full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string)
+        print('url: %s' % full_url)
+        # 提交HTTP GET请求
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            key = 'Token'
+            if key in root_obj:
+                token = root_obj[key]['Id']
+                expire_time = root_obj[key]['ExpireTime']
+                return token, expire_time
+        print(response.text)
+        return None, None
+
+
+class TTSProvider(TTSProviderBase):
+
+
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        
+        # 新增空值判断逻辑
+        access_key_id = config.get("access_key_id")
+        access_key_secret = config.get("access_key_secret")
+        if access_key_id and access_key_secret:
+            # 使用密钥对生成临时token
+            token, expire_time = AccessToken.create_token(access_key_id, access_key_secret)
+        else:
+            # 直接使用预生成的长期token
+            token = config.get("token")
+            expire_time = None
+        
+        print('token: %s, expire time(s): %s' % (token, expire_time))
+
+        self.appkey = config.get("appkey")
+        self.token = token
+        self.format = config.get("format", "wav")
+        self.sample_rate = config.get("sample_rate", 16000)
+        self.voice = config.get("voice", "xiaoyun")
+        self.volume = config.get("volume", 50)
+        self.speech_rate = config.get("speech_rate", 0)
+        self.pitch_rate = config.get("pitch_rate", 0)
+
+        self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
+        self.api_url = f"https://{self.host}/stream/v1/tts"
+        self.header = {
+            "Content-Type": "application/json"
+        }
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "appkey": self.appkey,
+            "token": self.token,
+            "text": text,
+            "format": self.format,
+            "sample_rate": self.sample_rate,
+            "voice": self.voice,
+            "volume": self.volume,
+            "speech_rate": self.speech_rate,
+            "pitch_rate": self.pitch_rate
+        }
+
+        print(self.api_url, json.dumps(request_json, ensure_ascii=False))
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
+            if resp.headers['Content-Type'].startswith('audio/'):
+                with open(output_file, 'wb') as f:
+                    f.write(resp.content)
+                return output_file
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/base.py

@@ -0,0 +1,123 @@
+import asyncio
+from config.logger import setup_logging
+import os
+import numpy as np
+import opuslib_next
+from pydub import AudioSegment
+from abc import ABC, abstractmethod
+import queue
+
+TAG = __name__
+logger = setup_logging()
+
+
+class TTSProviderBase(ABC):
+    def __init__(self, config, delete_audio_file):
+        self.delete_audio_file = delete_audio_file
+        self.output_file = config.get("output_file")
+
+    @abstractmethod
+    def generate_filename(self):
+        pass
+
+    def to_tts(self, text):
+        tmp_file = self.generate_filename()
+        try:
+            max_repeat_time = 5
+            while not os.path.exists(tmp_file) and max_repeat_time > 0:
+                asyncio.run(self.text_to_speak(text, tmp_file))
+                if not os.path.exists(tmp_file):
+                    max_repeat_time = max_repeat_time - 1
+                    logger.bind(tag=TAG).error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次")
+
+            if max_repeat_time > 0:
+                logger.bind(tag=TAG).info(f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次")
+
+            return tmp_file
+        except Exception as e:
+            logger.bind(tag=TAG).info(f": {e}")
+            return None
+
+    def to_tts_stream(self, text, queue: queue.Queue, text_index=0):
+        try:
+            asyncio.run(self.text_to_speak_stream(text, queue, text_index))
+        except Exception as e:
+            logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
+            return None
+
+    @abstractmethod
+    async def text_to_speak(self, text, output_file):
+        pass
+
+    async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
+        raise Exception("该TTS还没有实现stream模式")
+
+    def wav_to_opus_data(self, wav_file_path):
+        # 使用pydub加载PCM文件
+        # 获取文件后缀名
+        file_type = os.path.splitext(wav_file_path)[1]
+        if file_type:
+            file_type = file_type.lstrip('.')
+        audio = AudioSegment.from_file(wav_file_path, format=file_type)
+
+        duration = len(audio) / 1000.0
+
+        # 转换为单声道和16kHz采样率（确保与编码器匹配）
+        audio = audio.set_channels(1).set_frame_rate(16000)
+
+        # 获取原始PCM数据（16位小端）
+        raw_data = audio.raw_data
+
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+        opus_datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i:i + frame_size * 2]
+
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                chunk += b'\x00' * (frame_size * 2 - len(chunk))
+
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+
+            # 编码Opus数据
+            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
+            opus_datas.append(opus_data)
+
+        return opus_datas, duration
+
+    def wav_to_opus_data_audio_raw(self, raw_data):
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+        opus_datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i:i + frame_size * 2]
+
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                # logger.bind(tag=TAG).info("开始补0")
+                chunk += b'\x00' * (frame_size * 2 - len(chunk))
+
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+
+            # 编码Opus数据
+            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
+            opus_datas.append(opus_data)
+
+        return opus_datas

main/xiaozhi-server/core/providers/tts/cozecn.py

@@ -0,0 +1,38 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.model = config.get("model")
+        self.access_token = config.get("access_token")
+        self.voice = config.get("voice")
+        self.response_format = config.get("response_format")
+
+        self.host = "api.coze.cn"
+        self.api_url = f"https://{self.host}/v1/audio/speech"
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "input": text,
+            "voice_id": self.voice,
+            "response_format": self.response_format,
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
+        data = response.content
+        file_to_save = open(output_file, "wb")
+        file_to_save.write(data)

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -0,0 +1,62 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.appid = config.get("appid")
+        self.access_token = config.get("access_token")
+        self.cluster = config.get("cluster")
+        self.voice = config.get("voice")
+        self.api_url = config.get("api_url")
+        self.authorization = config.get("authorization")
+        self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
+        check_model_key("TTS", self.access_token)
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "app": {
+                "appid": f"{self.appid}",
+                "token": "access_token",
+                "cluster": self.cluster
+            },
+            "user": {
+                "uid": "1"
+            },
+            "audio": {
+                "voice_type": self.voice,
+                "encoding": "wav",
+                "speed_ratio": 1.0,
+                "volume_ratio": 1.0,
+                "pitch_ratio": 1.0,
+            },
+            "request": {
+                "reqid": str(uuid.uuid4()),
+                "text": text,
+                "text_type": "plain",
+                "operation": "query",
+                "with_frontend": 1,
+                "frontend_type": "unitTson"
+            }
+        }
+
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            if "data" in resp.json():
+                data = resp.json()["data"]
+                file_to_save = open(output_file, "wb")
+                file_to_save.write(base64.b64decode(data))
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/edge.py

@@ -0,0 +1,18 @@
+import os
+import uuid
+import edge_tts
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.voice = config.get("voice")
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        communicate = edge_tts.Communicate(text, voice=self.voice)  # Use your preferred voice
+        await communicate.save(output_file)

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -0,0 +1,268 @@
+import base64
+import os
+import traceback
+import uuid
+import queue
+import io
+
+import numpy as np
+import requests
+import ormsgpack
+from pathlib import Path
+
+import torch
+import torchaudio
+from pydantic import BaseModel, Field, conint, model_validator
+from pydub import AudioSegment
+from typing_extensions import Annotated
+from datetime import datetime
+from typing import Literal
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ServeReferenceAudio(BaseModel):
+    audio: bytes
+    text: str
+
+    @model_validator(mode="before")
+    def decode_audio(cls, values):
+        audio = values.get("audio")
+        if (
+                isinstance(audio, str) and len(audio) > 255
+        ):  # Check if audio is a string (Base64)
+            try:
+                values["audio"] = base64.b64decode(audio)
+            except Exception as e:
+                # If the audio is not a valid base64 string, we will just ignore it and let the server handle it
+                pass
+        return values
+
+    def __repr__(self) -> str:
+        return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
+
+
+class ServeTTSRequest(BaseModel):
+    text: str
+    chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
+    # Audio format
+    format: Literal["wav", "pcm", "mp3"] = "wav"
+    # References audios for in-context learning
+    references: list[ServeReferenceAudio] = []
+    # Reference id
+    # For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
+    # Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
+    reference_id: str | None = None
+    seed: int | None = None
+    use_memory_cache: Literal["on", "off"] = "off"
+    # Normalize text for en & zh, this increase stability for numbers
+    normalize: bool = True
+    # not usually used below
+    streaming: bool = False
+    max_new_tokens: int = 1024
+    top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+    repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
+    temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+
+    class Config:
+        # Allow arbitrary types for pytorch related types
+        arbitrary_types_allowed = True
+
+
+def audio_to_bytes(file_path):
+    if not file_path or not Path(file_path).exists():
+        return None
+    with open(file_path, "rb") as wav_file:
+        wav = wav_file.read()
+    return wav
+
+
+def read_ref_text(ref_text):
+    path = Path(ref_text)
+    if path.exists() and path.is_file():
+        with path.open("r", encoding="utf-8") as file:
+            return file.read()
+    return ref_text
+
+
+class TTSProvider(TTSProviderBase):
+
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        self.reference_id = config.get("reference_id")
+        self.reference_audio = config.get("reference_audio", [])
+        self.reference_text = config.get("reference_text", [])
+        self.format = config.get("format", "wav")
+        self.channels = config.get("channels", 1)
+        self.rate = config.get("rate", 44100)
+        self.api_key = config.get("api_key", "YOUR_API_KEY")
+        have_key = check_model_key("FishSpeech TTS", self.api_key)
+        if not have_key:
+            return
+        self.normalize = config.get("normalize", True)
+        self.max_new_tokens = config.get("max_new_tokens", 1024)
+        self.chunk_length = config.get("chunk_length", 200)
+        self.top_p = config.get("top_p", 0.7)
+        self.repetition_penalty = config.get("repetition_penalty", 1.2)
+        self.temperature = config.get("temperature", 0.7)
+        self.streaming = config.get("streaming", False)
+        self.use_memory_cache = config.get("use_memory_cache", "on")
+        self.seed = config.get("seed")
+        self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        # Prepare reference data
+        byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+        ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+
+        data = {
+            "text": text,
+            "references": [
+                ServeReferenceAudio(
+                    audio=audio if audio else b"", text=text
+                )
+                for text, audio in zip(ref_texts, byte_audios)
+            ],
+            "reference_id": self.reference_id,
+            "normalize": self.normalize,
+            "format": self.format,
+            "max_new_tokens": self.max_new_tokens,
+            "chunk_length": self.chunk_length,
+            "top_p": self.top_p,
+            "repetition_penalty": self.repetition_penalty,
+            "temperature": self.temperature,
+            "streaming": self.streaming,
+            "use_memory_cache": self.use_memory_cache,
+            "seed": self.seed,
+        }
+
+        pydantic_data = ServeTTSRequest(**data)
+
+        response = requests.post(
+            self.api_url,
+            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/msgpack",
+            },
+        )
+
+        if response.status_code == 200:
+            audio_content = response.content
+
+            with open(output_file, "wb") as audio_file:
+                audio_file.write(audio_content)
+
+
+
+        else:
+            print(f"Request failed with status code {response.status_code}")
+            print(response.json())
+
+    def _get_audio_from_tts(self, data_bytes):
+        tts_speech = torch.from_numpy(np.array(np.frombuffer(data_bytes, dtype=np.int16))).unsqueeze(dim=0)
+        with io.BytesIO() as bf:
+            torchaudio.save(bf, tts_speech, 44100, format="wav")
+            audio = AudioSegment.from_file(bf, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        return audio
+
+    async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
+        try:
+            # Prepare reference data
+            byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+            ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+
+            data = {
+                "text": text,
+                "references": [
+                    ServeReferenceAudio(
+                        audio=audio if audio else b"", text=text
+                    )
+                    for text, audio in zip(ref_texts, byte_audios)
+                ],
+                "reference_id": self.reference_id,
+                "normalize": self.normalize,
+                "format": self.format,
+                "max_new_tokens": self.max_new_tokens,
+                "chunk_length": self.chunk_length,
+                "top_p": self.top_p,
+                "repetition_penalty": self.repetition_penalty,
+                "temperature": self.temperature,
+                "streaming": self.streaming,
+                "use_memory_cache": self.use_memory_cache,
+                "seed": self.seed,
+            }
+
+            pydantic_data = ServeTTSRequest(**data)
+            audio_buff = None
+            chunk_total = b''
+            last_raw = b''
+            audio_raw = b''
+            print("请求tts")
+            with requests.post(
+                    self.api_url,
+                    data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+                    headers={
+                        "Authorization": f"Bearer {self.api_key}",
+                        "Content-Type": "application/msgpack",
+                    },
+            ) as response:
+                if response.status_code == 200:
+                    for chunk in response.iter_content():
+                        # 拼接当前块和上一块数据
+                        chunk_total += chunk
+                        # 最后一个是静音，说明是一个完整的音频
+                        if len(chunk_total) % 2 == 0 and chunk_total[-2:] == b'\x00\x00':
+                            audio = self._get_audio_from_tts(chunk_total)
+                            audio_raw = audio_raw + audio.raw_data
+                            #长度凑够2贞开始发送，60ms*4=240ms
+                            if len(audio_raw) >= 7680:
+                                duration = 60 * len(audio_raw) // 1920
+                                if (len(audio_raw) % 1920) > 0:
+                                    duration += 60
+                                duration = duration / 1000.0
+                                logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
+                                opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
+                                queue.put({
+                                    "data": opus_datas,
+                                    "duration": duration,
+                                    "end": False,
+                                    "text_index": text_index
+                                })
+                                audio_raw = b''
+                            chunk_total = b''
+                    if len(chunk_total) > 0:
+                        audio = self._get_audio_from_tts(chunk_total)
+                        audio_raw = audio_raw + audio.raw_data
+                        duration = 60 * len(audio_raw) // 1920
+                        if (len(audio_raw) % 1920) > 0:
+                            duration += 60
+                        duration = duration / 1000.0
+                        # 把 audio 转成 opus
+                        logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
+                        opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
+                        queue.put({
+                            "data": opus_datas,
+                            "duration": duration,
+                            "end": False
+                        })
+
+                else:
+                    print('请求失败:', response.status_code, response.text)
+                queue.put({
+                    "data": None,
+                    "end": True
+                })
+        except Exception as e:
+            logger.bind(tag=TAG).error("tts发生错误")
+            traceback.print_exc()
+            raise e

main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py

@@ -0,0 +1,67 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.text_lang = config.get("text_lang", "zh")
+        self.ref_audio_path = config.get("ref_audio_path")
+        self.prompt_text = config.get("prompt_text")
+        self.prompt_lang = config.get("prompt_lang", "zh")
+        self.top_k = config.get("top_k", 5)
+        self.top_p = config.get("top_p", 1)
+        self.temperature = config.get("temperature", 1)
+        self.text_split_method = config.get("text_split_method", "cut0")
+        self.batch_size = config.get("batch_size", 1)
+        self.batch_threshold = config.get("batch_threshold", 0.75)
+        self.split_bucket = config.get("split_bucket", True)
+        self.return_fragment = config.get("return_fragment", False)
+        self.speed_factor = config.get("speed_factor", 1.0)
+        self.streaming_mode = config.get("streaming_mode", False)
+        self.seed = config.get("seed", -1)
+        self.parallel_infer = config.get("parallel_infer", True)
+        self.repetition_penalty = config.get("repetition_penalty", 1.35)
+        self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "text": text,
+            "text_lang": self.text_lang,
+            "ref_audio_path": self.ref_audio_path,
+            "aux_ref_audio_paths": self.aux_ref_audio_paths,
+            "prompt_text": self.prompt_text,
+            "prompt_lang": self.prompt_lang,
+            "top_k": self.top_k,
+            "top_p": self.top_p,
+            "temperature": self.temperature,
+            "text_split_method": self.text_split_method,
+            "batch_size": self.batch_size,
+            "batch_threshold": self.batch_threshold,
+            "split_bucket": self.split_bucket,
+            "return_fragment": self.return_fragment,
+            "speed_factor": self.speed_factor,
+            "streaming_mode": self.streaming_mode,
+            "seed": self.seed,
+            "parallel_infer": self.parallel_infer,
+            "repetition_penalty": self.repetition_penalty
+        }
+
+        resp = requests.post(self.url, json=request_json)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -0,0 +1,52 @@
+import os
+import uuid
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.text_lang = config.get("text_lang", "audo")
+        self.ref_audio_path = config.get("ref_audio_path")
+        self.prompt_lang = config.get("prompt_lang")
+        self.prompt_text = config.get("prompt_text")
+        self.top_k = config.get("top_k", 5)
+        self.top_p = config.get("top_p", 1)
+        self.temperature = config.get("temperature", 1)
+        self.sample_steps = config.get("sample_steps", 16)
+        self.media_type = config.get("media_type", "wav")
+        self.streaming_mode = config.get("streaming_mode", False)
+        self.threshold = config.get("threshold", 30)
+
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_params = {
+            "text": text,
+            "text_lang": self.text_lang,
+            "ref_audio_path": self.ref_audio_path,
+            "prompt_lang": self.prompt_lang,
+            "prompt_text": self.prompt_text,
+            "top_k": self.top_k,
+            "top_p": self.top_p,
+            "temperature": self.temperature,
+            "sample_steps": self.sample_steps,
+            "media_type": self.media_type,
+            "streaming_mode": self.streaming_mode,
+            "threshold": self.threshold,
+        }
+
+        resp = requests.get(self.url, params=request_params)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}")

main/xiaozhi-server/core/providers/tts/minimax.py

@@ -0,0 +1,77 @@
+import os
+import uuid
+import json
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.group_id = config.get("group_id")
+        self.api_key = config.get("api_key")
+        self.model = config.get("model")
+        self.voice_id = config.get("voice_id")
+
+        default_voice_setting = {
+            "voice_id": "female-shaonv",
+            "speed": 1,
+            "vol": 1,
+            "pitch": 0,
+            "emotion": "happy"
+        }
+        default_pronunciation_dict = {
+            "tone": [
+                "处理/(chu3)(li3)", "危险/dangerous"
+            ]
+        }
+        defult_audio_setting = {
+            "sample_rate": 32000,
+            "bitrate": 128000,
+            "format": "mp3",
+            "channel": 1
+        }
+        self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})}
+        self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})}
+        self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
+        self.timber_weights = config.get("timber_weights", [])
+
+        if self.voice_id:
+            self.voice_setting["voice_id"] = self.voice_id
+
+        self.host = "api.minimax.chat"
+        self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
+        self.header = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}"
+        }
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "text": text,
+            "stream": False,
+            "voice_setting": self.voice_setting,
+            "pronunciation_dict": self.pronunciation_dict,
+            "audio_setting": self.audio_setting,
+        }
+
+        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
+            request_json["timber_weights"] = self.timber_weights
+            request_json["voice_setting"]["voice_id"] = ""
+
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            # 检查返回请求数据的status_code是否为0
+            if resp.json()["base_resp"]["status_code"] == 0:
+                data = resp.json()['data']['audio']
+                file_to_save = open(output_file, "wb")
+                file_to_save.write(bytes.fromhex(data))
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/openai.py

@@ -0,0 +1,40 @@
+import os
+import uuid
+import requests
+from datetime import datetime
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.api_key = config.get("api_key")
+        self.api_url = config.get("api_url", "https://api.openai.com/v1/audio/speech")
+        self.model = config.get("model", "tts-1")
+        self.voice = config.get("voice", "alloy")
+        self.response_format = "wav"
+        self.speed = config.get("speed", 1.0)
+        self.output_file = config.get("output_file", "tmp/")
+        check_model_key("TTS", self.api_key)
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        data = {
+            "model": self.model,
+            "input": text,
+            "voice": self.voice,
+            "response_format": "wav",
+            "speed": self.speed
+        }
+        response = requests.post(self.api_url, json=data, headers=headers)
+        if response.status_code == 200:
+            with open(output_file, "wb") as audio_file:
+                audio_file.write(response.content)
+        else:
+            raise Exception(f"OpenAI TTS请求失败: {response.status_code} - {response.text}")

main/xiaozhi-server/core/providers/tts/siliconflow.py

@@ -0,0 +1,39 @@
+import os
+import uuid
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.model = config.get("model")
+        self.access_token = config.get("access_token")
+        self.voice = config.get("voice")
+        self.response_format = config.get("response_format")
+        self.sample_rate = config.get("sample_rate")
+        self.speed = config.get("speed")
+        self.gain = config.get("gain")
+
+        self.host = "api.siliconflow.cn"
+        self.api_url = f"https://{self.host}/v1/audio/speech"
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "input": text,
+            "voice": self.voice,
+            "response_format": self.response_format,
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
+        data = response.content
+        file_to_save = open(output_file, "wb")
+        file_to_save.write(data)

main/xiaozhi-server/core/providers/tts/ttson.py

@@ -0,0 +1,64 @@
+import os
+import uuid
+import json
+import requests
+import shutil
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url", "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=")
+        self.voice_id = config.get("voice_id", 1695)
+        self.token = config.get("token")
+        self.to_lang = config.get("to_lang")
+        self.volume_change_dB = config.get("volume_change_dB", 0)
+        self.speed_factor = config.get("speed_factor", 1)
+        self.stream = config.get("stream", False)
+        self.output_file = config.get("output_file")
+        self.pitch_factor = config.get("pitch_factor", 0)
+        self.format = config.get("format", "mp3")
+        self.emotion = config.get("emotion", 1)
+        self.header = {
+            "Content-Type": "application/json"
+        }
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        url = f'{self.url}{self.token}'
+        result = "firefly"
+        payload = json.dumps({
+            "to_lang": self.to_lang,
+            "text": text,
+            "emotion": self.emotion,
+            "format": self.format,
+            "volume_change_dB": self.volume_change_dB,
+            "voice_id": self.voice_id,
+            "pitch_factor": self.pitch_factor,
+            "speed_factor": self.speed_factor,
+            "token": self.token
+        })
+
+        resp = requests.request("POST", url, data=payload)
+        if resp.status_code != 200:
+            return None
+        resp_json = resp.json()
+        try:
+            result = resp_json['url'] + ':' + str(
+                resp_json[
+                    'port']) + '/flashsummary/retrieveFileData?stream=True&token=' + self.token + '&voice_audio_path=' + \
+                     resp_json['voice_path']
+        except Exception as e:
+            print("error:", e)
+
+        audio_content = requests.get(result)
+        with open(output_file, "wb") as f:
+            f.write(audio_content.content)
+            return True
+        voice_path = resp_json.get("voice_path")
+        des_path = output_file
+        shutil.move(voice_path, des_path)