mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-30 05:13:59 +08:00
merge main
This commit is contained in:
lizhongxiang
@@ -0,0 +1,268 @@
|
||||
import base64
|
||||
import os
|
||||
import traceback
|
||||
import uuid
|
||||
import queue
|
||||
import io
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
import ormsgpack
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torchaudio
|
||||
from pydantic import BaseModel, Field, conint, model_validator
|
||||
from pydub import AudioSegment
|
||||
from typing_extensions import Annotated
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ServeReferenceAudio(BaseModel):
|
||||
audio: bytes
|
||||
text: str
|
||||
|
||||
@model_validator(mode="before")
|
||||
def decode_audio(cls, values):
|
||||
audio = values.get("audio")
|
||||
if (
|
||||
isinstance(audio, str) and len(audio) > 255
|
||||
): # Check if audio is a string (Base64)
|
||||
try:
|
||||
values["audio"] = base64.b64decode(audio)
|
||||
except Exception as e:
|
||||
# If the audio is not a valid base64 string, we will just ignore it and let the server handle it
|
||||
pass
|
||||
return values
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
|
||||
|
||||
|
||||
class ServeTTSRequest(BaseModel):
|
||||
text: str
|
||||
chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
|
||||
# Audio format
|
||||
format: Literal["wav", "pcm", "mp3"] = "wav"
|
||||
# References audios for in-context learning
|
||||
references: list[ServeReferenceAudio] = []
|
||||
# Reference id
|
||||
# For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
|
||||
# Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
|
||||
reference_id: str | None = None
|
||||
seed: int | None = None
|
||||
use_memory_cache: Literal["on", "off"] = "off"
|
||||
# Normalize text for en & zh, this increase stability for numbers
|
||||
normalize: bool = True
|
||||
# not usually used below
|
||||
streaming: bool = False
|
||||
max_new_tokens: int = 1024
|
||||
top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
|
||||
repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
|
||||
temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
|
||||
|
||||
class Config:
|
||||
# Allow arbitrary types for pytorch related types
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
|
||||
def audio_to_bytes(file_path):
|
||||
if not file_path or not Path(file_path).exists():
|
||||
return None
|
||||
with open(file_path, "rb") as wav_file:
|
||||
wav = wav_file.read()
|
||||
return wav
|
||||
|
||||
|
||||
def read_ref_text(ref_text):
|
||||
path = Path(ref_text)
|
||||
if path.exists() and path.is_file():
|
||||
with path.open("r", encoding="utf-8") as file:
|
||||
return file.read()
|
||||
return ref_text
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
|
||||
self.reference_id = config.get("reference_id")
|
||||
self.reference_audio = config.get("reference_audio", [])
|
||||
self.reference_text = config.get("reference_text", [])
|
||||
self.format = config.get("format", "wav")
|
||||
self.channels = config.get("channels", 1)
|
||||
self.rate = config.get("rate", 44100)
|
||||
self.api_key = config.get("api_key", "YOUR_API_KEY")
|
||||
have_key = check_model_key("FishSpeech TTS", self.api_key)
|
||||
if not have_key:
|
||||
return
|
||||
self.normalize = config.get("normalize", True)
|
||||
self.max_new_tokens = config.get("max_new_tokens", 1024)
|
||||
self.chunk_length = config.get("chunk_length", 200)
|
||||
self.top_p = config.get("top_p", 0.7)
|
||||
self.repetition_penalty = config.get("repetition_penalty", 1.2)
|
||||
self.temperature = config.get("temperature", 0.7)
|
||||
self.streaming = config.get("streaming", False)
|
||||
self.use_memory_cache = config.get("use_memory_cache", "on")
|
||||
self.seed = config.get("seed")
|
||||
self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
# Prepare reference data
|
||||
byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
|
||||
ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
|
||||
|
||||
data = {
|
||||
"text": text,
|
||||
"references": [
|
||||
ServeReferenceAudio(
|
||||
audio=audio if audio else b"", text=text
|
||||
)
|
||||
for text, audio in zip(ref_texts, byte_audios)
|
||||
],
|
||||
"reference_id": self.reference_id,
|
||||
"normalize": self.normalize,
|
||||
"format": self.format,
|
||||
"max_new_tokens": self.max_new_tokens,
|
||||
"chunk_length": self.chunk_length,
|
||||
"top_p": self.top_p,
|
||||
"repetition_penalty": self.repetition_penalty,
|
||||
"temperature": self.temperature,
|
||||
"streaming": self.streaming,
|
||||
"use_memory_cache": self.use_memory_cache,
|
||||
"seed": self.seed,
|
||||
}
|
||||
|
||||
pydantic_data = ServeTTSRequest(**data)
|
||||
|
||||
response = requests.post(
|
||||
self.api_url,
|
||||
data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/msgpack",
|
||||
},
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
audio_content = response.content
|
||||
|
||||
with open(output_file, "wb") as audio_file:
|
||||
audio_file.write(audio_content)
|
||||
|
||||
|
||||
|
||||
else:
|
||||
print(f"Request failed with status code {response.status_code}")
|
||||
print(response.json())
|
||||
|
||||
def _get_audio_from_tts(self, data_bytes):
|
||||
tts_speech = torch.from_numpy(np.array(np.frombuffer(data_bytes, dtype=np.int16))).unsqueeze(dim=0)
|
||||
with io.BytesIO() as bf:
|
||||
torchaudio.save(bf, tts_speech, 44100, format="wav")
|
||||
audio = AudioSegment.from_file(bf, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
return audio
|
||||
|
||||
async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
|
||||
try:
|
||||
# Prepare reference data
|
||||
byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
|
||||
ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
|
||||
|
||||
data = {
|
||||
"text": text,
|
||||
"references": [
|
||||
ServeReferenceAudio(
|
||||
audio=audio if audio else b"", text=text
|
||||
)
|
||||
for text, audio in zip(ref_texts, byte_audios)
|
||||
],
|
||||
"reference_id": self.reference_id,
|
||||
"normalize": self.normalize,
|
||||
"format": self.format,
|
||||
"max_new_tokens": self.max_new_tokens,
|
||||
"chunk_length": self.chunk_length,
|
||||
"top_p": self.top_p,
|
||||
"repetition_penalty": self.repetition_penalty,
|
||||
"temperature": self.temperature,
|
||||
"streaming": self.streaming,
|
||||
"use_memory_cache": self.use_memory_cache,
|
||||
"seed": self.seed,
|
||||
}
|
||||
|
||||
pydantic_data = ServeTTSRequest(**data)
|
||||
audio_buff = None
|
||||
chunk_total = b''
|
||||
last_raw = b''
|
||||
audio_raw = b''
|
||||
print("请求tts")
|
||||
with requests.post(
|
||||
self.api_url,
|
||||
data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/msgpack",
|
||||
},
|
||||
) as response:
|
||||
if response.status_code == 200:
|
||||
for chunk in response.iter_content():
|
||||
# 拼接当前块和上一块数据
|
||||
chunk_total += chunk
|
||||
# 最后一个是静音，说明是一个完整的音频
|
||||
if len(chunk_total) % 2 == 0 and chunk_total[-2:] == b'\x00\x00':
|
||||
audio = self._get_audio_from_tts(chunk_total)
|
||||
audio_raw = audio_raw + audio.raw_data
|
||||
#长度凑够2贞开始发送，60ms*4=240ms
|
||||
if len(audio_raw) >= 7680:
|
||||
duration = 60 * len(audio_raw) // 1920
|
||||
if (len(audio_raw) % 1920) > 0:
|
||||
duration += 60
|
||||
duration = duration / 1000.0
|
||||
logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
|
||||
queue.put({
|
||||
"data": opus_datas,
|
||||
"duration": duration,
|
||||
"end": False,
|
||||
"text_index": text_index
|
||||
})
|
||||
audio_raw = b''
|
||||
chunk_total = b''
|
||||
if len(chunk_total) > 0:
|
||||
audio = self._get_audio_from_tts(chunk_total)
|
||||
audio_raw = audio_raw + audio.raw_data
|
||||
duration = 60 * len(audio_raw) // 1920
|
||||
if (len(audio_raw) % 1920) > 0:
|
||||
duration += 60
|
||||
duration = duration / 1000.0
|
||||
# 把 audio 转成 opus
|
||||
logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
|
||||
queue.put({
|
||||
"data": opus_datas,
|
||||
"duration": duration,
|
||||
"end": False
|
||||
})
|
||||
|
||||
else:
|
||||
print('请求失败:', response.status_code, response.text)
|
||||
queue.put({
|
||||
"data": None,
|
||||
"end": True
|
||||
})
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error("tts发生错误")
|
||||
traceback.print_exc()
|
||||
raise e
|
||||
Reference in New Issue
Block a user