重新划分目录 (#214)

* 🦄 refactor(web): 修改zhikongtaiweb到web

* 🦄 refactor: 重写前端

路由守护尚未写完

* 🦄 refactor: 标准化路由

* update:前端重写，保留后端

* update:添加前端代码

* update:pip转成poetry启动

* update:增加mem0ai包依赖

* update:文档增加mem0ai的描述

* feat: play online mp3 (#181)

Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com>

* 修改前端代码

* update:调整项目目录

* update:优化

* update:配置文件去除8002端口

* update:增加开发说明

* update:更新开发协议

---------

Co-authored-by: kalicyh <34980061+kaliCYH@users.noreply.github.com>
Co-authored-by: hrz <1710360675@qq.com>
Co-authored-by: freshlife001 <talent@mises.site>
Co-authored-by: CGD <3030332422@qq.com>

core/providers/tts/aliyun.py

@@ -1,57 +0,0 @@
-import os
-import uuid
-import json
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-import http.client
-import urllib.parse
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.appkey = config.get("appkey")
-        self.token = config.get("token")
-        self.format = config.get("format", "wav")
-        self.sample_rate = config.get("sample_rate", 16000)
-        self.voice = config.get("voice", "xiaoyun")
-        self.volume = config.get("volume", 50)
-        self.speech_rate = config.get("speech_rate", 0)
-        self.pitch_rate = config.get("pitch_rate", 0)
-
-        self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
-        self.api_url = f"https://{self.host}/stream/v1/tts"
-        self.header = {
-            "Content-Type": "application/json"
-        }
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "appkey": self.appkey,
-            "token": self.token,
-            "text": text,
-            "format": self.format,
-            "sample_rate": self.sample_rate,
-            "voice": self.voice,
-            "volume": self.volume,
-            "speech_rate": self.speech_rate,
-            "pitch_rate": self.pitch_rate
-        }
-
-        print(self.api_url, json.dumps(request_json, ensure_ascii=False))
-        try:
-            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
-            # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
-            if resp.headers['Content-Type'].startswith('audio/'):
-                with open(output_file, 'wb') as f:
-                    f.write(resp.content)
-                return output_file
-            else:
-                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

core/providers/tts/base.py

@@ -1,84 +0,0 @@
-import asyncio
-from config.logger import setup_logging
-import os
-import numpy as np
-import opuslib_next
-from pydub import AudioSegment
-from abc import ABC, abstractmethod
-
-TAG = __name__
-logger = setup_logging()
-
-
-class TTSProviderBase(ABC):
-    def __init__(self, config, delete_audio_file):
-        self.delete_audio_file = delete_audio_file
-        self.output_file = config.get("output_file")
-
-    @abstractmethod
-    def generate_filename(self):
-        pass
-
-    def to_tts(self, text):
-        tmp_file = self.generate_filename()
-        try:
-            max_repeat_time = 5
-            while not os.path.exists(tmp_file) and max_repeat_time > 0:
-                asyncio.run(self.text_to_speak(text, tmp_file))
-                if not os.path.exists(tmp_file):
-                    max_repeat_time = max_repeat_time - 1
-                    logger.bind(tag=TAG).error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次")
-
-            if max_repeat_time > 0:
-                logger.bind(tag=TAG).info(f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次")
-
-            return tmp_file
-        except Exception as e:
-            logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
-            return None
-
-    @abstractmethod
-    async def text_to_speak(self, text, output_file):
-        pass
-
-    def wav_to_opus_data(self, wav_file_path):
-        # 使用pydub加载PCM文件
-        # 获取文件后缀名
-        file_type = os.path.splitext(wav_file_path)[1]
-        if file_type:
-            file_type = file_type.lstrip('.')
-        audio = AudioSegment.from_file(wav_file_path, format=file_type)
-
-        duration = len(audio) / 1000.0
-
-        # 转换为单声道和16kHz采样率（确保与编码器匹配）
-        audio = audio.set_channels(1).set_frame_rate(16000)
-
-        # 获取原始PCM数据（16位小端）
-        raw_data = audio.raw_data
-
-        # 初始化Opus编码器
-        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
-
-        # 编码参数
-        frame_duration = 60  # 60ms per frame
-        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
-
-        opus_datas = []
-        # 按帧处理所有音频数据（包括最后一帧可能补零）
-        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
-            # 获取当前帧的二进制数据
-            chunk = raw_data[i:i + frame_size * 2]
-
-            # 如果最后一帧不足，补零
-            if len(chunk) < frame_size * 2:
-                chunk += b'\x00' * (frame_size * 2 - len(chunk))
-
-            # 转换为numpy数组处理
-            np_frame = np.frombuffer(chunk, dtype=np.int16)
-
-            # 编码Opus数据
-            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
-            opus_datas.append(opus_data)
-
-        return opus_datas, duration

core/providers/tts/cozecn.py

@@ -1,38 +0,0 @@
-import os
-import uuid
-import json
-import base64
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.model = config.get("model")
-        self.access_token = config.get("access_token")
-        self.voice = config.get("voice")
-        self.response_format = config.get("response_format")
-
-        self.host = "api.coze.cn"
-        self.api_url = f"https://{self.host}/v1/audio/speech"
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "input": text,
-            "voice_id": self.voice,
-            "response_format": self.response_format,
-        }
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-            "Content-Type": "application/json"
-        }
-        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
-        data = response.content
-        file_to_save = open(output_file, "wb")
-        file_to_save.write(data)

core/providers/tts/doubao.py

@@ -1,60 +0,0 @@
-import os
-import uuid
-import json
-import base64
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.appid = config.get("appid")
-        self.access_token = config.get("access_token")
-        self.cluster = config.get("cluster")
-        self.voice = config.get("voice")
-        self.api_url = config.get("api_url")
-        self.authorization = config.get("authorization")
-        self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "app": {
-                "appid": f"{self.appid}",
-                "token": "access_token",
-                "cluster": self.cluster
-            },
-            "user": {
-                "uid": "1"
-            },
-            "audio": {
-                "voice_type": self.voice,
-                "encoding": "wav",
-                "speed_ratio": 1.0,
-                "volume_ratio": 1.0,
-                "pitch_ratio": 1.0,
-            },
-            "request": {
-                "reqid": str(uuid.uuid4()),
-                "text": text,
-                "text_type": "plain",
-                "operation": "query",
-                "with_frontend": 1,
-                "frontend_type": "unitTson"
-            }
-        }
-
-        try:
-            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
-            if "data" in resp.json():
-                data = resp.json()["data"]
-                file_to_save = open(output_file, "wb")
-                file_to_save.write(base64.b64decode(data))
-            else:
-                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

core/providers/tts/edge.py

@@ -1,18 +0,0 @@
-import os
-import uuid
-import edge_tts
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.voice = config.get("voice")
-
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        communicate = edge_tts.Communicate(text, voice=self.voice)  # Use your preferred voice
-        await communicate.save(output_file)

core/providers/tts/fishspeech.py

@@ -1,158 +0,0 @@
-
-import base64
-import os
-import uuid
-import requests
-import ormsgpack
-from pathlib import Path
-from pydantic import BaseModel, Field, conint, model_validator
-from typing_extensions import Annotated
-from datetime import datetime
-from typing import Literal
-from core.providers.tts.base import TTSProviderBase
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-
-class ServeReferenceAudio(BaseModel):
-    audio: bytes
-    text: str
-
-    @model_validator(mode="before")
-    def decode_audio(cls, values):
-        audio = values.get("audio")
-        if (
-            isinstance(audio, str) and len(audio) > 255
-        ):  # Check if audio is a string (Base64)
-            try:
-                values["audio"] = base64.b64decode(audio)
-            except Exception as e:
-                # If the audio is not a valid base64 string, we will just ignore it and let the server handle it
-                pass
-        return values
-
-    def __repr__(self) -> str:
-        return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
-
-class ServeTTSRequest(BaseModel):
-    text: str
-    chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
-    # Audio format
-    format: Literal["wav", "pcm", "mp3"] = "wav"
-    # References audios for in-context learning
-    references: list[ServeReferenceAudio] = []
-    # Reference id
-    # For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
-    # Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
-    reference_id: str | None = None
-    seed: int | None = None
-    use_memory_cache: Literal["on", "off"] = "off"
-    # Normalize text for en & zh, this increase stability for numbers
-    normalize: bool = True
-    # not usually used below
-    streaming: bool = False
-    max_new_tokens: int = 1024
-    top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
-    repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
-    temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
-
-    class Config:
-        # Allow arbitrary types for pytorch related types
-        arbitrary_types_allowed = True
-
-
-def audio_to_bytes(file_path):
-    if not file_path or not Path(file_path).exists():
-        return None
-    with open(file_path, "rb") as wav_file:
-        wav = wav_file.read()
-    return wav
-
-def read_ref_text(ref_text):
-    path = Path(ref_text)
-    if path.exists() and path.is_file():
-        with path.open("r", encoding="utf-8") as file:
-            return file.read()
-    return ref_text
-
-class TTSProvider(TTSProviderBase):
-
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-
-        self.reference_id = config.get("reference_id")
-        self.reference_audio = config.get("reference_audio",[]) 
-        self.reference_text = config.get("reference_text",[])
-        self.format =  config.get("format","wav")
-        self.channels = config.get("channels",1)
-        self.rate = config.get("rate",44100)
-        self.api_key = config.get("api_key","YOUR_API_KEY")
-        if "你" in self.api_key:
-            logger.bind(tag=TAG).error("你还没配置FishSpeech TTS的密钥，请在配置文件中配置密钥，否则无法正常工作")
-            return
-        self.normalize = config.get("normalize",True)
-        self.max_new_tokens = config.get("max_new_tokens",1024)
-        self.chunk_length = config.get("chunk_length",200)
-        self.top_p = config.get("top_p",0.7)
-        self.repetition_penalty = config.get("repetition_penalty",1.2)
-        self.temperature = config.get("temperature",0.7)
-        self.streaming = config.get("streaming",False)
-        self.use_memory_cache = config.get("use_memory_cache","on")
-        self.seed = config.get("seed")
-        self.api_url =   config.get("api_url","http://127.0.0.1:8080/v1/tts")
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        # Prepare reference data
-        byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
-        ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
-
-        data = {
-            "text": text,
-            "references": [
-                ServeReferenceAudio(
-                    audio=audio if audio else b"", text=text
-                )
-                for text, audio in zip(ref_texts, byte_audios)
-            ],
-            "reference_id": self.reference_id,
-            "normalize": self.normalize,
-            "format": self.format,
-            "max_new_tokens": self.max_new_tokens,
-            "chunk_length": self.chunk_length,
-            "top_p": self.top_p,
-            "repetition_penalty": self.repetition_penalty,
-            "temperature": self.temperature,
-            "streaming": self.streaming,
-            "use_memory_cache": self.use_memory_cache,
-            "seed": self.seed,
-        }
-
-        pydantic_data = ServeTTSRequest(**data)
-
-        response = requests.post(
-            self.api_url,
-            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
-            headers={
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/msgpack",
-            },
-        )
-
-        if response.status_code == 200:
-            audio_content = response.content
-
-            with open(output_file, "wb") as audio_file:
-                audio_file.write(audio_content)
-                
-
-
-        else:
-            print(f"Request failed with status code {response.status_code}")
-            print(response.json())
-
-

core/providers/tts/gpt_sovits_v2.py

@@ -1,67 +0,0 @@
-import os
-import uuid
-import json
-import base64
-import requests
-from config.logger import setup_logging
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.url = config.get("url")
-        self.text_lang = config.get("text_lang", "zh")
-        self.ref_audio_path = config.get("ref_audio_path")
-        self.prompt_text = config.get("prompt_text")
-        self.prompt_lang = config.get("prompt_lang", "zh")
-        self.top_k = config.get("top_k", 5)
-        self.top_p = config.get("top_p", 1)
-        self.temperature = config.get("temperature", 1)
-        self.text_split_method = config.get("text_split_method", "cut0")
-        self.batch_size = config.get("batch_size", 1)
-        self.batch_threshold = config.get("batch_threshold", 0.75)
-        self.split_bucket = config.get("split_bucket", True)
-        self.return_fragment = config.get("return_fragment", False)
-        self.speed_factor = config.get("speed_factor", 1.0)
-        self.streaming_mode = config.get("streaming_mode", False)
-        self.seed = config.get("seed", -1)
-        self.parallel_infer = config.get("parallel_infer", True)
-        self.repetition_penalty = config.get("repetition_penalty", 1.35)
-        self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "text": text,
-            "text_lang": self.text_lang,
-            "ref_audio_path": self.ref_audio_path,
-            "aux_ref_audio_paths": self.aux_ref_audio_paths,
-            "prompt_text": self.prompt_text,
-            "prompt_lang": self.prompt_lang,
-            "top_k": self.top_k,
-            "top_p": self.top_p,
-            "temperature": self.temperature,
-            "text_split_method": self.text_split_method,
-            "batch_size": self.batch_size,
-            "batch_threshold": self.batch_threshold,
-            "split_bucket": self.split_bucket,
-            "return_fragment": self.return_fragment,
-            "speed_factor": self.speed_factor,
-            "streaming_mode": self.streaming_mode,
-            "seed": self.seed,
-            "parallel_infer": self.parallel_infer,
-            "repetition_penalty": self.repetition_penalty
-        }
-
-        resp = requests.post(self.url, json=request_json)
-        if resp.status_code == 200:
-            with open(output_file, "wb") as file:
-                file.write(resp.content)
-        else:
-            logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")

core/providers/tts/minimax.py

@@ -1,77 +0,0 @@
-import os
-import uuid
-import json
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.group_id = config.get("group_id")
-        self.api_key = config.get("api_key")
-        self.model = config.get("model")
-        self.voice_id = config.get("voice_id")
-
-        default_voice_setting = {
-            "voice_id": "female-shaonv",
-            "speed": 1,
-            "vol": 1,
-            "pitch": 0,
-            "emotion": "happy"
-        }
-        default_pronunciation_dict = {
-            "tone": [
-                "处理/(chu3)(li3)", "危险/dangerous"
-            ]
-        }
-        defult_audio_setting = {
-            "sample_rate": 32000,
-            "bitrate": 128000,
-            "format": "mp3",
-            "channel": 1
-        }
-        self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})}
-        self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})}
-        self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
-        self.timber_weights = config.get("timber_weights", [])
-
-        if self.voice_id:
-            self.voice_setting["voice_id"] = self.voice_id
-
-        self.host = "api.minimax.chat"
-        self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
-        self.header = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {self.api_key}"
-        }
-
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "text": text,
-            "stream": False,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting,
-        }
-
-        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
-
-        try:
-            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
-            # 检查返回请求数据的status_code是否为0
-            if resp.json()["base_resp"]["status_code"] == 0:
-                data = resp.json()['data']['audio']
-                file_to_save = open(output_file, "wb")
-                file_to_save.write(bytes.fromhex(data))
-            else:
-                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

core/providers/tts/siliconflow.py

@@ -1,39 +0,0 @@
-import os
-import uuid
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.model = config.get("model")
-        self.access_token = config.get("access_token")
-        self.voice = config.get("voice")
-        self.response_format = config.get("response_format")
-        self.sample_rate = config.get("sample_rate")
-        self.speed = config.get("speed")
-        self.gain = config.get("gain")
-
-        self.host = "api.siliconflow.cn"
-        self.api_url = f"https://{self.host}/v1/audio/speech"
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "input": text,
-            "voice": self.voice,
-            "response_format": self.response_format,
-        }
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-            "Content-Type": "application/json"
-        }
-        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
-        data = response.content
-        file_to_save = open(output_file, "wb")
-        file_to_save.write(data)