merge from master

core/connection.py

@@ -15,8 +15,8 @@ from core.handle.textHandle import handleTextMessage
 from core.utils.util import get_string_no_punctuation_or_emoji
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.audioHandle import handleAudioMessage, sendAudioMessage
-from .auth import AuthMiddleware, AuthenticationError
-
+from config.private_config import PrivateConfig
+from core.auth import AuthMiddleware, AuthenticationError
 
 class ConnectionHandler:
     def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts):
@@ -73,6 +73,8 @@ class ConnectionHandler:
         for cmd in self.cmd_exit:
             if len(cmd) > self.max_cmd_length:
                 self.max_cmd_length = len(cmd)
+        
+        self.private_config = None
 
     async def handle_connection(self, ws):
         try:
@@ -83,6 +85,28 @@ class ConnectionHandler:
             # 进行认证
             await self.auth.authenticate(self.headers)
 
+            device_id = self.headers.get("device-id", None)
+            
+            # Load private configuration if device_id is provided
+            bUsePrivateConfig = self.config.get("use_private_config", False)
+            logging.info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
+            if bUsePrivateConfig and device_id:
+                self.private_config = PrivateConfig(device_id, self.config)
+                await self.private_config.load_or_create()
+                # Create private instances using private config
+                vad, asr, llm, tts = self.private_config.create_private_instances()
+                if vad is not None and asr is not None and llm is not None and tts is not None:
+                    self.vad = vad
+                    self.asr = asr
+                    self.llm = llm
+                    self.tts = tts
+
+                    self.logger.info(f"Loaded private config and instances for device {device_id}")
+                    self.private_config.update_last_chat_time()
+                else:
+                    self.logger.error(f"Failed to load private config for device {device_id}")
+                    self.private_config = None
+
             # 认证通过,继续处理
             self.websocket = ws
             self.session_id = str(uuid.uuid4())
@@ -121,6 +145,8 @@ class ConnectionHandler:
 
     def _initialize_components(self):
         self.prompt = self.config["prompt"]
+        if self.private_config:
+            self.prompt = self.private_config.private_config.get("prompt", self.prompt)
         # 赋予LLM时间观念
         if "{date_time}" in self.prompt:
             date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())

core/http_server.py

@@ -1,92 +0,0 @@
-import logging
-import os
-from aiohttp import web
-from core.utils.util import get_local_ip, get_project_dir, check_password
-from manager.api.prompt import PromptApi
-from manager.api.auth import AuthApi
-from aiohttp.web_middlewares import middleware
-
-logger = logging.getLogger(__name__)
-
-
-@middleware
-async def cors_middleware(request, handler):
-    # 预检请求处理
-    if request.method == 'OPTIONS':
-        return web.Response(
-            status=204,
-            headers={
-                'Access-Control-Allow-Origin': '*',
-                'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
-                'Access-Control-Allow-Headers': 'Content-Type, Authorization, Token',
-                'Access-Control-Max-Age': '86400',
-            }
-        )
-
-    try:
-        response = await handler(request)
-    except web.HTTPException as ex:
-        response = ex
-
-    # 添加CORS头到所有响应
-    cors_headers = {
-        'Access-Control-Allow-Origin': '*',
-        'Access-Control-Expose-Headers': 'Content-Type, Authorization, Token',
-        'Vary': 'Origin'  # 避免缓存问题
-    }
-    response.headers.update(cors_headers)
-
-    return response
-
-
-class ConfigServer:
-    def __init__(self, config: dict):
-        self.config = config
-        self.app = web.Application(middlewares=[cors_middleware])  # 注册中间件
-
-        # 初始化接口处理器
-        self.prompt_handler = PromptApi(config)
-        self.auth_handler = AuthApi(config)
-        self.setup_routes()
-
-    def setup_routes(self):
-        # 注册prompt接口
-        self.app.router.add_get('/api/prompt', self.prompt_handler.get_prompt)
-        self.app.router.add_post('/api/prompt', self.prompt_handler.update_prompt)
-
-        # 注册auth接口
-        self.app.router.add_post('/api/login', self.auth_handler.login)
-
-        # 注册静态文件路由
-        static_dir = os.path.join(get_project_dir(), 'manager/static')  # 获取static目录绝对路径
-        self.app.router.add_static(
-            prefix='/manager/',  # 匹配前缀
-            path=static_dir,  # 静态文件目录
-            name='static'
-        )
-        self.app.router.add_get('/manager', self.redirect_to_index)
-
-    async def redirect_to_index(self, _):
-        raise web.HTTPFound('/manager/')
-
-    async def start(self):
-        try:
-            http_config = self.config['manager']
-            if not http_config.get('enabled', False):
-                logger.info("HTTP server is disabled")
-                return
-            token = self.config['manager']['token']
-            if not check_password(token):
-                logger.info("您设置的后台密码太弱了，启动后台管理失败！")
-                return
-
-            runner = web.AppRunner(self.app)
-            await runner.setup()
-            site = web.TCPSite(runner, http_config['ip'], http_config['port'])
-            await site.start()
-            logger.info(
-                f"Config HTTP server is running at http://{get_local_ip()}:{http_config['port']}/manager/login.html")
-            return runner  # 返回runner以便后续清理
-        except Exception as e:
-            logger.error(f"Failed to start HTTP server: {e}")
-            raise

core/providers/tts/fishspeech.py

@@ -0,0 +1,152 @@
+
+import base64
+import os
+import uuid
+import requests
+import ormsgpack
+from pathlib import Path
+from pydantic import BaseModel, Field, conint, model_validator
+from typing_extensions import Annotated
+from datetime import datetime
+from typing import Literal
+# from base import TTSProviderBase
+from core.providers.tts.base import TTSProviderBase
+
+
+class ServeReferenceAudio(BaseModel):
+    audio: bytes
+    text: str
+
+    @model_validator(mode="before")
+    def decode_audio(cls, values):
+        audio = values.get("audio")
+        if (
+            isinstance(audio, str) and len(audio) > 255
+        ):  # Check if audio is a string (Base64)
+            try:
+                values["audio"] = base64.b64decode(audio)
+            except Exception as e:
+                # If the audio is not a valid base64 string, we will just ignore it and let the server handle it
+                pass
+        return values
+
+    def __repr__(self) -> str:
+        return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
+
+class ServeTTSRequest(BaseModel):
+    text: str
+    chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
+    # Audio format
+    format: Literal["wav", "pcm", "mp3"] = "wav"
+    # References audios for in-context learning
+    references: list[ServeReferenceAudio] = []
+    # Reference id
+    # For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
+    # Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
+    reference_id: str | None = None
+    seed: int | None = None
+    use_memory_cache: Literal["on", "off"] = "off"
+    # Normalize text for en & zh, this increase stability for numbers
+    normalize: bool = True
+    # not usually used below
+    streaming: bool = False
+    max_new_tokens: int = 1024
+    top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+    repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
+    temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+
+    class Config:
+        # Allow arbitrary types for pytorch related types
+        arbitrary_types_allowed = True
+
+
+def audio_to_bytes(file_path):
+    if not file_path or not Path(file_path).exists():
+        return None
+    with open(file_path, "rb") as wav_file:
+        wav = wav_file.read()
+    return wav
+
+def read_ref_text(ref_text):
+    path = Path(ref_text)
+    if path.exists() and path.is_file():
+        with path.open("r", encoding="utf-8") as file:
+            return file.read()
+    return ref_text
+
+class TTSProvider(TTSProviderBase):
+
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        self.reference_id = config.get("reference_id")
+        self.reference_audio = config.get("reference_audio",[]) 
+        self.reference_text = config.get("reference_text",[])
+        self.format =  config.get("format","wav")
+        self.channels = config.get("channels",1)
+        self.rate = config.get("rate",44100)
+        self.api_key = config.get("api_key","YOUR_API_KEY")
+        self.normalize = config.get("normalize",True)
+        self.max_new_tokens = config.get("max_new_tokens",1024)
+        self.chunk_length = config.get("chunk_length",200)
+        self.top_p = config.get("top_p",0.7)
+        self.repetition_penalty = config.get("repetition_penalty",1.2)
+        self.temperature = config.get("temperature",0.7)
+        self.streaming = config.get("streaming",False)
+        self.use_memory_cache = config.get("use_memory_cache","on")
+        self.seed = config.get("seed")
+        self.api_url =   config.get("api_url","http://127.0.0.1:8080/v1/tts")
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        # Prepare reference data
+        byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+        ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+
+        data = {
+            "text": text,
+            "references": [
+                ServeReferenceAudio(
+                    audio=audio if audio else b"", text=text
+                )
+                for text, audio in zip(ref_texts, byte_audios)
+            ],
+            "reference_id": self.reference_id,
+            "normalize": self.normalize,
+            "format": self.format,
+            "max_new_tokens": self.max_new_tokens,
+            "chunk_length": self.chunk_length,
+            "top_p": self.top_p,
+            "repetition_penalty": self.repetition_penalty,
+            "temperature": self.temperature,
+            "streaming": self.streaming,
+            "use_memory_cache": self.use_memory_cache,
+            "seed": self.seed,
+        }
+
+        pydantic_data = ServeTTSRequest(**data)
+
+        response = requests.post(
+            self.api_url,
+            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/msgpack",
+            },
+        )
+
+        if response.status_code == 200:
+            audio_content = response.content
+
+            with open(output_file, "wb") as audio_file:
+                audio_file.write(audio_content)
+                
+
+
+        else:
+            print(f"Request failed with status code {response.status_code}")
+            print(response.json())
+
+

core/utils/llm.py

@@ -16,7 +16,7 @@ def create_instance(class_name, *args, **kwargs):
         lib_name = f'core.providers.llm.{class_name}.{class_name}'
         if lib_name not in sys.modules:
             sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
-            return sys.modules[lib_name].LLMProvider(*args, **kwargs)
+        return sys.modules[lib_name].LLMProvider(*args, **kwargs)
 
     raise ValueError(f"不支持的LLM类型: {class_name}，请检查该配置的type是否设置正确")
 

core/utils/tts.py

@@ -14,7 +14,7 @@ def create_instance(class_name, *args, **kwargs):
         lib_name = f'core.providers.tts.{class_name}'
         if lib_name not in sys.modules:
             sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
-            return sys.modules[lib_name].TTSProvider(*args, **kwargs)
+        return sys.modules[lib_name].TTSProvider(*args, **kwargs)
 
     raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")