update:server连接api (#747)

* update:server连接manager-api

* update:读取智能体模型配置

* update:添加默认模型的按钮

* update:优化配置读取方式

* update：server兼容manager接口改造

* update:优化私有配置加载

* update:加载私有模型配置

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -4,15 +4,17 @@ from core.providers.llm.base import LLMProviderBase
 from config.logger import setup_logging
 import requests
 import json
+
 TAG = __name__
 logger = setup_logging()
 
+
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         """初始化Gemini LLM Provider"""
         self.model_name = config.get("model_name", "gemini-1.5-pro")
         self.api_key = config.get("api_key")
-        self.http_proxy=config.get("http_proxy")
+        self.http_proxy = config.get("http_proxy")
         self.https_proxy = config.get("https_proxy")
         have_key = check_model_key("LLM", self.api_key)
 
@@ -22,7 +24,7 @@ class LLMProvider(LLMProviderBase):
         try:
             # 初始化Gemini客户端
             # 配置代理（如果提供了代理配置）
-            self.proxies=None
+            self.proxies = None
             if self.http_proxy is not "" or self.https_proxy is not "":
 
                 self.proxies = {
@@ -62,19 +64,16 @@ class LLMProvider(LLMProviderBase):
                 role = "model" if msg["role"] == "assistant" else "user"
                 content = msg["content"].strip()
                 if content:
-                    chat_history.append({
-                        "role": role,
-                        "parts": [{"text":content}]
-
-                    })
+                    chat_history.append({"role": role, "parts": [{"text": content}]})
 
             # 获取当前消息
             current_msg = dialogue[-1]["content"]
 
             # 构建请求体
             request_body = {
-                "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
-                "generationConfig": self.generation_config
+                "contents": chat_history
+                + [{"role": "user", "parts": [{"text": current_msg}]}],
+                "generationConfig": self.generation_config,
             }
 
             # 构建请求URL
@@ -87,11 +86,17 @@ class LLMProvider(LLMProviderBase):
 
             # 发送POST请求,经测试手动 request 无法使用 stream 模式
             if self.proxies:
-                response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
+                response = requests.post(
+                    url,
+                    headers=headers,
+                    json=request_body,
+                    stream=False,
+                    proxies=self.proxies,
+                )
                 try:
                     data = response.json()  # 直接解析JSON
-                    if 'candidates' in data and data['candidates']:
-                        yield data['candidates'][0]['content']['parts'][0]['text']
+                    if "candidates" in data and data["candidates"]:
+                        yield data["candidates"][0]["content"]["parts"][0]["text"]
                     else:
                         yield "未找到候选回复。"
                 except json.JSONDecodeError as e:
@@ -104,13 +109,11 @@ class LLMProvider(LLMProviderBase):
 
                 # 发送消息并获取流式响应
                 response = chat.send_message(
-                    current_msg,
-                    stream=True,
-                    generation_config=self.generation_config
+                    current_msg, stream=True, generation_config=self.generation_config
                 )
                 # 处理流式响应
                 for chunk in response:
-                    if hasattr(chunk, 'text') and chunk.text:
+                    if hasattr(chunk, "text") and chunk.text:
                         yield chunk.text
 
         except Exception as e:
@@ -125,9 +128,6 @@ class LLMProvider(LLMProviderBase):
             else:
                 yield f"【Gemini服务响应异常: {error_msg}】"
 
-
-
-
         except requests.exceptions.RequestException as e:
             yield f"请求失败：{e}"
         except json.JSONDecodeError as e:

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -11,7 +11,7 @@ class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.model_name = config.get("model_name")
         self.api_key = config.get("api_key")
-        if 'base_url' in config:
+        if "base_url" in config:
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
@@ -33,18 +33,22 @@ class LLMProvider(LLMProviderBase):
             for chunk in responses:
                 try:
                     # 检查是否存在有效的choice且content不为空
-                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
-                    content = delta.content if hasattr(delta, 'content') else ''
+                    delta = (
+                        chunk.choices[0].delta
+                        if getattr(chunk, "choices", None)
+                        else None
+                    )
+                    content = delta.content if hasattr(delta, "content") else ""
                 except IndexError:
-                    content = ''
+                    content = ""
                 if content:
                     # 处理标签跨多个chunk的情况
-                    if '<think>' in content:
+                    if "<think>" in content:
                         is_active = False
-                        content = content.split('<think>')[0]
-                    if '</think>' in content:
+                        content = content.split("<think>")[0]
+                    if "</think>" in content:
                         is_active = True
-                        content = content.split('</think>')[-1]
+                        content = content.split("</think>")[-1]
                     if is_active:
                         yield content
 
@@ -54,10 +58,7 @@ class LLMProvider(LLMProviderBase):
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
             stream = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                tools=functions
+                model=self.model_name, messages=dialogue, stream=True, tools=functions
             )
 
             for chunk in stream:

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -6,13 +6,14 @@ from core.utils.util import check_model_key
 
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
     def __init__(self, config):
         super().__init__(config)
         self.api_key = config.get("api_key", "")
         self.api_version = config.get("api_version", "v1.1")
         have_key = check_model_key("Mem0ai", self.api_key)
-        if not have_key :
+        if not have_key:
             self.use_mem0 = False
             return
         else:
@@ -30,54 +31,55 @@ class MemoryProvider(MemoryProviderBase):
             return None
         if len(msgs) < 2:
             return None
-        
+
         try:
             # Format the content as a message list for mem0
             messages = [
                 {"role": message.role, "content": message.content}
-                for message in msgs if message.role != "system"
+                for message in msgs
+                if message.role != "system"
             ]
-            result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
+            result = self.client.add(
+                messages, user_id=self.role_id, output_format=self.api_version
+            )
             logger.bind(tag=TAG).debug(f"Save memory result: {result}")
         except Exception as e:
             logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
             return None
 
-    async def query_memory(self, query: str)-> str:
+    async def query_memory(self, query: str) -> str:
         if not self.use_mem0:
             return ""
         try:
             results = self.client.search(
-                query,
-                user_id=self.role_id,
-                output_format=self.api_version
+                query, user_id=self.role_id, output_format=self.api_version
             )
-            if not results or 'results' not in results:
+            if not results or "results" not in results:
                 return ""
-                
+
             # Format each memory entry with its update time up to minutes
             memories = []
-            for entry in results['results']:
-                timestamp = entry.get('updated_at', '')
+            for entry in results["results"]:
+                timestamp = entry.get("updated_at", "")
                 if timestamp:
                     try:
                         # Parse and reformat the timestamp
-                        dt = timestamp.split('.')[0]  # Remove milliseconds
-                        formatted_time = dt.replace('T', ' ')
+                        dt = timestamp.split(".")[0]  # Remove milliseconds
+                        formatted_time = dt.replace("T", " ")
                     except:
                         formatted_time = timestamp
-                memory = entry.get('memory', '')
+                memory = entry.get("memory", "")
                 if timestamp and memory:
                     # Store tuple of (timestamp, formatted_string) for sorting
                     memories.append((timestamp, f"[{formatted_time}] {memory}"))
-            
+
             # Sort by timestamp in descending order (newest first)
             memories.sort(key=lambda x: x[0], reverse=True)
-            
+
             # Extract only the formatted strings
             memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
             logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
             return memories_str
         except Exception as e:
             logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
-            return ""
+            return ""

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -3,7 +3,8 @@ import time
 import json
 import os
 import yaml
-from core.utils.util import get_project_dir
+from config.config_loader import get_project_dir
+
 
 short_term_memory_prompt = """
 # 时空记忆编织者
@@ -71,11 +72,12 @@ short_term_memory_prompt = """
 ```
 """
 
+
 def extract_json_data(json_code):
     start = json_code.find("```json")
     # 从start开始找到下一个```结束
-    end = json_code.find("```", start+1)
-    #print("start:", start, "end:", end)
+    end = json_code.find("```", start + 1)
+    # print("start:", start, "end:", end)
     if start == -1 or end == -1:
         try:
             jsonData = json.loads(json_code)
@@ -83,74 +85,76 @@ def extract_json_data(json_code):
         except Exception as e:
             print("Error:", e)
         return ""
-    jsonData = json_code[start+7:end]
+    jsonData = json_code[start + 7 : end]
     return jsonData
 
+
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
     def __init__(self, config):
         super().__init__(config)
         self.short_momery = ""
-        self.memory_path = get_project_dir() + 'data/.memory.yaml'
+        self.memory_path = get_project_dir() + "data/.memory.yaml"
         self.load_memory()
 
     def init_memory(self, role_id, llm):
         super().init_memory(role_id, llm)
         self.load_memory()
-    
+
     def load_memory(self):
         all_memory = {}
         if os.path.exists(self.memory_path):
-            with open(self.memory_path, 'r', encoding='utf-8') as f:
+            with open(self.memory_path, "r", encoding="utf-8") as f:
                 all_memory = yaml.safe_load(f) or {}
         if self.role_id in all_memory:
             self.short_momery = all_memory[self.role_id]
-    
+
     def save_memory_to_file(self):
         all_memory = {}
         if os.path.exists(self.memory_path):
-              with open(self.memory_path, 'r', encoding='utf-8') as f:
-                  all_memory = yaml.safe_load(f) or {}
+            with open(self.memory_path, "r", encoding="utf-8") as f:
+                all_memory = yaml.safe_load(f) or {}
         all_memory[self.role_id] = self.short_momery
-        with open(self.memory_path, 'w', encoding='utf-8') as f:
+        with open(self.memory_path, "w", encoding="utf-8") as f:
             yaml.dump(all_memory, f, allow_unicode=True)
-        
+
     async def save_memory(self, msgs):
         if self.llm is None:
             logger.bind(tag=TAG).error("LLM is not set for memory provider")
             return None
-        
+
         if len(msgs) < 2:
             return None
-        
+
         msgStr = ""
         for msg in msgs:
             if msg.role == "user":
                 msgStr += f"User: {msg.content}\n"
-            elif msg.role== "assistant":
+            elif msg.role == "assistant":
                 msgStr += f"Assistant: {msg.content}\n"
         if len(self.short_momery) > 0:
-            msgStr+="历史记忆：\n"
-            msgStr+=self.short_momery
-        
-        #当前时间
+            msgStr += "历史记忆：\n"
+            msgStr += self.short_momery
+
+        # 当前时间
         time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
         msgStr += f"当前时间：{time_str}"
 
         result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
- 
+
         json_str = extract_json_data(result)
         try:
-            json_data = json.loads(json_str) # 检查json格式是否正确
+            json_data = json.loads(json_str)  # 检查json格式是否正确
             self.short_momery = json_str
         except Exception as e:
             print("Error:", e)
-        
+
         self.save_memory_to_file()
         logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
 
         return self.short_momery
-    
-    async def query_memory(self, query: str)-> str:
-        return self.short_momery
+
+    async def query_memory(self, query: str) -> str:
+        return self.short_momery

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -6,6 +6,10 @@ import requests
 from datetime import datetime
 from core.utils.util import check_model_key
 from core.providers.tts.base import TTSProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
 
 
 class TTSProvider(TTSProviderBase):
@@ -21,18 +25,19 @@ class TTSProvider(TTSProviderBase):
         check_model_key("TTS", self.access_token)
 
     def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+        return os.path.join(
+            self.output_file,
+            f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        )
 
     async def text_to_speak(self, text, output_file):
         request_json = {
             "app": {
                 "appid": f"{self.appid}",
                 "token": "access_token",
-                "cluster": self.cluster
-            },
-            "user": {
-                "uid": "1"
+                "cluster": self.cluster,
             },
+            "user": {"uid": "1"},
             "audio": {
                 "voice_type": self.voice,
                 "encoding": "wav",
@@ -46,17 +51,21 @@ class TTSProvider(TTSProviderBase):
                 "text_type": "plain",
                 "operation": "query",
                 "with_frontend": 1,
-                "frontend_type": "unitTson"
-            }
+                "frontend_type": "unitTson",
+            },
         }
 
         try:
-            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            resp = requests.post(
+                self.api_url, json.dumps(request_json), headers=self.header
+            )
             if "data" in resp.json():
                 data = resp.json()["data"]
                 file_to_save = open(output_file, "wb")
                 file_to_save.write(base64.b64decode(data))
             else:
-                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+                raise Exception(
+                    f"{__name__} status_code: {resp.status_code} response: {resp.content}"
+                )
         except Exception as e:
             raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -24,7 +24,7 @@ class ServeReferenceAudio(BaseModel):
     def decode_audio(cls, values):
         audio = values.get("audio")
         if (
-                isinstance(audio, str) and len(audio) > 255
+            isinstance(audio, str) and len(audio) > 255
         ):  # Check if audio is a string (Base64)
             try:
                 values["audio"] = base64.b64decode(audio)
@@ -107,7 +107,10 @@ class TTSProvider(TTSProviderBase):
         self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
 
     def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+        return os.path.join(
+            self.output_file,
+            f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        )
 
     async def text_to_speak(self, text, output_file):
         # Prepare reference data
@@ -117,9 +120,7 @@ class TTSProvider(TTSProviderBase):
         data = {
             "text": text,
             "references": [
-                ServeReferenceAudio(
-                    audio=audio if audio else b"", text=text
-                )
+                ServeReferenceAudio(audio=audio if audio else b"", text=text)
                 for text, audio in zip(ref_texts, byte_audios)
             ],
             "reference_id": self.reference_id,
@@ -139,7 +140,9 @@ class TTSProvider(TTSProviderBase):
 
         response = requests.post(
             self.api_url,
-            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+            data=ormsgpack.packb(
+                pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC
+            ),
             headers={
                 "Authorization": f"Bearer {self.api_key}",
                 "Content-Type": "application/msgpack",
@@ -152,8 +155,6 @@ class TTSProvider(TTSProviderBase):
             with open(output_file, "wb") as audio_file:
                 audio_file.write(audio_content)
 
-
-
         else:
             print(f"Request failed with status code {response.status_code}")
             print(response.json())

main/xiaozhi-server/core/providers/tts/openai.py

@@ -4,6 +4,11 @@ import requests
 from datetime import datetime
 from core.utils.util import check_model_key
 from core.providers.tts.base import TTSProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
 
 class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
@@ -18,23 +23,28 @@ class TTSProvider(TTSProviderBase):
         check_model_key("TTS", self.api_key)
 
     def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+        return os.path.join(
+            self.output_file,
+            f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        )
 
     async def text_to_speak(self, text, output_file):
         headers = {
             "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json"
+            "Content-Type": "application/json",
         }
         data = {
             "model": self.model,
             "input": text,
             "voice": self.voice,
             "response_format": "wav",
-            "speed": self.speed
+            "speed": self.speed,
         }
         response = requests.post(self.api_url, json=data, headers=headers)
         if response.status_code == 200:
             with open(output_file, "wb") as audio_file:
                 audio_file.write(response.content)
         else:
-            raise Exception(f"OpenAI TTS请求失败: {response.status_code} - {response.text}")
+            raise Exception(
+                f"OpenAI TTS请求失败: {response.status_code} - {response.text}"
+            )

main/xiaozhi-server/core/providers/vad/base.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class VADProviderBase(ABC):
+    @abstractmethod
+    def is_vad(self, conn, data) -> bool:
+        """检测音频数据中的语音活动"""
+        pass

main/xiaozhi-server/core/providers/vad/silero.py

@@ -0,0 +1,63 @@
+import time
+import numpy as np
+import torch
+import opuslib_next
+from config.logger import setup_logging
+from core.providers.vad.base import VADProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class VADProvider(VADProviderBase):
+    def __init__(self, config):
+        logger.bind(tag=TAG).info("SileroVAD", config)
+        self.model, self.utils = torch.hub.load(
+            repo_or_dir=config["model_dir"],
+            source="local",
+            model="silero_vad",
+            force_reload=False,
+        )
+        (get_speech_timestamps, _, _, _, _) = self.utils
+
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.vad_threshold = config.get("threshold")
+        self.silence_threshold_ms = config.get("min_silence_duration_ms")
+
+    def is_vad(self, conn, opus_packet):
+        try:
+            pcm_frame = self.decoder.decode(opus_packet, 960)
+            conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
+
+            # 处理缓冲区中的完整帧（每次处理512采样点）
+            client_have_voice = False
+            while len(conn.client_audio_buffer) >= 512 * 2:
+                # 提取前512个采样点（1024字节）
+                chunk = conn.client_audio_buffer[: 512 * 2]
+                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :]
+
+                # 转换为模型需要的张量格式
+                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+                audio_float32 = audio_int16.astype(np.float32) / 32768.0
+                audio_tensor = torch.from_numpy(audio_float32)
+
+                # 检测语音活动
+                speech_prob = self.model(audio_tensor, 16000).item()
+                client_have_voice = speech_prob >= self.vad_threshold
+
+                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
+                if conn.client_have_voice and not client_have_voice:
+                    stop_duration = (
+                        time.time() * 1000 - conn.client_have_voice_last_time
+                    )
+                    if stop_duration >= self.silence_threshold_ms:
+                        conn.client_voice_stop = True
+                if client_have_voice:
+                    conn.client_have_voice = True
+                    conn.client_have_voice_last_time = time.time() * 1000
+
+            return client_have_voice
+        except opuslib_next.OpusError as e:
+            logger.bind(tag=TAG).info(f"解码错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")