mergin main

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -0,0 +1,164 @@
+import time
+import wave
+import os
+import sys
+import io
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+import uuid
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+import numpy as np
+import sherpa_onnx
+
+from modelscope.hub.file_download import model_file_download
+
+TAG = __name__
+logger = setup_logging()
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+        
+        # 初始化模型文件路径
+        model_files = {
+            "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
+            "tokens.txt": os.path.join(self.model_dir, "tokens.txt")
+        }
+
+        # 下载并检查模型文件
+        try:
+            for file_name, file_path in model_files.items():
+                if not os.path.isfile(file_path):
+                    logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}")
+                    model_file_download(
+                        model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
+                        file_path=file_name,
+                        local_dir=self.model_dir
+                    )
+                    
+                    if not os.path.isfile(file_path):
+                        raise FileNotFoundError(f"模型文件下载失败: {file_path}")
+                        
+            self.model_path = model_files["model.int8.onnx"]
+            self.tokens_path = model_files["tokens.txt"]
+            
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
+            raise
+
+        with CaptureOutput():
+            self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+                model=self.model_path,
+                tokens=self.tokens_path,
+                num_threads=2,
+                sample_rate=16000,
+                feature_dim=80,
+                decoding_method="greedy_search",
+                debug=False,
+                use_itn=True,
+            )
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
+        """
+        Args:
+        wave_filename:
+            Path to a wave file. It should be single channel and each sample should
+            be 16-bit. Its sample rate does not need to be 16kHz.
+        Returns:
+        Return a tuple containing:
+        - A 1-D array of dtype np.float32 containing the samples, which are
+        normalized to the range [-1, 1].
+        - sample rate of the wave file
+        """
+
+        with wave.open(wave_filename) as f:
+            assert f.getnchannels() == 1, f.getnchannels()
+            assert f.getsampwidth() == 2, f.getsampwidth()  # it is in bytes
+            num_samples = f.getnframes()
+            samples = f.readframes(num_samples)
+            samples_int16 = np.frombuffer(samples, dtype=np.int16)
+            samples_float32 = samples_int16.astype(np.float32)
+
+            samples_float32 = samples_float32 / 32768
+            return samples_float32, f.getframerate()
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        file_path = None
+        try:
+            # 保存音频文件
+            start_time = time.time()
+            file_path = self.save_audio_to_file(opus_data, session_id)
+            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+
+            # 语音识别
+            start_time = time.time()
+            s = self.model.create_stream()
+            samples, sample_rate = self.read_wave(file_path)
+            s.accept_waveform(sample_rate, samples)
+            self.model.decode_stream(s)
+            text = s.result.text
+            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+
+            return text, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None
+
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -9,6 +9,7 @@ logger = setup_logging()
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
+        self.mode = config.get("mode", "chat-messages")
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
 
@@ -19,27 +20,59 @@ class LLMProvider(LLMProviderBase):
             conversation_id = self.session_conversation_map.get(session_id)
 
             # 发起流式请求
-            with requests.post(
-                    f"{self.base_url}/chat-messages",
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json={
+            if self.mode == "chat-messages":
+                request_json = {
                         "query": last_msg["content"],
                         "response_mode": "streaming",
                         "user": session_id,
                         "inputs": {},
                         "conversation_id": conversation_id
-                    },
+                    }
+            elif self.mode == "workflows/run":
+                request_json = {
+                    "inputs": {"query": last_msg["content"]},
+                    "response_mode": "streaming",
+                    "user": session_id
+                }
+            elif self.mode == "completion-messages":
+                request_json = {
+                    "inputs": {"query": last_msg["content"]},
+                    "response_mode": "streaming",
+                    "user": session_id
+                }
+
+            with requests.post(
+                    f"{self.base_url}/{self.mode}",
+                    headers={"Authorization": f"Bearer {self.api_key}"},
+                    json=request_json,
                     stream=True
             ) as r:
-                for line in r.iter_lines():
-                    if line.startswith(b'data: '):
-                        event = json.loads(line[6:])
-                        # 如果没有找到conversation_id，则获取此次conversation_id
-                        if not conversation_id:
-                            conversation_id = event.get('conversation_id')
-                            self.session_conversation_map[session_id] = conversation_id  # 更新映射
-                        if event.get('answer'):
-                            yield event['answer']
+                if self.mode == "chat-messages":
+                    for line in r.iter_lines():
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            # 如果没有找到conversation_id，则获取此次conversation_id
+                            if not conversation_id:
+                                conversation_id = event.get('conversation_id')
+                                self.session_conversation_map[session_id] = conversation_id  # 更新映射
+                            if event.get('answer'):
+                                yield event['answer']
+                elif self.mode == "workflows/run":
+                    for line in r.iter_lines():
+                        # logger.bind(tag=TAG).info(f"chat message response: {line}")
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            if event.get('event') == "workflow_finished":
+                                if event['data']['status'] == "succeeded":
+                                    yield event['data']['outputs']['answer']
+                                else:
+                                    yield "【服务响应异常】"
+                elif self.mode == "completion-messages":
+                    for line in r.iter_lines():
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            if event.get('answer'):
+                                yield event['answer'] 
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -1,14 +1,19 @@
 import google.generativeai as genai
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
-
+from config.logger import setup_logging
+import requests
+import json
+TAG = __name__
+logger = setup_logging()
 
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         """初始化Gemini LLM Provider"""
         self.model_name = config.get("model_name", "gemini-1.5-pro")
         self.api_key = config.get("api_key")
-
+        self.http_proxy=config.get("http_proxy")
+        self.https_proxy = config.get("https_proxy")
         have_key = check_model_key("LLM", self.api_key)
 
         if not have_key:
@@ -16,6 +21,19 @@ class LLMProvider(LLMProviderBase):
 
         try:
             # 初始化Gemini客户端
+            # 配置代理（如果提供了代理配置）
+            self.proxies=None
+            if self.http_proxy is not "" or self.https_proxy is not "":
+
+                self.proxies = {
+                    "http": self.http_proxy,
+                    "https": self.https_proxy,
+                }
+                logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
+                # 使用猴子补丁修改 google-generativeai 库的请求会话
+
+                # 使用 session 对象配置 genai
+
             genai.configure(api_key=self.api_key)
             self.model = genai.GenerativeModel(self.model_name)
 
@@ -46,26 +64,54 @@ class LLMProvider(LLMProviderBase):
                 if content:
                     chat_history.append({
                         "role": role,
-                        "parts": [content]
+                        "parts": [{"text":content}]
+
                     })
 
             # 获取当前消息
             current_msg = dialogue[-1]["content"]
 
-            # 创建新的聊天会话
-            chat = self.model.start_chat(history=chat_history)
+            # 构建请求体
+            request_body = {
+                "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
+                "generationConfig": self.generation_config
+            }
 
-            # 发送消息并获取流式响应
-            response = chat.send_message(
-                current_msg,
-                stream=True,
-                generation_config=self.generation_config
-            )
+            # 构建请求URL
+            url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
 
-            # 处理流式响应
-            for chunk in response:
-                if hasattr(chunk, 'text') and chunk.text:
-                    yield chunk.text
+            # 构建请求头
+            headers = {
+                "Content-Type": "application/json",
+            }
+
+            # 发送POST请求,经测试手动 request 无法使用 stream 模式
+            if self.proxies:
+                response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
+                try:
+                    data = response.json()  # 直接解析JSON
+                    if 'candidates' in data and data['candidates']:
+                        yield data['candidates'][0]['content']['parts'][0]['text']
+                    else:
+                        yield "未找到候选回复。"
+                except json.JSONDecodeError as e:
+                    yield f"JSON解码错误：{e}"
+                except Exception as e:
+                    yield f"发生错误：{e}"
+            else:
+                logger.bind(tag=TAG).info(f"Gemini stream mode ")
+                chat = self.model.start_chat(history=chat_history)
+
+                # 发送消息并获取流式响应
+                response = chat.send_message(
+                    current_msg,
+                    stream=True,
+                    generation_config=self.generation_config
+                )
+                # 处理流式响应
+                for chunk in response:
+                    if hasattr(chunk, 'text') and chunk.text:
+                        yield chunk.text
 
         except Exception as e:
             error_msg = str(e)
@@ -78,3 +124,13 @@ class LLMProvider(LLMProviderBase):
                 yield "【Gemini API key无效】"
             else:
                 yield f"【Gemini服务响应异常: {error_msg}】"
+
+
+
+
+        except requests.exceptions.RequestException as e:
+            yield f"请求失败：{e}"
+        except json.JSONDecodeError as e:
+            yield f"JSON解码错误：{e}"
+        except Exception as e:
+            yield f"发生错误：{e}"

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -1,8 +1,11 @@
 import openai
+from config.logger import setup_logging
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
 
 TAG = __name__
+logger = setup_logging()
+
 
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
@@ -12,6 +15,8 @@ class LLMProvider(LLMProviderBase):
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
+        self.max_tokens = config.get("max_tokens", 500)
+
         check_model_key("LLM", self.api_key)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
 
@@ -20,9 +25,10 @@ class LLMProvider(LLMProviderBase):
             responses = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
-                stream=True
+                stream=True,
+                max_tokens=self.max_tokens,
             )
-            
+
             is_active = True
             for chunk in responses:
                 try:
@@ -43,7 +49,7 @@ class LLMProvider(LLMProviderBase):
                         yield content
 
         except Exception as e:
-            self.logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
@@ -51,12 +57,12 @@ class LLMProvider(LLMProviderBase):
                 model=self.model_name,
                 messages=dialogue,
                 stream=True,
-                tools=functions,
+                tools=functions
             )
-            
+
             for chunk in stream:
                 yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
-                
+
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
-            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
+            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}