update:合并非tts代码

main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py

@@ -2,10 +2,12 @@ from config.logger import setup_logging
 from http import HTTPStatus
 from dashscope import Application
 from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
 
+
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
@@ -13,27 +15,32 @@ class LLMProvider(LLMProviderBase):
         self.base_url = config.get("base_url")
         self.is_No_prompt = config.get("is_no_prompt")
         self.memory_id = config.get("ali_memory_id")
+        check_model_key("AliBLLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
             # 处理dialogue
             if self.is_No_prompt:
                 dialogue.pop(0)
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】处理后的dialogue: {dialogue}"
+                )
 
             # 构造调用参数
             call_params = {
                 "api_key": self.api_key,
                 "app_id": self.app_id,
                 "session_id": session_id,
-                "messages": dialogue
+                "messages": dialogue,
             }
             if self.memory_id != False:
                 # 百练memory需要prompt参数
                 prompt = dialogue[-1].get("content")
                 call_params["memory_id"] = self.memory_id
                 call_params["prompt"] = prompt
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】处理后的prompt: {prompt}"
+                )
 
             responses = Application.call(**call_params)
             if responses.status_code != HTTPStatus.OK:
@@ -44,9 +51,16 @@ class LLMProvider(LLMProviderBase):
                 )
                 yield "【阿里百练API服务响应异常】"
             else:
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】构造参数: {call_params}"
+                )
                 yield responses.output.text
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
             yield "【LLM服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"阿里百练暂未实现完整的工具调用（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/base.py

@@ -35,4 +35,5 @@ class LLMProviderBase(ABC):
         """
         # For providers that don't support functions, just return regular response
         for token in self.response(session_id, dialogue):
-            yield {"type": "content", "content": token}
+            yield token, None
+

main/xiaozhi-server/core/providers/llm/coze/coze.py

@@ -1,12 +1,17 @@
 from config.logger import setup_logging
-import requests
 import json
-import re
 from core.providers.llm.base import LLMProviderBase
-import os
+
 # official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
 from cozepy import COZE_CN_BASE_URL
-from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa
+from cozepy import (
+    Coze,
+    TokenAuth,
+    Message,
+    ChatEventType,
+)  # noqa
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -15,25 +20,23 @@ logger = setup_logging()
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.personal_access_token = config.get("personal_access_token")
-        self.bot_id = config.get("bot_id")
-        self.user_id = config.get("user_id")
+        self.bot_id = str(config.get("bot_id"))
+        self.user_id = str(config.get("user_id"))
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        check_model_key("CozeLLM", self.personal_access_token)
 
     def response(self, session_id, dialogue):
         coze_api_token = self.personal_access_token
         coze_api_base = COZE_CN_BASE_URL
 
         last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-        
+
         coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
         conversation_id = self.session_conversation_map.get(session_id)
 
         # 如果没有找到conversation_id，则创建新的对话
         if not conversation_id:
-            conversation = coze.conversations.create(
-                messages=[
-                ]
-            )
+            conversation = coze.conversations.create(messages=[])
             conversation_id = conversation.id
             self.session_conversation_map[session_id] = conversation_id  # 更新映射
 
@@ -47,4 +50,24 @@ class LLMProvider(LLMProviderBase):
         ):
             if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
                 print(event.message.content, end="", flush=True)
-                yield event.message.content
+                yield event.message.content
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -2,6 +2,8 @@ import json
 from config.logger import setup_logging
 import requests
 from core.providers.llm.base import LLMProviderBase
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -13,6 +15,7 @@ class LLMProvider(LLMProviderBase):
         self.mode = config.get("mode", "chat-messages")
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        check_model_key("DifyLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
@@ -58,7 +61,10 @@ class LLMProvider(LLMProviderBase):
                                 self.session_conversation_map[session_id] = (
                                     conversation_id  # 更新映射
                                 )
-                            if event.get("answer"):
+                            # 过滤 message_replace 事件，此事件会全量推一次
+                            if event.get("event") != "message_replace" and event.get(
+                                "answer"
+                            ):
                                 yield event["answer"]
                 elif self.mode == "workflows/run":
                     for line in r.iter_lines():
@@ -73,9 +79,32 @@ class LLMProvider(LLMProviderBase):
                     for line in r.iter_lines():
                         if line.startswith(b"data: "):
                             event = json.loads(line[6:])
-                            if event.get("answer"):
+                            # 过滤 message_replace 事件，此事件会全量推一次
+                            if event.get("event") != "message_replace" and event.get(
+                                "answer"
+                            ):
                                 yield event["answer"]
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
             yield "【服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py

@@ -2,6 +2,7 @@ import json
 from config.logger import setup_logging
 import requests
 from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -13,6 +14,7 @@ class LLMProvider(LLMProviderBase):
         self.base_url = config.get("base_url")
         self.detail = config.get("detail", False)
         self.variables = config.get("variables", {})
+        check_model_key("FastGPTLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
@@ -21,37 +23,36 @@ class LLMProvider(LLMProviderBase):
 
             # 发起流式请求
             with requests.post(
-                    f"{self.base_url}/chat/completions",
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json={
-                        "stream": True,
-                        "chatId": session_id,
-                        "detail": self.detail,
-                        "variables": self.variables,
-                        "messages": [
-                            {
-                                "role": "user",
-                                "content": last_msg["content"]
-                            }
-                        ]
-                    },
-                    stream=True
+                f"{self.base_url}/chat/completions",
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                json={
+                    "stream": True,
+                    "chatId": session_id,
+                    "detail": self.detail,
+                    "variables": self.variables,
+                    "messages": [{"role": "user", "content": last_msg["content"]}],
+                },
+                stream=True,
             ) as r:
                 for line in r.iter_lines():
                     if line:
                         try:
-                            if line.startswith(b'data: '):
-                                if line[6:].decode('utf-8') == '[DONE]':
+                            if line.startswith(b"data: "):
+                                if line[6:].decode("utf-8") == "[DONE]":
                                     break
 
                                 data = json.loads(line[6:])
-                                if 'choices' in data and len(data['choices']) > 0:
-                                    delta = data['choices'][0].get('delta', {})
-                                    if delta and 'content' in delta and delta['content'] is not None:
-                                        content = delta['content']
-                                        if '<think>' in content:
+                                if "choices" in data and len(data["choices"]) > 0:
+                                    delta = data["choices"][0].get("delta", {})
+                                    if (
+                                        delta
+                                        and "content" in delta
+                                        and delta["content"] is not None
+                                    ):
+                                        content = delta["content"]
+                                        if "<think>" in content:
                                             continue
-                                        if '</think>' in content:
+                                        if "</think>" in content:
                                             continue
                                         yield content
 
@@ -62,4 +63,9 @@ class LLMProvider(LLMProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
-            yield "【服务响应异常】"
+            yield "【服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"fastgpt暂未实现完整的工具调用（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -1,136 +1,205 @@
-import google.generativeai as genai
-from core.utils.util import check_model_key
-from core.providers.llm.base import LLMProviderBase
-from config.logger import setup_logging
+import os, json, uuid
+from types import SimpleNamespace
+from typing import Any, Dict, List
+
 import requests
-import json
+from google import generativeai as genai
+from google.generativeai import types, GenerationConfig
+
+from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
+from config.logger import setup_logging
+from google.generativeai.types import GenerateContentResponse
+from requests import RequestException
+
+log = setup_logging()
 TAG = __name__
-logger = setup_logging()
+
+
+def test_proxy(proxy_url: str, test_url: str) -> bool:
+    try:
+        resp = requests.get(test_url, proxies={"http": proxy_url, "https": proxy_url})
+        return 200 <= resp.status_code < 400
+    except RequestException:
+        return False
+
+
+def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
+    """
+    分别测试 HTTP 和 HTTPS 代理是否可用，并设置环境变量。
+    如果 HTTPS 代理不可用但 HTTP 可用，会将 HTTPS_PROXY 也指向 HTTP。
+    """
+    test_http_url = "http://www.google.com"
+    test_https_url = "https://www.google.com"
+
+    ok_http = ok_https = False
+
+    if http_proxy:
+        ok_http = test_proxy(http_proxy, test_http_url)
+        if ok_http:
+            os.environ["HTTP_PROXY"] = http_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}")
+        else:
+            log.bind(tag=TAG).warning(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
+
+    if https_proxy:
+        ok_https = test_proxy(https_proxy, test_https_url)
+        if ok_https:
+            os.environ["HTTPS_PROXY"] = https_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}")
+        else:
+            log.bind(tag=TAG).warning(
+                f"配置提供的Gemini HTTPS代理不可用: {https_proxy}"
+            )
+
+    # 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy
+    if ok_http and not ok_https:
+        if test_proxy(http_proxy, test_https_url):
+            os.environ["HTTPS_PROXY"] = http_proxy
+            ok_https = True
+            log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}")
+
+    if not ok_http and not ok_https:
+        log.bind(tag=TAG).error(
+            f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置"
+        )
+        raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置")
+
 
 class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        """初始化Gemini LLM Provider"""
-        self.model_name = config.get("model_name", "gemini-1.5-pro")
-        self.api_key = config.get("api_key")
-        self.http_proxy=config.get("http_proxy")
-        self.https_proxy = config.get("https_proxy")
-        have_key = check_model_key("LLM", self.api_key)
+    def __init__(self, cfg: Dict[str, Any]):
+        self.model_name = cfg.get("model_name", "gemini-2.0-flash")
+        self.api_key = cfg["api_key"]
+        http_proxy = cfg.get("http_proxy")
+        https_proxy = cfg.get("https_proxy")
 
-        if not have_key:
-            return
+        if not check_model_key("LLM", self.api_key):
+            raise ValueError("无效的Gemini API Key，请检查是否配置正确")
 
-        try:
-            # 初始化Gemini客户端
-            # 配置代理（如果提供了代理配置）
-            self.proxies=None
-            if self.http_proxy is not "" or self.https_proxy is not "":
+        if http_proxy or https_proxy:
+            log.bind(tag=TAG).info(
+                f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境..."
+            )
+            setup_proxy_env(http_proxy, https_proxy)
+            log.bind(tag=TAG).info(
+                f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}"
+            )
+        genai.configure(api_key=self.api_key)
+        self.model = genai.GenerativeModel(self.model_name)
 
-                self.proxies = {
-                    "http": self.http_proxy,
-                    "https": self.https_proxy,
-                }
-                logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
-                # 使用猴子补丁修改 google-generativeai 库的请求会话
+        self.gen_cfg = GenerationConfig(
+            temperature=0.7,
+            top_p=0.9,
+            top_k=40,
+            max_output_tokens=2048,
+        )
 
-                # 使用 session 对象配置 genai
-
-            genai.configure(api_key=self.api_key)
-            self.model = genai.GenerativeModel(self.model_name)
-
-            # 设置生成参数
-            self.generation_config = {
-                "temperature": 0.7,
-                "top_p": 0.9,
-                "top_k": 40,
-                "max_output_tokens": 2048,
-            }
-            self.chat = None
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
-            self.model = None
+    @staticmethod
+    def _build_tools(funcs: List[Dict[str, Any]] | None):
+        if not funcs:
+            return None
+        return [
+            types.Tool(
+                function_declarations=[
+                    types.FunctionDeclaration(
+                        name=f["function"]["name"],
+                        description=f["function"]["description"],
+                        parameters=f["function"]["parameters"],
+                    )
+                    for f in funcs
+                ]
+            )
+        ]
 
+    # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
     def response(self, session_id, dialogue):
-        """生成Gemini对话响应"""
-        if not self.model:
-            yield "【Gemini服务未正确初始化】"
-            return
+        yield from self._generate(dialogue, None)
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        yield from self._generate(dialogue, self._build_tools(functions))
+
+    def _generate(self, dialogue, tools):
+        role_map = {"assistant": "model", "user": "user"}
+        contents: list = []
+        # 拼接对话
+        for m in dialogue:
+            r = m["role"]
+
+            if r == "assistant" and "tool_calls" in m:
+                tc = m["tool_calls"][0]
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [
+                            {
+                                "function_call": {
+                                    "name": tc["function"]["name"],
+                                    "args": json.loads(tc["function"]["arguments"]),
+                                }
+                            }
+                        ],
+                    }
+                )
+                continue
+
+            if r == "tool":
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [{"text": str(m.get("content", ""))}],
+                    }
+                )
+                continue
+
+            contents.append(
+                {
+                    "role": role_map.get(r, "user"),
+                    "parts": [{"text": str(m.get("content", ""))}],
+                }
+            )
+
+        stream: GenerateContentResponse = self.model.generate_content(
+            contents=contents,
+            generation_config=self.gen_cfg,
+            tools=tools,
+            stream=True,
+        )
 
         try:
-            # 处理对话历史
-            chat_history = []
-            for msg in dialogue[:-1]:  # 历史对话
-                role = "model" if msg["role"] == "assistant" else "user"
-                content = msg["content"].strip()
-                if content:
-                    chat_history.append({
-                        "role": role,
-                        "parts": [{"text":content}]
+            for chunk in stream:
+                cand = chunk.candidates[0]
+                for part in cand.content.parts:
+                    # a) 函数调用-通常是最后一段话才是函数调用
+                    if getattr(part, "function_call", None):
+                        fc = part.function_call
+                        yield None, [
+                            SimpleNamespace(
+                                id=uuid.uuid4().hex,
+                                type="function",
+                                function=SimpleNamespace(
+                                    name=fc.name,
+                                    arguments=json.dumps(
+                                        dict(fc.args), ensure_ascii=False
+                                    ),
+                                ),
+                            )
+                        ]
+                        return
+                    # b) 普通文本
+                    if getattr(part, "text", None):
+                        yield part.text if tools is None else (part.text, None)
 
-                    })
+        finally:
+            if tools is not None:
+                yield None, None  # function‑mode 结束，返回哑包
 
-            # 获取当前消息
-            current_msg = dialogue[-1]["content"]
-
-            # 构建请求体
-            request_body = {
-                "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
-                "generationConfig": self.generation_config
-            }
-
-            # 构建请求URL
-            url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
-
-            # 构建请求头
-            headers = {
-                "Content-Type": "application/json",
-            }
-
-            # 发送POST请求,经测试手动 request 无法使用 stream 模式
-            if self.proxies:
-                response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
-                try:
-                    data = response.json()  # 直接解析JSON
-                    if 'candidates' in data and data['candidates']:
-                        yield data['candidates'][0]['content']['parts'][0]['text']
-                    else:
-                        yield "未找到候选回复。"
-                except json.JSONDecodeError as e:
-                    yield f"JSON解码错误：{e}"
-                except Exception as e:
-                    yield f"发生错误：{e}"
-            else:
-                logger.bind(tag=TAG).info(f"Gemini stream mode ")
-                chat = self.model.start_chat(history=chat_history)
-
-                # 发送消息并获取流式响应
-                response = chat.send_message(
-                    current_msg,
-                    stream=True,
-                    generation_config=self.generation_config
-                )
-                # 处理流式响应
-                for chunk in response:
-                    if hasattr(chunk, 'text') and chunk.text:
-                        yield chunk.text
-
-        except Exception as e:
-            error_msg = str(e)
-            logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
-
-            # 针对不同错误返回友好提示
-            if "Rate limit" in error_msg:
-                yield "【Gemini服务请求太频繁,请稍后再试】"
-            elif "Invalid API key" in error_msg:
-                yield "【Gemini API key无效】"
-            else:
-                yield f"【Gemini服务响应异常: {error_msg}】"
-
-
-
-
-        except requests.exceptions.RequestException as e:
-            yield f"请求失败：{e}"
-        except json.JSONDecodeError as e:
-            yield f"JSON解码错误：{e}"
-        except Exception as e:
-            yield f"发生错误：{e}"
+    # 关闭stream，预留后续打断对话功能的功能方法，官方文档推荐打断对话要关闭上一个流，可以有效减少配额计费和资源占用
+    @staticmethod
+    def _safe_finish_stream(stream: GenerateContentResponse):
+        if hasattr(stream, "resolve"):
+            stream.resolve()  # Gemini SDK version ≥ 0.5.0
+        elif hasattr(stream, "close"):
+            stream.close()  # Gemini SDK version < 0.5.0
+        else:
+            for _ in stream:  # 兜底耗尽
+                pass

main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py

@@ -0,0 +1,71 @@
+import requests
+from requests.exceptions import RequestException
+from config.logger import setup_logging
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.agent_id = config.get("agent_id")  # 对应 agent_id
+        self.api_key = config.get("api_key")
+        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
+        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
+
+    def response(self, session_id, dialogue):
+        try:
+            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
+
+            # 提取最后一个 role 为 'user' 的 content
+            input_text = None
+            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
+                # 逆序遍历，找到最后一个 role 为 'user' 的消息
+                for message in reversed(dialogue):
+                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
+                        input_text = message.get("content", "")
+                        break  # 找到后立即退出循环
+
+            # 构造请求数据
+            payload = {
+                "text": input_text,
+                "agent_id": self.agent_id,
+                "conversation_id": session_id,  # 使用 session_id 作为 conversation_id
+            }
+            # 设置请求头
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            }
+
+            # 发起 POST 请求
+            response = requests.post(self.api_url, json=payload, headers=headers)
+
+            # 检查请求是否成功
+            response.raise_for_status()
+
+            # 解析返回数据
+            data = response.json()
+            speech = (
+                data.get("response", {})
+                .get("speech", {})
+                .get("plain", {})
+                .get("speech", "")
+            )
+
+            # 返回生成的内容
+            if speech:
+                yield speech
+            else:
+                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
+
+        except RequestException as e:
+            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"homeassistant不支持（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -21,27 +21,67 @@ class LLMProvider(LLMProviderBase):
             api_key="ollama"  # Ollama doesn't need an API key but OpenAI client requires one
         )
 
+        # 检查是否是qwen3模型
+        self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
+
     def response(self, session_id, dialogue):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             responses = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
                 stream=True
             )
-            is_active=True
+            is_active = True
+            # 用于处理跨chunk的标签
+            buffer = ""
+
             for chunk in responses:
                 try:
                     delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
                     content = delta.content if hasattr(delta, 'content') else ''
+
                     if content:
-                        if '<think>' in content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
                             is_active = False
-                            content = content.split('<think>')[0]
-                        if '</think>' in content:
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
                             is_active = True
-                            content = content.split('</think>')[-1]
-                        if is_active:
-                            yield content
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer
+                            buffer = ""  # 清空缓冲区
+
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
 
@@ -51,6 +91,22 @@ class LLMProvider(LLMProviderBase):
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             stream = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
@@ -58,9 +114,50 @@ class LLMProvider(LLMProviderBase):
                 tools=functions,
             )
 
+            is_active = True
+            buffer = ""
+
             for chunk in stream:
-                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else None
+                    tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None
+
+                    # 如果是工具调用，直接传递
+                    if tool_calls:
+                        yield None, tool_calls
+                        continue
+
+                    # 处理文本内容
+                    if content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
+                            is_active = False
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
+                            is_active = True
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer, None
+                            buffer = ""  # 清空缓冲区
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
+                    continue
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
-            yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}
+            yield f"【Ollama服务响应异常: {str(e)}】", None

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -1,4 +1,5 @@
 import openai
+from openai.types import CompletionUsage
 from config.logger import setup_logging
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
@@ -11,11 +12,19 @@ class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.model_name = config.get("model_name")
         self.api_key = config.get("api_key")
-        if 'base_url' in config:
+        if "base_url" in config:
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
-        self.max_tokens = config.get("max_tokens", 500)
+        max_tokens = config.get("max_tokens")
+        if max_tokens is None or max_tokens == "":
+            max_tokens = 500
+
+        try:
+            max_tokens = int(max_tokens)
+        except (ValueError, TypeError):
+            max_tokens = 500
+        self.max_tokens = max_tokens
 
         check_model_key("LLM", self.api_key)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
@@ -33,18 +42,22 @@ class LLMProvider(LLMProviderBase):
             for chunk in responses:
                 try:
                     # 检查是否存在有效的choice且content不为空
-                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
-                    content = delta.content if hasattr(delta, 'content') else ''
+                    delta = (
+                        chunk.choices[0].delta
+                        if getattr(chunk, "choices", None)
+                        else None
+                    )
+                    content = delta.content if hasattr(delta, "content") else ""
                 except IndexError:
-                    content = ''
+                    content = ""
                 if content:
                     # 处理标签跨多个chunk的情况
-                    if '<think>' in content:
+                    if "<think>" in content:
                         is_active = False
-                        content = content.split('<think>')[0]
-                    if '</think>' in content:
+                        content = content.split("<think>")[0]
+                    if "</think>" in content:
                         is_active = True
-                        content = content.split('</think>')[-1]
+                        content = content.split("</think>")[-1]
                     if is_active:
                         yield content
 
@@ -54,15 +67,22 @@ class LLMProvider(LLMProviderBase):
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
             stream = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                tools=functions
+                model=self.model_name, messages=dialogue, stream=True, tools=functions
             )
 
             for chunk in stream:
-                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                # 检查是否存在有效的choice且content不为空
+                if getattr(chunk, "choices", None):
+                    yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                # 存在 CompletionUsage 消息时，生成 Token 消耗 log
+                elif isinstance(getattr(chunk, 'usage', None), CompletionUsage):
+                    usage_info = getattr(chunk, 'usage', None)
+                    logger.bind(tag=TAG).info(
+                        f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，" 
+                        f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
+                        f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
+                    )
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
-            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
+            yield f"【OpenAI服务响应异常: {e}】", None

main/xiaozhi-server/core/providers/llm/system_prompt.py

@@ -0,0 +1,103 @@
+def get_system_prompt_for_function(functions: str) -> str:
+    """
+    生成系统提示信息
+    :param functions: 可用的函数列表
+    :return: 系统提示信息
+    """
+
+    SYSTEM_PROMPT = f"""
+====
+
+TOOL USE
+
+You have access to a set of tools that are executed upon the user's approval. You can use one tool per message, and will receive the result of that tool use in the user's response. 
+You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use.
+
+# Tool Use Formatting
+
+Tool use is formatted using JSON-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags. 
+Here's the structure:
+
+<tool_call>
+{{
+    "name": "function name",
+    "arguments": {{
+        "param1": "value1",
+        "param2": "value2",
+        // Add more parameters as needed, if parameters are required, you must provide them
+    }}
+}}
+<tool_call>
+
+For example:
+if you got tool as follow
+
+{{
+    "type": "function",
+    "function": {{
+        "name": "handle_exit_intent",
+        "description": "当用户想结束对话或需要退出系统时调用",
+        "parameters": {{
+            "type": "object",
+            "properties": {{
+                "say_goodbye": {{
+                    "type": "string",
+                    "description": "和用户友好结束对话的告别语",
+                }}
+            }},
+            "required": ["say_goodbye"],
+        }},
+    }},
+}}
+
+you should respond with the following format:
+
+<tool_call>
+{{
+    "name": "handle_exit_intent",
+    "arguments": {{
+        "say_goodbye": "再见，祝您生活愉快！"
+    }}
+}}
+</tool_call>
+
+
+Always adhere to this format for the tool use to ensure proper parsing and execution.
+
+# Tools
+
+{functions}
+
+# Tool Use Guidelines
+
+1. Tools must be called in a separate message, Do not add thoughts when calling tools. The message must start with <tool_call> and end with </tool_call>, with the tool invocation JSON data in between. No additional response content is needed.
+2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information. 
+   For example using the list_files tool is more effective than running a command like \`ls\` in the terminal. It's critical that you think about each available tool and use the one that best fits the current step in the task.
+3. If multiple actions are needed, use one tool at a time per message to accomplish the task iteratively, with each tool use being informed by the result of the previous tool use. Do not assume the outcome of any tool use. 
+   Each step must be informed by the previous step's result.
+4. Formulate your tool use using the JSON format specified for each tool.
+5. After each tool use, the user will respond with the result of that tool use. This result will provide you with the necessary information to continue your task or make further decisions. This response may include:
+- Information about whether the tool succeeded or failed, along with any reasons for failure.
+- Linter errors that may have arisen due to the changes you made, which you'll need to address.
+- New terminal output in reaction to the changes, which you may need to consider or act upon.
+- Any other relevant feedback or information related to the tool use.
+6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user.
+7. Tool calls should contain no extra information. Only after receiving the tool's response should you integrate it into a complete reply.
+
+It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to:
+1. Confirm the success of each step before proceeding.
+2. Address any issues or errors that arise immediately.
+3. Adapt your approach based on new information or unexpected results.
+4. Ensure that each action builds correctly on the previous ones.
+
+By waiting for and carefully considering the user's response after each tool use, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work.
+
+====
+
+USER CHAT CONTENT
+
+The following additional message is the user's chat message, and should be followed to the best of your ability without interfering with the TOOL USE guidelines.
+
+"""
+
+    return SYSTEM_PROMPT