Merge branch 'main' into mcp-client

# Conflicts:
#	main/xiaozhi-server/core/connection.py
#	main/xiaozhi-server/requirements.txt

main/xiaozhi-server/core/connection.py

@@ -9,7 +9,7 @@ import traceback
 import threading
 import websockets
 from typing import Dict, Any
-import plugins_func.loadplugins
+from plugins_func.loadplugins import auto_import_modules
 from config.logger import setup_logging
 from core.utils.dialogue import Message, Dialogue
 from core.handle.textHandle import handleTextMessage
@@ -26,6 +26,8 @@ from core.mcp.manager import MCPManager
 
 TAG = __name__
 
+auto_import_modules('plugins_func.functions')
+
 
 class TTSException(RuntimeError):
     pass
@@ -112,11 +114,15 @@ class ConnectionHandler:
 
             # 进行认证
             await self.auth.authenticate(self.headers)
-
             device_id = self.headers.get("device-id", None)
-            self.memory.init_memory(device_id, self.llm)
-            self.intent.set_llm(self.llm)
 
+            # 认证通过,继续处理
+            self.websocket = ws
+            self.session_id = str(uuid.uuid4())
+
+            self.welcome_msg = self.config["xiaozhi"]
+            self.welcome_msg["session_id"] = self.session_id
+            await self.websocket.send(json.dumps(self.welcome_msg))
             # Load private configuration if device_id is provided
             bUsePrivateConfig = self.config.get("use_private_config", False)
             self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
@@ -144,19 +150,8 @@ class ConnectionHandler:
                     self.private_config = None
                     raise
 
-            # 认证通过,继续处理
-            self.websocket = ws
-            self.session_id = str(uuid.uuid4())
-
-            self.welcome_msg = self.config["xiaozhi"]
-            self.welcome_msg["session_id"] = self.session_id
-            await self.websocket.send(json.dumps(self.welcome_msg))
-
             # 异步初始化
-            await self.loop.run_in_executor(None, self._initialize_components)
-			# 初始化MCP服务
-            await self.mcp_manager.initialize_servers()
-
+            self.executor.submit(self._initialize_components)
             # tts 消化线程
             tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
             tts_priority.start()
@@ -192,16 +187,26 @@ class ConnectionHandler:
             await handleAudioMessage(self, message)
 
     def _initialize_components(self):
+        """加载插件"""
+        self.func_handler = FunctionHandler(self)
+
+        """加载提示词"""
         self.prompt = self.config["prompt"]
         if self.private_config:
             self.prompt = self.private_config.private_config.get("prompt", self.prompt)
-
-        #self.client_ip_info = get_ip_info(self.client_ip)
-        #self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
-        #self.prompt = self.prompt + f"\n我在:{self.client_ip_info}"
         self.dialogue.put(Message(role="system", content=self.prompt))
 
-        self.func_handler = FunctionHandler(self.config)
+        """加载记忆"""
+        device_id = self.headers.get("device-id", None)
+        self.memory.init_memory(device_id, self.llm)
+        self.intent.set_llm(self.llm)
+
+        """加载位置信息"""
+        self.client_ip_info = get_ip_info(self.client_ip)
+        if self.client_ip_info is not None and "city" in self.client_ip_info:
+            self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
+            self.prompt = self.prompt + f"\nuser location:{self.client_ip_info}"
+            self.dialogue.update_system_message(self.prompt)
 
     def change_system_prompt(self, prompt):
         self.prompt = prompt

main/xiaozhi-server/core/handle/functionHandler.py

@@ -1,16 +1,16 @@
-import asyncio
-from enum import Enum
-
 from config.logger import setup_logging
 import json
 from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+from plugins_func.functions.hass_init import append_devices_to_prompt
+
 TAG = __name__
 logger = setup_logging()
 
 
 class FunctionHandler:
-    def __init__(self, config):
-        self.config = config
+    def __init__(self, conn):
+        self.conn = conn
+        self.config = conn.config
         self.function_registry = FunctionRegistry()
         self.register_nessary_functions()
         self.register_config_functions()
@@ -26,9 +26,10 @@ class FunctionHandler:
         func_names = ",".join(surport_plugins)
         for function_desc in self.functions_desc:
             if function_desc["function"]["name"] == "plugin_loader":
-                function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names)
+                function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]",
+                                                                                                            func_names)
                 break
-        
+
     def upload_functions_desc(self):
         self.functions_desc = self.function_registry.get_all_function_desc()
 
@@ -47,7 +48,6 @@ class FunctionHandler:
     def register_nessary_functions(self):
         """注册必要的函数"""
         self.function_registry.register_function("handle_exit_intent")
-        self.function_registry.register_function("play_music")
         self.function_registry.register_function("plugin_loader")
         self.function_registry.register_function("get_time")
         self.function_registry.register_function("raise_and_lower_the_volume")
@@ -57,6 +57,9 @@ class FunctionHandler:
         for func in self.config["Intent"]["function_call"].get("functions", []):
             self.function_registry.register_function(func)
 
+        """home assistant需要初始化提示词"""
+        append_devices_to_prompt(self.conn)
+
     def get_function(self, name):
         return self.function_registry.get_function(name)
 
@@ -81,4 +84,4 @@ class FunctionHandler:
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理function call错误: {e}")
 
-        return None
+        return None

main/xiaozhi-server/core/handle/helloHandle.py

@@ -1,8 +1,72 @@
 import json
 from config.logger import setup_logging
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils.util import remove_punctuation_and_length
+import shutil
+import asyncio
+import os
+import random
+import time
 
 logger = setup_logging()
 
+WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10,
+                 "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]}
+
 
 async def handleHelloMessage(conn):
     await conn.websocket.send(json.dumps(conn.welcome_msg))
+
+
+async def checkWakeupWords(conn, text):
+    enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"]
+    """是否开启唤醒词加速"""
+    if not enable_wakeup_words_response_cache:
+        return False
+    """检查是否是唤醒词"""
+    _, text = remove_punctuation_and_length(text)
+    if text in conn.config.get("wakeup_words"):
+        await send_stt_message(conn, text)
+        conn.tts_first_text_index = 0
+        conn.tts_last_text_index = 0
+        conn.llm_finish_task = True
+
+        file = getWakeupWordFile(WAKEUP_CONFIG["file_name"])
+        if file is None:
+            asyncio.create_task(wakeupWordsResponse(conn))
+            return False
+        opus_packets, duration = conn.tts.audio_to_opus_data(file)
+        conn.audio_play_queue.put((opus_packets, text, 0))
+        if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
+            asyncio.create_task(wakeupWordsResponse(conn))
+        return True
+    return False
+
+
+def getWakeupWordFile(file_name):
+    for file in os.listdir(WAKEUP_CONFIG["dir"]):
+        if "my_"+file_name in file:
+            return f"config/assets/{file}"
+
+    """查找config/assets/目录下名称为wakeup_words的文件"""
+    for file in os.listdir(WAKEUP_CONFIG["dir"]):
+        if file_name in file:
+            return f"config/assets/{file}"
+    return None
+
+
+async def wakeupWordsResponse(conn):
+    """唤醒词响应"""
+    wakeup_word = random.choice(WAKEUP_CONFIG["words"])
+    result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
+    tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
+    if tts_file is not None and os.path.exists(tts_file):
+        file_type = os.path.splitext(tts_file)[1]
+        if file_type:
+            file_type = file_type.lstrip('.')
+        old_file = getWakeupWordFile("my_" +WAKEUP_CONFIG["file_name"])
+        if old_file is not None:
+            os.remove(old_file)
+        """将文件挪到"wakeup_words.mp3"""
+        shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" +WAKEUP_CONFIG["file_name"] + "." + file_type)
+        WAKEUP_CONFIG["create_time"] = time.time()

main/xiaozhi-server/core/handle/intentHandler.py

@@ -2,6 +2,7 @@ from config.logger import setup_logging
 import json
 import uuid
 from core.handle.sendAudioHandle import send_stt_message
+from core.handle.helloHandle import checkWakeupWords
 from core.utils.util import remove_punctuation_and_length
 
 TAG = __name__
@@ -12,6 +13,10 @@ async def handle_user_intent(conn, text):
     # 检查是否有明确的退出命令
     if await check_direct_exit(conn, text):
         return True
+    # 检查是否是唤醒词
+    if await checkWakeupWords(conn, text):
+        return True
+
     if conn.use_function_call_mode:
         # 使用支持function calling的聊天方法,不再进行意图分析
         return False
@@ -35,6 +40,7 @@ async def check_direct_exit(conn, text):
     return False
 
 
+
 async def analyze_intent_with_llm(conn, text):
     """使用LLM分析用户意图"""
     if not hasattr(conn, 'intent') or not conn.intent:

main/xiaozhi-server/core/handle/iotHandle.py

@@ -51,8 +51,6 @@ def create_iot_function(device_name, method_name, method_info):
             # 生成结果信息
             result = f"{device_name}的{method_name}操作执行成功"
 
-            # 根据方法名尝试自动更新状态
-            await update_state_by_method(conn, device_name, method_name, params)
 
             # 处理响应中可能的占位符
             response = response_success
@@ -114,35 +112,6 @@ def create_iot_query_function(device_name, prop_name, prop_info):
     return wrap_async_function(iot_query_function)
 
 
-async def update_state_by_method(conn, device_name, method_name, params):
-    """根据方法和参数自动更新设备状态"""
-    try:
-        # 规则1: 方法名为TurnOn，设置power为True
-        if method_name == "TurnOn":
-            await set_iot_status(conn, device_name, "power", True)
-
-        # 规则2: 方法名为TurnOff，设置power为False
-        elif method_name == "TurnOff":
-            await set_iot_status(conn, device_name, "power", False)
-
-        # 规则3: Set开头的方法，尝试更新对应参数
-        elif method_name.startswith("Set"):
-            # 从参数中找到可能的状态值
-            for param_name, param_value in params.items():
-                # 尝试更新对应名称的属性
-                await set_iot_status(conn, device_name, param_name, param_value)
-
-        # 其他方法，尝试直接从参数更新状态
-        else:
-            for param_name, param_value in params.items():
-                # 检查设备是否有此属性
-                status = await get_iot_status(conn, device_name, param_name)
-                if status is not None:  # 属性存在
-                    await set_iot_status(conn, device_name, param_name, param_value)
-    except Exception as e:
-        logger.bind(tag=TAG).warning(f"自动更新状态失败: {e}")
-
-
 class IotDescriptor:
     """
     A class to represent an IoT descriptor.

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -7,12 +7,26 @@ from core.utils.util import remove_punctuation_and_length, get_string_no_punctua
 TAG = __name__
 logger = setup_logging()
 
+
 async def sendAudioMessage(conn, audios, text, text_index=0):
     # 发送句子开始消息
     if text_index == conn.tts_first_text_index:
         logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
     await send_tts_message(conn, "sentence_start", text)
 
+    # 播放音频
+    await sendAudio(conn, audios)
+
+    await send_tts_message(conn, "sentence_end", text)
+
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        await send_tts_message(conn, 'stop', None)
+        if conn.close_after_chat:
+            await conn.close()
+
+# 播放音频
+async def sendAudio(conn, audios):
     # 流控参数优化
     original_frame_duration = 60  # 原始帧时长（毫秒）
     adjusted_frame_duration = int(original_frame_duration * 0.8)  # 缩短20%
@@ -42,13 +56,6 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
     if compensation > 0:
         await asyncio.sleep(compensation)
 
-    await send_tts_message(conn, "sentence_end", text)
-
-    # 发送结束消息（如果是最后一个文本）
-    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
-        await send_tts_message(conn, 'stop', None)
-        if conn.close_after_chat:
-            await conn.close()
 
 async def send_tts_message(conn, state, text=None):
     """发送 TTS 状态消息"""
@@ -60,10 +67,20 @@ async def send_tts_message(conn, state, text=None):
     if text is not None:
         message["text"] = text
 
-    await conn.websocket.send(json.dumps(message))
+    # TTS播放结束
     if state == "stop":
+        # 播放提示音
+        tts_notify = conn.config.get("enable_stop_tts_notify", False)
+        if tts_notify:
+            stop_tts_notify_voice = conn.config.get("stop_tts_notify_voice", "config/assets/tts_notify.mp3")
+            audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
+            await sendAudio(conn, audios)
+        # 清除服务端讲话状态
         conn.clearSpeakStatus()
 
+    # 发送消息到客户端
+    await conn.websocket.send(json.dumps(message))
+
 
 async def send_stt_message(conn, text):
     """发送 STT 状态消息"""

main/xiaozhi-server/core/handle/textHandle.py

@@ -2,7 +2,9 @@ from config.logger import setup_logging
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
+from core.utils.util import remove_punctuation_and_length
 from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
+from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
 
 TAG = __name__
@@ -24,7 +26,7 @@ async def handleTextMessage(conn, message):
         elif msg_json["type"] == "listen":
             if "mode" in msg_json:
                 conn.client_listen_mode = msg_json["mode"]
-                logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn. client_listen_mode}")
+                logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
             if msg_json["state"] == "start":
                 conn.client_have_voice = True
                 conn.client_voice_stop = False
@@ -38,11 +40,25 @@ async def handleTextMessage(conn, message):
                 conn.client_have_voice = False
                 conn.asr_audio.clear()
                 if "text" in msg_json:
-                    await startToChat(conn, msg_json["text"])
+                    text = msg_json["text"]
+                    _, text = remove_punctuation_and_length(text)
+
+                    # 识别是否是唤醒词
+                    is_wakeup_words = text in conn.config.get("wakeup_words")
+                    # 是否开启唤醒词回复
+                    enable_greeting = conn.config.get("enable_greeting", True)
+
+                    if is_wakeup_words and not enable_greeting:
+                        # 如果是唤醒词，且关闭了唤醒词回复，就不用回答
+                        await send_stt_message(conn, text)
+                        await send_tts_message(conn, "stop", None)
+                    else:
+                        # 否则需要LLM对文字内容进行答复
+                        await startToChat(conn, text)
         elif msg_json["type"] == "iot":
             if "descriptors" in msg_json:
                 await handleIotDescriptors(conn, msg_json["descriptors"])
             if "states" in msg_json:
-                await handleIotStatus(conn, msg_json["states"])  
+                await handleIotStatus(conn, msg_json["states"])
     except json.JSONDecodeError:
         await conn.websocket.send(message)

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -0,0 +1,164 @@
+import time
+import wave
+import os
+import sys
+import io
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+import uuid
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+import numpy as np
+import sherpa_onnx
+
+from modelscope.hub.file_download import model_file_download
+
+TAG = __name__
+logger = setup_logging()
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+        
+        # 初始化模型文件路径
+        model_files = {
+            "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
+            "tokens.txt": os.path.join(self.model_dir, "tokens.txt")
+        }
+
+        # 下载并检查模型文件
+        try:
+            for file_name, file_path in model_files.items():
+                if not os.path.isfile(file_path):
+                    logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}")
+                    model_file_download(
+                        model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
+                        file_path=file_name,
+                        local_dir=self.model_dir
+                    )
+                    
+                    if not os.path.isfile(file_path):
+                        raise FileNotFoundError(f"模型文件下载失败: {file_path}")
+                        
+            self.model_path = model_files["model.int8.onnx"]
+            self.tokens_path = model_files["tokens.txt"]
+            
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
+            raise
+
+        with CaptureOutput():
+            self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+                model=self.model_path,
+                tokens=self.tokens_path,
+                num_threads=2,
+                sample_rate=16000,
+                feature_dim=80,
+                decoding_method="greedy_search",
+                debug=False,
+                use_itn=True,
+            )
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
+        """
+        Args:
+        wave_filename:
+            Path to a wave file. It should be single channel and each sample should
+            be 16-bit. Its sample rate does not need to be 16kHz.
+        Returns:
+        Return a tuple containing:
+        - A 1-D array of dtype np.float32 containing the samples, which are
+        normalized to the range [-1, 1].
+        - sample rate of the wave file
+        """
+
+        with wave.open(wave_filename) as f:
+            assert f.getnchannels() == 1, f.getnchannels()
+            assert f.getsampwidth() == 2, f.getsampwidth()  # it is in bytes
+            num_samples = f.getnframes()
+            samples = f.readframes(num_samples)
+            samples_int16 = np.frombuffer(samples, dtype=np.int16)
+            samples_float32 = samples_int16.astype(np.float32)
+
+            samples_float32 = samples_float32 / 32768
+            return samples_float32, f.getframerate()
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        file_path = None
+        try:
+            # 保存音频文件
+            start_time = time.time()
+            file_path = self.save_audio_to_file(opus_data, session_id)
+            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+
+            # 语音识别
+            start_time = time.time()
+            s = self.model.create_stream()
+            samples, sample_rate = self.read_wave(file_path)
+            s.accept_waveform(sample_rate, samples)
+            self.model.decode_stream(s)
+            text = s.result.text
+            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+
+            return text, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None
+
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py

@@ -0,0 +1,52 @@
+from config.logger import setup_logging
+from http import HTTPStatus
+from dashscope import Application
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.app_id = config["app_id"]
+        self.base_url = config.get("base_url")
+        self.is_No_prompt = config.get("is_no_prompt")
+        self.memory_id = config.get("ali_memory_id")
+
+    def response(self, session_id, dialogue):
+        try:
+            # 处理dialogue
+            if self.is_No_prompt:
+                dialogue.pop(0)
+                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
+
+            # 构造调用参数
+            call_params = {
+                "api_key": self.api_key,
+                "app_id": self.app_id,
+                "session_id": session_id,
+                "messages": dialogue
+            }
+            if self.memory_id != False:
+                # 百练memory需要prompt参数
+                prompt = dialogue[-1].get("content")
+                call_params["memory_id"] = self.memory_id
+                call_params["prompt"] = prompt
+                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
+
+            responses = Application.call(**call_params)
+            if responses.status_code != HTTPStatus.OK:
+                logger.bind(tag=TAG).error(
+                    f"code={responses.status_code}, "
+                    f"message={responses.message}, "
+                    f"请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
+                )
+                yield "【阿里百练API服务响应异常】"
+            else:
+                logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
+                yield responses.output.text
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
+            yield "【LLM服务响应异常】"

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -9,6 +9,7 @@ logger = setup_logging()
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
+        self.mode = config.get("mode", "chat-messages")
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
 
@@ -19,27 +20,59 @@ class LLMProvider(LLMProviderBase):
             conversation_id = self.session_conversation_map.get(session_id)
 
             # 发起流式请求
-            with requests.post(
-                    f"{self.base_url}/chat-messages",
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json={
+            if self.mode == "chat-messages":
+                request_json = {
                         "query": last_msg["content"],
                         "response_mode": "streaming",
                         "user": session_id,
                         "inputs": {},
                         "conversation_id": conversation_id
-                    },
+                    }
+            elif self.mode == "workflows/run":
+                request_json = {
+                    "inputs": {"query": last_msg["content"]},
+                    "response_mode": "streaming",
+                    "user": session_id
+                }
+            elif self.mode == "completion-messages":
+                request_json = {
+                    "inputs": {"query": last_msg["content"]},
+                    "response_mode": "streaming",
+                    "user": session_id
+                }
+
+            with requests.post(
+                    f"{self.base_url}/{self.mode}",
+                    headers={"Authorization": f"Bearer {self.api_key}"},
+                    json=request_json,
                     stream=True
             ) as r:
-                for line in r.iter_lines():
-                    if line.startswith(b'data: '):
-                        event = json.loads(line[6:])
-                        # 如果没有找到conversation_id，则获取此次conversation_id
-                        if not conversation_id:
-                            conversation_id = event.get('conversation_id')
-                            self.session_conversation_map[session_id] = conversation_id  # 更新映射
-                        if event.get('answer'):
-                            yield event['answer']
+                if self.mode == "chat-messages":
+                    for line in r.iter_lines():
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            # 如果没有找到conversation_id，则获取此次conversation_id
+                            if not conversation_id:
+                                conversation_id = event.get('conversation_id')
+                                self.session_conversation_map[session_id] = conversation_id  # 更新映射
+                            if event.get('answer'):
+                                yield event['answer']
+                elif self.mode == "workflows/run":
+                    for line in r.iter_lines():
+                        # logger.bind(tag=TAG).info(f"chat message response: {line}")
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            if event.get('event') == "workflow_finished":
+                                if event['data']['status'] == "succeeded":
+                                    yield event['data']['outputs']['answer']
+                                else:
+                                    yield "【服务响应异常】"
+                elif self.mode == "completion-messages":
+                    for line in r.iter_lines():
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            if event.get('answer'):
+                                yield event['answer'] 
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -1,14 +1,19 @@
 import google.generativeai as genai
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
-
+from config.logger import setup_logging
+import requests
+import json
+TAG = __name__
+logger = setup_logging()
 
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         """初始化Gemini LLM Provider"""
         self.model_name = config.get("model_name", "gemini-1.5-pro")
         self.api_key = config.get("api_key")
-
+        self.http_proxy=config.get("http_proxy")
+        self.https_proxy = config.get("https_proxy")
         have_key = check_model_key("LLM", self.api_key)
 
         if not have_key:
@@ -16,6 +21,19 @@ class LLMProvider(LLMProviderBase):
 
         try:
             # 初始化Gemini客户端
+            # 配置代理（如果提供了代理配置）
+            self.proxies=None
+            if self.http_proxy is not "" or self.https_proxy is not "":
+
+                self.proxies = {
+                    "http": self.http_proxy,
+                    "https": self.https_proxy,
+                }
+                logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
+                # 使用猴子补丁修改 google-generativeai 库的请求会话
+
+                # 使用 session 对象配置 genai
+
             genai.configure(api_key=self.api_key)
             self.model = genai.GenerativeModel(self.model_name)
 
@@ -46,26 +64,54 @@ class LLMProvider(LLMProviderBase):
                 if content:
                     chat_history.append({
                         "role": role,
-                        "parts": [content]
+                        "parts": [{"text":content}]
+
                     })
 
             # 获取当前消息
             current_msg = dialogue[-1]["content"]
 
-            # 创建新的聊天会话
-            chat = self.model.start_chat(history=chat_history)
+            # 构建请求体
+            request_body = {
+                "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
+                "generationConfig": self.generation_config
+            }
 
-            # 发送消息并获取流式响应
-            response = chat.send_message(
-                current_msg,
-                stream=True,
-                generation_config=self.generation_config
-            )
+            # 构建请求URL
+            url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
 
-            # 处理流式响应
-            for chunk in response:
-                if hasattr(chunk, 'text') and chunk.text:
-                    yield chunk.text
+            # 构建请求头
+            headers = {
+                "Content-Type": "application/json",
+            }
+
+            # 发送POST请求,经测试手动 request 无法使用 stream 模式
+            if self.proxies:
+                response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
+                try:
+                    data = response.json()  # 直接解析JSON
+                    if 'candidates' in data and data['candidates']:
+                        yield data['candidates'][0]['content']['parts'][0]['text']
+                    else:
+                        yield "未找到候选回复。"
+                except json.JSONDecodeError as e:
+                    yield f"JSON解码错误：{e}"
+                except Exception as e:
+                    yield f"发生错误：{e}"
+            else:
+                logger.bind(tag=TAG).info(f"Gemini stream mode ")
+                chat = self.model.start_chat(history=chat_history)
+
+                # 发送消息并获取流式响应
+                response = chat.send_message(
+                    current_msg,
+                    stream=True,
+                    generation_config=self.generation_config
+                )
+                # 处理流式响应
+                for chunk in response:
+                    if hasattr(chunk, 'text') and chunk.text:
+                        yield chunk.text
 
         except Exception as e:
             error_msg = str(e)
@@ -78,3 +124,13 @@ class LLMProvider(LLMProviderBase):
                 yield "【Gemini API key无效】"
             else:
                 yield f"【Gemini服务响应异常: {error_msg}】"
+
+
+
+
+        except requests.exceptions.RequestException as e:
+            yield f"请求失败：{e}"
+        except json.JSONDecodeError as e:
+            yield f"JSON解码错误：{e}"
+        except Exception as e:
+            yield f"发生错误：{e}"

main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py

@@ -1,62 +0,0 @@
-import requests
-from requests.exceptions import RequestException
-from config.logger import setup_logging
-from core.providers.llm.base import LLMProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-
-class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        self.agent_id = config.get("agent_id")  # 对应 agent_id
-        self.api_key = config.get("api_key")
-        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
-        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
-
-    def response(self, session_id, dialogue):
-        print(dialogue)
-        try:
-            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
-
-            # 提取最后一个 role 为 'user' 的 content
-            input_text = None
-            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
-                # 逆序遍历，找到最后一个 role 为 'user' 的消息
-                for message in reversed(dialogue):
-                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
-                        input_text = message.get("content", "")
-                        break  # 找到后立即退出循环
-
-            # 构造请求数据
-            payload = {
-                "text": input_text,
-                "agent_id": self.agent_id,
-                "conversation_id": session_id  # 使用 session_id 作为 conversation_id
-            }
-            # 设置请求头
-            headers = {
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json"
-            }
-
-            # 发起 POST 请求
-            response = requests.post(self.api_url, json=payload, headers=headers)
-
-            # 检查请求是否成功
-            response.raise_for_status()
-
-            # 解析返回数据
-            data = response.json()
-            speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
-
-            # 返回生成的内容
-            if speech:
-                yield speech
-            else:
-                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
-
-        except RequestException as e:
-            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -1,7 +1,11 @@
 import openai
+from config.logger import setup_logging
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
 
+TAG = __name__
+logger = setup_logging()
+
 
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
@@ -11,6 +15,8 @@ class LLMProvider(LLMProviderBase):
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
+        self.max_tokens = config.get("max_tokens", 500)
+
         check_model_key("LLM", self.api_key)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
 
@@ -19,9 +25,10 @@ class LLMProvider(LLMProviderBase):
             responses = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
-                stream=True
+                stream=True,
+                max_tokens=self.max_tokens,
             )
-            
+
             is_active = True
             for chunk in responses:
                 try:
@@ -50,12 +57,12 @@ class LLMProvider(LLMProviderBase):
                 model=self.model_name,
                 messages=dialogue,
                 stream=True,
-                tools=functions,
+                tools=functions
             )
-            
+
             for chunk in stream:
                 yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
-                
+
         except Exception as e:
-            self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
-            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
+            logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
+            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}

main/xiaozhi-server/core/providers/tts/base.py

@@ -5,6 +5,7 @@ import numpy as np
 import opuslib_next
 from pydub import AudioSegment
 from abc import ABC, abstractmethod
+from core.utils.tts import MarkdownCleaner
 
 TAG = __name__
 logger = setup_logging()
@@ -23,6 +24,7 @@ class TTSProviderBase(ABC):
         tmp_file = self.generate_filename()
         try:
             max_repeat_time = 5
+            text = MarkdownCleaner.clean_markdown(text)
             while not os.path.exists(tmp_file) and max_repeat_time > 0:
                 asyncio.run(self.text_to_speak(text, tmp_file))
                 if not os.path.exists(tmp_file):
@@ -34,7 +36,7 @@ class TTSProviderBase(ABC):
 
             return tmp_file
         except Exception as e:
-            logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
+            logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
             return None
 
     @abstractmethod
@@ -47,7 +49,8 @@ class TTSProviderBase(ABC):
         file_type = os.path.splitext(audio_file_path)[1]
         if file_type:
             file_type = file_type.lstrip('.')
-        audio = AudioSegment.from_file(audio_file_path, format=file_type)
+        # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
+        audio = AudioSegment.from_file(audio_file_path, format=file_type, parameters=["-nostdin"])
 
         # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
         audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)

main/xiaozhi-server/core/utils/dialogue.py

@@ -35,6 +35,15 @@ class Dialogue:
             self.getMessages(m, dialogue)
         return dialogue
 
+    def update_system_message(self, new_content: str):
+        """更新或添加系统消息"""
+        # 查找第一个系统消息
+        system_msg = next((msg for msg in self.dialogue if msg.role == "system"), None)
+        if system_msg:
+            system_msg.content = new_content
+        else:
+            self.put(Message(role="system", content=new_content))
+
     def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
         if memory_str is None or len(memory_str) == 0:
             return self.get_llm_dialogue()

main/xiaozhi-server/core/utils/tts.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from config.logger import setup_logging
 import importlib
@@ -14,4 +15,98 @@ def create_instance(class_name, *args, **kwargs):
             sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
         return sys.modules[lib_name].TTSProvider(*args, **kwargs)
 
-    raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
+    raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
+
+
+class MarkdownCleaner:
+    """
+    封装 Markdown 清理逻辑：直接用 MarkdownCleaner.clean_markdown(text) 即可
+    """
+    # 公式字符
+    NORMAL_FORMULA_CHARS = re.compile(r'[a-zA-Z\\^_{}\+\-\(\)\[\]=]')
+
+    @staticmethod
+    def _replace_inline_dollar(m: re.Match) -> str:
+        """
+        只要捕获到完整的 "$...$":
+          - 如果内部有典型公式字符 => 去掉两侧 $
+          - 否则 (纯数字/货币等) => 保留 "$...$"
+        """
+        content = m.group(1)
+        if MarkdownCleaner.NORMAL_FORMULA_CHARS.search(content):
+            return content
+        else:
+            return m.group(0)
+
+    @staticmethod
+    def _replace_table_block(match: re.Match) -> str:
+        """
+        当匹配到一个整段表格块时，回调该函数。
+        """
+        block_text = match.group('table_block')
+        lines = block_text.strip('\n').split('\n')
+
+        parsed_table = []
+        for line in lines:
+            line_stripped = line.strip()
+            if re.match(r'^\|\s*[-:]+\s*(\|\s*[-:]+\s*)+\|?$', line_stripped):
+                continue
+            columns = [col.strip() for col in line_stripped.split('|') if col.strip() != '']
+            if columns:
+                parsed_table.append(columns)
+
+        if not parsed_table:
+            return ""
+
+        headers = parsed_table[0]
+        data_rows = parsed_table[1:] if len(parsed_table) > 1 else []
+
+        lines_for_tts = []
+        if len(parsed_table) == 1:
+            # 只有一行
+            only_line_str = ", ".join(parsed_table[0])
+            lines_for_tts.append(f"单行表格：{only_line_str}")
+        else:
+            lines_for_tts.append(f"表头是：{', '.join(headers)}")
+            for i, row in enumerate(data_rows, start=1):
+                row_str_list = []
+                for col_index, cell_val in enumerate(row):
+                    if col_index < len(headers):
+                        row_str_list.append(f"{headers[col_index]} = {cell_val}")
+                    else:
+                        row_str_list.append(cell_val)
+                lines_for_tts.append(f"第 {i} 行：{', '.join(row_str_list)}")
+
+        return "\n".join(lines_for_tts) + "\n"
+
+    # 预编译所有正则表达式（按执行频率排序）
+    # 这里要把 replace_xxx 的静态方法放在最前定义，以便在列表里能正确引用它们。
+    REGEXES = [
+        (re.compile(r'```.*?```', re.DOTALL), ''),  # 代码块
+        (re.compile(r'^#+\s*', re.MULTILINE), ''),  # 标题
+        (re.compile(r'(\*\*|__)(.*?)\1'), r'\2'),  # 粗体
+        (re.compile(r'(\*|_)(?=\S)(.*?)(?<=\S)\1'), r'\2'),  # 斜体
+        (re.compile(r'!\[.*?\]\(.*?\)'), ''),  # 图片
+        (re.compile(r'\[(.*?)\]\(.*?\)'), r'\1'),  # 链接
+        (re.compile(r'^\s*>+\s*', re.MULTILINE), ''),  # 引用
+        (
+            re.compile(r'(?P<table_block>(?:^[^\n]*\|[^\n]*\n)+)', re.MULTILINE),
+            _replace_table_block
+        ),
+        (re.compile(r'^\s*[*+-]\s*', re.MULTILINE), '- '),  # 列表
+        (re.compile(r'\$\$.*?\$\$', re.DOTALL), ''),  # 块级公式
+        (
+            re.compile(r'(?<![A-Za-z0-9])\$([^\n$]+)\$(?![A-Za-z0-9])'),
+            _replace_inline_dollar
+        ),
+        (re.compile(r'\n{2,}'), '\n'),  # 多余空行
+    ]
+
+    @staticmethod
+    def clean_markdown(text: str) -> str:
+        """
+        主入口方法：依序执行所有正则，移除或替换 Markdown 元素
+        """
+        for regex, replacement in MarkdownCleaner.REGEXES:
+            text = regex.sub(replacement, text)
+        return text.strip()

main/xiaozhi-server/core/utils/util.py

@@ -73,9 +73,7 @@ def get_ip_info(ip_addr):
         resp = requests.get(url).json()
 
         ip_info = {
-            "city": resp.get("cityName"),
-            "region": resp.get("regionName"), 
-            "country": resp.get("countryName")
+            "city": resp.get("cityName")
         }
         return ip_info
     except Exception as e: