update:合并非tts代码

main/xiaozhi-server/core/handle/abortHandle.py

@@ -3,14 +3,16 @@ import queue
 from config.logger import setup_logging
 
 TAG = __name__
-logger = setup_logging()
 
 
 async def handleAbortMessage(conn):
-    logger.bind(tag=TAG).info("Abort message received")
+    conn.logger.bind(tag=TAG).info("Abort message received")
     # 设置成打断状态，会自动打断llm、tts任务
     conn.client_abort = True
+    conn.clear_queues()
     # 打断客户端说话状态
-    await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
+    await conn.websocket.send(
+        json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})
+    )
     conn.clearSpeakStatus()
-    logger.bind(tag=TAG).info("Abort message received-end")
+    conn.logger.bind(tag=TAG).info("Abort message received-end")

main/xiaozhi-server/core/handle/functionHandler.py

@@ -4,7 +4,6 @@ from plugins_func.register import FunctionRegistry, ActionResponse, Action, Tool
 from plugins_func.functions.hass_init import append_devices_to_prompt
 
 TAG = __name__
-logger = setup_logging()
 
 
 class FunctionHandler:
@@ -40,7 +39,9 @@ class FunctionHandler:
         for func in self.functions_desc:
             func_names.append(func["function"]["name"])
         # 打印当前支持的函数列表
-        logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
+        self.conn.logger.bind(tag=TAG, session_id=self.conn.session_id).info(
+            f"当前支持的函数列表: {func_names}"
+        )
         return func_names
 
     def get_functions(self):
@@ -57,7 +58,9 @@ class FunctionHandler:
 
     def register_config_functions(self):
         """注册配置中的函数,可以不同客户端使用不同的配置"""
-        for func in self.config["Intent"]["function_call"].get("functions", []):
+        for func in self.config["Intent"][self.config["selected_module"]["Intent"]].get(
+            "functions", []
+        ):
             self.function_registry.register_function(func)
 
         """home assistant需要初始化提示词"""
@@ -77,7 +80,9 @@ class FunctionHandler:
             func = funcItem.func
             arguments = function_call_data["arguments"]
             arguments = json.loads(arguments) if arguments else {}
-            logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
+            self.conn.logger.bind(tag=TAG).debug(
+                f"调用函数: {function_name}, 参数: {arguments}"
+            )
             if (
                 funcItem.type == ToolType.SYSTEM_CTL
                 or funcItem.type == ToolType.IOT_CTL
@@ -92,6 +97,6 @@ class FunctionHandler:
                     action=Action.NOTFOUND, result="没有找到对应的函数", response=""
                 )
         except Exception as e:
-            logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+            self.conn.logger.bind(tag=TAG).error(f"处理function call错误: {e}")
 
         return None

main/xiaozhi-server/core/handle/helloHandle.py

@@ -1,6 +1,4 @@
 import json
-from config.logger import setup_logging
-from core.providers.tts.dto.dto import TTSMessageDTO, MsgType
 from core.handle.sendAudioHandle import send_stt_message
 from core.utils.util import remove_punctuation_and_length
 import shutil
@@ -9,7 +7,7 @@ import os
 import random
 import time
 
-logger = setup_logging()
+TAG = __name__
 
 WAKEUP_CONFIG = {
     "dir": "config/assets/",
@@ -21,7 +19,16 @@ WAKEUP_CONFIG = {
 }
 
 
-async def handleHelloMessage(conn):
+async def handleHelloMessage(conn, msg_json):
+    """处理hello消息"""
+    audio_params = msg_json.get("audio_params")
+    if audio_params:
+        format = audio_params.get("format")
+        conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
+        conn.audio_format = format
+        conn.asr.set_audio_format(format)
+        conn.welcome_msg["audio_params"] = audio_params
+
     await conn.websocket.send(json.dumps(conn.welcome_msg))
 
 
@@ -44,21 +51,11 @@ async def checkWakeupWords(conn, text):
         if file is None:
             asyncio.create_task(wakeupWordsResponse(conn))
             return False
-        opus_packets, duration = conn.tts.audio_to_opus_data(file)
+        opus_packets, _ = conn.tts.audio_to_opus_data(file)
         text_hello = WAKEUP_CONFIG["text"]
         if not text_hello:
             text_hello = text
-
-        conn.tts.tts_audio_queue.put(
-            TTSMessageDTO(
-                u_id=conn.u_id,
-                msg_type=MsgType.TTS_TEXT_RESPONSE,
-                content=opus_packets,
-                tts_finish_text="",
-                sentence_type=None,
-                duration=0,
-            )
-        )
+        conn.audio_play_queue.put((opus_packets, text_hello, 0))
         if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
             asyncio.create_task(wakeupWordsResponse(conn))
         return True
@@ -80,11 +77,20 @@ def getWakeupWordFile(file_name):
 
 
 async def wakeupWordsResponse(conn):
+    wait_max_time = 5
+    while conn.llm is None or not conn.llm.response_no_stream:
+        await asyncio.sleep(1)
+        wait_max_time -= 1
+        if wait_max_time <= 0:
+            conn.logger.bind(tag=TAG).error("连接对象没有llm")
+            return
+
     """唤醒词响应"""
     wakeup_word = random.choice(WAKEUP_CONFIG["words"])
     result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
-    # TODO 将result转换为tts_message_dto
-    tts_file = None
+    if result is None or result == "":
+        return
+    tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
 
     if tts_file is not None and os.path.exists(tts_file):
         file_type = os.path.splitext(tts_file)[1]

main/xiaozhi-server/core/handle/iotHandle.py

@@ -10,7 +10,6 @@ from plugins_func.register import (
 )
 
 TAG = __name__
-logger = setup_logging()
 
 
 def wrap_async_function(async_func):
@@ -21,7 +20,7 @@ def wrap_async_function(async_func):
             # 获取连接对象（第一个参数）
             conn = args[0]
             if not hasattr(conn, "loop"):
-                logger.bind(tag=TAG).error("Connection对象没有loop属性")
+                conn.logger.bind(tag=TAG).error("Connection对象没有loop属性")
                 return ActionResponse(
                     Action.ERROR,
                     "Connection对象没有loop属性",
@@ -35,7 +34,7 @@ def wrap_async_function(async_func):
             # 等待结果返回
             return future.result()
         except Exception as e:
-            logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
+            conn.logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
             return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
 
     return wrapper
@@ -57,7 +56,7 @@ def create_iot_function(device_name, method_name, method_info):
                 response_failure = "操作失败"
 
             # 打印响应参数
-            logger.bind(tag=TAG).info(
+            conn.logger.bind(tag=TAG).debug(
                 f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
             )
 
@@ -86,7 +85,9 @@ def create_iot_function(device_name, method_name, method_info):
 
             return ActionResponse(Action.RESPONSE, result, response)
         except Exception as e:
-            logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
+            conn.logger.bind(tag=TAG).error(
+                f"执行{device_name}的{method_name}操作失败: {e}"
+            )
 
             # 操作失败时使用大模型提供的失败响应
             response = response_failure
@@ -104,7 +105,7 @@ def create_iot_query_function(device_name, prop_name, prop_info):
     async def iot_query_function(conn, response_success=None, response_failure=None):
         try:
             # 打印响应参数
-            logger.bind(tag=TAG).info(
+            conn.logger.bind(tag=TAG).info(
                 f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
             )
 
@@ -122,7 +123,9 @@ def create_iot_query_function(device_name, prop_name, prop_info):
 
                 return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
         except Exception as e:
-            logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
+            conn.logger.bind(tag=TAG).error(
+                f"查询{device_name}的{prop_name}时出错: {e}"
+            )
 
             # 查询出错时使用大模型提供的失败响应
             response = response_failure
@@ -144,33 +147,34 @@ class IotDescriptor:
         self.methods = []
 
         # 根据描述创建属性
-        for key, value in properties.items():
-            property_item = globals()[key] = {}
-            property_item["name"] = key
-            property_item["description"] = value["description"]
-            if value["type"] == "number":
-                property_item["value"] = 0
-            elif value["type"] == "boolean":
-                property_item["value"] = False
-            else:
-                property_item["value"] = ""
-            self.properties.append(property_item)
+        if properties is not None:
+            for key, value in properties.items():
+                property_item = {}
+                property_item["name"] = key
+                property_item["description"] = value["description"]
+                if value["type"] == "number":
+                    property_item["value"] = 0
+                elif value["type"] == "boolean":
+                    property_item["value"] = False
+                else:
+                    property_item["value"] = ""
+                self.properties.append(property_item)
 
         # 根据描述创建方法
-        for key, value in methods.items():
-            method = globals()[key] = {}
-            method["description"] = value["description"]
-            method["name"] = key
-            for k, v in value["parameters"].items():
-                method[k] = {}
-                method[k]["description"] = v["description"]
-                if v["type"] == "number":
-                    method[k]["value"] = 0
-                elif v["type"] == "boolean":
-                    method[k]["value"] = False
-                else:
-                    method[k]["value"] = ""
-            self.methods.append(method)
+        if methods is not None:
+            for key, value in methods.items():
+                method = {}
+                method["description"] = value["description"]
+                method["name"] = key
+                # 检查方法是否有参数
+                if "parameters" in value:
+                    method["parameters"] = {}
+                    for k, v in value["parameters"].items():
+                        method["parameters"][k] = {
+                            "description": v["description"],
+                            "type": v["type"],
+                        }
+                self.methods.append(method)
 
 
 def register_device_type(descriptor):
@@ -219,13 +223,19 @@ def register_device_type(descriptor):
         func_name = f"{device_name.lower()}_{method_name.lower()}"
 
         # 创建参数字典，添加原有参数
-        parameters = {
-            param_name: {
-                "type": param_info["type"],
-                "description": param_info["description"],
+        parameters = {}
+        required_params = []
+
+        # 如果方法有参数，则添加参数信息
+        if "parameters" in method_info:
+            parameters = {
+                param_name: {
+                    "type": param_info["type"],
+                    "description": param_info["description"],
+                }
+                for param_name, param_info in method_info["parameters"].items()
             }
-            for param_name, param_info in method_info["parameters"].items()
-        }
+            required_params = list(method_info["parameters"].keys())
 
         # 添加响应参数
         parameters.update(
@@ -242,7 +252,6 @@ def register_device_type(descriptor):
         )
 
         # 构建必须参数列表（原有参数 + 响应参数）
-        required_params = list(method_info["parameters"].keys())
         required_params.extend(["response_success", "response_failure"])
 
         func_desc = {
@@ -269,19 +278,36 @@ def register_device_type(descriptor):
 
 # 用于接受前端设备推送的搜索iot描述
 async def handleIotDescriptors(conn, descriptors):
-    if not conn.use_function_call_mode:
-        return
     wait_max_time = 5
     while conn.func_handler is None or not conn.func_handler.finish_init:
         await asyncio.sleep(1)
         wait_max_time -= 1
         if wait_max_time <= 0:
-            logger.bind(tag=TAG).error("连接对象没有func_handler")
+            conn.logger.bind(tag=TAG).debug("连接对象没有func_handler")
             return
     """处理物联网描述"""
     functions_changed = False
 
     for descriptor in descriptors:
+
+        # 如果descriptor没有properties和methods，则直接跳过
+        if "properties" not in descriptor and "methods" not in descriptor:
+            continue
+
+        # 处理缺失properties的情况
+        if "properties" not in descriptor:
+            descriptor["properties"] = {}
+            # 从methods中提取所有参数作为properties
+            if "methods" in descriptor:
+                for method_name, method_info in descriptor["methods"].items():
+                    if "parameters" in method_info:
+                        for param_name, param_info in method_info["parameters"].items():
+                            # 将参数信息转换为属性信息
+                            descriptor["properties"][param_name] = {
+                                "description": param_info["description"],
+                                "type": param_info["type"],
+                            }
+
         # 创建IOT设备描述符
         iot_descriptor = IotDescriptor(
             descriptor["name"],
@@ -291,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors):
         )
         conn.iot_descriptors[descriptor["name"]] = iot_descriptor
 
-        if conn.use_function_call_mode:
+        if conn.load_function_plugin:
             # 注册或获取设备类型
             type_id = register_device_type(descriptor)
             device_functions = device_type_registry.get_device_functions(type_id)
@@ -300,7 +326,7 @@ async def handleIotDescriptors(conn, descriptors):
             if hasattr(conn, "func_handler"):
                 for func_name in device_functions:
                     conn.func_handler.function_registry.register_function(func_name)
-                    logger.bind(tag=TAG).info(
+                    conn.logger.bind(tag=TAG).info(
                         f"注册IOT函数到function handler: {func_name}"
                     )
                     functions_changed = True
@@ -309,8 +335,8 @@ async def handleIotDescriptors(conn, descriptors):
     if functions_changed and hasattr(conn, "func_handler"):
         conn.func_handler.upload_functions_desc()
         func_names = conn.func_handler.current_support_functions()
-        logger.bind(tag=TAG).info(f"设备类型: {type_id}")
-        logger.bind(tag=TAG).info(
+        conn.logger.bind(tag=TAG).info(f"设备类型: {type_id}")
+        conn.logger.bind(tag=TAG).info(
             f"更新function描述列表完成，当前支持的函数: {func_names}"
         )
 
@@ -324,13 +350,13 @@ async def handleIotStatus(conn, states):
                     for k, v in state["state"].items():
                         if property_item["name"] == k:
                             if type(v) != type(property_item["value"]):
-                                logger.bind(tag=TAG).error(
+                                conn.logger.bind(tag=TAG).error(
                                     f"属性{property_item['name']}的值类型不匹配"
                                 )
                                 break
                             else:
                                 property_item["value"] = v
-                                logger.bind(tag=TAG).info(
+                                conn.logger.bind(tag=TAG).info(
                                     f"物联网状态更新: {key} , {property_item['name']} = {v}"
                                 )
                             break
@@ -344,7 +370,7 @@ async def get_iot_status(conn, name, property_name):
             for property_item in value.properties:
                 if property_item["name"] == property_name:
                     return property_item["value"]
-    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
+    conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
     return None
 
 
@@ -355,16 +381,16 @@ async def set_iot_status(conn, name, property_name, value):
             for property_item in iot_descriptor.properties:
                 if property_item["name"] == property_name:
                     if type(value) != type(property_item["value"]):
-                        logger.bind(tag=TAG).error(
+                        conn.logger.bind(tag=TAG).error(
                             f"属性{property_item['name']}的值类型不匹配"
                         )
                         return
                     property_item["value"] = value
-                    logger.bind(tag=TAG).info(
+                    conn.logger.bind(tag=TAG).info(
                         f"物联网状态更新: {name} , {property_name} = {value}"
                     )
                     return
-    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
+    conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
 
 
 async def send_iot_conn(conn, name, method_name, parameters):
@@ -375,19 +401,17 @@ async def send_iot_conn(conn, name, method_name, parameters):
             for method in value.methods:
                 # 找到了方法
                 if method["name"] == method_name:
-                    await conn.websocket.send(
-                        json.dumps(
-                            {
-                                "type": "iot",
-                                "commands": [
-                                    {
-                                        "name": name,
-                                        "method": method_name,
-                                        "parameters": parameters,
-                                    }
-                                ],
-                            }
-                        )
-                    )
+                    # 构建命令对象
+                    command = {
+                        "name": name,
+                        "method": method_name,
+                    }
+
+                    # 只有当参数不为空时才添加parameters字段
+                    if parameters:
+                        command["parameters"] = parameters
+                    send_message = json.dumps({"type": "iot", "commands": [command]})
+                    await conn.websocket.send(send_message)
+                    conn.logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}")
                     return
-    logger.bind(tag=TAG).error(f"未找到方法{method_name}")
+    conn.logger.bind(tag=TAG).error(f"未找到方法{method_name}")

main/xiaozhi-server/core/handle/reportHandle.py

@@ -0,0 +1,145 @@
+"""
+TTS上报功能已集成到ConnectionHandler类中。
+
+上报功能包括：
+1. 每个连接对象拥有自己的上报队列和处理线程
+2. 上报线程的生命周期与连接对象绑定
+3. 使用ConnectionHandler.enqueue_tts_report方法进行上报
+
+具体实现请参考core/connection.py中的相关代码。
+"""
+
+import opuslib_next
+
+from config.manage_api_client import report as manage_report
+
+TAG = __name__
+
+
+def report(conn, type, text, opus_data):
+    """执行聊天记录上报操作
+
+    Args:
+        conn: 连接对象
+        type: 上报类型，1为用户，2为智能体
+        text: 合成文本
+        opus_data: opus音频数据
+    """
+    try:
+        if opus_data:
+            audio_data = opus_to_wav(conn, opus_data)
+        else:
+            audio_data = None
+        # 执行上报
+        manage_report(
+            mac_address=conn.device_id,
+            session_id=conn.session_id,
+            chat_type=type,
+            content=text,
+            audio=audio_data,
+        )
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}")
+
+
+def opus_to_wav(conn, opus_data):
+    """将Opus数据转换为WAV格式的字节流
+
+    Args:
+        output_dir: 输出目录（保留参数以保持接口兼容）
+        opus_data: opus音频数据
+
+    Returns:
+        bytes: WAV格式的音频数据
+    """
+    decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+    pcm_data = []
+
+    for opus_packet in opus_data:
+        try:
+            pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+            pcm_data.append(pcm_frame)
+        except opuslib_next.OpusError as e:
+            conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+    if not pcm_data:
+        raise ValueError("没有有效的PCM数据")
+
+    # 创建WAV文件头
+    pcm_data_bytes = b"".join(pcm_data)
+    num_samples = len(pcm_data_bytes) // 2  # 16-bit samples
+
+    # WAV文件头
+    wav_header = bytearray()
+    wav_header.extend(b"RIFF")  # ChunkID
+    wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little"))  # ChunkSize
+    wav_header.extend(b"WAVE")  # Format
+    wav_header.extend(b"fmt ")  # Subchunk1ID
+    wav_header.extend((16).to_bytes(4, "little"))  # Subchunk1Size
+    wav_header.extend((1).to_bytes(2, "little"))  # AudioFormat (PCM)
+    wav_header.extend((1).to_bytes(2, "little"))  # NumChannels
+    wav_header.extend((16000).to_bytes(4, "little"))  # SampleRate
+    wav_header.extend((32000).to_bytes(4, "little"))  # ByteRate
+    wav_header.extend((2).to_bytes(2, "little"))  # BlockAlign
+    wav_header.extend((16).to_bytes(2, "little"))  # BitsPerSample
+    wav_header.extend(b"data")  # Subchunk2ID
+    wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little"))  # Subchunk2Size
+
+    # 返回完整的WAV数据
+    return bytes(wav_header) + pcm_data_bytes
+
+
+def enqueue_tts_report(conn, text, opus_data):
+    if not conn.read_config_from_api or conn.need_bind or not conn.report_tts_enable:
+        return
+    if conn.chat_history_conf == 0:
+        return
+    """将TTS数据加入上报队列
+
+    Args:
+        conn: 连接对象
+        text: 合成文本
+        opus_data: opus音频数据
+    """
+    try:
+        # 使用连接对象的队列，传入文本和二进制数据而非文件路径
+        if conn.chat_history_conf == 2:
+            conn.report_queue.put((2, text, opus_data))
+            conn.logger.bind(tag=TAG).debug(
+                f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
+            )
+        else:
+            conn.report_queue.put((2, text, None))
+            conn.logger.bind(tag=TAG).debug(
+                f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频"
+            )
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
+
+
+def enqueue_asr_report(conn, text, opus_data):
+    if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
+        return
+    if conn.chat_history_conf == 0:
+        return
+    """将ASR数据加入上报队列
+
+    Args:
+        conn: 连接对象
+        text: 合成文本
+        opus_data: opus音频数据
+    """
+    try:
+        # 使用连接对象的队列，传入文本和二进制数据而非文件路径
+        if conn.chat_history_conf == 2:
+            conn.report_queue.put((1, text, opus_data))
+            conn.logger.bind(tag=TAG).debug(
+                f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
+            )
+        else:
+            conn.report_queue.put((1, text, None))
+            conn.logger.bind(tag=TAG).debug(
+                f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
+            )
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"加入ASR上报队列失败: {text}, {e}")

main/xiaozhi-server/core/handle/textHandle.py

@@ -1,33 +1,37 @@
-from config.logger import setup_logging
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.utils.util import remove_punctuation_and_length
+from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
 from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
+from core.handle.reportHandle import enqueue_asr_report
 import asyncio
 
 TAG = __name__
-logger = setup_logging()
 
 
 async def handleTextMessage(conn, message):
     """处理文本消息"""
-    logger.bind(tag=TAG).info(f"收到文本消息：{message}")
     try:
         msg_json = json.loads(message)
         if isinstance(msg_json, int):
+            conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}")
             await conn.websocket.send(message)
             return
         if msg_json["type"] == "hello":
-            await handleHelloMessage(conn)
+            conn.logger.bind(tag=TAG).info(f"收到hello消息：{message}")
+            await handleHelloMessage(conn, msg_json)
         elif msg_json["type"] == "abort":
+            conn.logger.bind(tag=TAG).info(f"收到abort消息：{message}")
             await handleAbortMessage(conn)
         elif msg_json["type"] == "listen":
+            conn.logger.bind(tag=TAG).info(f"收到listen消息：{message}")
             if "mode" in msg_json:
                 conn.client_listen_mode = msg_json["mode"]
-                logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
+                conn.logger.bind(tag=TAG).debug(
+                    f"客户端拾音模式：{conn.client_listen_mode}"
+                )
             if msg_json["state"] == "start":
                 conn.client_have_voice = True
                 conn.client_voice_stop = False
@@ -53,13 +57,99 @@ async def handleTextMessage(conn, message):
                         # 如果是唤醒词，且关闭了唤醒词回复，就不用回答
                         await send_stt_message(conn, text)
                         await send_tts_message(conn, "stop", None)
+                    elif is_wakeup_words:
+                        # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
+                        enqueue_asr_report(conn, "嘿，你好呀", [])
+                        await startToChat(conn, "嘿，你好呀")
                     else:
+                        # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
+                        enqueue_asr_report(conn, text, [])
                         # 否则需要LLM对文字内容进行答复
                         await startToChat(conn, text)
         elif msg_json["type"] == "iot":
+            conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}")
             if "descriptors" in msg_json:
                 asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
             if "states" in msg_json:
                 asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
+        elif msg_json["type"] == "server":
+            # 记录日志时过滤敏感信息
+            conn.logger.bind(tag=TAG).info(
+                f"收到服务器消息：{filter_sensitive_info(msg_json)}"
+            )
+            # 如果配置是从API读取的，则需要验证secret
+            if not conn.read_config_from_api:
+                return
+            # 获取post请求的secret
+            post_secret = msg_json.get("content", {}).get("secret", "")
+            secret = conn.config["manager-api"].get("secret", "")
+            # 如果secret不匹配，则返回
+            if post_secret != secret:
+                await conn.websocket.send(
+                    json.dumps(
+                        {
+                            "type": "server",
+                            "status": "error",
+                            "message": "服务器密钥验证失败",
+                        }
+                    )
+                )
+                return
+            # 动态更新配置
+            if msg_json["action"] == "update_config":
+                try:
+                    # 更新WebSocketServer的配置
+                    if not conn.server:
+                        await conn.websocket.send(
+                            json.dumps(
+                                {
+                                    "type": "server",
+                                    "status": "error",
+                                    "message": "无法获取服务器实例",
+                                    "content": {"action": "update_config"},
+                                }
+                            )
+                        )
+                        return
+
+                    if not await conn.server.update_config():
+                        await conn.websocket.send(
+                            json.dumps(
+                                {
+                                    "type": "server",
+                                    "status": "error",
+                                    "message": "更新服务器配置失败",
+                                    "content": {"action": "update_config"},
+                                }
+                            )
+                        )
+                        return
+
+                    # 发送成功响应
+                    await conn.websocket.send(
+                        json.dumps(
+                            {
+                                "type": "server",
+                                "status": "success",
+                                "message": "配置更新成功",
+                                "content": {"action": "update_config"},
+                            }
+                        )
+                    )
+                except Exception as e:
+                    conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
+                    await conn.websocket.send(
+                        json.dumps(
+                            {
+                                "type": "server",
+                                "status": "error",
+                                "message": f"更新配置失败: {str(e)}",
+                                "content": {"action": "update_config"},
+                            }
+                        )
+                    )
+            # 重启服务器
+            elif msg_json["action"] == "restart":
+                await conn.handle_restart(msg_json)
     except json.JSONDecodeError:
         await conn.websocket.send(message)

main/xiaozhi-server/core/mcp/MCPClient.py

@@ -1,86 +1,125 @@
+from __future__ import annotations
+
 from datetime import timedelta
-from typing import Optional
+import asyncio, os, shutil, concurrent.futures
 from contextlib import AsyncExitStack
-import os, shutil
+from typing import Optional, List, Dict, Any
+
 from mcp import ClientSession, StdioServerParameters
 from mcp.client.stdio import stdio_client
-
+from mcp.client.sse import sse_client
 from config.logger import setup_logging
 
 TAG = __name__
 
+
 class MCPClient:
-    def __init__(self, config):
-        # Initialize session and client objects
-        self.session: Optional[ClientSession] = None
-        self.exit_stack = AsyncExitStack()
+    def __init__(self, config: Dict[str, Any]):
         self.logger = setup_logging()
         self.config = config
-        self.tolls = []
+
+        self._worker_task: Optional[asyncio.Task] = None
+        self._ready_evt = asyncio.Event()
+        self._shutdown_evt = asyncio.Event()
+
+        self.session: Optional[ClientSession] = None
+        self.tools: List = []
 
     async def initialize(self):
-        args = self.config.get("args", [])
+        if self._worker_task:
+            return
+        self._worker_task = asyncio.create_task(self._worker(), name="MCPClientWorker")
+        await self._ready_evt.wait()
 
-        command = (
-            shutil.which("npx")
-            if self.config["command"] == "npx"
-            else self.config["command"]
+        self.logger.bind(tag=TAG).info(
+            f"Connected, tools = {[t.name for t in self.tools]}"
         )
-        
-        env={**os.environ}
-        if self.config.get("env"):
-            env.update(self.config["env"])
-        
-        server_params = StdioServerParameters(
-            command=command,
-            args=args,
-            env=env
-        )
-        
-        stdio_transport = await self.exit_stack.enter_async_context(stdio_client(server_params))
-        self.stdio, self.write = stdio_transport
-        time_out_delta =  timedelta(seconds=15)
-        self.session = await self.exit_stack.enter_async_context(ClientSession(read_stream=self.stdio, write_stream=self.write, read_timeout_seconds=time_out_delta))
-        
-        await self.session.initialize()
-        
-        # List available tools
-        response = await self.session.list_tools()
-        tools = response.tools
-        self.tools = tools
-        self.logger.bind(tag=TAG).info(f"Connected to server with tools:{[tool.name for tool in tools]}")
-    
-    def has_tool(self, tool_name):
-        return any(tool.name == tool_name for tool in self.tools)
-    
-    def get_available_tools(self):
-        available_tools = [{"type": "function", "function":{ 
-            "name": tool.name,
-            "description": tool.description,
-            "parameters": tool.inputSchema
-        } } for tool in self.tools]
-
-        return available_tools
-    
-    async def call_tool(self, tool_name: str, tool_args: dict):
-        self.logger.bind(tag=TAG).info(f"MCPClient Calling tool {tool_name} with args: {tool_args}")
-        try:
-            response = await self.session.call_tool(tool_name, tool_args)
-        except Exception as e:
-            self.logger.bind(tag=TAG).error(f"Error calling tool {tool_name}: {e}")
-            from types import SimpleNamespace
-            error_content = SimpleNamespace(
-                type='text',
-                text=f"Error calling tool {tool_name}: {e}"
-            )
-            error_response = SimpleNamespace(
-                content=[error_content],
-                isError=True
-            )
-            return error_response
-        self.logger.bind(tag=TAG).info(f"MCPClient Response from tool {tool_name}: {response}")
-        return response
 
     async def cleanup(self):
-        """Clean up resources"""
-        await self.exit_stack.aclose()
+        if not self._worker_task:
+            return
+
+        self._shutdown_evt.set()
+        try:
+            await asyncio.wait_for(self._worker_task, timeout=20)
+        except (asyncio.TimeoutError, Exception) as e:
+            self.logger.bind(tag=TAG).error(f"worker shutdown err: {e}")
+        finally:
+            self._worker_task = None
+
+    def has_tool(self, name: str) -> bool:
+        return any(t.name == name for t in self.tools)
+
+    def get_available_tools(self):
+        return [
+            {
+                "type": "function",
+                "function": {
+                    "name": t.name,
+                    "description": t.description,
+                    "parameters": t.inputSchema,
+                },
+            }
+            for t in self.tools
+        ]
+
+    async def call_tool(self, name: str, args: dict):
+        if not self.session:
+            raise RuntimeError("MCPClient not initialized")
+
+        loop = self._worker_task.get_loop()
+        coro = self.session.call_tool(name, args)
+
+        if loop is asyncio.get_running_loop():
+            return await coro
+
+        fut: concurrent.futures.Future = asyncio.run_coroutine_threadsafe(coro, loop)
+        return await asyncio.wrap_future(fut)
+
+    async def _worker(self):
+        async with AsyncExitStack() as stack:
+            try:
+                # 建立 StdioClient
+                if "command" in self.config:
+                    cmd = (
+                        shutil.which("npx")
+                        if self.config["command"] == "npx"
+                        else self.config["command"]
+                    )
+                    env = {**os.environ, **self.config.get("env", {})}
+                    params = StdioServerParameters(
+                        command=cmd,
+                        args=self.config.get("args", []),
+                        env=env,
+                    )
+                    stdio_r, stdio_w = await stack.enter_async_context(stdio_client(params))
+                    read_stream, write_stream = stdio_r, stdio_w
+                # 建立SSEClient
+                elif "url" in self.config:
+                    sse_r, sse_w = await stack.enter_async_context(sse_client(self.config["url"]))
+                    read_stream, write_stream = sse_r, sse_w
+
+                else:
+                    raise ValueError("MCPClient config must include 'command' or 'url'")
+
+                self.session = await stack.enter_async_context(
+                    ClientSession(
+                        read_stream=read_stream,
+                        write_stream=write_stream,
+                        read_timeout_seconds=timedelta(seconds=15),
+                    )
+                )
+                await self.session.initialize()
+
+                # 获取工具
+                self.tools = (await self.session.list_tools()).tools
+
+                self._ready_evt.set()
+
+                # 挂起等待关闭
+                await self._shutdown_evt.wait()
+
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"worker error: {e}")
+                self._ready_evt.set()
+                raise

main/xiaozhi-server/core/mcp/manager.py

@@ -1,26 +1,29 @@
 """MCP服务管理器"""
+
+import asyncio
 import os, json
 from typing import Dict, Any, List
 from .MCPClient import MCPClient
-from config.logger import setup_logging
-from core.utils.util import get_project_dir
-from plugins_func.register import register_function, ActionResponse, Action, ToolType
+from plugins_func.register import register_function, ToolType
+from config.config_loader import get_project_dir
 
 TAG = __name__
 
+
 class MCPManager:
     """管理多个MCP服务的集中管理器"""
 
-    def __init__(self,conn) -> None:
+    def __init__(self, conn) -> None:
         """
         初始化MCP管理器
         """
         self.conn = conn
-        self.logger = setup_logging()
-        self.config_path = get_project_dir() + 'data/.mcp_server_settings.json'
+        self.config_path = get_project_dir() + "data/.mcp_server_settings.json"
         if os.path.exists(self.config_path) == False:
             self.config_path = ""
-            self.logger.bind(tag=TAG).warning(f"请检查mcp服务配置文件：data/.mcp_server_settings.json")
+            self.conn.logger.bind(tag=TAG).warning(
+                f"请检查mcp服务配置文件：data/.mcp_server_settings.json"
+            )
         self.client: Dict[str, MCPClient] = {}
         self.tools = []
 
@@ -31,37 +34,47 @@ class MCPManager:
         """
         if len(self.config_path) == 0:
             return {}
-        
+
         try:
-            with open(self.config_path, 'r', encoding='utf-8') as f:
+            with open(self.config_path, "r", encoding="utf-8") as f:
                 config = json.load(f)
-            return config.get('mcpServers', {})
+            return config.get("mcpServers", {})
         except Exception as e:
-            self.logger.bind(tag=TAG).error(f"Error loading MCP config from {self.config_path}: {e}")
+            self.conn.logger.bind(tag=TAG).error(
+                f"Error loading MCP config from {self.config_path}: {e}"
+            )
             return {}
 
     async def initialize_servers(self) -> None:
         """初始化所有MCP服务"""
         config = self.load_config()
         for name, srv_config in config.items():
-            if not srv_config.get("command"):
-                self.logger.bind(tag=TAG).warning(f"Skipping server {name}: command not specified")
+            if not srv_config.get("command") and not srv_config.get("url"):
+                self.conn.logger.bind(tag=TAG).warning(
+                    f"Skipping server {name}: neither command nor url specified"
+                )
                 continue
 
             try:
                 client = MCPClient(srv_config)
                 await client.initialize()
                 self.client[name] = client
-                self.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}")
+                self.conn.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}")
                 client_tools = client.get_available_tools()
                 self.tools.extend(client_tools)
                 for tool in client_tools:
-                    func_name = "mcp_"+tool["function"]["name"]
-                    register_function(func_name, tool, ToolType.MCP_CLIENT)(self.execute_tool)
-                    self.conn.func_handler.function_registry.register_function(func_name)
+                    func_name = "mcp_" + tool["function"]["name"]
+                    register_function(func_name, tool, ToolType.MCP_CLIENT)(
+                        self.execute_tool
+                    )
+                    self.conn.func_handler.function_registry.register_function(
+                        func_name
+                    )
 
             except Exception as e:
-                self.logger.bind(tag=TAG).error(f"Failed to initialize MCP server {name}: {e}")
+                self.conn.logger.bind(tag=TAG).error(
+                    f"Failed to initialize MCP server {name}: {e}"
+                )
         self.conn.func_handler.upload_functions_desc()
 
     def get_all_tools(self) -> List[Dict[str, Any]]:
@@ -79,7 +92,10 @@ class MCPManager:
             bool: 是否是MCP工具
         """
         for tool in self.tools:
-            if tool.get("function") != None and tool["function"].get("name") == tool_name:
+            if (
+                tool.get("function") != None
+                and tool["function"].get("name") == tool_name
+            ):
                 return True
         return False
 
@@ -87,24 +103,29 @@ class MCPManager:
         """执行工具调用
         Args:
             tool_name: 工具名称
-            arguments: 工具参数 
+            arguments: 工具参数
         Returns:
             Any: 工具执行结果
         Raises:
             ValueError: 工具未找到时抛出
         """
-        self.logger.bind(tag=TAG).info(f"Executing tool {tool_name} with arguments: {arguments}")
+        self.conn.logger.bind(tag=TAG).info(
+            f"Executing tool {tool_name} with arguments: {arguments}"
+        )
         for client in self.client.values():
             if client.has_tool(tool_name):
                 return await client.call_tool(tool_name, arguments)
-            
+
         raise ValueError(f"Tool {tool_name} not found in any MCP server")
 
     async def cleanup_all(self) -> None:
-        for name, client in self.client.items():
+        """依次关闭所有 MCPClient，不让异常阻断整体流程。"""
+        for name, client in list(self.client.items()):
             try:
-                await client.cleanup()
-                self.logger.bind(tag=TAG).info(f"Cleaned up MCP client: {name}")
-            except Exception as e:
-                self.logger.bind(tag=TAG).error(f"Error cleaning up MCP client {name}: {e}")
+                await asyncio.wait_for(client.cleanup(), timeout=20)
+                self.conn.logger.bind(tag=TAG).info(f"MCP client closed: {name}")
+            except (asyncio.TimeoutError, Exception) as e:
+                self.conn.logger.bind(tag=TAG).error(
+                    f"Error closing MCP client {name}: {e}"
+                )
         self.client.clear()

main/xiaozhi-server/core/ota_server.py

@@ -0,0 +1,135 @@
+import json
+import time
+import asyncio
+from aiohttp import web
+from config.logger import setup_logging
+from core.connection import ConnectionHandler
+from core.utils.util import get_local_ip, initialize_modules
+
+TAG = __name__
+
+
+class SimpleOtaServer:
+    def __init__(self, config: dict):
+        self.config = config
+        self.logger = setup_logging()
+
+    def _get_websocket_url(self, local_ip: str, port: int) -> str:
+        """获取websocket地址
+
+        Args:
+            local_ip: 本地IP地址
+            port: 端口号
+
+        Returns:
+            str: websocket地址
+        """
+        server_config = self.config["server"]
+        websocket_config = server_config.get("websocket")
+
+        if websocket_config and "你" not in websocket_config:
+            return websocket_config
+        else:
+            return f"ws://{local_ip}:{port}/xiaozhi/v1/"
+
+    async def start(self):
+        server_config = self.config["server"]
+        host = server_config.get("ip", "0.0.0.0")
+        port = int(server_config.get("ota_port"))
+
+        if port:
+            app = web.Application()
+            # 添加路由
+            app.add_routes(
+                [
+                    web.get("/xiaozhi/ota/", self._handle_ota_get_request),
+                    web.post("/xiaozhi/ota/", self._handle_ota_request),
+                    web.options("/xiaozhi/ota/", self._handle_ota_request),
+                ]
+            )
+
+            # 运行服务
+            runner = web.AppRunner(app)
+            await runner.setup()
+            site = web.TCPSite(runner, host, port)
+            await site.start()
+
+            # 保持服务运行
+            while True:
+                await asyncio.sleep(3600)  # 每隔 1 小时检查一次
+
+    async def _handle_ota_request(self, request):
+        """处理 /xiaozhi/ota/ 的 POST 请求"""
+        try:
+            data = await request.text()
+            self.logger.bind(tag=TAG).debug(f"OTA请求方法: {request.method}")
+            self.logger.bind(tag=TAG).debug(f"OTA请求头: {request.headers}")
+            self.logger.bind(tag=TAG).debug(f"OTA请求数据: {data}")
+
+            device_id = request.headers.get("device-id", "")
+            if device_id:
+                self.logger.bind(tag=TAG).info(f"OTA请求设备ID: {device_id}")
+            else:
+                raise Exception("OTA请求设备ID为空")
+
+            data_json = json.loads(data)
+
+            server_config = self.config["server"]
+            host = server_config.get("ip", "0.0.0.0")
+            port = int(server_config.get("port", 8000))
+            local_ip = get_local_ip()
+
+            # OTA基础信息
+            return_json = {
+                "server_time": {
+                    "timestamp": int(round(time.time() * 1000)),
+                    "timezone_offset": server_config.get("timezone_offset", 8) * 60,
+                },
+                "firmware": {
+                    "version": data_json["application"].get("version", "1.0.0"),
+                    "url": "",
+                },
+                "websocket": {
+                    "url": self._get_websocket_url(local_ip, port),
+                },
+            }
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"OTA请求异常: {e}")
+            return_json = {"success": False, "message": "request error."}
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        finally:
+            # 添加header，允许跨域访问
+            response.headers["Access-Control-Allow-Headers"] = (
+                "client-id, content-type, device-id"
+            )
+            response.headers["Access-Control-Allow-Credentials"] = "true"
+            response.headers["Access-Control-Allow-Origin"] = "*"
+            return response
+
+    async def _handle_ota_get_request(self, request):
+        """处理 /xiaozhi/ota/ 的 GET 请求"""
+        try:
+            server_config = self.config["server"]
+            local_ip = get_local_ip()
+            port = int(server_config.get("port", 8000))
+            websocket_url = self._get_websocket_url(local_ip, port)
+            message = f"OTA接口运行正常，向设备发送的websocket地址是：{websocket_url}"
+            response = web.Response(text=message, content_type="text/plain")
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"OTA GET请求异常: {e}")
+            response = web.Response(text="OTA接口异常", content_type="text/plain")
+        finally:
+            # 添加header，允许跨域访问
+            response.headers["Access-Control-Allow-Headers"] = (
+                "client-id, content-type, device-id"
+            )
+            response.headers["Access-Control-Allow-Credentials"] = "true"
+            response.headers["Access-Control-Allow-Origin"] = "*"
+            return response

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -0,0 +1,270 @@
+import http.client
+import json
+import asyncio
+from typing import Optional, Tuple, List
+import opuslib_next
+import wave
+import io
+import os
+import uuid
+import hmac
+import hashlib
+import base64
+import requests
+from urllib import parse
+import time
+from datetime import datetime
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {
+            "AccessKeyId": access_key_id,
+            "Action": "CreateToken",
+            "Format": "JSON",
+            "RegionId": "cn-shanghai",
+            "SignatureMethod": "HMAC-SHA1",
+            "SignatureNonce": str(uuid.uuid1()),
+            "SignatureVersion": "1.0",
+            "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "Version": "2019-02-28",
+        }
+        # 构造规范化的请求字符串
+        query_string = AccessToken._encode_dict(parameters)
+        # print('规范化的请求字符串: %s' % query_string)
+        # 构造待签名字符串
+        string_to_sign = (
+            "GET"
+            + "&"
+            + AccessToken._encode_text("/")
+            + "&"
+            + AccessToken._encode_text(query_string)
+        )
+        # print('待签名的字符串: %s' % string_to_sign)
+        # 计算签名
+        secreted_string = hmac.new(
+            bytes(access_key_secret + "&", encoding="utf-8"),
+            bytes(string_to_sign, encoding="utf-8"),
+            hashlib.sha1,
+        ).digest()
+        signature = base64.b64encode(secreted_string)
+        # print('签名: %s' % signature)
+        # 进行URL编码
+        signature = AccessToken._encode_text(signature)
+        # print('URL编码后的签名: %s' % signature)
+        # 调用服务
+        full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
+            signature,
+            query_string,
+        )
+        # print('url: %s' % full_url)
+        # 提交HTTP GET请求
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            key = "Token"
+            if key in root_obj:
+                token = root_obj[key]["Id"]
+                expire_time = root_obj[key]["ExpireTime"]
+                return token, expire_time
+        # print(response.text)
+        return None, None
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        """阿里云ASR初始化"""
+        # 新增空值判断逻辑
+        self.access_key_id = config.get("access_key_id")
+        self.access_key_secret = config.get("access_key_secret")
+
+        self.app_key = config.get("appkey")
+        self.host = "nls-gateway-cn-shanghai.aliyuncs.com"
+        self.base_url = f"https://{self.host}/stream/v1/asr"
+        self.sample_rate = 16000
+        self.format = "wav"
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+
+        if self.access_key_id and self.access_key_secret:
+            # 使用密钥对生成临时token
+            self._refresh_token()
+        else:
+            # 直接使用预生成的长期token
+            self.token = config.get("token")
+            self.expire_time = None
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _refresh_token(self):
+        """刷新Token并记录过期时间"""
+        if self.access_key_id and self.access_key_secret:
+            self.token, expire_time_str = AccessToken.create_token(
+                self.access_key_id, self.access_key_secret
+            )
+            if not expire_time_str:
+                raise ValueError("无法获取有效的Token过期时间")
+
+            try:
+                # 统一转换为字符串处理
+                expire_str = str(expire_time_str).strip()
+
+                if expire_str.isdigit():
+                    expire_time = datetime.fromtimestamp(int(expire_str))
+                else:
+                    expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
+                self.expire_time = expire_time.timestamp() - 60
+            except Exception as e:
+                raise ValueError(f"无效的过期时间格式: {expire_str}") from e
+
+        else:
+            self.expire_time = None
+
+        if not self.token:
+            raise ValueError("无法获取有效的访问Token")
+
+    def _is_token_expired(self):
+        """检查Token是否过期"""
+        if not self.expire_time:
+            return False  # 长期Token不过期
+        # 新增调试日志
+        # current_time = time.time()
+        # remaining = self.expire_time - current_time
+        # print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | "
+        #              f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
+        #              f"剩余 {remaining:.2f}秒")
+        return time.time() > self.expire_time
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(
+            self.output_file,
+            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        )
+
+    def _construct_request_url(self) -> str:
+        """构造请求URL，包含参数"""
+        request = f"{self.base_url}?appkey={self.app_key}"
+        request += f"&format={self.format}"
+        request += f"&sample_rate={self.sample_rate}"
+        request += "&enable_punctuation_prediction=true"
+        request += "&enable_inverse_text_normalization=true"
+        request += "&enable_voice_detection=false"
+        return request
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)  # 单声道
+            wf.setsampwidth(2)  # 16-bit
+            wf.setframerate(self.sample_rate)
+            wf.writeframes(b"".join(pcm_data))
+
+        logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}")
+        return file_path
+
+    async def _send_request(self, pcm_data: bytes) -> Optional[str]:
+        """发送请求到阿里云ASR服务"""
+        try:
+            # 设置HTTP头
+            headers = {
+                "X-NLS-Token": self.token,
+                "Content-type": "application/octet-stream",
+                "Content-Length": str(len(pcm_data)),
+            }
+
+            # 创建连接并发送请求
+            conn = http.client.HTTPSConnection(self.host)
+            request_url = self._construct_request_url()
+
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                None,
+                lambda: conn.request(
+                    method="POST", url=request_url, body=pcm_data, headers=headers
+                ),
+            )
+
+            # 获取响应
+            response = await loop.run_in_executor(None, conn.getresponse)
+            body = await loop.run_in_executor(None, response.read)
+            conn.close()
+
+            # 解析响应
+            try:
+                body_json = json.loads(body)
+                status = body_json.get("status")
+
+                if status == 20000000:
+                    result = body_json.get("result", "")
+                    logger.bind(tag=TAG).debug(f"ASR结果: {result}")
+                    return result
+                else:
+                    logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
+                    return None
+
+            except ValueError:
+                logger.bind(tag=TAG).error("响应不是JSON格式")
+                return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True)
+            return None
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if self._is_token_expired():
+            logger.warning("Token已过期，正在自动刷新...")
+            self._refresh_token()
+
+        file_path = None
+        try:
+            # 解码Opus为PCM
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+
+            # 发送请求并获取文本
+            text = await self._send_request(combined_pcm_data)
+
+            if text:
+                return text, file_path
+
+            return "", file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", file_path

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -0,0 +1,106 @@
+import base64
+import hashlib
+import hmac
+import json
+import time
+from datetime import datetime, timezone
+import os
+import uuid
+from typing import Optional, Tuple, List
+import wave
+import opuslib_next
+
+from aip import AipSpeech
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.secret_key = config.get("secret_key")
+
+        dev_pid = config.get("dev_pid", "1537")
+        self.dev_pid = int(dev_pid) if dev_pid else 1537
+
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        self.client = AipSpeech(str(self.app_id), self.api_key, self.secret_key)
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if not opus_data:
+            logger.bind(tag=TAG).warning("音频数据为空！")
+            return None, None
+
+        file_path = None
+        try:
+            # 检查配置是否已设置
+            if not self.app_id or not self.api_key or not self.secret_key:
+                logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
+                return None, file_path
+
+            # 将Opus音频数据解码为PCM
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                self.save_audio_to_file(pcm_data, session_id)
+
+            start_time = time.time()
+            # 识别本地文件
+            result = self.client.asr(
+                combined_pcm_data,
+                "pcm",
+                16000,
+                {
+                    "dev_pid": str(self.dev_pid),
+                },
+            )
+
+            if result and result["err_no"] == 0:
+                logger.bind(tag=TAG).debug(
+                    f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+                result = result["result"][0]
+                return result, file_path
+            else:
+                raise Exception(
+                    f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
+                )
+                return None, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
+            return None, file_path

main/xiaozhi-server/core/providers/asr/base.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from typing import Optional, Tuple, List
-
+import opuslib_next
 from config.logger import setup_logging
 
 TAG = __name__
@@ -8,12 +8,37 @@ logger = setup_logging()
 
 
 class ASRProviderBase(ABC):
+    def __init__(self):
+        self.audio_format = "opus"
+
     @abstractmethod
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """解码Opus数据并保存为WAV文件"""
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
         pass
 
     @abstractmethod
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         pass
+
+    def set_audio_format(self, format: str) -> None:
+        """设置音频格式"""
+        self.audio_format = format
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes]) -> bytes:
+        """将Opus音频数据解码为PCM数据"""
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -45,14 +45,14 @@ def parse_response(res):
     payload 类似与http 请求体
     """
     protocol_version = res[0] >> 4
-    header_size = res[0] & 0x0f
+    header_size = res[0] & 0x0F
     message_type = res[1] >> 4
-    message_type_specific_flags = res[1] & 0x0f
+    message_type_specific_flags = res[1] & 0x0F
     serialization_method = res[2] >> 4
-    message_compression = res[2] & 0x0f
+    message_compression = res[2] & 0x0F
     reserved = res[3]
-    header_extensions = res[4:header_size * 4]
-    payload = res[header_size * 4:]
+    header_extensions = res[4 : header_size * 4]
+    payload = res[header_size * 4 :]
     result = {}
     payload_msg = None
     payload_size = 0
@@ -61,13 +61,13 @@ def parse_response(res):
         payload_msg = payload[4:]
     elif message_type == SERVER_ACK:
         seq = int.from_bytes(payload[:4], "big", signed=True)
-        result['seq'] = seq
+        result["seq"] = seq
         if len(payload) >= 8:
             payload_size = int.from_bytes(payload[4:8], "big", signed=False)
             payload_msg = payload[8:]
     elif message_type == SERVER_ERROR_RESPONSE:
         code = int.from_bytes(payload[:4], "big", signed=False)
-        result['code'] = code
+        result["code"] = code
         payload_size = int.from_bytes(payload[4:8], "big", signed=False)
         payload_msg = payload[8:]
     if payload_msg is None:
@@ -78,17 +78,21 @@ def parse_response(res):
         payload_msg = json.loads(str(payload_msg, "utf-8"))
     elif serialization_method != NO_SERIALIZATION:
         payload_msg = str(payload_msg, "utf-8")
-    result['payload_msg'] = payload_msg
-    result['payload_size'] = payload_size
+    result["payload_msg"] = payload_msg
+    result["payload_size"] = payload_size
     return result
 
 
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
         self.appid = config.get("appid")
         self.cluster = config.get("cluster")
         self.access_token = config.get("access_token")
+        self.boosting_table_name = config.get("boosting_table_name", "")
+        self.correct_table_name = config.get("correct_table_name", "")
         self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
 
         self.host = "openspeech.bytedance.com"
         self.ws_url = f"wss://{self.host}/api/v2/asr"
@@ -98,21 +102,12 @@ class ASRProvider(ASRProviderBase):
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -122,7 +117,9 @@ class ASRProvider(ASRProviderBase):
         return file_path
 
     @staticmethod
-    def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
+    def _generate_header(
+        message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE
+    ) -> bytearray:
         """Generate protocol header."""
         header = bytearray()
         header_size = 1
@@ -146,10 +143,12 @@ class ASRProvider(ASRProviderBase):
             "request": {
                 "reqid": reqid,
                 "show_utterances": False,
-                "sequence": 1
+                "sequence": 1,
+                "boosting_table_name": self.boosting_table_name,
+                "correct_table_name": self.correct_table_name,
             },
             "audio": {
-                "format": "wav",
+                "format": "raw",
                 "rate": 16000,
                 "language": "zh-CN",
                 "bits": 16,
@@ -158,18 +157,23 @@ class ASRProvider(ASRProviderBase):
             },
         }
 
-    async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
+    async def _send_request(
+        self, audio_data: List[bytes], segment_size: int
+    ) -> Optional[str]:
         """Send request to Volcano ASR service."""
         try:
-            auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
-            async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
+            auth_header = {"Authorization": "Bearer; {}".format(self.access_token)}
+            async with websockets.connect(
+                self.ws_url, additional_headers=auth_header
+            ) as websocket:
                 # Prepare request data
                 request_params = self._construct_request(str(uuid.uuid4()))
-                print(request_params)
                 payload_bytes = str.encode(json.dumps(request_params))
                 payload_bytes = gzip.compress(payload_bytes)
                 full_client_request = self._generate_header()
-                full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                full_client_request.extend(
+                    (len(payload_bytes)).to_bytes(4, "big")
+                )  # payload size(4 bytes)
                 full_client_request.extend(payload_bytes)  # payload
 
                 # Send header and metadata
@@ -177,22 +181,29 @@ class ASRProvider(ASRProviderBase):
                 await websocket.send(full_client_request)
                 res = await websocket.recv()
                 result = parse_response(res)
-                if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
+                if (
+                    "payload_msg" in result
+                    and result["payload_msg"]["code"] != self.success_code
+                ):
                     logger.bind(tag=TAG).error(f"ASR error: {result}")
                     return None
 
-                for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
+                for seq, (chunk, last) in enumerate(
+                    self.slice_data(audio_data, segment_size), 1
+                ):
                     if last:
                         audio_only_request = self._generate_header(
                             message_type=CLIENT_AUDIO_ONLY_REQUEST,
-                            message_type_specific_flags=NEG_SEQUENCE
+                            message_type_specific_flags=NEG_SEQUENCE,
                         )
                     else:
                         audio_only_request = self._generate_header(
                             message_type=CLIENT_AUDIO_ONLY_REQUEST
                         )
                     payload_bytes = gzip.compress(chunk)
-                    audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                    audio_only_request.extend(
+                        (len(payload_bytes)).to_bytes(4, "big")
+                    )  # payload size(4 bytes)
                     audio_only_request.extend(payload_bytes)  # payload
                     # Send audio data
                     await websocket.send(audio_only_request)
@@ -201,9 +212,12 @@ class ASRProvider(ASRProviderBase):
                 response = await websocket.recv()
                 result = parse_response(response)
 
-                if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
-                    if len(result['payload_msg']['result']) > 0:
-                        return result['payload_msg']['result'][0]["text"]
+                if (
+                    "payload_msg" in result
+                    and result["payload_msg"]["code"] == self.success_code
+                ):
+                    if len(result["payload_msg"]["result"]) > 0:
+                        return result["payload_msg"]["result"][0]["text"]
                     return None
                 else:
                     logger.bind(tag=TAG).error(f"ASR error: {result}")
@@ -213,29 +227,6 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
             return None
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
-    @staticmethod
-    def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
-        with io.BytesIO(data) as _f:
-            wave_fp = wave.open(_f, 'rb')
-            nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
-            wave_bytes = wave_fp.readframes(nframes)
-        return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
-
     @staticmethod
     def slice_data(data: bytes, chunk_size: int) -> (list, bool):
         """
@@ -247,40 +238,46 @@ class ASRProvider(ASRProviderBase):
         data_len = len(data)
         offset = 0
         while offset + chunk_size < data_len:
-            yield data[offset: offset + chunk_size], False
+            yield data[offset : offset + chunk_size], False
             offset += chunk_size
         else:
-            yield data[offset: data_len], True
+            yield data[offset:data_len], True
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
+
+        file_path = None
         try:
             # 合并所有opus数据包
-            pcm_data = self.decode_opus(opus_data, session_id)
-            combined_pcm_data = b''.join(pcm_data)
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
 
-            wav_buffer = io.BytesIO()
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
-            with wave.open(wav_buffer, "wb") as wav_file:
-                wav_file.setnchannels(1)  # 设置声道数
-                wav_file.setsampwidth(2)  # 设置采样宽度
-                wav_file.setframerate(16000)  # 设置采样率
-                wav_file.writeframes(combined_pcm_data)  # 写入 PCM 数据
-
-            # 获取封装后的 WAV 数据
-            wav_data = wav_buffer.getvalue()
-            nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
-            size_per_sec = nchannels * sampwidth * framerate
+            # 直接使用PCM数据
+            # 计算分段大小 (单声道, 16bit, 16kHz采样率)
+            size_per_sec = 1 * 2 * 16000  # nchannels * sampwidth * framerate
             segment_size = int(size_per_sec * self.seg_duration / 1000)
 
             # 语音识别
             start_time = time.time()
-            text = await self._send_request(wav_data, segment_size)
+            text = await self._send_request(combined_pcm_data, segment_size)
             if text:
-                logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
-                return text, None
-            return "", None
+                logger.bind(tag=TAG).debug(
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+                )
+                return text, file_path
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -6,9 +6,7 @@ import io
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
 import uuid
-import opuslib_next
 from core.providers.asr.base import ASRProviderBase
-
 from funasr import AutoModel
 from funasr.utils.postprocess_utils import rich_transcription_postprocess
 
@@ -35,6 +33,7 @@ class CaptureOutput:
 
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
         self.model_dir = config.get("model_dir")
         self.output_dir = config.get("output_dir")  # 修正配置键名
         self.delete_audio_file = delete_audio_file
@@ -46,25 +45,16 @@ class ASRProvider(ASRProviderBase):
                 model=self.model_dir,
                 vad_kwargs={"max_single_segment_time": 30000},
                 disable_update=True,
-                hub="hf"
+                hub="hf",
                 # device="cuda:0",  # 启用GPU加速
             )
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -73,38 +63,51 @@ class ASRProvider(ASRProviderBase):
 
         return file_path
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
         try:
-            # 保存音频文件
-            start_time = time.time()
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+            # 合并所有opus数据包
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 语音识别
             start_time = time.time()
             result = self.model.generate(
-                input=file_path,
+                input=combined_pcm_data,
                 cache={},
                 language="auto",
                 use_itn=True,
                 batch_size_s=60,
             )
             text = rich_transcription_postprocess(result[0]["text"])
-            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+            logger.bind(tag=TAG).debug(
+                f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+            )
 
             return text, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
 
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
+        # finally:
+        #     # 文件清理逻辑
+        #     if self.delete_audio_file and file_path and os.path.exists(file_path):
+        #         try:
+        #             os.remove(file_path)
+        #             logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+        #         except Exception as e:
+        #             logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -0,0 +1,185 @@
+from typing import Optional, Tuple, List
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+import os
+import ssl
+import json
+import uuid
+import wave
+import websockets
+from config.logger import setup_logging
+import asyncio
+import re
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        """
+        Initialize the ASRProvider with server configuration.
+        :param config: Dictionary containing 'host', 'port', and 'is_ssl'.
+        :param delete_audio_file: Boolean to indicate whether to delete audio files after processing.
+        """
+        super().__init__()
+        self.host = config.get("host", "localhost")
+        self.port = config.get("port", 10095)
+        self.api_key = config.get("api_key", "none")
+        self.is_ssl = str(config.get("is_ssl", True)).lower() in (
+            "true",
+            "1",
+            "yes",
+        )
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+        self.uri = (
+            f"wss://{self.host}:{self.port}"
+            if self.is_ssl
+            else f"ws://{self.host}:{self.port}"
+        )
+        self.ssl_context = ssl.SSLContext() if self.is_ssl else None
+        if self.ssl_context:
+            self.ssl_context.check_hostname = False
+            self.ssl_context.verify_mode = ssl.CERT_NONE
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def _receive_responses(self, ws) -> None:
+        """
+        Asynchronous generator to receive messages from the WebSocket.
+        Yields each message as it is received.
+        """
+        text = ""
+        while True:
+            try:
+                response = await asyncio.wait_for(ws.recv(), timeout=5)
+                response_data = json.loads(response)
+                logger.bind(tag=TAG).debug(f"Received response: {response_data}")
+                if response_data.get("is_final", True):
+                    text += response_data.get("text", "")
+                    break
+                else:
+                    text += response_data.get("text", "")
+            except asyncio.TimeoutError:
+                logger.bind(tag=TAG).error(
+                    "Timeout while waiting for response from WebSocket."
+                )
+                break
+            except websockets.exceptions.ConnectionClosed as e:
+                logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
+                break
+        return text
+
+    async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple:
+        """
+        Internal method to handle WebSocket communication.
+        Reuses the persistent WebSocket connection if available.
+        :param pcm_data: PCM audio data to send.
+        :param session_id: Unique session identifier.
+        :return: Tuple containing recognized text and optional timestamp.
+        """
+
+        # Send initial configuration message
+        config_message = json.dumps(
+            {
+                "mode": "offline",
+                "chunk_size": [5, 10, 5],
+                "chunk_interval": 10,
+                "wav_name": session_id,
+                "is_speaking": True,
+                "itn": False,
+            }
+        )
+        await ws.send(config_message)
+        logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
+
+        # Send PCM data
+        await ws.send(pcm_data)
+        logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
+
+        # Indicate end of speech
+        end_message = json.dumps({"is_speaking": False})
+        await ws.send(end_message)
+        logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Convert speech data to text using FunASR.
+        :param opus_data: List of Opus-encoded audio data chunks.
+        :param session_id: Unique session identifier.
+        :return: Tuple containing recognized text and optional timestamp.
+        """
+        file_path = None
+        if self.audio_format == "pcm":
+            pcm_data = opus_data
+        else:
+            pcm_data = self.decode_opus(opus_data)
+        combined_pcm_data = b"".join(pcm_data)
+
+        # 判断是否保存为WAV文件
+        if self.delete_audio_file:
+            pass
+        else:
+            file_path = self.save_audio_to_file(pcm_data, session_id)
+        auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
+        async with websockets.connect(
+            self.uri,
+            additional_headers=auth_header,
+            subprotocols=["binary"],
+            ping_interval=None,
+            ssl=self.ssl_context,
+        ) as ws:
+            try:
+                # Use asyncio to handle WebSocket communication
+                send_task = asyncio.create_task(
+                    self._send_data(ws, combined_pcm_data, session_id)
+                )
+                receive_task = asyncio.create_task(self._receive_responses(ws))
+
+                # Gather tasks with error handling
+                done, pending = await asyncio.wait(
+                    [send_task, receive_task], return_when=asyncio.FIRST_EXCEPTION
+                )
+
+                # Cancel any pending tasks
+                for task in pending:
+                    task.cancel()
+
+                # Check for exceptions in completed tasks
+                for task in done:
+                    if task.exception():
+                        raise task.exception()
+
+                # Get the result from the receive task
+                result = receive_task.result()
+                match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
+                if match:
+                    result = match.group(4).strip()
+                return (
+                    result,
+                    file_path,
+                )  # Return the recognized text and timestamp (if any)
+
+            except websockets.exceptions.ConnectionClosed as e:
+                logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
+                return "", file_path
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"Error during speech-to-text conversion: {e}", exc_info=True
+                )
+                return "", file_path

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -37,17 +37,18 @@ class CaptureOutput:
 
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
         self.model_dir = config.get("model_dir")
         self.output_dir = config.get("output_dir")
         self.delete_audio_file = delete_audio_file
 
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
-        
+
         # 初始化模型文件路径
         model_files = {
             "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
-            "tokens.txt": os.path.join(self.model_dir, "tokens.txt")
+            "tokens.txt": os.path.join(self.model_dir, "tokens.txt"),
         }
 
         # 下载并检查模型文件
@@ -58,15 +59,15 @@ class ASRProvider(ASRProviderBase):
                     model_file_download(
                         model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
                         file_path=file_name,
-                        local_dir=self.model_dir
+                        local_dir=self.model_dir,
                     )
-                    
+
                     if not os.path.isfile(file_path):
                         raise FileNotFoundError(f"模型文件下载失败: {file_path}")
-                        
+
             self.model_path = model_files["model.int8.onnx"]
             self.tokens_path = model_files["tokens.txt"]
-            
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
             raise
@@ -83,21 +84,12 @@ class ASRProvider(ASRProviderBase):
                 use_itn=True,
             )
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -130,14 +122,22 @@ class ASRProvider(ASRProviderBase):
             samples_float32 = samples_float32 / 32768
             return samples_float32, f.getframerate()
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
         try:
             # 保存音频文件
             start_time = time.time()
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            file_path = self.save_audio_to_file(pcm_data, session_id)
+            logger.bind(tag=TAG).debug(
+                f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
+            )
 
             # 语音识别
             start_time = time.time()
@@ -146,14 +146,15 @@ class ASRProvider(ASRProviderBase):
             s.accept_waveform(sample_rate, samples)
             self.model.decode_stream(s)
             text = s.result.text
-            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+            logger.bind(tag=TAG).debug(
+                f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+            )
 
             return text, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
-
+            return "", file_path
         finally:
             # 文件清理逻辑
             if self.delete_audio_file and file_path and os.path.exists(file_path):

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -17,77 +17,66 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+
 class ASRProvider(ASRProviderBase):
     API_URL = "https://asr.tencentcloudapi.com"
     API_VERSION = "2019-06-14"
     FORMAT = "pcm"  # 支持的音频格式：pcm, wav, mp3
 
     def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
         self.secret_id = config.get("secret_id")
         self.secret_key = config.get("secret_key")
         self.output_dir = config.get("output_dir")
-        
+        self.delete_audio_file = delete_audio_file
+
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-
-        file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
-        
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-        
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
+
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
             wf.setframerate(16000)
             wf.writeframes(b"".join(pcm_data))
-        
+
         return file_path
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes]) -> bytes:
-        """将Opus音频数据解码为PCM数据"""
-        import opuslib_next
-        
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-        
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
-        return b"".join(pcm_data)
-
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
-            logger.bind(tag=TAG).warn("音频数据为空！")
+            logger.bind(tag=TAG).warning("音频数据为空！")
             return None, None
 
+        file_path = None
         try:
             # 检查配置是否已设置
             if not self.secret_id or not self.secret_key:
                 logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
-                return None, None
+                return None, file_path
 
             # 将Opus音频数据解码为PCM
-            pcm_data = self.decode_opus(opus_data)
-            
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                self.save_audio_to_file(pcm_data, session_id)
+
             # 将音频数据转换为Base64编码
-            base64_audio = base64.b64encode(pcm_data).decode('utf-8')
+            base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8")
 
             # 构建请求体
             request_body = self._build_request_body(base64_audio)
@@ -98,15 +87,17 @@ class ASRProvider(ASRProviderBase):
             # 发送请求
             start_time = time.time()
             result = self._send_request(request_body, timestamp, authorization)
-            
+
             if result:
-                logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
-            
-            return result, None
+                logger.bind(tag=TAG).debug(
+                    f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+
+            return result, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
-            return None, None
+            return None, file_path
 
     def _build_request_body(self, base64_audio: str) -> str:
         """构建请求体"""
@@ -117,7 +108,7 @@ class ASRProvider(ASRProviderBase):
             "SourceType": 1,  # 音频数据来源为语音文件
             "VoiceFormat": self.FORMAT,  # 音频格式
             "Data": base64_audio,  # Base64编码的音频数据
-            "DataLen": len(base64_audio)  # 数据长度
+            "DataLen": len(base64_audio),  # 数据长度
         }
         return json.dumps(request_map)
 
@@ -150,9 +141,11 @@ class ASRProvider(ASRProviderBase):
             action = "SentenceRecognition"  # 接口名称
 
             # 构建规范头部信息，注意顺序和格式
-            canonical_headers = f"content-type:{content_type.lower()}\n" + \
-                               f"host:{host.lower()}\n" + \
-                               f"x-tc-action:{action.lower()}\n"
+            canonical_headers = (
+                f"content-type:{content_type.lower()}\n"
+                + f"host:{host.lower()}\n"
+                + f"x-tc-action:{action.lower()}\n"
+            )
 
             signed_headers = "content-type;host;x-tc-action"
 
@@ -160,21 +153,25 @@ class ASRProvider(ASRProviderBase):
             payload_hash = self._sha256_hex(request_body)
 
             # 构建规范请求字符串
-            canonical_request = f"{http_request_method}\n" + \
-                               f"{canonical_uri}\n" + \
-                               f"{canonical_query_string}\n" + \
-                               f"{canonical_headers}\n" + \
-                               f"{signed_headers}\n" + \
-                               f"{payload_hash}"
+            canonical_request = (
+                f"{http_request_method}\n"
+                + f"{canonical_uri}\n"
+                + f"{canonical_query_string}\n"
+                + f"{canonical_headers}\n"
+                + f"{signed_headers}\n"
+                + f"{payload_hash}"
+            )
 
             # 计算规范请求的哈希值
             hashed_canonical_request = self._sha256_hex(canonical_request)
 
             # 构建待签名字符串
-            string_to_sign = f"{algorithm}\n" + \
-                            f"{timestamp}\n" + \
-                            f"{credential_scope}\n" + \
-                            f"{hashed_canonical_request}"
+            string_to_sign = (
+                f"{algorithm}\n"
+                + f"{timestamp}\n"
+                + f"{credential_scope}\n"
+                + f"{hashed_canonical_request}"
+            )
 
             # 计算签名密钥
             secret_date = self._hmac_sha256(f"TC3{self.secret_key}", date)
@@ -182,13 +179,17 @@ class ASRProvider(ASRProviderBase):
             secret_signing = self._hmac_sha256(secret_service, "tc3_request")
 
             # 计算签名
-            signature = self._bytes_to_hex(self._hmac_sha256(secret_signing, string_to_sign))
+            signature = self._bytes_to_hex(
+                self._hmac_sha256(secret_signing, string_to_sign)
+            )
 
             # 构建授权头
-            authorization = f"{algorithm} " + \
-                           f"Credential={self.secret_id}/{credential_scope}, " + \
-                           f"SignedHeaders={signed_headers}, " + \
-                           f"Signature={signature}"
+            authorization = (
+                f"{algorithm} "
+                + f"Credential={self.secret_id}/{credential_scope}, "
+                + f"SignedHeaders={signed_headers}, "
+                + f"Signature={signature}"
+            )
 
             return timestamp, authorization
 
@@ -196,7 +197,9 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).error(f"生成认证头失败: {e}", exc_info=True)
             raise RuntimeError(f"生成认证头失败: {e}")
 
-    def _send_request(self, request_body: str, timestamp: str, authorization: str) -> Optional[str]:
+    def _send_request(
+        self, request_body: str, timestamp: str, authorization: str
+    ) -> Optional[str]:
         """发送请求到腾讯云API"""
         headers = {
             "Content-Type": "application/json; charset=utf-8",
@@ -205,47 +208,47 @@ class ASRProvider(ASRProviderBase):
             "X-TC-Action": "SentenceRecognition",
             "X-TC-Version": self.API_VERSION,
             "X-TC-Timestamp": timestamp,
-            "X-TC-Region": "ap-shanghai"
+            "X-TC-Region": "ap-shanghai",
         }
 
         try:
             response = requests.post(self.API_URL, headers=headers, data=request_body)
-            
+
             if not response.ok:
                 raise IOError(f"请求失败: {response.status_code} {response.reason}")
-            
+
             response_json = response.json()
-            
+
             # 检查是否有错误
             if "Response" in response_json and "Error" in response_json["Response"]:
                 error = response_json["Response"]["Error"]
                 error_code = error["Code"]
                 error_message = error["Message"]
                 raise IOError(f"API返回错误: {error_code}: {error_message}")
-            
+
             # 提取识别结果
             if "Response" in response_json and "Result" in response_json["Response"]:
                 return response_json["Response"]["Result"]
             else:
-                logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
+                logger.bind(tag=TAG).warning(f"响应中没有识别结果: {response_json}")
                 return ""
-                
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
             return None
 
     def _sha256_hex(self, data: str) -> str:
         """计算字符串的SHA256哈希值"""
-        digest = hashlib.sha256(data.encode('utf-8')).digest()
+        digest = hashlib.sha256(data.encode("utf-8")).digest()
         return self._bytes_to_hex(digest)
 
     def _hmac_sha256(self, key, data: str) -> bytes:
         """计算HMAC-SHA256"""
         if isinstance(key, str):
-            key = key.encode('utf-8')
-        
-        return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest()
+            key = key.encode("utf-8")
+
+        return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest()
 
     def _bytes_to_hex(self, bytes_data: bytes) -> str:
         """字节数组转十六进制字符串"""
-        return ''.join(f"{b:02x}" for b in bytes_data)
+        return "".join(f"{b:02x}" for b in bytes_data)

main/xiaozhi-server/core/providers/intent/base.py

@@ -9,18 +9,6 @@ logger = setup_logging()
 class IntentProviderBase(ABC):
     def __init__(self, config):
         self.config = config
-        self.intent_options = [
-            {
-                "name": "handle_exit_intent",
-                "desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
-            },
-            {
-                "name": "play_music",
-                "desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
-            },
-            {"name": "get_time", "desc": "获取今天日期或者当前时间信息"},
-            {"name": "continue_chat", "desc": "继续聊天"},
-        ]
 
     def set_llm(self, llm):
         self.llm = llm

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -15,57 +15,72 @@ class IntentProvider(IntentProviderBase):
     def __init__(self, config):
         super().__init__(config)
         self.llm = None
-        self.promot = self.get_intent_system_prompt()
+        self.promot = ""
         # 添加缓存管理
         self.intent_cache = {}  # 缓存意图识别结果
         self.cache_expiry = 600  # 缓存有效期10分钟
         self.cache_max_size = 100  # 最多缓存100个意图
+        self.history_count = 4  # 默认使用最近4条对话记录
 
-    def get_intent_system_prompt(self) -> str:
+    def get_intent_system_prompt(self, functions_list: str) -> str:
         """
-        根据配置的意图选项动态生成系统提示词
+        根据配置的意图选项和可用函数动态生成系统提示词
+        Args:
+            functions: 可用的函数列表，JSON格式字符串
         Returns:
             格式化后的系统提示词
         """
 
+        # 构建函数说明部分
+        functions_desc = "可用的函数列表：\n"
+        for func in functions_list:
+            func_info = func.get("function", {})
+            name = func_info.get("name", "")
+            desc = func_info.get("description", "")
+            params = func_info.get("parameters", {})
+
+            functions_desc += f"\n函数名: {name}\n"
+            functions_desc += f"描述: {desc}\n"
+
+            if params:
+                functions_desc += "参数:\n"
+                for param_name, param_info in params.get("properties", {}).items():
+                    param_desc = param_info.get("description", "")
+                    param_type = param_info.get("type", "")
+                    functions_desc += f"- {param_name} ({param_type}): {param_desc}\n"
+
+            functions_desc += "---\n"
+
         prompt = (
-            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
-            "<start>"
-            f"{str(self.intent_options)}"
-            "<end>\n"
-            "处理步骤:"
-            "1. 思考意图类型，生成function_call格式"
-            "\n\n"
-            "返回格式示例：\n"
-            '1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n'
-            '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
-            '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n'
-            '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n'
-            "\n"
-            "注意:\n"
-            '- 播放音乐：无歌名时，song_name设为"random"\n'
-            "- 如果没有明显的意图，应按照继续聊天意图处理\n"
-            "- 只返回纯JSON，不要任何其他内容\n"
-            "\n"
-            "示例分析:\n"
+            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            f"{functions_desc}\n"
+            "处理步骤:\n"
+            "1. 分析用户输入，确定用户意图\n"
+            "2. 从可用函数列表中选择最匹配的函数\n"
+            "3. 如果找到匹配的函数，生成对应的function_call 格式\n"
+            '4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
+            "返回格式要求：\n"
+            "1. 必须返回纯JSON格式\n"
+            "2. 必须包含function_call字段\n"
+            "3. function_call必须包含name字段\n"
+            "4. 如果函数需要参数，必须包含arguments字段\n\n"
+            "示例：\n"
             "```\n"
-            "用户: 你也太搞笑了\n"
-            '返回: {"function_call": {"name": "continue_chat"}}\n'
-            "```\n"
-            "```\n"
-            "用户: 现在是几号了?现在几点了？\n"
+            "用户: 现在几点了？\n"
             '返回: {"function_call": {"name": "get_time"}}\n'
             "```\n"
             "```\n"
-            "用户: 我们明天再聊吧\n"
-            '返回: {"function_call": {"name": "handle_exit_intent"}}\n'
+            "用户: 我想结束对话\n"
+            '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
             "```\n"
             "```\n"
-            "用户: 播放中秋月\n"
-            '返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n'
-            "```\n"
-            "```\n"
-            "可用的音乐名称:\n"
+            "用户: 你好啊\n"
+            '返回: {"function_call": {"name": "continue_chat"}}\n'
+            "```\n\n"
+            "注意：\n"
+            "1. 只返回JSON格式，不要包含任何其他文字\n"
+            '2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
+            "3. 确保返回的JSON格式正确，包含所有必要的字段\n"
         )
         return prompt
 
@@ -90,6 +105,14 @@ class IntentProvider(IntentProviderBase):
             for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]:
                 del self.intent_cache[key]
 
+    def replyResult(self, text: str, original_text: str):
+        llm_result = self.llm.response_no_stream(
+            system_prompt=text,
+            user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+            + original_text,
+        )
+        return llm_result
+
     async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
@@ -118,22 +141,35 @@ class IntentProvider(IntentProviderBase):
         # 清理缓存
         self.clean_cache()
 
-        # 构建用户最后一句话的提示
-        msgStr = ""
+        if self.promot == "":
+            if hasattr(conn, "func_handler"):
+                functions = conn.func_handler.get_functions()
+                self.promot = self.get_intent_system_prompt(functions)
 
-        # 只使用最后两句即可
-        if len(dialogue_history) >= 2:
-            # 保证最少有两句话的时候处理
-            msgStr += f"{dialogue_history[-2].role}: {dialogue_history[-2].content}\n"
-        msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n"
-
-        msgStr += f"User: {text}\n"
-        user_prompt = f"当前的对话如下：\n{msgStr}"
         music_config = initialize_music_handler(conn)
         music_file_names = music_config["music_file_names"]
-        prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
+        prompt_music = f"{self.promot}\n<musicNames>{music_file_names}\n</musicNames>"
+
+        devices = conn.config["plugins"]["home_assistant"].get("devices", [])
+        if len(devices) > 0:
+            hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
+            for device in devices:
+                hass_prompt += device + "\n"
+            prompt_music += hass_prompt
+
         logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
 
+        # 构建用户对话历史的提示
+        msgStr = ""
+
+        # 获取最近的对话历史
+        start_idx = max(0, len(dialogue_history) - self.history_count)
+        for i in range(start_idx, len(dialogue_history)):
+            msgStr += f"{dialogue_history[i].role}: {dialogue_history[i].content}\n"
+
+        msgStr += f"User: {text}\n"
+        user_prompt = f"current dialogue:\n{msgStr}"
+
         # 记录预处理完成时间
         preprocess_time = time.time() - total_start_time
         logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")
@@ -179,9 +215,18 @@ class IntentProvider(IntentProviderBase):
 
                 # 记录识别到的function call
                 logger.bind(tag=TAG).info(
-                    f"识别到function call: {function_name}, 参数: {function_args}"
+                    f"llm 识别到意图: {function_name}, 参数: {function_args}"
                 )
 
+                # 如果是继续聊天，清理工具调用相关的历史消息
+                if function_name == "continue_chat":
+                    # 保留非工具相关的消息
+                    clean_history = [
+                        msg for msg in conn.dialogue.dialogue
+                        if msg.role not in ["tool", "function"]
+                    ]
+                    conn.dialogue.dialogue = clean_history
+
                 # 添加到缓存
                 self.intent_cache[cache_key] = {
                     "intent": intent,

main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py

@@ -2,10 +2,12 @@ from config.logger import setup_logging
 from http import HTTPStatus
 from dashscope import Application
 from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
 
+
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
@@ -13,27 +15,32 @@ class LLMProvider(LLMProviderBase):
         self.base_url = config.get("base_url")
         self.is_No_prompt = config.get("is_no_prompt")
         self.memory_id = config.get("ali_memory_id")
+        check_model_key("AliBLLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
             # 处理dialogue
             if self.is_No_prompt:
                 dialogue.pop(0)
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】处理后的dialogue: {dialogue}"
+                )
 
             # 构造调用参数
             call_params = {
                 "api_key": self.api_key,
                 "app_id": self.app_id,
                 "session_id": session_id,
-                "messages": dialogue
+                "messages": dialogue,
             }
             if self.memory_id != False:
                 # 百练memory需要prompt参数
                 prompt = dialogue[-1].get("content")
                 call_params["memory_id"] = self.memory_id
                 call_params["prompt"] = prompt
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】处理后的prompt: {prompt}"
+                )
 
             responses = Application.call(**call_params)
             if responses.status_code != HTTPStatus.OK:
@@ -44,9 +51,16 @@ class LLMProvider(LLMProviderBase):
                 )
                 yield "【阿里百练API服务响应异常】"
             else:
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】构造参数: {call_params}"
+                )
                 yield responses.output.text
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
             yield "【LLM服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"阿里百练暂未实现完整的工具调用（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/base.py

@@ -35,4 +35,5 @@ class LLMProviderBase(ABC):
         """
         # For providers that don't support functions, just return regular response
         for token in self.response(session_id, dialogue):
-            yield {"type": "content", "content": token}
+            yield token, None
+

main/xiaozhi-server/core/providers/llm/coze/coze.py

@@ -1,12 +1,17 @@
 from config.logger import setup_logging
-import requests
 import json
-import re
 from core.providers.llm.base import LLMProviderBase
-import os
+
 # official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
 from cozepy import COZE_CN_BASE_URL
-from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa
+from cozepy import (
+    Coze,
+    TokenAuth,
+    Message,
+    ChatEventType,
+)  # noqa
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -15,25 +20,23 @@ logger = setup_logging()
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.personal_access_token = config.get("personal_access_token")
-        self.bot_id = config.get("bot_id")
-        self.user_id = config.get("user_id")
+        self.bot_id = str(config.get("bot_id"))
+        self.user_id = str(config.get("user_id"))
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        check_model_key("CozeLLM", self.personal_access_token)
 
     def response(self, session_id, dialogue):
         coze_api_token = self.personal_access_token
         coze_api_base = COZE_CN_BASE_URL
 
         last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-        
+
         coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
         conversation_id = self.session_conversation_map.get(session_id)
 
         # 如果没有找到conversation_id，则创建新的对话
         if not conversation_id:
-            conversation = coze.conversations.create(
-                messages=[
-                ]
-            )
+            conversation = coze.conversations.create(messages=[])
             conversation_id = conversation.id
             self.session_conversation_map[session_id] = conversation_id  # 更新映射
 
@@ -47,4 +50,24 @@ class LLMProvider(LLMProviderBase):
         ):
             if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
                 print(event.message.content, end="", flush=True)
-                yield event.message.content
+                yield event.message.content
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -2,6 +2,8 @@ import json
 from config.logger import setup_logging
 import requests
 from core.providers.llm.base import LLMProviderBase
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -13,6 +15,7 @@ class LLMProvider(LLMProviderBase):
         self.mode = config.get("mode", "chat-messages")
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        check_model_key("DifyLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
@@ -58,7 +61,10 @@ class LLMProvider(LLMProviderBase):
                                 self.session_conversation_map[session_id] = (
                                     conversation_id  # 更新映射
                                 )
-                            if event.get("answer"):
+                            # 过滤 message_replace 事件，此事件会全量推一次
+                            if event.get("event") != "message_replace" and event.get(
+                                "answer"
+                            ):
                                 yield event["answer"]
                 elif self.mode == "workflows/run":
                     for line in r.iter_lines():
@@ -73,9 +79,32 @@ class LLMProvider(LLMProviderBase):
                     for line in r.iter_lines():
                         if line.startswith(b"data: "):
                             event = json.loads(line[6:])
-                            if event.get("answer"):
+                            # 过滤 message_replace 事件，此事件会全量推一次
+                            if event.get("event") != "message_replace" and event.get(
+                                "answer"
+                            ):
                                 yield event["answer"]
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
             yield "【服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py

@@ -2,6 +2,7 @@ import json
 from config.logger import setup_logging
 import requests
 from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -13,6 +14,7 @@ class LLMProvider(LLMProviderBase):
         self.base_url = config.get("base_url")
         self.detail = config.get("detail", False)
         self.variables = config.get("variables", {})
+        check_model_key("FastGPTLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
@@ -21,37 +23,36 @@ class LLMProvider(LLMProviderBase):
 
             # 发起流式请求
             with requests.post(
-                    f"{self.base_url}/chat/completions",
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json={
-                        "stream": True,
-                        "chatId": session_id,
-                        "detail": self.detail,
-                        "variables": self.variables,
-                        "messages": [
-                            {
-                                "role": "user",
-                                "content": last_msg["content"]
-                            }
-                        ]
-                    },
-                    stream=True
+                f"{self.base_url}/chat/completions",
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                json={
+                    "stream": True,
+                    "chatId": session_id,
+                    "detail": self.detail,
+                    "variables": self.variables,
+                    "messages": [{"role": "user", "content": last_msg["content"]}],
+                },
+                stream=True,
             ) as r:
                 for line in r.iter_lines():
                     if line:
                         try:
-                            if line.startswith(b'data: '):
-                                if line[6:].decode('utf-8') == '[DONE]':
+                            if line.startswith(b"data: "):
+                                if line[6:].decode("utf-8") == "[DONE]":
                                     break
 
                                 data = json.loads(line[6:])
-                                if 'choices' in data and len(data['choices']) > 0:
-                                    delta = data['choices'][0].get('delta', {})
-                                    if delta and 'content' in delta and delta['content'] is not None:
-                                        content = delta['content']
-                                        if '<think>' in content:
+                                if "choices" in data and len(data["choices"]) > 0:
+                                    delta = data["choices"][0].get("delta", {})
+                                    if (
+                                        delta
+                                        and "content" in delta
+                                        and delta["content"] is not None
+                                    ):
+                                        content = delta["content"]
+                                        if "<think>" in content:
                                             continue
-                                        if '</think>' in content:
+                                        if "</think>" in content:
                                             continue
                                         yield content
 
@@ -62,4 +63,9 @@ class LLMProvider(LLMProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
-            yield "【服务响应异常】"
+            yield "【服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"fastgpt暂未实现完整的工具调用（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -1,136 +1,205 @@
-import google.generativeai as genai
-from core.utils.util import check_model_key
-from core.providers.llm.base import LLMProviderBase
-from config.logger import setup_logging
+import os, json, uuid
+from types import SimpleNamespace
+from typing import Any, Dict, List
+
 import requests
-import json
+from google import generativeai as genai
+from google.generativeai import types, GenerationConfig
+
+from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
+from config.logger import setup_logging
+from google.generativeai.types import GenerateContentResponse
+from requests import RequestException
+
+log = setup_logging()
 TAG = __name__
-logger = setup_logging()
+
+
+def test_proxy(proxy_url: str, test_url: str) -> bool:
+    try:
+        resp = requests.get(test_url, proxies={"http": proxy_url, "https": proxy_url})
+        return 200 <= resp.status_code < 400
+    except RequestException:
+        return False
+
+
+def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
+    """
+    分别测试 HTTP 和 HTTPS 代理是否可用，并设置环境变量。
+    如果 HTTPS 代理不可用但 HTTP 可用，会将 HTTPS_PROXY 也指向 HTTP。
+    """
+    test_http_url = "http://www.google.com"
+    test_https_url = "https://www.google.com"
+
+    ok_http = ok_https = False
+
+    if http_proxy:
+        ok_http = test_proxy(http_proxy, test_http_url)
+        if ok_http:
+            os.environ["HTTP_PROXY"] = http_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}")
+        else:
+            log.bind(tag=TAG).warning(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
+
+    if https_proxy:
+        ok_https = test_proxy(https_proxy, test_https_url)
+        if ok_https:
+            os.environ["HTTPS_PROXY"] = https_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}")
+        else:
+            log.bind(tag=TAG).warning(
+                f"配置提供的Gemini HTTPS代理不可用: {https_proxy}"
+            )
+
+    # 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy
+    if ok_http and not ok_https:
+        if test_proxy(http_proxy, test_https_url):
+            os.environ["HTTPS_PROXY"] = http_proxy
+            ok_https = True
+            log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}")
+
+    if not ok_http and not ok_https:
+        log.bind(tag=TAG).error(
+            f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置"
+        )
+        raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置")
+
 
 class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        """初始化Gemini LLM Provider"""
-        self.model_name = config.get("model_name", "gemini-1.5-pro")
-        self.api_key = config.get("api_key")
-        self.http_proxy=config.get("http_proxy")
-        self.https_proxy = config.get("https_proxy")
-        have_key = check_model_key("LLM", self.api_key)
+    def __init__(self, cfg: Dict[str, Any]):
+        self.model_name = cfg.get("model_name", "gemini-2.0-flash")
+        self.api_key = cfg["api_key"]
+        http_proxy = cfg.get("http_proxy")
+        https_proxy = cfg.get("https_proxy")
 
-        if not have_key:
-            return
+        if not check_model_key("LLM", self.api_key):
+            raise ValueError("无效的Gemini API Key，请检查是否配置正确")
 
-        try:
-            # 初始化Gemini客户端
-            # 配置代理（如果提供了代理配置）
-            self.proxies=None
-            if self.http_proxy is not "" or self.https_proxy is not "":
+        if http_proxy or https_proxy:
+            log.bind(tag=TAG).info(
+                f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境..."
+            )
+            setup_proxy_env(http_proxy, https_proxy)
+            log.bind(tag=TAG).info(
+                f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}"
+            )
+        genai.configure(api_key=self.api_key)
+        self.model = genai.GenerativeModel(self.model_name)
 
-                self.proxies = {
-                    "http": self.http_proxy,
-                    "https": self.https_proxy,
-                }
-                logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
-                # 使用猴子补丁修改 google-generativeai 库的请求会话
+        self.gen_cfg = GenerationConfig(
+            temperature=0.7,
+            top_p=0.9,
+            top_k=40,
+            max_output_tokens=2048,
+        )
 
-                # 使用 session 对象配置 genai
-
-            genai.configure(api_key=self.api_key)
-            self.model = genai.GenerativeModel(self.model_name)
-
-            # 设置生成参数
-            self.generation_config = {
-                "temperature": 0.7,
-                "top_p": 0.9,
-                "top_k": 40,
-                "max_output_tokens": 2048,
-            }
-            self.chat = None
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
-            self.model = None
+    @staticmethod
+    def _build_tools(funcs: List[Dict[str, Any]] | None):
+        if not funcs:
+            return None
+        return [
+            types.Tool(
+                function_declarations=[
+                    types.FunctionDeclaration(
+                        name=f["function"]["name"],
+                        description=f["function"]["description"],
+                        parameters=f["function"]["parameters"],
+                    )
+                    for f in funcs
+                ]
+            )
+        ]
 
+    # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
     def response(self, session_id, dialogue):
-        """生成Gemini对话响应"""
-        if not self.model:
-            yield "【Gemini服务未正确初始化】"
-            return
+        yield from self._generate(dialogue, None)
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        yield from self._generate(dialogue, self._build_tools(functions))
+
+    def _generate(self, dialogue, tools):
+        role_map = {"assistant": "model", "user": "user"}
+        contents: list = []
+        # 拼接对话
+        for m in dialogue:
+            r = m["role"]
+
+            if r == "assistant" and "tool_calls" in m:
+                tc = m["tool_calls"][0]
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [
+                            {
+                                "function_call": {
+                                    "name": tc["function"]["name"],
+                                    "args": json.loads(tc["function"]["arguments"]),
+                                }
+                            }
+                        ],
+                    }
+                )
+                continue
+
+            if r == "tool":
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [{"text": str(m.get("content", ""))}],
+                    }
+                )
+                continue
+
+            contents.append(
+                {
+                    "role": role_map.get(r, "user"),
+                    "parts": [{"text": str(m.get("content", ""))}],
+                }
+            )
+
+        stream: GenerateContentResponse = self.model.generate_content(
+            contents=contents,
+            generation_config=self.gen_cfg,
+            tools=tools,
+            stream=True,
+        )
 
         try:
-            # 处理对话历史
-            chat_history = []
-            for msg in dialogue[:-1]:  # 历史对话
-                role = "model" if msg["role"] == "assistant" else "user"
-                content = msg["content"].strip()
-                if content:
-                    chat_history.append({
-                        "role": role,
-                        "parts": [{"text":content}]
+            for chunk in stream:
+                cand = chunk.candidates[0]
+                for part in cand.content.parts:
+                    # a) 函数调用-通常是最后一段话才是函数调用
+                    if getattr(part, "function_call", None):
+                        fc = part.function_call
+                        yield None, [
+                            SimpleNamespace(
+                                id=uuid.uuid4().hex,
+                                type="function",
+                                function=SimpleNamespace(
+                                    name=fc.name,
+                                    arguments=json.dumps(
+                                        dict(fc.args), ensure_ascii=False
+                                    ),
+                                ),
+                            )
+                        ]
+                        return
+                    # b) 普通文本
+                    if getattr(part, "text", None):
+                        yield part.text if tools is None else (part.text, None)
 
-                    })
+        finally:
+            if tools is not None:
+                yield None, None  # function‑mode 结束，返回哑包
 
-            # 获取当前消息
-            current_msg = dialogue[-1]["content"]
-
-            # 构建请求体
-            request_body = {
-                "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
-                "generationConfig": self.generation_config
-            }
-
-            # 构建请求URL
-            url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
-
-            # 构建请求头
-            headers = {
-                "Content-Type": "application/json",
-            }
-
-            # 发送POST请求,经测试手动 request 无法使用 stream 模式
-            if self.proxies:
-                response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
-                try:
-                    data = response.json()  # 直接解析JSON
-                    if 'candidates' in data and data['candidates']:
-                        yield data['candidates'][0]['content']['parts'][0]['text']
-                    else:
-                        yield "未找到候选回复。"
-                except json.JSONDecodeError as e:
-                    yield f"JSON解码错误：{e}"
-                except Exception as e:
-                    yield f"发生错误：{e}"
-            else:
-                logger.bind(tag=TAG).info(f"Gemini stream mode ")
-                chat = self.model.start_chat(history=chat_history)
-
-                # 发送消息并获取流式响应
-                response = chat.send_message(
-                    current_msg,
-                    stream=True,
-                    generation_config=self.generation_config
-                )
-                # 处理流式响应
-                for chunk in response:
-                    if hasattr(chunk, 'text') and chunk.text:
-                        yield chunk.text
-
-        except Exception as e:
-            error_msg = str(e)
-            logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
-
-            # 针对不同错误返回友好提示
-            if "Rate limit" in error_msg:
-                yield "【Gemini服务请求太频繁,请稍后再试】"
-            elif "Invalid API key" in error_msg:
-                yield "【Gemini API key无效】"
-            else:
-                yield f"【Gemini服务响应异常: {error_msg}】"
-
-
-
-
-        except requests.exceptions.RequestException as e:
-            yield f"请求失败：{e}"
-        except json.JSONDecodeError as e:
-            yield f"JSON解码错误：{e}"
-        except Exception as e:
-            yield f"发生错误：{e}"
+    # 关闭stream，预留后续打断对话功能的功能方法，官方文档推荐打断对话要关闭上一个流，可以有效减少配额计费和资源占用
+    @staticmethod
+    def _safe_finish_stream(stream: GenerateContentResponse):
+        if hasattr(stream, "resolve"):
+            stream.resolve()  # Gemini SDK version ≥ 0.5.0
+        elif hasattr(stream, "close"):
+            stream.close()  # Gemini SDK version < 0.5.0
+        else:
+            for _ in stream:  # 兜底耗尽
+                pass

main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py

@@ -0,0 +1,71 @@
+import requests
+from requests.exceptions import RequestException
+from config.logger import setup_logging
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.agent_id = config.get("agent_id")  # 对应 agent_id
+        self.api_key = config.get("api_key")
+        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
+        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
+
+    def response(self, session_id, dialogue):
+        try:
+            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
+
+            # 提取最后一个 role 为 'user' 的 content
+            input_text = None
+            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
+                # 逆序遍历，找到最后一个 role 为 'user' 的消息
+                for message in reversed(dialogue):
+                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
+                        input_text = message.get("content", "")
+                        break  # 找到后立即退出循环
+
+            # 构造请求数据
+            payload = {
+                "text": input_text,
+                "agent_id": self.agent_id,
+                "conversation_id": session_id,  # 使用 session_id 作为 conversation_id
+            }
+            # 设置请求头
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            }
+
+            # 发起 POST 请求
+            response = requests.post(self.api_url, json=payload, headers=headers)
+
+            # 检查请求是否成功
+            response.raise_for_status()
+
+            # 解析返回数据
+            data = response.json()
+            speech = (
+                data.get("response", {})
+                .get("speech", {})
+                .get("plain", {})
+                .get("speech", "")
+            )
+
+            # 返回生成的内容
+            if speech:
+                yield speech
+            else:
+                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
+
+        except RequestException as e:
+            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"homeassistant不支持（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -21,27 +21,67 @@ class LLMProvider(LLMProviderBase):
             api_key="ollama"  # Ollama doesn't need an API key but OpenAI client requires one
         )
 
+        # 检查是否是qwen3模型
+        self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
+
     def response(self, session_id, dialogue):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             responses = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
                 stream=True
             )
-            is_active=True
+            is_active = True
+            # 用于处理跨chunk的标签
+            buffer = ""
+
             for chunk in responses:
                 try:
                     delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
                     content = delta.content if hasattr(delta, 'content') else ''
+
                     if content:
-                        if '<think>' in content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
                             is_active = False
-                            content = content.split('<think>')[0]
-                        if '</think>' in content:
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
                             is_active = True
-                            content = content.split('</think>')[-1]
-                        if is_active:
-                            yield content
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer
+                            buffer = ""  # 清空缓冲区
+
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
 
@@ -51,6 +91,22 @@ class LLMProvider(LLMProviderBase):
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             stream = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
@@ -58,9 +114,50 @@ class LLMProvider(LLMProviderBase):
                 tools=functions,
             )
 
+            is_active = True
+            buffer = ""
+
             for chunk in stream:
-                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else None
+                    tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None
+
+                    # 如果是工具调用，直接传递
+                    if tool_calls:
+                        yield None, tool_calls
+                        continue
+
+                    # 处理文本内容
+                    if content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
+                            is_active = False
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
+                            is_active = True
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer, None
+                            buffer = ""  # 清空缓冲区
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
+                    continue
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
-            yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}
+            yield f"【Ollama服务响应异常: {str(e)}】", None

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -1,4 +1,5 @@
 import openai
+from openai.types import CompletionUsage
 from config.logger import setup_logging
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
@@ -11,11 +12,19 @@ class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.model_name = config.get("model_name")
         self.api_key = config.get("api_key")
-        if 'base_url' in config:
+        if "base_url" in config:
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
-        self.max_tokens = config.get("max_tokens", 500)
+        max_tokens = config.get("max_tokens")
+        if max_tokens is None or max_tokens == "":
+            max_tokens = 500
+
+        try:
+            max_tokens = int(max_tokens)
+        except (ValueError, TypeError):
+            max_tokens = 500
+        self.max_tokens = max_tokens
 
         check_model_key("LLM", self.api_key)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
@@ -33,18 +42,22 @@ class LLMProvider(LLMProviderBase):
             for chunk in responses:
                 try:
                     # 检查是否存在有效的choice且content不为空
-                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
-                    content = delta.content if hasattr(delta, 'content') else ''
+                    delta = (
+                        chunk.choices[0].delta
+                        if getattr(chunk, "choices", None)
+                        else None
+                    )
+                    content = delta.content if hasattr(delta, "content") else ""
                 except IndexError:
-                    content = ''
+                    content = ""
                 if content:
                     # 处理标签跨多个chunk的情况
-                    if '<think>' in content:
+                    if "<think>" in content:
                         is_active = False
-                        content = content.split('<think>')[0]
-                    if '</think>' in content:
+                        content = content.split("<think>")[0]
+                    if "</think>" in content:
                         is_active = True
-                        content = content.split('</think>')[-1]
+                        content = content.split("</think>")[-1]
                     if is_active:
                         yield content
 
@@ -54,15 +67,22 @@ class LLMProvider(LLMProviderBase):
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
             stream = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                tools=functions
+                model=self.model_name, messages=dialogue, stream=True, tools=functions
             )
 
             for chunk in stream:
-                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                # 检查是否存在有效的choice且content不为空
+                if getattr(chunk, "choices", None):
+                    yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                # 存在 CompletionUsage 消息时，生成 Token 消耗 log
+                elif isinstance(getattr(chunk, 'usage', None), CompletionUsage):
+                    usage_info = getattr(chunk, 'usage', None)
+                    logger.bind(tag=TAG).info(
+                        f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，" 
+                        f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
+                        f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
+                    )
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
-            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
+            yield f"【OpenAI服务响应异常: {e}】", None

main/xiaozhi-server/core/providers/llm/system_prompt.py

@@ -0,0 +1,103 @@
+def get_system_prompt_for_function(functions: str) -> str:
+    """
+    生成系统提示信息
+    :param functions: 可用的函数列表
+    :return: 系统提示信息
+    """
+
+    SYSTEM_PROMPT = f"""
+====
+
+TOOL USE
+
+You have access to a set of tools that are executed upon the user's approval. You can use one tool per message, and will receive the result of that tool use in the user's response. 
+You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use.
+
+# Tool Use Formatting
+
+Tool use is formatted using JSON-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags. 
+Here's the structure:
+
+<tool_call>
+{{
+    "name": "function name",
+    "arguments": {{
+        "param1": "value1",
+        "param2": "value2",
+        // Add more parameters as needed, if parameters are required, you must provide them
+    }}
+}}
+<tool_call>
+
+For example:
+if you got tool as follow
+
+{{
+    "type": "function",
+    "function": {{
+        "name": "handle_exit_intent",
+        "description": "当用户想结束对话或需要退出系统时调用",
+        "parameters": {{
+            "type": "object",
+            "properties": {{
+                "say_goodbye": {{
+                    "type": "string",
+                    "description": "和用户友好结束对话的告别语",
+                }}
+            }},
+            "required": ["say_goodbye"],
+        }},
+    }},
+}}
+
+you should respond with the following format:
+
+<tool_call>
+{{
+    "name": "handle_exit_intent",
+    "arguments": {{
+        "say_goodbye": "再见，祝您生活愉快！"
+    }}
+}}
+</tool_call>
+
+
+Always adhere to this format for the tool use to ensure proper parsing and execution.
+
+# Tools
+
+{functions}
+
+# Tool Use Guidelines
+
+1. Tools must be called in a separate message, Do not add thoughts when calling tools. The message must start with <tool_call> and end with </tool_call>, with the tool invocation JSON data in between. No additional response content is needed.
+2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information. 
+   For example using the list_files tool is more effective than running a command like \`ls\` in the terminal. It's critical that you think about each available tool and use the one that best fits the current step in the task.
+3. If multiple actions are needed, use one tool at a time per message to accomplish the task iteratively, with each tool use being informed by the result of the previous tool use. Do not assume the outcome of any tool use. 
+   Each step must be informed by the previous step's result.
+4. Formulate your tool use using the JSON format specified for each tool.
+5. After each tool use, the user will respond with the result of that tool use. This result will provide you with the necessary information to continue your task or make further decisions. This response may include:
+- Information about whether the tool succeeded or failed, along with any reasons for failure.
+- Linter errors that may have arisen due to the changes you made, which you'll need to address.
+- New terminal output in reaction to the changes, which you may need to consider or act upon.
+- Any other relevant feedback or information related to the tool use.
+6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user.
+7. Tool calls should contain no extra information. Only after receiving the tool's response should you integrate it into a complete reply.
+
+It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to:
+1. Confirm the success of each step before proceeding.
+2. Address any issues or errors that arise immediately.
+3. Adapt your approach based on new information or unexpected results.
+4. Ensure that each action builds correctly on the previous ones.
+
+By waiting for and carefully considering the user's response after each tool use, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work.
+
+====
+
+USER CHAT CONTENT
+
+The following additional message is the user's chat message, and should be followed to the best of your ability without interfering with the TOOL USE guidelines.
+
+"""
+
+    return SYSTEM_PROMPT

main/xiaozhi-server/core/providers/memory/base.py

@@ -4,6 +4,7 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+
 class MemoryProviderBase(ABC):
     def __init__(self, config):
         self.config = config
@@ -20,6 +21,6 @@ class MemoryProviderBase(ABC):
         """Query memories for specific role based on similarity"""
         return "please implement query method"
 
-    def init_memory(self, role_id, llm):
-        self.role_id = role_id    
+    def init_memory(self, role_id, llm, **kwargs):
+        self.role_id = role_id
         self.llm = llm

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -6,13 +6,14 @@ from core.utils.util import check_model_key
 
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory=None):
         super().__init__(config)
         self.api_key = config.get("api_key", "")
         self.api_version = config.get("api_version", "v1.1")
         have_key = check_model_key("Mem0ai", self.api_key)
-        if not have_key :
+        if not have_key:
             self.use_mem0 = False
             return
         else:
@@ -30,54 +31,55 @@ class MemoryProvider(MemoryProviderBase):
             return None
         if len(msgs) < 2:
             return None
-        
+
         try:
             # Format the content as a message list for mem0
             messages = [
                 {"role": message.role, "content": message.content}
-                for message in msgs if message.role != "system"
+                for message in msgs
+                if message.role != "system"
             ]
-            result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
+            result = self.client.add(
+                messages, user_id=self.role_id, output_format=self.api_version
+            )
             logger.bind(tag=TAG).debug(f"Save memory result: {result}")
         except Exception as e:
             logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
             return None
 
-    async def query_memory(self, query: str)-> str:
+    async def query_memory(self, query: str) -> str:
         if not self.use_mem0:
             return ""
         try:
             results = self.client.search(
-                query,
-                user_id=self.role_id,
-                output_format=self.api_version
+                query, user_id=self.role_id, output_format=self.api_version
             )
-            if not results or 'results' not in results:
+            if not results or "results" not in results:
                 return ""
-                
+
             # Format each memory entry with its update time up to minutes
             memories = []
-            for entry in results['results']:
-                timestamp = entry.get('updated_at', '')
+            for entry in results["results"]:
+                timestamp = entry.get("updated_at", "")
                 if timestamp:
                     try:
                         # Parse and reformat the timestamp
-                        dt = timestamp.split('.')[0]  # Remove milliseconds
-                        formatted_time = dt.replace('T', ' ')
+                        dt = timestamp.split(".")[0]  # Remove milliseconds
+                        formatted_time = dt.replace("T", " ")
                     except:
                         formatted_time = timestamp
-                memory = entry.get('memory', '')
+                memory = entry.get("memory", "")
                 if timestamp and memory:
                     # Store tuple of (timestamp, formatted_string) for sorting
                     memories.append((timestamp, f"[{formatted_time}] {memory}"))
-            
+
             # Sort by timestamp in descending order (newest first)
             memories.sort(key=lambda x: x[0], reverse=True)
-            
+
             # Extract only the formatted strings
             memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
             logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
             return memories_str
         except Exception as e:
             logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
-            return ""
+            return ""

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -3,7 +3,9 @@ import time
 import json
 import os
 import yaml
-from core.utils.util import get_project_dir
+from config.config_loader import get_project_dir
+from config.manage_api_client import save_mem_local_short
+
 
 short_term_memory_prompt = """
 # 时空记忆编织者
@@ -71,11 +73,23 @@ short_term_memory_prompt = """
 ```
 """
 
+short_term_memory_prompt_only_content = """
+你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
+1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
+2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
+3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
+4、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
+5、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
+6、只需要返回总结摘要，严格控制在1800字内
+7、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
+"""
+
+
 def extract_json_data(json_code):
     start = json_code.find("```json")
     # 从start开始找到下一个```结束
-    end = json_code.find("```", start+1)
-    #print("start:", start, "end:", end)
+    end = json_code.find("```", start + 1)
+    # print("start:", start, "end:", end)
     if start == -1 or end == -1:
         try:
             jsonData = json.loads(json_code)
@@ -83,74 +97,89 @@ def extract_json_data(json_code):
         except Exception as e:
             print("Error:", e)
         return ""
-    jsonData = json_code[start+7:end]
+    jsonData = json_code[start + 7 : end]
     return jsonData
 
+
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory):
         super().__init__(config)
         self.short_momery = ""
-        self.memory_path = get_project_dir() + 'data/.memory.yaml'
-        self.load_memory()
+        self.save_to_file = True
+        self.memory_path = get_project_dir() + "data/.memory.yaml"
+        self.load_memory(summary_memory)
+
+    def init_memory(
+        self, role_id, llm, summary_memory=None, save_to_file=True, **kwargs
+    ):
+        super().init_memory(role_id, llm, **kwargs)
+        self.save_to_file = save_to_file
+        self.load_memory(summary_memory)
+
+    def load_memory(self, summary_memory):
+        # api获取到总结记忆后直接返回
+        if summary_memory or not self.save_to_file:
+            self.short_momery = summary_memory
+            return
 
-    def init_memory(self, role_id, llm):
-        super().init_memory(role_id, llm)
-        self.load_memory()
-    
-    def load_memory(self):
         all_memory = {}
         if os.path.exists(self.memory_path):
-            with open(self.memory_path, 'r', encoding='utf-8') as f:
+            with open(self.memory_path, "r", encoding="utf-8") as f:
                 all_memory = yaml.safe_load(f) or {}
         if self.role_id in all_memory:
             self.short_momery = all_memory[self.role_id]
-    
+
     def save_memory_to_file(self):
         all_memory = {}
         if os.path.exists(self.memory_path):
-              with open(self.memory_path, 'r', encoding='utf-8') as f:
-                  all_memory = yaml.safe_load(f) or {}
+            with open(self.memory_path, "r", encoding="utf-8") as f:
+                all_memory = yaml.safe_load(f) or {}
         all_memory[self.role_id] = self.short_momery
-        with open(self.memory_path, 'w', encoding='utf-8') as f:
+        with open(self.memory_path, "w", encoding="utf-8") as f:
             yaml.dump(all_memory, f, allow_unicode=True)
-        
+
     async def save_memory(self, msgs):
         if self.llm is None:
             logger.bind(tag=TAG).error("LLM is not set for memory provider")
             return None
-        
+
         if len(msgs) < 2:
             return None
-        
+
         msgStr = ""
         for msg in msgs:
             if msg.role == "user":
                 msgStr += f"User: {msg.content}\n"
-            elif msg.role== "assistant":
+            elif msg.role == "assistant":
                 msgStr += f"Assistant: {msg.content}\n"
-        if len(self.short_momery) > 0:
-            msgStr+="历史记忆：\n"
-            msgStr+=self.short_momery
-        
-        #当前时间
+        if self.short_momery and len(self.short_momery) > 0:
+            msgStr += "历史记忆：\n"
+            msgStr += self.short_momery
+
+        # 当前时间
         time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
         msgStr += f"当前时间：{time_str}"
 
-        result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
- 
-        json_str = extract_json_data(result)
-        try:
-            json_data = json.loads(json_str) # 检查json格式是否正确
-            self.short_momery = json_str
-        except Exception as e:
-            print("Error:", e)
-        
-        self.save_memory_to_file()
+        if self.save_to_file:
+            result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
+            json_str = extract_json_data(result)
+            try:
+                json.loads(json_str)  # 检查json格式是否正确
+                self.short_momery = json_str
+                self.save_memory_to_file()
+            except Exception as e:
+                print("Error:", e)
+        else:
+            result = self.llm.response_no_stream(
+                short_term_memory_prompt_only_content, msgStr
+            )
+            save_mem_local_short(self.role_id, result)
         logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
 
         return self.short_momery
-    
-    async def query_memory(self, query: str)-> str:
-        return self.short_momery
+
+    async def query_memory(self, query: str) -> str:
+        return self.short_momery

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -1,18 +1,20 @@
-'''
+"""
 不使用记忆，可以选择此模块
-'''
+"""
+
 from ..base import MemoryProviderBase, logger
 
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory=None):
         super().__init__(config)
-      
+
     async def save_memory(self, msgs):
         logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
         return None
 
-    async def query_memory(self, query: str)-> str:
+    async def query_memory(self, query: str) -> str:
         logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.")
-        return ""
+        return ""

main/xiaozhi-server/core/providers/vad/base.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class VADProviderBase(ABC):
+    @abstractmethod
+    def is_vad(self, conn, data) -> bool:
+        """检测音频数据中的语音活动"""
+        pass

main/xiaozhi-server/core/providers/vad/silero.py

@@ -0,0 +1,71 @@
+import time
+import numpy as np
+import torch
+import opuslib_next
+from config.logger import setup_logging
+from core.providers.vad.base import VADProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class VADProvider(VADProviderBase):
+    def __init__(self, config):
+        logger.bind(tag=TAG).info("SileroVAD", config)
+        self.model, self.utils = torch.hub.load(
+            repo_or_dir=config["model_dir"],
+            source="local",
+            model="silero_vad",
+            force_reload=False,
+        )
+        (get_speech_timestamps, _, _, _, _) = self.utils
+
+        self.decoder = opuslib_next.Decoder(16000, 1)
+
+        # 处理空字符串的情况
+        threshold = config.get("threshold", "0.5")
+        min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
+
+        self.vad_threshold = float(threshold) if threshold else 0.5
+        self.silence_threshold_ms = (
+            int(min_silence_duration_ms) if min_silence_duration_ms else 1000
+        )
+
+    def is_vad(self, conn, opus_packet):
+        try:
+            pcm_frame = self.decoder.decode(opus_packet, 960)
+            conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
+
+            # 处理缓冲区中的完整帧（每次处理512采样点）
+            client_have_voice = False
+            while len(conn.client_audio_buffer) >= 512 * 2:
+                # 提取前512个采样点（1024字节）
+                chunk = conn.client_audio_buffer[: 512 * 2]
+                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :]
+
+                # 转换为模型需要的张量格式
+                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+                audio_float32 = audio_int16.astype(np.float32) / 32768.0
+                audio_tensor = torch.from_numpy(audio_float32)
+
+                # 检测语音活动
+                with torch.no_grad():
+                    speech_prob = self.model(audio_tensor, 16000).item()
+                client_have_voice = speech_prob >= self.vad_threshold
+
+                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
+                if conn.client_have_voice and not client_have_voice:
+                    stop_duration = (
+                        time.time() * 1000 - conn.client_have_voice_last_time
+                    )
+                    if stop_duration >= self.silence_threshold_ms:
+                        conn.client_voice_stop = True
+                if client_have_voice:
+                    conn.client_have_voice = True
+                    conn.client_have_voice_last_time = time.time() * 1000
+
+            return client_have_voice
+        except opuslib_next.OpusError as e:
+            logger.bind(tag=TAG).info(f"解码错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")

main/xiaozhi-server/core/utils/auth_code_gen.py

@@ -1,97 +0,0 @@
-import random
-import threading
-import time
-from typing import Set
-
-class AuthCodeGenerator:
-    _instance = None
-    _instance_lock = threading.Lock()
-    
-    def __new__(cls):
-        if not cls._instance:
-            with cls._instance_lock:
-                if not cls._instance:
-                    cls._instance = super(AuthCodeGenerator, cls).__new__(cls)
-                    # 初始化随机种子
-                    random.seed(time.time())
-        return cls._instance
-
-    def __init__(self):
-        # 确保 __init__ 只被调用一次
-        if not hasattr(self, '_initialized'):
-            self._used_codes: Set[str] = set()
-            self._code_timestamps = {}
-            self._lock = threading.Lock()
-            self._code_timeout = 3 * 24 * 60 * 60
-            self._initialized = True
-    
-    @classmethod
-    def get_instance(cls):
-        """获取AuthCodeGenerator的单例实例"""
-        return cls()
-
-    def generate_code(self) -> str:
-        """
-        生成6位数字认证码，确保不重复
-        返回: 6位数字字符串
-        """
-        with self._lock:
-            self._clean_expired_codes()  # 清理过期code
-            while True:
-                # 使用时间戳和已用码数量作为种子，确保每次生成不同的随机数
-                seed = int(time.time() * 1000) + len(self._used_codes)
-                random.seed(seed)
-                
-                # 生成6位随机数字
-                code = ''.join(str(random.randint(0, 9)) for _ in range(6))
-                
-                # 检查是否已存在
-                if code not in self._used_codes:
-                    self._used_codes.add(code)
-                    self._code_timestamps[code] = time.time()
-                    return code
-    
-    def remove_code(self, code: str) -> bool:
-        """
-        删除已使用的认证码
-        参数:
-            code: 要删除的认证码
-        返回:
-            bool: 删除成功返回True，码不存在返回False
-        """
-        print('remove_code', code)
-        with self._lock:
-            if code in self._used_codes:
-                self._used_codes.remove(code)
-                if code in self._code_timestamps:
-                    del self._code_timestamps[code]
-                return True
-            return False
-    
-    def is_code_used(self, code: str) -> bool:
-        """
-        检查认证码是否已被使用
-        参数:
-            code: 要检查的认证码
-        返回:
-            bool: 如果码存在返回True，否则返回False
-        """
-        with self._lock:
-            return code in self._used_codes
-    
-    def clear_codes(self):
-        """清空所有已使用的认证码"""
-        with self._lock:
-            self._used_codes.clear()
-            self._code_timestamps.clear()
-    
-    def _clean_expired_codes(self):
-        """清理过期的认证码"""
-        current_time = time.time()
-        expired_codes = [
-            code for code, timestamp in self._code_timestamps.items()
-            if (current_time - timestamp) > self._code_timeout
-        ]
-        for code in expired_codes:
-            self._used_codes.remove(code)
-            del self._code_timestamps[code]

main/xiaozhi-server/core/utils/dialogue.py

@@ -4,7 +4,14 @@ from datetime import datetime
 
 
 class Message:
-    def __init__(self, role: str, content: str = None, uniq_id: str = None, tool_calls = None, tool_call_id=None):
+    def __init__(
+        self,
+        role: str,
+        content: str = None,
+        uniq_id: str = None,
+        tool_calls=None,
+        tool_call_id=None,
+    ):
         self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
         self.role = role
         self.content = content
@@ -16,7 +23,7 @@ class Dialogue:
     def __init__(self):
         self.dialogue: List[Message] = []
         # 获取当前时间
-        self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        self.current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
     def put(self, message: Message):
         self.dialogue.append(message)
@@ -25,7 +32,15 @@ class Dialogue:
         if m.tool_calls is not None:
             dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
         elif m.role == "tool":
-            dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content})
+            dialogue.append(
+                {
+                    "role": m.role,
+                    "tool_call_id": (
+                        str(uuid.uuid4()) if m.tool_call_id is None else m.tool_call_id
+                    ),
+                    "content": m.content,
+                }
+            )
         else:
             dialogue.append({"role": m.role, "content": m.content})
 
@@ -44,23 +59,24 @@ class Dialogue:
         else:
             self.put(Message(role="system", content=new_content))
 
-    def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
+    def get_llm_dialogue_with_memory(
+        self, memory_str: str = None
+    ) -> List[Dict[str, str]]:
         if memory_str is None or len(memory_str) == 0:
             return self.get_llm_dialogue()
-        
+
         # 构建带记忆的对话
         dialogue = []
-        
+
         # 添加系统提示和记忆
         system_message = next(
             (msg for msg in self.dialogue if msg.role == "system"), None
         )
 
-
         if system_message:
             enhanced_system_prompt = (
                 f"{system_message.content}\n\n"
-                f"相关记忆：\n{memory_str}"
+                f"以下是用户的历史记忆：\n```\n{memory_str}\n```"
             )
             dialogue.append({"role": "system", "content": enhanced_system_prompt})
 

main/xiaozhi-server/core/utils/lock_manager.py

@@ -1,39 +0,0 @@
-import asyncio
-from typing import Dict
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-class FileLockManager:
-    _instance = None
-    _locks: Dict[str, asyncio.Lock] = {}
-
-    def __new__(cls):
-        if cls._instance is None:
-            cls._instance = super(FileLockManager, cls).__new__(cls)
-        return cls._instance
-
-    @classmethod
-    def get_lock(cls, file_path: str) -> asyncio.Lock:
-        """获取指定文件的锁"""
-        if file_path not in cls._locks:
-            cls._locks[file_path] = asyncio.Lock()
-        return cls._locks[file_path]
-
-    @classmethod
-    async def acquire_lock(cls, file_path: str):
-        """获取锁"""
-        lock = cls.get_lock(file_path)
-        await lock.acquire()
-        logger.bind(tag=TAG).debug(f"Acquired lock for {file_path}")
-
-    @classmethod
-    def release_lock(cls, file_path: str):
-        """释放锁"""
-        if file_path in cls._locks:
-            try:
-                cls._locks[file_path].release()
-                logger.bind(tag=TAG).debug(f"Released lock for {file_path}")
-            except RuntimeError as e:
-                logger.bind(tag=TAG).warning(f"Failed to release lock for {file_path}: {e}")

main/xiaozhi-server/core/utils/memory.py

@@ -2,16 +2,17 @@ import os
 import sys
 import importlib
 from config.logger import setup_logging
-from core.utils.util import read_config, get_project_dir
 
 logger = setup_logging()
 
+
 def create_instance(class_name, *args, **kwargs):
-    if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')):
-        lib_name = f'core.providers.memory.{class_name}.{class_name}'
+    if os.path.exists(
+        os.path.join("core", "providers", "memory", class_name, f"{class_name}.py")
+    ):
+        lib_name = f"core.providers.memory.{class_name}.{class_name}"
         if lib_name not in sys.modules:
-            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+            sys.modules[lib_name] = importlib.import_module(f"{lib_name}")
         return sys.modules[lib_name].MemoryProvider(*args, **kwargs)
 
     raise ValueError(f"不支持的记忆服务类型: {class_name}")
-

main/xiaozhi-server/core/utils/output_counter.py

@@ -0,0 +1,50 @@
+import datetime
+from typing import Dict, Tuple
+
+# 全局字典，用于存储每个设备的每日输出字数
+_device_daily_output: Dict[Tuple[str, datetime.date], int] = {}
+# 记录最后一次检查的日期
+_last_check_date: datetime.date = None
+
+
+def reset_device_output():
+    """
+    重置所有设备的每日输出字数
+    每天0点调用此函数
+    """
+    _device_daily_output.clear()
+
+
+def get_device_output(device_id: str) -> int:
+    """
+    获取设备当日的输出字数
+    """
+    current_date = datetime.datetime.now().date()
+    return _device_daily_output.get((device_id, current_date), 0)
+
+
+def add_device_output(device_id: str, char_count: int):
+    """
+    增加设备的输出字数
+    """
+    current_date = datetime.datetime.now().date()
+    global _last_check_date
+
+    # 如果是第一次调用或者日期发生变化，清空计数器
+    if _last_check_date is None or _last_check_date != current_date:
+        _device_daily_output.clear()
+        _last_check_date = current_date
+
+    current_count = _device_daily_output.get((device_id, current_date), 0)
+    _device_daily_output[(device_id, current_date)] = current_count + char_count
+
+
+def check_device_output_limit(device_id: str, max_output_size: int) -> bool:
+    """
+    检查设备是否超过输出限制
+    :return: True 如果超过限制，False 如果未超过
+    """
+    if not device_id:
+        return False
+    current_output = get_device_output(device_id)
+    return current_output >= max_output_size

main/xiaozhi-server/core/utils/util.py

@@ -1,19 +1,40 @@
-import os
 import json
-import yaml
 import socket
 import subprocess
-import logging
 import re
+import os
+import numpy as np
 import requests
+import opuslib_next
+from pydub import AudioSegment
+from typing import Dict, Any
+from core.utils import tts, llm, intent, memory, vad, asr
+import copy
 
-
-def get_project_dir():
-    """获取项目根目录"""
-    return (
-        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-        + "/"
-    )
+TAG = __name__
+emoji_map = {
+    "neutral": "😶",
+    "happy": "🙂",
+    "laughing": "😆",
+    "funny": "😂",
+    "sad": "😔",
+    "angry": "😠",
+    "crying": "😭",
+    "loving": "😍",
+    "embarrassed": "😳",
+    "surprised": "😲",
+    "shocked": "😱",
+    "thinking": "🤔",
+    "winking": "😉",
+    "cool": "😎",
+    "relaxed": "😌",
+    "delicious": "🤤",
+    "kissy": "😘",
+    "confident": "😏",
+    "sleepy": "😴",
+    "silly": "😜",
+    "confused": "🙄",
+}
 
 
 def get_local_ip():
@@ -72,7 +93,7 @@ def is_private_ip(ip_addr):
         return False  # IP address format error or insufficient segments
 
 
-def get_ip_info(ip_addr):
+def get_ip_info(ip_addr, logger):
     try:
         if is_private_ip(ip_addr):
             ip_addr = ""
@@ -81,16 +102,10 @@ def get_ip_info(ip_addr):
         ip_info = {"city": resp.get("city")}
         return ip_info
     except Exception as e:
-        logging.error(f"Error getting client ip info: {e}")
+        logger.bind(tag=TAG).error(f"Error getting client ip info: {e}")
         return {}
 
 
-def read_config(config_path):
-    with open(config_path, "r", encoding="utf-8") as file:
-        config = yaml.safe_load(file)
-    return config
-
-
 def write_json_file(file_path, data):
     """将数据写入 JSON 文件"""
     with open(file_path, "w", encoding="utf-8") as file:
@@ -103,13 +118,14 @@ def is_punctuation_or_emoji(char):
     punctuation_set = {
         "，",
         ",",  # 中文逗号 + 英文逗号
-        "。",
-        ".",  # 中文句号 + 英文句号
-        "！",
-        "!",  # 中文感叹号 + 英文感叹号
         "-",
         "－",  # 英文连字符 + 中文全角横线
         "、",  # 中文顿号
+        "“",
+        "”",
+        '"',  # 中文双引号 + 英文引号
+        "：",
+        ":",  # 中文冒号 + 英文冒号
     }
     if char.isspace() or char in punctuation_set:
         return True
@@ -169,15 +185,30 @@ def remove_punctuation_and_length(text):
 
 def check_model_key(modelType, modelKey):
     if "你" in modelKey:
-        logging.error(
-            "你还没配置"
-            + modelType
-            + "的密钥，请在配置文件中配置密钥，否则无法正常工作"
+        raise ValueError(
+            "你还没配置" + modelType + "的密钥，请检查一下所使用的LLM是否配置了密钥"
         )
-        return False
     return True
 
 
+def parse_string_to_list(value, separator=";"):
+    """
+    将输入值转换为列表
+    Args:
+        value: 输入值，可以是 None、字符串或列表
+        separator: 分隔符，默认为分号
+    Returns:
+        list: 处理后的列表
+    """
+    if value is None or value == "":
+        return []
+    elif isinstance(value, str):
+        return [item.strip() for item in value.split(separator) if item.strip()]
+    elif isinstance(value, list):
+        return value
+    return []
+
+
 def check_ffmpeg_installed():
     ffmpeg_installed = False
     try:
@@ -208,7 +239,755 @@ def check_ffmpeg_installed():
 def extract_json_from_string(input_string):
     """提取字符串中的 JSON 部分"""
     pattern = r"(\{.*\})"
-    match = re.search(pattern, input_string)
+    match = re.search(pattern, input_string, re.DOTALL)  # 添加 re.DOTALL
     if match:
         return match.group(1)  # 返回提取的 JSON 字符串
     return None
+
+
+def initialize_modules(
+    logger,
+    config: Dict[str, Any],
+    init_vad=False,
+    init_asr=False,
+    init_llm=False,
+    init_tts=False,
+    init_memory=False,
+    init_intent=False,
+) -> Dict[str, Any]:
+    """
+    初始化所有模块组件
+
+    Args:
+        config: 配置字典
+
+    Returns:
+        Dict[str, Any]: 包含所有初始化后的模块的字典
+    """
+    modules = {}
+
+    # 初始化TTS模块
+    if init_tts:
+        select_tts_module = config["selected_module"]["TTS"]
+        tts_type = (
+            select_tts_module
+            if "type" not in config["TTS"][select_tts_module]
+            else config["TTS"][select_tts_module]["type"]
+        )
+        modules["tts"] = tts.create_instance(
+            tts_type,
+            config["TTS"][select_tts_module],
+            str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
+        )
+        logger.bind(tag=TAG).info(f"初始化组件: tts成功 {select_tts_module}")
+
+    # 初始化LLM模块
+    if init_llm:
+        select_llm_module = config["selected_module"]["LLM"]
+        llm_type = (
+            select_llm_module
+            if "type" not in config["LLM"][select_llm_module]
+            else config["LLM"][select_llm_module]["type"]
+        )
+        modules["llm"] = llm.create_instance(
+            llm_type,
+            config["LLM"][select_llm_module],
+        )
+        logger.bind(tag=TAG).info(f"初始化组件: llm成功 {select_llm_module}")
+
+    # 初始化Intent模块
+    if init_intent:
+        select_intent_module = config["selected_module"]["Intent"]
+        intent_type = (
+            select_intent_module
+            if "type" not in config["Intent"][select_intent_module]
+            else config["Intent"][select_intent_module]["type"]
+        )
+        modules["intent"] = intent.create_instance(
+            intent_type,
+            config["Intent"][select_intent_module],
+        )
+        logger.bind(tag=TAG).info(f"初始化组件: intent成功 {select_intent_module}")
+
+    # 初始化Memory模块
+    if init_memory:
+        select_memory_module = config["selected_module"]["Memory"]
+        memory_type = (
+            select_memory_module
+            if "type" not in config["Memory"][select_memory_module]
+            else config["Memory"][select_memory_module]["type"]
+        )
+        modules["memory"] = memory.create_instance(
+            memory_type,
+            config["Memory"][select_memory_module],
+            config.get("summaryMemory", None),
+        )
+        logger.bind(tag=TAG).info(f"初始化组件: memory成功 {select_memory_module}")
+
+    # 初始化VAD模块
+    if init_vad:
+        select_vad_module = config["selected_module"]["VAD"]
+        vad_type = (
+            select_vad_module
+            if "type" not in config["VAD"][select_vad_module]
+            else config["VAD"][select_vad_module]["type"]
+        )
+        modules["vad"] = vad.create_instance(
+            vad_type,
+            config["VAD"][select_vad_module],
+        )
+        logger.bind(tag=TAG).info(f"初始化组件: vad成功 {select_vad_module}")
+
+    # 初始化ASR模块
+    if init_asr:
+        select_asr_module = config["selected_module"]["ASR"]
+        asr_type = (
+            select_asr_module
+            if "type" not in config["ASR"][select_asr_module]
+            else config["ASR"][select_asr_module]["type"]
+        )
+        modules["asr"] = asr.create_instance(
+            asr_type,
+            config["ASR"][select_asr_module],
+            str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
+        )
+        logger.bind(tag=TAG).info(f"初始化组件: asr成功 {select_asr_module}")
+    return modules
+
+
+def analyze_emotion(text):
+    """
+    分析文本情感并返回对应的emoji名称（支持中英文）
+    """
+    if not text or not isinstance(text, str):
+        return "neutral"
+
+    original_text = text
+    text = text.lower().strip()
+
+    # 检查是否包含现有emoji
+    for emotion, emoji in emoji_map.items():
+        if emoji in original_text:
+            return emotion
+
+    # 标点符号分析
+    has_exclamation = "!" in original_text or "！" in original_text
+    has_question = "?" in original_text or "？" in original_text
+    has_ellipsis = "..." in original_text or "…" in original_text
+
+    # 定义情感关键词映射（中英文扩展版）
+    emotion_keywords = {
+        "happy": [
+            "开心",
+            "高兴",
+            "快乐",
+            "愉快",
+            "幸福",
+            "满意",
+            "棒",
+            "好",
+            "不错",
+            "完美",
+            "棒极了",
+            "太好了",
+            "好呀",
+            "好的",
+            "happy",
+            "joy",
+            "great",
+            "good",
+            "nice",
+            "awesome",
+            "fantastic",
+            "wonderful",
+        ],
+        "laughing": [
+            "哈哈",
+            "哈哈哈",
+            "呵呵",
+            "嘿嘿",
+            "嘻嘻",
+            "笑死",
+            "太好笑了",
+            "笑死我了",
+            "lol",
+            "lmao",
+            "haha",
+            "hahaha",
+            "hehe",
+            "rofl",
+            "funny",
+            "laugh",
+        ],
+        "funny": [
+            "搞笑",
+            "滑稽",
+            "逗",
+            "幽默",
+            "笑点",
+            "段子",
+            "笑话",
+            "太逗了",
+            "hilarious",
+            "joke",
+            "comedy",
+        ],
+        "sad": [
+            "伤心",
+            "难过",
+            "悲哀",
+            "悲伤",
+            "忧郁",
+            "郁闷",
+            "沮丧",
+            "失望",
+            "想哭",
+            "难受",
+            "不开心",
+            "唉",
+            "呜呜",
+            "sad",
+            "upset",
+            "unhappy",
+            "depressed",
+            "sorrow",
+            "gloomy",
+        ],
+        "angry": [
+            "生气",
+            "愤怒",
+            "气死",
+            "讨厌",
+            "烦人",
+            "可恶",
+            "烦死了",
+            "恼火",
+            "暴躁",
+            "火大",
+            "愤怒",
+            "气炸了",
+            "angry",
+            "mad",
+            "annoyed",
+            "furious",
+            "pissed",
+            "hate",
+        ],
+        "crying": [
+            "哭泣",
+            "泪流",
+            "大哭",
+            "伤心欲绝",
+            "泪目",
+            "流泪",
+            "哭死",
+            "哭晕",
+            "想哭",
+            "泪崩",
+            "cry",
+            "crying",
+            "tears",
+            "sob",
+            "weep",
+        ],
+        "loving": [
+            "爱你",
+            "喜欢",
+            "爱",
+            "亲爱的",
+            "宝贝",
+            "么么哒",
+            "抱抱",
+            "想你",
+            "思念",
+            "最爱",
+            "亲亲",
+            "喜欢你",
+            "love",
+            "like",
+            "adore",
+            "darling",
+            "sweetie",
+            "honey",
+            "miss you",
+            "heart",
+        ],
+        "embarrassed": [
+            "尴尬",
+            "不好意思",
+            "害羞",
+            "脸红",
+            "难为情",
+            "社死",
+            "丢脸",
+            "出丑",
+            "embarrassed",
+            "awkward",
+            "shy",
+            "blush",
+        ],
+        "surprised": [
+            "惊讶",
+            "吃惊",
+            "天啊",
+            "哇塞",
+            "哇",
+            "居然",
+            "竟然",
+            "没想到",
+            "出乎意料",
+            "surprise",
+            "wow",
+            "omg",
+            "oh my god",
+            "amazing",
+            "unbelievable",
+        ],
+        "shocked": [
+            "震惊",
+            "吓到",
+            "惊呆了",
+            "不敢相信",
+            "震撼",
+            "吓死",
+            "恐怖",
+            "害怕",
+            "吓人",
+            "shocked",
+            "shocking",
+            "scared",
+            "frightened",
+            "terrified",
+            "horror",
+        ],
+        "thinking": [
+            "思考",
+            "考虑",
+            "想一下",
+            "琢磨",
+            "沉思",
+            "冥想",
+            "想",
+            "思考中",
+            "在想",
+            "think",
+            "thinking",
+            "consider",
+            "ponder",
+            "meditate",
+        ],
+        "winking": [
+            "调皮",
+            "眨眼",
+            "你懂的",
+            "坏笑",
+            "邪恶",
+            "奸笑",
+            "使眼色",
+            "wink",
+            "teasing",
+            "naughty",
+            "mischievous",
+        ],
+        "cool": [
+            "酷",
+            "帅",
+            "厉害",
+            "棒极了",
+            "真棒",
+            "牛逼",
+            "强",
+            "优秀",
+            "杰出",
+            "出色",
+            "完美",
+            "cool",
+            "awesome",
+            "amazing",
+            "great",
+            "impressive",
+            "perfect",
+        ],
+        "relaxed": [
+            "放松",
+            "舒服",
+            "惬意",
+            "悠闲",
+            "轻松",
+            "舒适",
+            "安逸",
+            "自在",
+            "relax",
+            "relaxed",
+            "comfortable",
+            "cozy",
+            "chill",
+            "peaceful",
+        ],
+        "delicious": [
+            "好吃",
+            "美味",
+            "香",
+            "馋",
+            "可口",
+            "香甜",
+            "大餐",
+            "大快朵颐",
+            "流口水",
+            "垂涎",
+            "delicious",
+            "yummy",
+            "tasty",
+            "yum",
+            "appetizing",
+            "mouthwatering",
+        ],
+        "kissy": [
+            "亲亲",
+            "么么",
+            "吻",
+            "mua",
+            "muah",
+            "亲一下",
+            "飞吻",
+            "kiss",
+            "xoxo",
+            "hug",
+            "muah",
+            "smooch",
+        ],
+        "confident": [
+            "自信",
+            "肯定",
+            "确定",
+            "毫无疑问",
+            "当然",
+            "必须的",
+            "毫无疑问",
+            "确信",
+            "坚信",
+            "confident",
+            "sure",
+            "certain",
+            "definitely",
+            "positive",
+        ],
+        "sleepy": [
+            "困",
+            "睡觉",
+            "晚安",
+            "想睡",
+            "好累",
+            "疲惫",
+            "疲倦",
+            "困了",
+            "想休息",
+            "睡意",
+            "sleep",
+            "sleepy",
+            "tired",
+            "exhausted",
+            "bedtime",
+            "good night",
+        ],
+        "silly": [
+            "傻",
+            "笨",
+            "呆",
+            "憨",
+            "蠢",
+            "二",
+            "憨憨",
+            "傻乎乎",
+            "呆萌",
+            "silly",
+            "stupid",
+            "dumb",
+            "foolish",
+            "goofy",
+            "ridiculous",
+        ],
+        "confused": [
+            "疑惑",
+            "不明白",
+            "不懂",
+            "困惑",
+            "疑问",
+            "为什么",
+            "怎么回事",
+            "啥意思",
+            "不清楚",
+            "confused",
+            "puzzled",
+            "doubt",
+            "question",
+            "what",
+            "why",
+            "how",
+        ],
+    }
+
+    # 特殊句型判断（中英文）
+    # 赞美他人
+    if any(
+        phrase in text
+        for phrase in [
+            "你真",
+            "你好",
+            "您真",
+            "你真棒",
+            "你好厉害",
+            "你太强了",
+            "你真好",
+            "你真聪明",
+            "you are",
+            "you're",
+            "you look",
+            "you seem",
+            "so smart",
+            "so kind",
+        ]
+    ):
+        return "loving"
+    # 自我赞美
+    if any(
+        phrase in text
+        for phrase in [
+            "我真",
+            "我最",
+            "我太棒了",
+            "我厉害",
+            "我聪明",
+            "我优秀",
+            "i am",
+            "i'm",
+            "i feel",
+            "so good",
+            "so happy",
+        ]
+    ):
+        return "cool"
+    # 晚安/睡觉相关
+    if any(
+        phrase in text
+        for phrase in [
+            "睡觉",
+            "晚安",
+            "睡了",
+            "好梦",
+            "休息了",
+            "去睡了",
+            "sleep",
+            "good night",
+            "bedtime",
+            "go to bed",
+        ]
+    ):
+        return "sleepy"
+    # 疑问句
+    if has_question and not has_exclamation:
+        return "thinking"
+    # 强烈情感（感叹号）
+    if has_exclamation and not has_question:
+        # 检查是否是积极内容
+        positive_words = (
+            emotion_keywords["happy"]
+            + emotion_keywords["laughing"]
+            + emotion_keywords["cool"]
+        )
+        if any(word in text for word in positive_words):
+            return "laughing"
+        # 检查是否是消极内容
+        negative_words = (
+            emotion_keywords["angry"]
+            + emotion_keywords["sad"]
+            + emotion_keywords["crying"]
+        )
+        if any(word in text for word in negative_words):
+            return "angry"
+        return "surprised"
+    # 省略号（表示犹豫或思考）
+    if has_ellipsis:
+        return "thinking"
+
+    # 关键词匹配（带权重）
+    emotion_scores = {emotion: 0 for emotion in emoji_map.keys()}
+
+    # 给匹配到的关键词加分
+    for emotion, keywords in emotion_keywords.items():
+        for keyword in keywords:
+            if keyword in text:
+                emotion_scores[emotion] += 1
+
+    # 给长文本中的重复关键词额外加分
+    if len(text) > 20:  # 长文本
+        for emotion, keywords in emotion_keywords.items():
+            for keyword in keywords:
+                emotion_scores[emotion] += text.count(keyword) * 0.5
+
+    # 根据分数选择最可能的情感
+    max_score = max(emotion_scores.values())
+    if max_score == 0:
+        return "happy"  # 默认
+
+    # 可能有多个情感同分，根据上下文选择最合适的
+    top_emotions = [e for e, s in emotion_scores.items() if s == max_score]
+
+    # 如果多个情感同分，使用以下优先级
+    priority_order = [
+        "laughing",
+        "crying",
+        "angry",
+        "surprised",
+        "shocked",  # 强烈情感优先
+        "loving",
+        "happy",
+        "funny",
+        "cool",  # 积极情感
+        "sad",
+        "embarrassed",
+        "confused",  # 消极情感
+        "thinking",
+        "winking",
+        "relaxed",  # 中性情感
+        "delicious",
+        "kissy",
+        "confident",
+        "sleepy",
+        "silly",  # 特殊场景
+    ]
+
+    for emotion in priority_order:
+        if emotion in top_emotions:
+            return emotion
+
+    return top_emotions[0]  # 如果都不在优先级列表里，返回第一个
+
+
+def audio_to_data(audio_file_path, is_opus=True):
+    # 获取文件后缀名
+    file_type = os.path.splitext(audio_file_path)[1]
+    if file_type:
+        file_type = file_type.lstrip(".")
+    # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
+    audio = AudioSegment.from_file(
+        audio_file_path, format=file_type, parameters=["-nostdin"]
+    )
+
+    # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
+    audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+
+    # 音频时长(秒)
+    duration = len(audio) / 1000.0
+
+    # 获取原始PCM数据（16位小端）
+    raw_data = audio.raw_data
+
+    # 初始化Opus编码器
+    encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+    # 编码参数
+    frame_duration = 60  # 60ms per frame
+    frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+    datas = []
+    # 按帧处理所有音频数据（包括最后一帧可能补零）
+    for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+        # 获取当前帧的二进制数据
+        chunk = raw_data[i : i + frame_size * 2]
+
+        # 如果最后一帧不足，补零
+        if len(chunk) < frame_size * 2:
+            chunk += b"\x00" * (frame_size * 2 - len(chunk))
+
+        if is_opus:
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+            # 编码Opus数据
+            frame_data = encoder.encode(np_frame.tobytes(), frame_size)
+        else:
+            frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
+
+        datas.append(frame_data)
+
+    return datas, duration
+
+
+def check_vad_update(before_config, new_config):
+    if (
+        new_config.get("selected_module") is None
+        or new_config["selected_module"].get("VAD") is None
+    ):
+        return False
+    update_vad = False
+    current_vad_module = before_config["selected_module"]["VAD"]
+    new_vad_module = new_config["selected_module"]["VAD"]
+    current_vad_type = (
+        current_vad_module
+        if "type" not in before_config["VAD"][current_vad_module]
+        else before_config["VAD"][current_vad_module]["type"]
+    )
+    new_vad_type = (
+        new_vad_module
+        if "type" not in new_config["VAD"][new_vad_module]
+        else new_config["VAD"][new_vad_module]["type"]
+    )
+    update_vad = current_vad_type != new_vad_type
+    return update_vad
+
+
+def check_asr_update(before_config, new_config):
+    if (
+        new_config.get("selected_module") is None
+        or new_config["selected_module"].get("ASR") is None
+    ):
+        return False
+    update_asr = False
+    current_asr_module = before_config["selected_module"]["ASR"]
+    new_asr_module = new_config["selected_module"]["ASR"]
+    current_asr_type = (
+        current_asr_module
+        if "type" not in before_config["ASR"][current_asr_module]
+        else before_config["ASR"][current_asr_module]["type"]
+    )
+    new_asr_type = (
+        new_asr_module
+        if "type" not in new_config["ASR"][new_asr_module]
+        else new_config["ASR"][new_asr_module]["type"]
+    )
+    update_asr = current_asr_type != new_asr_type
+    return update_asr
+
+
+def filter_sensitive_info(config: dict) -> dict:
+    """
+    过滤配置中的敏感信息
+    Args:
+        config: 原始配置字典
+    Returns:
+        过滤后的配置字典
+    """
+    sensitive_keys = [
+        "api_key",
+        "personal_access_token",
+        "access_token",
+        "token",
+        "secret",
+        "access_key_secret",
+        "secret_key",
+    ]
+
+    def _filter_dict(d: dict) -> dict:
+        filtered = {}
+        for k, v in d.items():
+            if any(sensitive in k.lower() for sensitive in sensitive_keys):
+                filtered[k] = "***"
+            elif isinstance(v, dict):
+                filtered[k] = _filter_dict(v)
+            elif isinstance(v, list):
+                filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
+            else:
+                filtered[k] = v
+        return filtered
+
+    return _filter_dict(copy.deepcopy(config))

main/xiaozhi-server/core/utils/vad.py

@@ -1,77 +1,19 @@
-from abc import ABC, abstractmethod
+import importlib
+import os
+import sys
+from core.providers.vad.base import VADProviderBase
 from config.logger import setup_logging
-import opuslib_next
-import time
-import numpy as np
-import torch
 
 TAG = __name__
 logger = setup_logging()
 
-class VAD(ABC):
-    @abstractmethod
-    def is_vad(self, conn, data):
-        """检测音频数据中的语音活动"""
-        pass
 
+def create_instance(class_name: str, *args, **kwargs) -> VADProviderBase:
+    """工厂方法创建VAD实例"""
+    if os.path.exists(os.path.join("core", "providers", "vad", f"{class_name}.py")):
+        lib_name = f"core.providers.vad.{class_name}"
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f"{lib_name}")
+        return sys.modules[lib_name].VADProvider(*args, **kwargs)
 
-class SileroVAD(VAD):
-    def __init__(self, config):
-        logger.bind(tag=TAG).info("SileroVAD", config)
-        self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
-                                                source='local',
-                                                model='silero_vad',
-                                                force_reload=False)
-        (get_speech_timestamps, _, _, _, _) = self.utils
-
-        self.decoder = opuslib_next.Decoder(16000, 1)
-        self.vad_threshold = config.get("threshold")
-        self.silence_threshold_ms = config.get("min_silence_duration_ms")
-
-    def is_vad(self, conn, opus_packet):
-        try:
-            pcm_frame = self.decoder.decode(opus_packet, 960)
-            conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
-
-            # 处理缓冲区中的完整帧（每次处理512采样点）
-            client_have_voice = False
-            while len(conn.client_audio_buffer) >= 512 * 2:
-                # 提取前512个采样点（1024字节）
-                chunk = conn.client_audio_buffer[:512 * 2]
-                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
-
-                # 转换为模型需要的张量格式
-                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
-                audio_float32 = audio_int16.astype(np.float32) / 32768.0
-                audio_tensor = torch.from_numpy(audio_float32)
-
-                # 检测语音活动
-                speech_prob = self.model(audio_tensor, 16000).item()
-                client_have_voice = speech_prob >= self.vad_threshold
-
-                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
-                if conn.client_have_voice and not client_have_voice:
-                    stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
-                    if stop_duration >= self.silence_threshold_ms:
-                        conn.client_voice_stop = True
-                if client_have_voice:
-                    conn.client_have_voice = True
-                    conn.client_have_voice_last_time = time.time() * 1000
-
-            return client_have_voice
-        except opuslib_next.OpusError as e:
-            logger.bind(tag=TAG).info(f"解码错误: {e}")
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
-
-
-def create_instance(class_name, *args, **kwargs) -> VAD:
-    # 获取类对象
-    cls_map = {
-        "SileroVAD": SileroVAD,
-        # 可扩展其他SileroVAD实现
-    }
-
-    if cls := cls_map.get(class_name):
-        return cls(*args, **kwargs)
-    raise ValueError(f"不支持的SileroVAD类型: {class_name}")
+    raise ValueError(f"不支持的VAD类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/websocket_server.py

@@ -2,8 +2,8 @@ import asyncio
 import websockets
 from config.logger import setup_logging
 from core.connection import ConnectionHandler
-from core.utils.util import get_local_ip
-from core.utils import asr, vad, llm, tts, memory, intent
+from core.utils.util import initialize_modules, check_vad_update, check_asr_update
+from config.config_loader import get_config_from_api
 
 TAG = __name__
 
@@ -12,109 +12,112 @@ class WebSocketServer:
     def __init__(self, config: dict):
         self.config = config
         self.logger = setup_logging()
-        self._vad, self._asr, self._llm, self._memory, self.intent = (
-            self._create_processing_instances()
-        )
-        self.active_connections = set()  # 添加全局连接记录
-
-    def _create_processing_instances(self):
-        memory_cls_name = self.config["selected_module"].get(
-            "Memory", "nomem"
-        )  # 默认使用nomem
-        has_memory_cfg = (
-                self.config.get("Memory") and memory_cls_name in self.config["Memory"]
-        )
-        memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
-
-        """创建处理模块实例"""
-        return (
-            vad.create_instance(
-                self.config["selected_module"]["VAD"],
-                self.config["VAD"][self.config["selected_module"]["VAD"]],
-            ),
-            asr.create_instance(
-                (
-                    self.config["selected_module"]["ASR"]
-                    if not "type"
-                           in self.config["ASR"][self.config["selected_module"]["ASR"]]
-                    else self.config["ASR"][self.config["selected_module"]["ASR"]][
-                        "type"
-                    ]
-                ),
-                self.config["ASR"][self.config["selected_module"]["ASR"]],
-                self.config["delete_audio"],
-            ),
-            llm.create_instance(
-                (
-                    self.config["selected_module"]["LLM"]
-                    if not "type"
-                           in self.config["LLM"][self.config["selected_module"]["LLM"]]
-                    else self.config["LLM"][self.config["selected_module"]["LLM"]][
-                        "type"
-                    ]
-                ),
-                self.config["LLM"][self.config["selected_module"]["LLM"]],
-            ),
-            memory.create_instance(memory_cls_name, memory_cfg),
-            intent.create_instance(
-                (
-                    self.config["selected_module"]["Intent"]
-                    if not "type"
-                           in self.config["Intent"][self.config["selected_module"]["Intent"]]
-                    else self.config["Intent"][
-                        self.config["selected_module"]["Intent"]
-                    ]["type"]
-                ),
-                self.config["Intent"][self.config["selected_module"]["Intent"]],
-            ),
+        self.config_lock = asyncio.Lock()
+        modules = initialize_modules(
+            self.logger,
+            self.config,
+            "VAD" in self.config["selected_module"],
+            "ASR" in self.config["selected_module"],
+            "LLM" in self.config["selected_module"],
+            "TTS" in self.config["selected_module"],
+            "Memory" in self.config["selected_module"],
+            "Intent" in self.config["selected_module"],
         )
+        self._vad = modules["vad"] if "vad" in modules else None
+        self._asr = modules["asr"] if "asr" in modules else None
+        self._tts = modules["tts"] if "tts" in modules else None
+        self._llm = modules["llm"] if "llm" in modules else None
+        self._intent = modules["intent"] if "intent" in modules else None
+        self._memory = modules["memory"] if "memory" in modules else None
+        self.active_connections = set()
 
     async def start(self):
         server_config = self.config["server"]
-        host = server_config["ip"]
-        port = server_config["port"]
+        host = server_config.get("ip", "0.0.0.0")
+        port = int(server_config.get("port", 8000))
 
-        self.logger.bind(tag=TAG).info(
-            "Server is running at ws://{}:{}/xiaozhi/v1/", get_local_ip(), port
-        )
-        self.logger.bind(tag=TAG).info(
-            "=======上面的地址是websocket协议地址，请勿用浏览器访问======="
-        )
-        self.logger.bind(tag=TAG).info(
-            "如想测试websocket请用谷歌浏览器打开test目录下的test_page.html"
-        )
-        self.logger.bind(tag=TAG).info(
-            "=============================================================\n"
-        )
-        async with websockets.serve(self._handle_connection, host, port):
+        async with websockets.serve(
+            self._handle_connection, host, port, process_request=self._http_response
+        ):
             await asyncio.Future()
 
     async def _handle_connection(self, websocket):
         """处理新连接，每次创建独立的ConnectionHandler"""
         # 创建ConnectionHandler时传入当前server实例
-        _tts = tts.create_instance(
-            (
-                self.config["selected_module"]["TTS"]
-                if not "type"
-                       in self.config["TTS"][self.config["selected_module"]["TTS"]]
-                else self.config["TTS"][self.config["selected_module"]["TTS"]][
-                    "type"
-                ]
-            ),
-            self.config["TTS"][self.config["selected_module"]["TTS"]],
-            self.config["delete_audio"],
-        )
         handler = ConnectionHandler(
             self.config,
             self._vad,
             self._asr,
             self._llm,
-            _tts,
+            self._tts,
             self._memory,
-            self.intent,
+            self._intent,
+            self,  # 传入server实例
         )
         self.active_connections.add(handler)
         try:
             await handler.handle_connection(websocket)
         finally:
             self.active_connections.discard(handler)
+
+    async def _http_response(self, websocket, request_headers):
+        # 检查是否为 WebSocket 升级请求
+        if request_headers.headers.get("connection", "").lower() == "upgrade":
+            # 如果是 WebSocket 请求，返回 None 允许握手继续
+            return None
+        else:
+            # 如果是普通 HTTP 请求，返回 "server is running"
+            return websocket.respond(200, "Server is running\n")
+
+    async def update_config(self) -> bool:
+        """更新服务器配置并重新初始化组件
+
+        Returns:
+            bool: 更新是否成功
+        """
+        try:
+            async with self.config_lock:
+                # 重新获取配置
+                new_config = get_config_from_api(self.config)
+                if new_config is None:
+                    self.logger.bind(tag=TAG).error("获取新配置失败")
+                    return False
+                self.logger.bind(tag=TAG).info(f"获取新配置成功")
+                # 检查 VAD 和 ASR 类型是否需要更新
+                update_vad = check_vad_update(self.config, new_config)
+                update_asr = check_asr_update(self.config, new_config)
+                self.logger.bind(tag=TAG).info(
+                    f"检查VAD和ASR类型是否需要更新: {update_vad} {update_asr}"
+                )
+                # 更新配置
+                self.config = new_config
+                # 重新初始化组件
+                modules = initialize_modules(
+                    self.logger,
+                    new_config,
+                    update_vad,
+                    update_asr,
+                    "LLM" in new_config["selected_module"],
+                    "TTS" in new_config["selected_module"],
+                    "Memory" in new_config["selected_module"],
+                    "Intent" in new_config["selected_module"],
+                )
+
+                # 更新组件实例
+                if "vad" in modules:
+                    self._vad = modules["vad"]
+                if "asr" in modules:
+                    self._asr = modules["asr"]
+                if "tts" in modules:
+                    self._tts = modules["tts"]
+                if "llm" in modules:
+                    self._llm = modules["llm"]
+                if "intent" in modules:
+                    self._intent = modules["intent"]
+                if "memory" in modules:
+                    self._memory = modules["memory"]
+                self.logger.bind(tag=TAG).info(f"更新配置任务执行完毕")
+                return True
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")
+            return False