update:合并非tts代码

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -0,0 +1,270 @@
+import http.client
+import json
+import asyncio
+from typing import Optional, Tuple, List
+import opuslib_next
+import wave
+import io
+import os
+import uuid
+import hmac
+import hashlib
+import base64
+import requests
+from urllib import parse
+import time
+from datetime import datetime
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {
+            "AccessKeyId": access_key_id,
+            "Action": "CreateToken",
+            "Format": "JSON",
+            "RegionId": "cn-shanghai",
+            "SignatureMethod": "HMAC-SHA1",
+            "SignatureNonce": str(uuid.uuid1()),
+            "SignatureVersion": "1.0",
+            "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "Version": "2019-02-28",
+        }
+        # 构造规范化的请求字符串
+        query_string = AccessToken._encode_dict(parameters)
+        # print('规范化的请求字符串: %s' % query_string)
+        # 构造待签名字符串
+        string_to_sign = (
+            "GET"
+            + "&"
+            + AccessToken._encode_text("/")
+            + "&"
+            + AccessToken._encode_text(query_string)
+        )
+        # print('待签名的字符串: %s' % string_to_sign)
+        # 计算签名
+        secreted_string = hmac.new(
+            bytes(access_key_secret + "&", encoding="utf-8"),
+            bytes(string_to_sign, encoding="utf-8"),
+            hashlib.sha1,
+        ).digest()
+        signature = base64.b64encode(secreted_string)
+        # print('签名: %s' % signature)
+        # 进行URL编码
+        signature = AccessToken._encode_text(signature)
+        # print('URL编码后的签名: %s' % signature)
+        # 调用服务
+        full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
+            signature,
+            query_string,
+        )
+        # print('url: %s' % full_url)
+        # 提交HTTP GET请求
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            key = "Token"
+            if key in root_obj:
+                token = root_obj[key]["Id"]
+                expire_time = root_obj[key]["ExpireTime"]
+                return token, expire_time
+        # print(response.text)
+        return None, None
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        """阿里云ASR初始化"""
+        # 新增空值判断逻辑
+        self.access_key_id = config.get("access_key_id")
+        self.access_key_secret = config.get("access_key_secret")
+
+        self.app_key = config.get("appkey")
+        self.host = "nls-gateway-cn-shanghai.aliyuncs.com"
+        self.base_url = f"https://{self.host}/stream/v1/asr"
+        self.sample_rate = 16000
+        self.format = "wav"
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+
+        if self.access_key_id and self.access_key_secret:
+            # 使用密钥对生成临时token
+            self._refresh_token()
+        else:
+            # 直接使用预生成的长期token
+            self.token = config.get("token")
+            self.expire_time = None
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _refresh_token(self):
+        """刷新Token并记录过期时间"""
+        if self.access_key_id and self.access_key_secret:
+            self.token, expire_time_str = AccessToken.create_token(
+                self.access_key_id, self.access_key_secret
+            )
+            if not expire_time_str:
+                raise ValueError("无法获取有效的Token过期时间")
+
+            try:
+                # 统一转换为字符串处理
+                expire_str = str(expire_time_str).strip()
+
+                if expire_str.isdigit():
+                    expire_time = datetime.fromtimestamp(int(expire_str))
+                else:
+                    expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
+                self.expire_time = expire_time.timestamp() - 60
+            except Exception as e:
+                raise ValueError(f"无效的过期时间格式: {expire_str}") from e
+
+        else:
+            self.expire_time = None
+
+        if not self.token:
+            raise ValueError("无法获取有效的访问Token")
+
+    def _is_token_expired(self):
+        """检查Token是否过期"""
+        if not self.expire_time:
+            return False  # 长期Token不过期
+        # 新增调试日志
+        # current_time = time.time()
+        # remaining = self.expire_time - current_time
+        # print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | "
+        #              f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
+        #              f"剩余 {remaining:.2f}秒")
+        return time.time() > self.expire_time
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(
+            self.output_file,
+            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        )
+
+    def _construct_request_url(self) -> str:
+        """构造请求URL，包含参数"""
+        request = f"{self.base_url}?appkey={self.app_key}"
+        request += f"&format={self.format}"
+        request += f"&sample_rate={self.sample_rate}"
+        request += "&enable_punctuation_prediction=true"
+        request += "&enable_inverse_text_normalization=true"
+        request += "&enable_voice_detection=false"
+        return request
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)  # 单声道
+            wf.setsampwidth(2)  # 16-bit
+            wf.setframerate(self.sample_rate)
+            wf.writeframes(b"".join(pcm_data))
+
+        logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}")
+        return file_path
+
+    async def _send_request(self, pcm_data: bytes) -> Optional[str]:
+        """发送请求到阿里云ASR服务"""
+        try:
+            # 设置HTTP头
+            headers = {
+                "X-NLS-Token": self.token,
+                "Content-type": "application/octet-stream",
+                "Content-Length": str(len(pcm_data)),
+            }
+
+            # 创建连接并发送请求
+            conn = http.client.HTTPSConnection(self.host)
+            request_url = self._construct_request_url()
+
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                None,
+                lambda: conn.request(
+                    method="POST", url=request_url, body=pcm_data, headers=headers
+                ),
+            )
+
+            # 获取响应
+            response = await loop.run_in_executor(None, conn.getresponse)
+            body = await loop.run_in_executor(None, response.read)
+            conn.close()
+
+            # 解析响应
+            try:
+                body_json = json.loads(body)
+                status = body_json.get("status")
+
+                if status == 20000000:
+                    result = body_json.get("result", "")
+                    logger.bind(tag=TAG).debug(f"ASR结果: {result}")
+                    return result
+                else:
+                    logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
+                    return None
+
+            except ValueError:
+                logger.bind(tag=TAG).error("响应不是JSON格式")
+                return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True)
+            return None
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if self._is_token_expired():
+            logger.warning("Token已过期，正在自动刷新...")
+            self._refresh_token()
+
+        file_path = None
+        try:
+            # 解码Opus为PCM
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+
+            # 发送请求并获取文本
+            text = await self._send_request(combined_pcm_data)
+
+            if text:
+                return text, file_path
+
+            return "", file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", file_path

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -0,0 +1,106 @@
+import base64
+import hashlib
+import hmac
+import json
+import time
+from datetime import datetime, timezone
+import os
+import uuid
+from typing import Optional, Tuple, List
+import wave
+import opuslib_next
+
+from aip import AipSpeech
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.secret_key = config.get("secret_key")
+
+        dev_pid = config.get("dev_pid", "1537")
+        self.dev_pid = int(dev_pid) if dev_pid else 1537
+
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        self.client = AipSpeech(str(self.app_id), self.api_key, self.secret_key)
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if not opus_data:
+            logger.bind(tag=TAG).warning("音频数据为空！")
+            return None, None
+
+        file_path = None
+        try:
+            # 检查配置是否已设置
+            if not self.app_id or not self.api_key or not self.secret_key:
+                logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
+                return None, file_path
+
+            # 将Opus音频数据解码为PCM
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                self.save_audio_to_file(pcm_data, session_id)
+
+            start_time = time.time()
+            # 识别本地文件
+            result = self.client.asr(
+                combined_pcm_data,
+                "pcm",
+                16000,
+                {
+                    "dev_pid": str(self.dev_pid),
+                },
+            )
+
+            if result and result["err_no"] == 0:
+                logger.bind(tag=TAG).debug(
+                    f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+                result = result["result"][0]
+                return result, file_path
+            else:
+                raise Exception(
+                    f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
+                )
+                return None, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
+            return None, file_path

main/xiaozhi-server/core/providers/asr/base.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from typing import Optional, Tuple, List
-
+import opuslib_next
 from config.logger import setup_logging
 
 TAG = __name__
@@ -8,12 +8,37 @@ logger = setup_logging()
 
 
 class ASRProviderBase(ABC):
+    def __init__(self):
+        self.audio_format = "opus"
+
     @abstractmethod
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """解码Opus数据并保存为WAV文件"""
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
         pass
 
     @abstractmethod
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         pass
+
+    def set_audio_format(self, format: str) -> None:
+        """设置音频格式"""
+        self.audio_format = format
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes]) -> bytes:
+        """将Opus音频数据解码为PCM数据"""
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -45,14 +45,14 @@ def parse_response(res):
     payload 类似与http 请求体
     """
     protocol_version = res[0] >> 4
-    header_size = res[0] & 0x0f
+    header_size = res[0] & 0x0F
     message_type = res[1] >> 4
-    message_type_specific_flags = res[1] & 0x0f
+    message_type_specific_flags = res[1] & 0x0F
     serialization_method = res[2] >> 4
-    message_compression = res[2] & 0x0f
+    message_compression = res[2] & 0x0F
     reserved = res[3]
-    header_extensions = res[4:header_size * 4]
-    payload = res[header_size * 4:]
+    header_extensions = res[4 : header_size * 4]
+    payload = res[header_size * 4 :]
     result = {}
     payload_msg = None
     payload_size = 0
@@ -61,13 +61,13 @@ def parse_response(res):
         payload_msg = payload[4:]
     elif message_type == SERVER_ACK:
         seq = int.from_bytes(payload[:4], "big", signed=True)
-        result['seq'] = seq
+        result["seq"] = seq
         if len(payload) >= 8:
             payload_size = int.from_bytes(payload[4:8], "big", signed=False)
             payload_msg = payload[8:]
     elif message_type == SERVER_ERROR_RESPONSE:
         code = int.from_bytes(payload[:4], "big", signed=False)
-        result['code'] = code
+        result["code"] = code
         payload_size = int.from_bytes(payload[4:8], "big", signed=False)
         payload_msg = payload[8:]
     if payload_msg is None:
@@ -78,17 +78,21 @@ def parse_response(res):
         payload_msg = json.loads(str(payload_msg, "utf-8"))
     elif serialization_method != NO_SERIALIZATION:
         payload_msg = str(payload_msg, "utf-8")
-    result['payload_msg'] = payload_msg
-    result['payload_size'] = payload_size
+    result["payload_msg"] = payload_msg
+    result["payload_size"] = payload_size
     return result
 
 
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
         self.appid = config.get("appid")
         self.cluster = config.get("cluster")
         self.access_token = config.get("access_token")
+        self.boosting_table_name = config.get("boosting_table_name", "")
+        self.correct_table_name = config.get("correct_table_name", "")
         self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
 
         self.host = "openspeech.bytedance.com"
         self.ws_url = f"wss://{self.host}/api/v2/asr"
@@ -98,21 +102,12 @@ class ASRProvider(ASRProviderBase):
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -122,7 +117,9 @@ class ASRProvider(ASRProviderBase):
         return file_path
 
     @staticmethod
-    def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
+    def _generate_header(
+        message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE
+    ) -> bytearray:
         """Generate protocol header."""
         header = bytearray()
         header_size = 1
@@ -146,10 +143,12 @@ class ASRProvider(ASRProviderBase):
             "request": {
                 "reqid": reqid,
                 "show_utterances": False,
-                "sequence": 1
+                "sequence": 1,
+                "boosting_table_name": self.boosting_table_name,
+                "correct_table_name": self.correct_table_name,
             },
             "audio": {
-                "format": "wav",
+                "format": "raw",
                 "rate": 16000,
                 "language": "zh-CN",
                 "bits": 16,
@@ -158,18 +157,23 @@ class ASRProvider(ASRProviderBase):
             },
         }
 
-    async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
+    async def _send_request(
+        self, audio_data: List[bytes], segment_size: int
+    ) -> Optional[str]:
         """Send request to Volcano ASR service."""
         try:
-            auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
-            async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
+            auth_header = {"Authorization": "Bearer; {}".format(self.access_token)}
+            async with websockets.connect(
+                self.ws_url, additional_headers=auth_header
+            ) as websocket:
                 # Prepare request data
                 request_params = self._construct_request(str(uuid.uuid4()))
-                print(request_params)
                 payload_bytes = str.encode(json.dumps(request_params))
                 payload_bytes = gzip.compress(payload_bytes)
                 full_client_request = self._generate_header()
-                full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                full_client_request.extend(
+                    (len(payload_bytes)).to_bytes(4, "big")
+                )  # payload size(4 bytes)
                 full_client_request.extend(payload_bytes)  # payload
 
                 # Send header and metadata
@@ -177,22 +181,29 @@ class ASRProvider(ASRProviderBase):
                 await websocket.send(full_client_request)
                 res = await websocket.recv()
                 result = parse_response(res)
-                if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
+                if (
+                    "payload_msg" in result
+                    and result["payload_msg"]["code"] != self.success_code
+                ):
                     logger.bind(tag=TAG).error(f"ASR error: {result}")
                     return None
 
-                for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
+                for seq, (chunk, last) in enumerate(
+                    self.slice_data(audio_data, segment_size), 1
+                ):
                     if last:
                         audio_only_request = self._generate_header(
                             message_type=CLIENT_AUDIO_ONLY_REQUEST,
-                            message_type_specific_flags=NEG_SEQUENCE
+                            message_type_specific_flags=NEG_SEQUENCE,
                         )
                     else:
                         audio_only_request = self._generate_header(
                             message_type=CLIENT_AUDIO_ONLY_REQUEST
                         )
                     payload_bytes = gzip.compress(chunk)
-                    audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                    audio_only_request.extend(
+                        (len(payload_bytes)).to_bytes(4, "big")
+                    )  # payload size(4 bytes)
                     audio_only_request.extend(payload_bytes)  # payload
                     # Send audio data
                     await websocket.send(audio_only_request)
@@ -201,9 +212,12 @@ class ASRProvider(ASRProviderBase):
                 response = await websocket.recv()
                 result = parse_response(response)
 
-                if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
-                    if len(result['payload_msg']['result']) > 0:
-                        return result['payload_msg']['result'][0]["text"]
+                if (
+                    "payload_msg" in result
+                    and result["payload_msg"]["code"] == self.success_code
+                ):
+                    if len(result["payload_msg"]["result"]) > 0:
+                        return result["payload_msg"]["result"][0]["text"]
                     return None
                 else:
                     logger.bind(tag=TAG).error(f"ASR error: {result}")
@@ -213,29 +227,6 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
             return None
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
-    @staticmethod
-    def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
-        with io.BytesIO(data) as _f:
-            wave_fp = wave.open(_f, 'rb')
-            nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
-            wave_bytes = wave_fp.readframes(nframes)
-        return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
-
     @staticmethod
     def slice_data(data: bytes, chunk_size: int) -> (list, bool):
         """
@@ -247,40 +238,46 @@ class ASRProvider(ASRProviderBase):
         data_len = len(data)
         offset = 0
         while offset + chunk_size < data_len:
-            yield data[offset: offset + chunk_size], False
+            yield data[offset : offset + chunk_size], False
             offset += chunk_size
         else:
-            yield data[offset: data_len], True
+            yield data[offset:data_len], True
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
+
+        file_path = None
         try:
             # 合并所有opus数据包
-            pcm_data = self.decode_opus(opus_data, session_id)
-            combined_pcm_data = b''.join(pcm_data)
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
 
-            wav_buffer = io.BytesIO()
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
-            with wave.open(wav_buffer, "wb") as wav_file:
-                wav_file.setnchannels(1)  # 设置声道数
-                wav_file.setsampwidth(2)  # 设置采样宽度
-                wav_file.setframerate(16000)  # 设置采样率
-                wav_file.writeframes(combined_pcm_data)  # 写入 PCM 数据
-
-            # 获取封装后的 WAV 数据
-            wav_data = wav_buffer.getvalue()
-            nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
-            size_per_sec = nchannels * sampwidth * framerate
+            # 直接使用PCM数据
+            # 计算分段大小 (单声道, 16bit, 16kHz采样率)
+            size_per_sec = 1 * 2 * 16000  # nchannels * sampwidth * framerate
             segment_size = int(size_per_sec * self.seg_duration / 1000)
 
             # 语音识别
             start_time = time.time()
-            text = await self._send_request(wav_data, segment_size)
+            text = await self._send_request(combined_pcm_data, segment_size)
             if text:
-                logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
-                return text, None
-            return "", None
+                logger.bind(tag=TAG).debug(
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+                )
+                return text, file_path
+            return "", file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -6,9 +6,7 @@ import io
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
 import uuid
-import opuslib_next
 from core.providers.asr.base import ASRProviderBase
-
 from funasr import AutoModel
 from funasr.utils.postprocess_utils import rich_transcription_postprocess
 
@@ -35,6 +33,7 @@ class CaptureOutput:
 
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
         self.model_dir = config.get("model_dir")
         self.output_dir = config.get("output_dir")  # 修正配置键名
         self.delete_audio_file = delete_audio_file
@@ -46,25 +45,16 @@ class ASRProvider(ASRProviderBase):
                 model=self.model_dir,
                 vad_kwargs={"max_single_segment_time": 30000},
                 disable_update=True,
-                hub="hf"
+                hub="hf",
                 # device="cuda:0",  # 启用GPU加速
             )
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -73,38 +63,51 @@ class ASRProvider(ASRProviderBase):
 
         return file_path
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
         try:
-            # 保存音频文件
-            start_time = time.time()
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+            # 合并所有opus数据包
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
 
             # 语音识别
             start_time = time.time()
             result = self.model.generate(
-                input=file_path,
+                input=combined_pcm_data,
                 cache={},
                 language="auto",
                 use_itn=True,
                 batch_size_s=60,
             )
             text = rich_transcription_postprocess(result[0]["text"])
-            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+            logger.bind(tag=TAG).debug(
+                f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+            )
 
             return text, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
+            return "", file_path
 
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
+        # finally:
+        #     # 文件清理逻辑
+        #     if self.delete_audio_file and file_path and os.path.exists(file_path):
+        #         try:
+        #             os.remove(file_path)
+        #             logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+        #         except Exception as e:
+        #             logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -0,0 +1,185 @@
+from typing import Optional, Tuple, List
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+import os
+import ssl
+import json
+import uuid
+import wave
+import websockets
+from config.logger import setup_logging
+import asyncio
+import re
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        """
+        Initialize the ASRProvider with server configuration.
+        :param config: Dictionary containing 'host', 'port', and 'is_ssl'.
+        :param delete_audio_file: Boolean to indicate whether to delete audio files after processing.
+        """
+        super().__init__()
+        self.host = config.get("host", "localhost")
+        self.port = config.get("port", 10095)
+        self.api_key = config.get("api_key", "none")
+        self.is_ssl = str(config.get("is_ssl", True)).lower() in (
+            "true",
+            "1",
+            "yes",
+        )
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+        self.uri = (
+            f"wss://{self.host}:{self.port}"
+            if self.is_ssl
+            else f"ws://{self.host}:{self.port}"
+        )
+        self.ssl_context = ssl.SSLContext() if self.is_ssl else None
+        if self.ssl_context:
+            self.ssl_context.check_hostname = False
+            self.ssl_context.verify_mode = ssl.CERT_NONE
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def _receive_responses(self, ws) -> None:
+        """
+        Asynchronous generator to receive messages from the WebSocket.
+        Yields each message as it is received.
+        """
+        text = ""
+        while True:
+            try:
+                response = await asyncio.wait_for(ws.recv(), timeout=5)
+                response_data = json.loads(response)
+                logger.bind(tag=TAG).debug(f"Received response: {response_data}")
+                if response_data.get("is_final", True):
+                    text += response_data.get("text", "")
+                    break
+                else:
+                    text += response_data.get("text", "")
+            except asyncio.TimeoutError:
+                logger.bind(tag=TAG).error(
+                    "Timeout while waiting for response from WebSocket."
+                )
+                break
+            except websockets.exceptions.ConnectionClosed as e:
+                logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
+                break
+        return text
+
+    async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple:
+        """
+        Internal method to handle WebSocket communication.
+        Reuses the persistent WebSocket connection if available.
+        :param pcm_data: PCM audio data to send.
+        :param session_id: Unique session identifier.
+        :return: Tuple containing recognized text and optional timestamp.
+        """
+
+        # Send initial configuration message
+        config_message = json.dumps(
+            {
+                "mode": "offline",
+                "chunk_size": [5, 10, 5],
+                "chunk_interval": 10,
+                "wav_name": session_id,
+                "is_speaking": True,
+                "itn": False,
+            }
+        )
+        await ws.send(config_message)
+        logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
+
+        # Send PCM data
+        await ws.send(pcm_data)
+        logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
+
+        # Indicate end of speech
+        end_message = json.dumps({"is_speaking": False})
+        await ws.send(end_message)
+        logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Convert speech data to text using FunASR.
+        :param opus_data: List of Opus-encoded audio data chunks.
+        :param session_id: Unique session identifier.
+        :return: Tuple containing recognized text and optional timestamp.
+        """
+        file_path = None
+        if self.audio_format == "pcm":
+            pcm_data = opus_data
+        else:
+            pcm_data = self.decode_opus(opus_data)
+        combined_pcm_data = b"".join(pcm_data)
+
+        # 判断是否保存为WAV文件
+        if self.delete_audio_file:
+            pass
+        else:
+            file_path = self.save_audio_to_file(pcm_data, session_id)
+        auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
+        async with websockets.connect(
+            self.uri,
+            additional_headers=auth_header,
+            subprotocols=["binary"],
+            ping_interval=None,
+            ssl=self.ssl_context,
+        ) as ws:
+            try:
+                # Use asyncio to handle WebSocket communication
+                send_task = asyncio.create_task(
+                    self._send_data(ws, combined_pcm_data, session_id)
+                )
+                receive_task = asyncio.create_task(self._receive_responses(ws))
+
+                # Gather tasks with error handling
+                done, pending = await asyncio.wait(
+                    [send_task, receive_task], return_when=asyncio.FIRST_EXCEPTION
+                )
+
+                # Cancel any pending tasks
+                for task in pending:
+                    task.cancel()
+
+                # Check for exceptions in completed tasks
+                for task in done:
+                    if task.exception():
+                        raise task.exception()
+
+                # Get the result from the receive task
+                result = receive_task.result()
+                match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
+                if match:
+                    result = match.group(4).strip()
+                return (
+                    result,
+                    file_path,
+                )  # Return the recognized text and timestamp (if any)
+
+            except websockets.exceptions.ConnectionClosed as e:
+                logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
+                return "", file_path
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"Error during speech-to-text conversion: {e}", exc_info=True
+                )
+                return "", file_path

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -37,17 +37,18 @@ class CaptureOutput:
 
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
         self.model_dir = config.get("model_dir")
         self.output_dir = config.get("output_dir")
         self.delete_audio_file = delete_audio_file
 
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
-        
+
         # 初始化模型文件路径
         model_files = {
             "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
-            "tokens.txt": os.path.join(self.model_dir, "tokens.txt")
+            "tokens.txt": os.path.join(self.model_dir, "tokens.txt"),
         }
 
         # 下载并检查模型文件
@@ -58,15 +59,15 @@ class ASRProvider(ASRProviderBase):
                     model_file_download(
                         model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
                         file_path=file_name,
-                        local_dir=self.model_dir
+                        local_dir=self.model_dir,
                     )
-                    
+
                     if not os.path.isfile(file_path):
                         raise FileNotFoundError(f"模型文件下载失败: {file_path}")
-                        
+
             self.model_path = model_files["model.int8.onnx"]
             self.tokens_path = model_files["tokens.txt"]
-            
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
             raise
@@ -83,21 +84,12 @@ class ASRProvider(ASRProviderBase):
                 use_itn=True,
             )
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
@@ -130,14 +122,22 @@ class ASRProvider(ASRProviderBase):
             samples_float32 = samples_float32 / 32768
             return samples_float32, f.getframerate()
 
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
         try:
             # 保存音频文件
             start_time = time.time()
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            file_path = self.save_audio_to_file(pcm_data, session_id)
+            logger.bind(tag=TAG).debug(
+                f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
+            )
 
             # 语音识别
             start_time = time.time()
@@ -146,14 +146,15 @@ class ASRProvider(ASRProviderBase):
             s.accept_waveform(sample_rate, samples)
             self.model.decode_stream(s)
             text = s.result.text
-            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+            logger.bind(tag=TAG).debug(
+                f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+            )
 
             return text, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
-
+            return "", file_path
         finally:
             # 文件清理逻辑
             if self.delete_audio_file and file_path and os.path.exists(file_path):

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -17,77 +17,66 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+
 class ASRProvider(ASRProviderBase):
     API_URL = "https://asr.tencentcloudapi.com"
     API_VERSION = "2019-06-14"
     FORMAT = "pcm"  # 支持的音频格式：pcm, wav, mp3
 
     def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
         self.secret_id = config.get("secret_id")
         self.secret_key = config.get("secret_key")
         self.output_dir = config.get("output_dir")
-        
+        self.delete_audio_file = delete_audio_file
+
         # 确保输出目录存在
         os.makedirs(self.output_dir, exist_ok=True)
 
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-
-        file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
         file_path = os.path.join(self.output_dir, file_name)
-        
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-        
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
+
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
             wf.setsampwidth(2)  # 2 bytes = 16-bit
             wf.setframerate(16000)
             wf.writeframes(b"".join(pcm_data))
-        
+
         return file_path
 
-    @staticmethod
-    def decode_opus(opus_data: List[bytes]) -> bytes:
-        """将Opus音频数据解码为PCM数据"""
-        import opuslib_next
-        
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-        
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-        
-        return b"".join(pcm_data)
-
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str
+    ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
-            logger.bind(tag=TAG).warn("音频数据为空！")
+            logger.bind(tag=TAG).warning("音频数据为空！")
             return None, None
 
+        file_path = None
         try:
             # 检查配置是否已设置
             if not self.secret_id or not self.secret_key:
                 logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
-                return None, None
+                return None, file_path
 
             # 将Opus音频数据解码为PCM
-            pcm_data = self.decode_opus(opus_data)
-            
+            if self.audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 判断是否保存为WAV文件
+            if self.delete_audio_file:
+                pass
+            else:
+                self.save_audio_to_file(pcm_data, session_id)
+
             # 将音频数据转换为Base64编码
-            base64_audio = base64.b64encode(pcm_data).decode('utf-8')
+            base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8")
 
             # 构建请求体
             request_body = self._build_request_body(base64_audio)
@@ -98,15 +87,17 @@ class ASRProvider(ASRProviderBase):
             # 发送请求
             start_time = time.time()
             result = self._send_request(request_body, timestamp, authorization)
-            
+
             if result:
-                logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
-            
-            return result, None
+                logger.bind(tag=TAG).debug(
+                    f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+
+            return result, file_path
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
-            return None, None
+            return None, file_path
 
     def _build_request_body(self, base64_audio: str) -> str:
         """构建请求体"""
@@ -117,7 +108,7 @@ class ASRProvider(ASRProviderBase):
             "SourceType": 1,  # 音频数据来源为语音文件
             "VoiceFormat": self.FORMAT,  # 音频格式
             "Data": base64_audio,  # Base64编码的音频数据
-            "DataLen": len(base64_audio)  # 数据长度
+            "DataLen": len(base64_audio),  # 数据长度
         }
         return json.dumps(request_map)
 
@@ -150,9 +141,11 @@ class ASRProvider(ASRProviderBase):
             action = "SentenceRecognition"  # 接口名称
 
             # 构建规范头部信息，注意顺序和格式
-            canonical_headers = f"content-type:{content_type.lower()}\n" + \
-                               f"host:{host.lower()}\n" + \
-                               f"x-tc-action:{action.lower()}\n"
+            canonical_headers = (
+                f"content-type:{content_type.lower()}\n"
+                + f"host:{host.lower()}\n"
+                + f"x-tc-action:{action.lower()}\n"
+            )
 
             signed_headers = "content-type;host;x-tc-action"
 
@@ -160,21 +153,25 @@ class ASRProvider(ASRProviderBase):
             payload_hash = self._sha256_hex(request_body)
 
             # 构建规范请求字符串
-            canonical_request = f"{http_request_method}\n" + \
-                               f"{canonical_uri}\n" + \
-                               f"{canonical_query_string}\n" + \
-                               f"{canonical_headers}\n" + \
-                               f"{signed_headers}\n" + \
-                               f"{payload_hash}"
+            canonical_request = (
+                f"{http_request_method}\n"
+                + f"{canonical_uri}\n"
+                + f"{canonical_query_string}\n"
+                + f"{canonical_headers}\n"
+                + f"{signed_headers}\n"
+                + f"{payload_hash}"
+            )
 
             # 计算规范请求的哈希值
             hashed_canonical_request = self._sha256_hex(canonical_request)
 
             # 构建待签名字符串
-            string_to_sign = f"{algorithm}\n" + \
-                            f"{timestamp}\n" + \
-                            f"{credential_scope}\n" + \
-                            f"{hashed_canonical_request}"
+            string_to_sign = (
+                f"{algorithm}\n"
+                + f"{timestamp}\n"
+                + f"{credential_scope}\n"
+                + f"{hashed_canonical_request}"
+            )
 
             # 计算签名密钥
             secret_date = self._hmac_sha256(f"TC3{self.secret_key}", date)
@@ -182,13 +179,17 @@ class ASRProvider(ASRProviderBase):
             secret_signing = self._hmac_sha256(secret_service, "tc3_request")
 
             # 计算签名
-            signature = self._bytes_to_hex(self._hmac_sha256(secret_signing, string_to_sign))
+            signature = self._bytes_to_hex(
+                self._hmac_sha256(secret_signing, string_to_sign)
+            )
 
             # 构建授权头
-            authorization = f"{algorithm} " + \
-                           f"Credential={self.secret_id}/{credential_scope}, " + \
-                           f"SignedHeaders={signed_headers}, " + \
-                           f"Signature={signature}"
+            authorization = (
+                f"{algorithm} "
+                + f"Credential={self.secret_id}/{credential_scope}, "
+                + f"SignedHeaders={signed_headers}, "
+                + f"Signature={signature}"
+            )
 
             return timestamp, authorization
 
@@ -196,7 +197,9 @@ class ASRProvider(ASRProviderBase):
             logger.bind(tag=TAG).error(f"生成认证头失败: {e}", exc_info=True)
             raise RuntimeError(f"生成认证头失败: {e}")
 
-    def _send_request(self, request_body: str, timestamp: str, authorization: str) -> Optional[str]:
+    def _send_request(
+        self, request_body: str, timestamp: str, authorization: str
+    ) -> Optional[str]:
         """发送请求到腾讯云API"""
         headers = {
             "Content-Type": "application/json; charset=utf-8",
@@ -205,47 +208,47 @@ class ASRProvider(ASRProviderBase):
             "X-TC-Action": "SentenceRecognition",
             "X-TC-Version": self.API_VERSION,
             "X-TC-Timestamp": timestamp,
-            "X-TC-Region": "ap-shanghai"
+            "X-TC-Region": "ap-shanghai",
         }
 
         try:
             response = requests.post(self.API_URL, headers=headers, data=request_body)
-            
+
             if not response.ok:
                 raise IOError(f"请求失败: {response.status_code} {response.reason}")
-            
+
             response_json = response.json()
-            
+
             # 检查是否有错误
             if "Response" in response_json and "Error" in response_json["Response"]:
                 error = response_json["Response"]["Error"]
                 error_code = error["Code"]
                 error_message = error["Message"]
                 raise IOError(f"API返回错误: {error_code}: {error_message}")
-            
+
             # 提取识别结果
             if "Response" in response_json and "Result" in response_json["Response"]:
                 return response_json["Response"]["Result"]
             else:
-                logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
+                logger.bind(tag=TAG).warning(f"响应中没有识别结果: {response_json}")
                 return ""
-                
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
             return None
 
     def _sha256_hex(self, data: str) -> str:
         """计算字符串的SHA256哈希值"""
-        digest = hashlib.sha256(data.encode('utf-8')).digest()
+        digest = hashlib.sha256(data.encode("utf-8")).digest()
         return self._bytes_to_hex(digest)
 
     def _hmac_sha256(self, key, data: str) -> bytes:
         """计算HMAC-SHA256"""
         if isinstance(key, str):
-            key = key.encode('utf-8')
-        
-        return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest()
+            key = key.encode("utf-8")
+
+        return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest()
 
     def _bytes_to_hex(self, bytes_data: bytes) -> str:
         """字节数组转十六进制字符串"""
-        return ''.join(f"{b:02x}" for b in bytes_data)
+        return "".join(f"{b:02x}" for b in bytes_data)

main/xiaozhi-server/core/providers/intent/base.py

@@ -9,18 +9,6 @@ logger = setup_logging()
 class IntentProviderBase(ABC):
     def __init__(self, config):
         self.config = config
-        self.intent_options = [
-            {
-                "name": "handle_exit_intent",
-                "desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
-            },
-            {
-                "name": "play_music",
-                "desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
-            },
-            {"name": "get_time", "desc": "获取今天日期或者当前时间信息"},
-            {"name": "continue_chat", "desc": "继续聊天"},
-        ]
 
     def set_llm(self, llm):
         self.llm = llm

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -15,57 +15,72 @@ class IntentProvider(IntentProviderBase):
     def __init__(self, config):
         super().__init__(config)
         self.llm = None
-        self.promot = self.get_intent_system_prompt()
+        self.promot = ""
         # 添加缓存管理
         self.intent_cache = {}  # 缓存意图识别结果
         self.cache_expiry = 600  # 缓存有效期10分钟
         self.cache_max_size = 100  # 最多缓存100个意图
+        self.history_count = 4  # 默认使用最近4条对话记录
 
-    def get_intent_system_prompt(self) -> str:
+    def get_intent_system_prompt(self, functions_list: str) -> str:
         """
-        根据配置的意图选项动态生成系统提示词
+        根据配置的意图选项和可用函数动态生成系统提示词
+        Args:
+            functions: 可用的函数列表，JSON格式字符串
         Returns:
             格式化后的系统提示词
         """
 
+        # 构建函数说明部分
+        functions_desc = "可用的函数列表：\n"
+        for func in functions_list:
+            func_info = func.get("function", {})
+            name = func_info.get("name", "")
+            desc = func_info.get("description", "")
+            params = func_info.get("parameters", {})
+
+            functions_desc += f"\n函数名: {name}\n"
+            functions_desc += f"描述: {desc}\n"
+
+            if params:
+                functions_desc += "参数:\n"
+                for param_name, param_info in params.get("properties", {}).items():
+                    param_desc = param_info.get("description", "")
+                    param_type = param_info.get("type", "")
+                    functions_desc += f"- {param_name} ({param_type}): {param_desc}\n"
+
+            functions_desc += "---\n"
+
         prompt = (
-            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
-            "<start>"
-            f"{str(self.intent_options)}"
-            "<end>\n"
-            "处理步骤:"
-            "1. 思考意图类型，生成function_call格式"
-            "\n\n"
-            "返回格式示例：\n"
-            '1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n'
-            '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
-            '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n'
-            '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n'
-            "\n"
-            "注意:\n"
-            '- 播放音乐：无歌名时，song_name设为"random"\n'
-            "- 如果没有明显的意图，应按照继续聊天意图处理\n"
-            "- 只返回纯JSON，不要任何其他内容\n"
-            "\n"
-            "示例分析:\n"
+            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            f"{functions_desc}\n"
+            "处理步骤:\n"
+            "1. 分析用户输入，确定用户意图\n"
+            "2. 从可用函数列表中选择最匹配的函数\n"
+            "3. 如果找到匹配的函数，生成对应的function_call 格式\n"
+            '4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
+            "返回格式要求：\n"
+            "1. 必须返回纯JSON格式\n"
+            "2. 必须包含function_call字段\n"
+            "3. function_call必须包含name字段\n"
+            "4. 如果函数需要参数，必须包含arguments字段\n\n"
+            "示例：\n"
             "```\n"
-            "用户: 你也太搞笑了\n"
-            '返回: {"function_call": {"name": "continue_chat"}}\n'
-            "```\n"
-            "```\n"
-            "用户: 现在是几号了?现在几点了？\n"
+            "用户: 现在几点了？\n"
             '返回: {"function_call": {"name": "get_time"}}\n'
             "```\n"
             "```\n"
-            "用户: 我们明天再聊吧\n"
-            '返回: {"function_call": {"name": "handle_exit_intent"}}\n'
+            "用户: 我想结束对话\n"
+            '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
             "```\n"
             "```\n"
-            "用户: 播放中秋月\n"
-            '返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n'
-            "```\n"
-            "```\n"
-            "可用的音乐名称:\n"
+            "用户: 你好啊\n"
+            '返回: {"function_call": {"name": "continue_chat"}}\n'
+            "```\n\n"
+            "注意：\n"
+            "1. 只返回JSON格式，不要包含任何其他文字\n"
+            '2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
+            "3. 确保返回的JSON格式正确，包含所有必要的字段\n"
         )
         return prompt
 
@@ -90,6 +105,14 @@ class IntentProvider(IntentProviderBase):
             for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]:
                 del self.intent_cache[key]
 
+    def replyResult(self, text: str, original_text: str):
+        llm_result = self.llm.response_no_stream(
+            system_prompt=text,
+            user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+            + original_text,
+        )
+        return llm_result
+
     async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
@@ -118,22 +141,35 @@ class IntentProvider(IntentProviderBase):
         # 清理缓存
         self.clean_cache()
 
-        # 构建用户最后一句话的提示
-        msgStr = ""
+        if self.promot == "":
+            if hasattr(conn, "func_handler"):
+                functions = conn.func_handler.get_functions()
+                self.promot = self.get_intent_system_prompt(functions)
 
-        # 只使用最后两句即可
-        if len(dialogue_history) >= 2:
-            # 保证最少有两句话的时候处理
-            msgStr += f"{dialogue_history[-2].role}: {dialogue_history[-2].content}\n"
-        msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n"
-
-        msgStr += f"User: {text}\n"
-        user_prompt = f"当前的对话如下：\n{msgStr}"
         music_config = initialize_music_handler(conn)
         music_file_names = music_config["music_file_names"]
-        prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
+        prompt_music = f"{self.promot}\n<musicNames>{music_file_names}\n</musicNames>"
+
+        devices = conn.config["plugins"]["home_assistant"].get("devices", [])
+        if len(devices) > 0:
+            hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
+            for device in devices:
+                hass_prompt += device + "\n"
+            prompt_music += hass_prompt
+
         logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
 
+        # 构建用户对话历史的提示
+        msgStr = ""
+
+        # 获取最近的对话历史
+        start_idx = max(0, len(dialogue_history) - self.history_count)
+        for i in range(start_idx, len(dialogue_history)):
+            msgStr += f"{dialogue_history[i].role}: {dialogue_history[i].content}\n"
+
+        msgStr += f"User: {text}\n"
+        user_prompt = f"current dialogue:\n{msgStr}"
+
         # 记录预处理完成时间
         preprocess_time = time.time() - total_start_time
         logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")
@@ -179,9 +215,18 @@ class IntentProvider(IntentProviderBase):
 
                 # 记录识别到的function call
                 logger.bind(tag=TAG).info(
-                    f"识别到function call: {function_name}, 参数: {function_args}"
+                    f"llm 识别到意图: {function_name}, 参数: {function_args}"
                 )
 
+                # 如果是继续聊天，清理工具调用相关的历史消息
+                if function_name == "continue_chat":
+                    # 保留非工具相关的消息
+                    clean_history = [
+                        msg for msg in conn.dialogue.dialogue
+                        if msg.role not in ["tool", "function"]
+                    ]
+                    conn.dialogue.dialogue = clean_history
+
                 # 添加到缓存
                 self.intent_cache[cache_key] = {
                     "intent": intent,

main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py

@@ -2,10 +2,12 @@ from config.logger import setup_logging
 from http import HTTPStatus
 from dashscope import Application
 from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
 
+
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
@@ -13,27 +15,32 @@ class LLMProvider(LLMProviderBase):
         self.base_url = config.get("base_url")
         self.is_No_prompt = config.get("is_no_prompt")
         self.memory_id = config.get("ali_memory_id")
+        check_model_key("AliBLLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
             # 处理dialogue
             if self.is_No_prompt:
                 dialogue.pop(0)
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】处理后的dialogue: {dialogue}"
+                )
 
             # 构造调用参数
             call_params = {
                 "api_key": self.api_key,
                 "app_id": self.app_id,
                 "session_id": session_id,
-                "messages": dialogue
+                "messages": dialogue,
             }
             if self.memory_id != False:
                 # 百练memory需要prompt参数
                 prompt = dialogue[-1].get("content")
                 call_params["memory_id"] = self.memory_id
                 call_params["prompt"] = prompt
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】处理后的prompt: {prompt}"
+                )
 
             responses = Application.call(**call_params)
             if responses.status_code != HTTPStatus.OK:
@@ -44,9 +51,16 @@ class LLMProvider(LLMProviderBase):
                 )
                 yield "【阿里百练API服务响应异常】"
             else:
-                logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
+                logger.bind(tag=TAG).debug(
+                    f"【阿里百练API服务】构造参数: {call_params}"
+                )
                 yield responses.output.text
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
             yield "【LLM服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"阿里百练暂未实现完整的工具调用（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/base.py

@@ -35,4 +35,5 @@ class LLMProviderBase(ABC):
         """
         # For providers that don't support functions, just return regular response
         for token in self.response(session_id, dialogue):
-            yield {"type": "content", "content": token}
+            yield token, None
+

main/xiaozhi-server/core/providers/llm/coze/coze.py

@@ -1,12 +1,17 @@
 from config.logger import setup_logging
-import requests
 import json
-import re
 from core.providers.llm.base import LLMProviderBase
-import os
+
 # official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
 from cozepy import COZE_CN_BASE_URL
-from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa
+from cozepy import (
+    Coze,
+    TokenAuth,
+    Message,
+    ChatEventType,
+)  # noqa
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -15,25 +20,23 @@ logger = setup_logging()
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.personal_access_token = config.get("personal_access_token")
-        self.bot_id = config.get("bot_id")
-        self.user_id = config.get("user_id")
+        self.bot_id = str(config.get("bot_id"))
+        self.user_id = str(config.get("user_id"))
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        check_model_key("CozeLLM", self.personal_access_token)
 
     def response(self, session_id, dialogue):
         coze_api_token = self.personal_access_token
         coze_api_base = COZE_CN_BASE_URL
 
         last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-        
+
         coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
         conversation_id = self.session_conversation_map.get(session_id)
 
         # 如果没有找到conversation_id，则创建新的对话
         if not conversation_id:
-            conversation = coze.conversations.create(
-                messages=[
-                ]
-            )
+            conversation = coze.conversations.create(messages=[])
             conversation_id = conversation.id
             self.session_conversation_map[session_id] = conversation_id  # 更新映射
 
@@ -47,4 +50,24 @@ class LLMProvider(LLMProviderBase):
         ):
             if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
                 print(event.message.content, end="", flush=True)
-                yield event.message.content
+                yield event.message.content
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -2,6 +2,8 @@ import json
 from config.logger import setup_logging
 import requests
 from core.providers.llm.base import LLMProviderBase
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -13,6 +15,7 @@ class LLMProvider(LLMProviderBase):
         self.mode = config.get("mode", "chat-messages")
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        check_model_key("DifyLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
@@ -58,7 +61,10 @@ class LLMProvider(LLMProviderBase):
                                 self.session_conversation_map[session_id] = (
                                     conversation_id  # 更新映射
                                 )
-                            if event.get("answer"):
+                            # 过滤 message_replace 事件，此事件会全量推一次
+                            if event.get("event") != "message_replace" and event.get(
+                                "answer"
+                            ):
                                 yield event["answer"]
                 elif self.mode == "workflows/run":
                     for line in r.iter_lines():
@@ -73,9 +79,32 @@ class LLMProvider(LLMProviderBase):
                     for line in r.iter_lines():
                         if line.startswith(b"data: "):
                             event = json.loads(line[6:])
-                            if event.get("answer"):
+                            # 过滤 message_replace 事件，此事件会全量推一次
+                            if event.get("event") != "message_replace" and event.get(
+                                "answer"
+                            ):
                                 yield event["answer"]
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
             yield "【服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py

@@ -2,6 +2,7 @@ import json
 from config.logger import setup_logging
 import requests
 from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
 
 TAG = __name__
 logger = setup_logging()
@@ -13,6 +14,7 @@ class LLMProvider(LLMProviderBase):
         self.base_url = config.get("base_url")
         self.detail = config.get("detail", False)
         self.variables = config.get("variables", {})
+        check_model_key("FastGPTLLM", self.api_key)
 
     def response(self, session_id, dialogue):
         try:
@@ -21,37 +23,36 @@ class LLMProvider(LLMProviderBase):
 
             # 发起流式请求
             with requests.post(
-                    f"{self.base_url}/chat/completions",
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json={
-                        "stream": True,
-                        "chatId": session_id,
-                        "detail": self.detail,
-                        "variables": self.variables,
-                        "messages": [
-                            {
-                                "role": "user",
-                                "content": last_msg["content"]
-                            }
-                        ]
-                    },
-                    stream=True
+                f"{self.base_url}/chat/completions",
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                json={
+                    "stream": True,
+                    "chatId": session_id,
+                    "detail": self.detail,
+                    "variables": self.variables,
+                    "messages": [{"role": "user", "content": last_msg["content"]}],
+                },
+                stream=True,
             ) as r:
                 for line in r.iter_lines():
                     if line:
                         try:
-                            if line.startswith(b'data: '):
-                                if line[6:].decode('utf-8') == '[DONE]':
+                            if line.startswith(b"data: "):
+                                if line[6:].decode("utf-8") == "[DONE]":
                                     break
 
                                 data = json.loads(line[6:])
-                                if 'choices' in data and len(data['choices']) > 0:
-                                    delta = data['choices'][0].get('delta', {})
-                                    if delta and 'content' in delta and delta['content'] is not None:
-                                        content = delta['content']
-                                        if '<think>' in content:
+                                if "choices" in data and len(data["choices"]) > 0:
+                                    delta = data["choices"][0].get("delta", {})
+                                    if (
+                                        delta
+                                        and "content" in delta
+                                        and delta["content"] is not None
+                                    ):
+                                        content = delta["content"]
+                                        if "<think>" in content:
                                             continue
-                                        if '</think>' in content:
+                                        if "</think>" in content:
                                             continue
                                         yield content
 
@@ -62,4 +63,9 @@ class LLMProvider(LLMProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
-            yield "【服务响应异常】"
+            yield "【服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"fastgpt暂未实现完整的工具调用（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -1,136 +1,205 @@
-import google.generativeai as genai
-from core.utils.util import check_model_key
-from core.providers.llm.base import LLMProviderBase
-from config.logger import setup_logging
+import os, json, uuid
+from types import SimpleNamespace
+from typing import Any, Dict, List
+
 import requests
-import json
+from google import generativeai as genai
+from google.generativeai import types, GenerationConfig
+
+from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
+from config.logger import setup_logging
+from google.generativeai.types import GenerateContentResponse
+from requests import RequestException
+
+log = setup_logging()
 TAG = __name__
-logger = setup_logging()
+
+
+def test_proxy(proxy_url: str, test_url: str) -> bool:
+    try:
+        resp = requests.get(test_url, proxies={"http": proxy_url, "https": proxy_url})
+        return 200 <= resp.status_code < 400
+    except RequestException:
+        return False
+
+
+def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
+    """
+    分别测试 HTTP 和 HTTPS 代理是否可用，并设置环境变量。
+    如果 HTTPS 代理不可用但 HTTP 可用，会将 HTTPS_PROXY 也指向 HTTP。
+    """
+    test_http_url = "http://www.google.com"
+    test_https_url = "https://www.google.com"
+
+    ok_http = ok_https = False
+
+    if http_proxy:
+        ok_http = test_proxy(http_proxy, test_http_url)
+        if ok_http:
+            os.environ["HTTP_PROXY"] = http_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}")
+        else:
+            log.bind(tag=TAG).warning(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
+
+    if https_proxy:
+        ok_https = test_proxy(https_proxy, test_https_url)
+        if ok_https:
+            os.environ["HTTPS_PROXY"] = https_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}")
+        else:
+            log.bind(tag=TAG).warning(
+                f"配置提供的Gemini HTTPS代理不可用: {https_proxy}"
+            )
+
+    # 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy
+    if ok_http and not ok_https:
+        if test_proxy(http_proxy, test_https_url):
+            os.environ["HTTPS_PROXY"] = http_proxy
+            ok_https = True
+            log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}")
+
+    if not ok_http and not ok_https:
+        log.bind(tag=TAG).error(
+            f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置"
+        )
+        raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置")
+
 
 class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        """初始化Gemini LLM Provider"""
-        self.model_name = config.get("model_name", "gemini-1.5-pro")
-        self.api_key = config.get("api_key")
-        self.http_proxy=config.get("http_proxy")
-        self.https_proxy = config.get("https_proxy")
-        have_key = check_model_key("LLM", self.api_key)
+    def __init__(self, cfg: Dict[str, Any]):
+        self.model_name = cfg.get("model_name", "gemini-2.0-flash")
+        self.api_key = cfg["api_key"]
+        http_proxy = cfg.get("http_proxy")
+        https_proxy = cfg.get("https_proxy")
 
-        if not have_key:
-            return
+        if not check_model_key("LLM", self.api_key):
+            raise ValueError("无效的Gemini API Key，请检查是否配置正确")
 
-        try:
-            # 初始化Gemini客户端
-            # 配置代理（如果提供了代理配置）
-            self.proxies=None
-            if self.http_proxy is not "" or self.https_proxy is not "":
+        if http_proxy or https_proxy:
+            log.bind(tag=TAG).info(
+                f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境..."
+            )
+            setup_proxy_env(http_proxy, https_proxy)
+            log.bind(tag=TAG).info(
+                f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}"
+            )
+        genai.configure(api_key=self.api_key)
+        self.model = genai.GenerativeModel(self.model_name)
 
-                self.proxies = {
-                    "http": self.http_proxy,
-                    "https": self.https_proxy,
-                }
-                logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
-                # 使用猴子补丁修改 google-generativeai 库的请求会话
+        self.gen_cfg = GenerationConfig(
+            temperature=0.7,
+            top_p=0.9,
+            top_k=40,
+            max_output_tokens=2048,
+        )
 
-                # 使用 session 对象配置 genai
-
-            genai.configure(api_key=self.api_key)
-            self.model = genai.GenerativeModel(self.model_name)
-
-            # 设置生成参数
-            self.generation_config = {
-                "temperature": 0.7,
-                "top_p": 0.9,
-                "top_k": 40,
-                "max_output_tokens": 2048,
-            }
-            self.chat = None
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
-            self.model = None
+    @staticmethod
+    def _build_tools(funcs: List[Dict[str, Any]] | None):
+        if not funcs:
+            return None
+        return [
+            types.Tool(
+                function_declarations=[
+                    types.FunctionDeclaration(
+                        name=f["function"]["name"],
+                        description=f["function"]["description"],
+                        parameters=f["function"]["parameters"],
+                    )
+                    for f in funcs
+                ]
+            )
+        ]
 
+    # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
     def response(self, session_id, dialogue):
-        """生成Gemini对话响应"""
-        if not self.model:
-            yield "【Gemini服务未正确初始化】"
-            return
+        yield from self._generate(dialogue, None)
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        yield from self._generate(dialogue, self._build_tools(functions))
+
+    def _generate(self, dialogue, tools):
+        role_map = {"assistant": "model", "user": "user"}
+        contents: list = []
+        # 拼接对话
+        for m in dialogue:
+            r = m["role"]
+
+            if r == "assistant" and "tool_calls" in m:
+                tc = m["tool_calls"][0]
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [
+                            {
+                                "function_call": {
+                                    "name": tc["function"]["name"],
+                                    "args": json.loads(tc["function"]["arguments"]),
+                                }
+                            }
+                        ],
+                    }
+                )
+                continue
+
+            if r == "tool":
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [{"text": str(m.get("content", ""))}],
+                    }
+                )
+                continue
+
+            contents.append(
+                {
+                    "role": role_map.get(r, "user"),
+                    "parts": [{"text": str(m.get("content", ""))}],
+                }
+            )
+
+        stream: GenerateContentResponse = self.model.generate_content(
+            contents=contents,
+            generation_config=self.gen_cfg,
+            tools=tools,
+            stream=True,
+        )
 
         try:
-            # 处理对话历史
-            chat_history = []
-            for msg in dialogue[:-1]:  # 历史对话
-                role = "model" if msg["role"] == "assistant" else "user"
-                content = msg["content"].strip()
-                if content:
-                    chat_history.append({
-                        "role": role,
-                        "parts": [{"text":content}]
+            for chunk in stream:
+                cand = chunk.candidates[0]
+                for part in cand.content.parts:
+                    # a) 函数调用-通常是最后一段话才是函数调用
+                    if getattr(part, "function_call", None):
+                        fc = part.function_call
+                        yield None, [
+                            SimpleNamespace(
+                                id=uuid.uuid4().hex,
+                                type="function",
+                                function=SimpleNamespace(
+                                    name=fc.name,
+                                    arguments=json.dumps(
+                                        dict(fc.args), ensure_ascii=False
+                                    ),
+                                ),
+                            )
+                        ]
+                        return
+                    # b) 普通文本
+                    if getattr(part, "text", None):
+                        yield part.text if tools is None else (part.text, None)
 
-                    })
+        finally:
+            if tools is not None:
+                yield None, None  # function‑mode 结束，返回哑包
 
-            # 获取当前消息
-            current_msg = dialogue[-1]["content"]
-
-            # 构建请求体
-            request_body = {
-                "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
-                "generationConfig": self.generation_config
-            }
-
-            # 构建请求URL
-            url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
-
-            # 构建请求头
-            headers = {
-                "Content-Type": "application/json",
-            }
-
-            # 发送POST请求,经测试手动 request 无法使用 stream 模式
-            if self.proxies:
-                response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
-                try:
-                    data = response.json()  # 直接解析JSON
-                    if 'candidates' in data and data['candidates']:
-                        yield data['candidates'][0]['content']['parts'][0]['text']
-                    else:
-                        yield "未找到候选回复。"
-                except json.JSONDecodeError as e:
-                    yield f"JSON解码错误：{e}"
-                except Exception as e:
-                    yield f"发生错误：{e}"
-            else:
-                logger.bind(tag=TAG).info(f"Gemini stream mode ")
-                chat = self.model.start_chat(history=chat_history)
-
-                # 发送消息并获取流式响应
-                response = chat.send_message(
-                    current_msg,
-                    stream=True,
-                    generation_config=self.generation_config
-                )
-                # 处理流式响应
-                for chunk in response:
-                    if hasattr(chunk, 'text') and chunk.text:
-                        yield chunk.text
-
-        except Exception as e:
-            error_msg = str(e)
-            logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
-
-            # 针对不同错误返回友好提示
-            if "Rate limit" in error_msg:
-                yield "【Gemini服务请求太频繁,请稍后再试】"
-            elif "Invalid API key" in error_msg:
-                yield "【Gemini API key无效】"
-            else:
-                yield f"【Gemini服务响应异常: {error_msg}】"
-
-
-
-
-        except requests.exceptions.RequestException as e:
-            yield f"请求失败：{e}"
-        except json.JSONDecodeError as e:
-            yield f"JSON解码错误：{e}"
-        except Exception as e:
-            yield f"发生错误：{e}"
+    # 关闭stream，预留后续打断对话功能的功能方法，官方文档推荐打断对话要关闭上一个流，可以有效减少配额计费和资源占用
+    @staticmethod
+    def _safe_finish_stream(stream: GenerateContentResponse):
+        if hasattr(stream, "resolve"):
+            stream.resolve()  # Gemini SDK version ≥ 0.5.0
+        elif hasattr(stream, "close"):
+            stream.close()  # Gemini SDK version < 0.5.0
+        else:
+            for _ in stream:  # 兜底耗尽
+                pass

main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py

@@ -0,0 +1,71 @@
+import requests
+from requests.exceptions import RequestException
+from config.logger import setup_logging
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.agent_id = config.get("agent_id")  # 对应 agent_id
+        self.api_key = config.get("api_key")
+        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
+        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
+
+    def response(self, session_id, dialogue):
+        try:
+            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
+
+            # 提取最后一个 role 为 'user' 的 content
+            input_text = None
+            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
+                # 逆序遍历，找到最后一个 role 为 'user' 的消息
+                for message in reversed(dialogue):
+                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
+                        input_text = message.get("content", "")
+                        break  # 找到后立即退出循环
+
+            # 构造请求数据
+            payload = {
+                "text": input_text,
+                "agent_id": self.agent_id,
+                "conversation_id": session_id,  # 使用 session_id 作为 conversation_id
+            }
+            # 设置请求头
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            }
+
+            # 发起 POST 请求
+            response = requests.post(self.api_url, json=payload, headers=headers)
+
+            # 检查请求是否成功
+            response.raise_for_status()
+
+            # 解析返回数据
+            data = response.json()
+            speech = (
+                data.get("response", {})
+                .get("speech", {})
+                .get("plain", {})
+                .get("speech", "")
+            )
+
+            # 返回生成的内容
+            if speech:
+                yield speech
+            else:
+                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
+
+        except RequestException as e:
+            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"homeassistant不支持（function call），建议使用其他意图识别"
+        )

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -21,27 +21,67 @@ class LLMProvider(LLMProviderBase):
             api_key="ollama"  # Ollama doesn't need an API key but OpenAI client requires one
         )
 
+        # 检查是否是qwen3模型
+        self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
+
     def response(self, session_id, dialogue):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             responses = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
                 stream=True
             )
-            is_active=True
+            is_active = True
+            # 用于处理跨chunk的标签
+            buffer = ""
+
             for chunk in responses:
                 try:
                     delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
                     content = delta.content if hasattr(delta, 'content') else ''
+
                     if content:
-                        if '<think>' in content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
                             is_active = False
-                            content = content.split('<think>')[0]
-                        if '</think>' in content:
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
                             is_active = True
-                            content = content.split('</think>')[-1]
-                        if is_active:
-                            yield content
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer
+                            buffer = ""  # 清空缓冲区
+
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
 
@@ -51,6 +91,22 @@ class LLMProvider(LLMProviderBase):
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             stream = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
@@ -58,9 +114,50 @@ class LLMProvider(LLMProviderBase):
                 tools=functions,
             )
 
+            is_active = True
+            buffer = ""
+
             for chunk in stream:
-                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else None
+                    tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None
+
+                    # 如果是工具调用，直接传递
+                    if tool_calls:
+                        yield None, tool_calls
+                        continue
+
+                    # 处理文本内容
+                    if content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
+                            is_active = False
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
+                            is_active = True
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer, None
+                            buffer = ""  # 清空缓冲区
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
+                    continue
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
-            yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}
+            yield f"【Ollama服务响应异常: {str(e)}】", None

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -1,4 +1,5 @@
 import openai
+from openai.types import CompletionUsage
 from config.logger import setup_logging
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
@@ -11,11 +12,19 @@ class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.model_name = config.get("model_name")
         self.api_key = config.get("api_key")
-        if 'base_url' in config:
+        if "base_url" in config:
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
-        self.max_tokens = config.get("max_tokens", 500)
+        max_tokens = config.get("max_tokens")
+        if max_tokens is None or max_tokens == "":
+            max_tokens = 500
+
+        try:
+            max_tokens = int(max_tokens)
+        except (ValueError, TypeError):
+            max_tokens = 500
+        self.max_tokens = max_tokens
 
         check_model_key("LLM", self.api_key)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
@@ -33,18 +42,22 @@ class LLMProvider(LLMProviderBase):
             for chunk in responses:
                 try:
                     # 检查是否存在有效的choice且content不为空
-                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
-                    content = delta.content if hasattr(delta, 'content') else ''
+                    delta = (
+                        chunk.choices[0].delta
+                        if getattr(chunk, "choices", None)
+                        else None
+                    )
+                    content = delta.content if hasattr(delta, "content") else ""
                 except IndexError:
-                    content = ''
+                    content = ""
                 if content:
                     # 处理标签跨多个chunk的情况
-                    if '<think>' in content:
+                    if "<think>" in content:
                         is_active = False
-                        content = content.split('<think>')[0]
-                    if '</think>' in content:
+                        content = content.split("<think>")[0]
+                    if "</think>" in content:
                         is_active = True
-                        content = content.split('</think>')[-1]
+                        content = content.split("</think>")[-1]
                     if is_active:
                         yield content
 
@@ -54,15 +67,22 @@ class LLMProvider(LLMProviderBase):
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
             stream = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                tools=functions
+                model=self.model_name, messages=dialogue, stream=True, tools=functions
             )
 
             for chunk in stream:
-                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                # 检查是否存在有效的choice且content不为空
+                if getattr(chunk, "choices", None):
+                    yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                # 存在 CompletionUsage 消息时，生成 Token 消耗 log
+                elif isinstance(getattr(chunk, 'usage', None), CompletionUsage):
+                    usage_info = getattr(chunk, 'usage', None)
+                    logger.bind(tag=TAG).info(
+                        f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，" 
+                        f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
+                        f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
+                    )
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
-            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
+            yield f"【OpenAI服务响应异常: {e}】", None

main/xiaozhi-server/core/providers/llm/system_prompt.py

@@ -0,0 +1,103 @@
+def get_system_prompt_for_function(functions: str) -> str:
+    """
+    生成系统提示信息
+    :param functions: 可用的函数列表
+    :return: 系统提示信息
+    """
+
+    SYSTEM_PROMPT = f"""
+====
+
+TOOL USE
+
+You have access to a set of tools that are executed upon the user's approval. You can use one tool per message, and will receive the result of that tool use in the user's response. 
+You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use.
+
+# Tool Use Formatting
+
+Tool use is formatted using JSON-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags. 
+Here's the structure:
+
+<tool_call>
+{{
+    "name": "function name",
+    "arguments": {{
+        "param1": "value1",
+        "param2": "value2",
+        // Add more parameters as needed, if parameters are required, you must provide them
+    }}
+}}
+<tool_call>
+
+For example:
+if you got tool as follow
+
+{{
+    "type": "function",
+    "function": {{
+        "name": "handle_exit_intent",
+        "description": "当用户想结束对话或需要退出系统时调用",
+        "parameters": {{
+            "type": "object",
+            "properties": {{
+                "say_goodbye": {{
+                    "type": "string",
+                    "description": "和用户友好结束对话的告别语",
+                }}
+            }},
+            "required": ["say_goodbye"],
+        }},
+    }},
+}}
+
+you should respond with the following format:
+
+<tool_call>
+{{
+    "name": "handle_exit_intent",
+    "arguments": {{
+        "say_goodbye": "再见，祝您生活愉快！"
+    }}
+}}
+</tool_call>
+
+
+Always adhere to this format for the tool use to ensure proper parsing and execution.
+
+# Tools
+
+{functions}
+
+# Tool Use Guidelines
+
+1. Tools must be called in a separate message, Do not add thoughts when calling tools. The message must start with <tool_call> and end with </tool_call>, with the tool invocation JSON data in between. No additional response content is needed.
+2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information. 
+   For example using the list_files tool is more effective than running a command like \`ls\` in the terminal. It's critical that you think about each available tool and use the one that best fits the current step in the task.
+3. If multiple actions are needed, use one tool at a time per message to accomplish the task iteratively, with each tool use being informed by the result of the previous tool use. Do not assume the outcome of any tool use. 
+   Each step must be informed by the previous step's result.
+4. Formulate your tool use using the JSON format specified for each tool.
+5. After each tool use, the user will respond with the result of that tool use. This result will provide you with the necessary information to continue your task or make further decisions. This response may include:
+- Information about whether the tool succeeded or failed, along with any reasons for failure.
+- Linter errors that may have arisen due to the changes you made, which you'll need to address.
+- New terminal output in reaction to the changes, which you may need to consider or act upon.
+- Any other relevant feedback or information related to the tool use.
+6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user.
+7. Tool calls should contain no extra information. Only after receiving the tool's response should you integrate it into a complete reply.
+
+It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to:
+1. Confirm the success of each step before proceeding.
+2. Address any issues or errors that arise immediately.
+3. Adapt your approach based on new information or unexpected results.
+4. Ensure that each action builds correctly on the previous ones.
+
+By waiting for and carefully considering the user's response after each tool use, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work.
+
+====
+
+USER CHAT CONTENT
+
+The following additional message is the user's chat message, and should be followed to the best of your ability without interfering with the TOOL USE guidelines.
+
+"""
+
+    return SYSTEM_PROMPT

main/xiaozhi-server/core/providers/memory/base.py

@@ -4,6 +4,7 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+
 class MemoryProviderBase(ABC):
     def __init__(self, config):
         self.config = config
@@ -20,6 +21,6 @@ class MemoryProviderBase(ABC):
         """Query memories for specific role based on similarity"""
         return "please implement query method"
 
-    def init_memory(self, role_id, llm):
-        self.role_id = role_id    
+    def init_memory(self, role_id, llm, **kwargs):
+        self.role_id = role_id
         self.llm = llm

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -6,13 +6,14 @@ from core.utils.util import check_model_key
 
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory=None):
         super().__init__(config)
         self.api_key = config.get("api_key", "")
         self.api_version = config.get("api_version", "v1.1")
         have_key = check_model_key("Mem0ai", self.api_key)
-        if not have_key :
+        if not have_key:
             self.use_mem0 = False
             return
         else:
@@ -30,54 +31,55 @@ class MemoryProvider(MemoryProviderBase):
             return None
         if len(msgs) < 2:
             return None
-        
+
         try:
             # Format the content as a message list for mem0
             messages = [
                 {"role": message.role, "content": message.content}
-                for message in msgs if message.role != "system"
+                for message in msgs
+                if message.role != "system"
             ]
-            result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
+            result = self.client.add(
+                messages, user_id=self.role_id, output_format=self.api_version
+            )
             logger.bind(tag=TAG).debug(f"Save memory result: {result}")
         except Exception as e:
             logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
             return None
 
-    async def query_memory(self, query: str)-> str:
+    async def query_memory(self, query: str) -> str:
         if not self.use_mem0:
             return ""
         try:
             results = self.client.search(
-                query,
-                user_id=self.role_id,
-                output_format=self.api_version
+                query, user_id=self.role_id, output_format=self.api_version
             )
-            if not results or 'results' not in results:
+            if not results or "results" not in results:
                 return ""
-                
+
             # Format each memory entry with its update time up to minutes
             memories = []
-            for entry in results['results']:
-                timestamp = entry.get('updated_at', '')
+            for entry in results["results"]:
+                timestamp = entry.get("updated_at", "")
                 if timestamp:
                     try:
                         # Parse and reformat the timestamp
-                        dt = timestamp.split('.')[0]  # Remove milliseconds
-                        formatted_time = dt.replace('T', ' ')
+                        dt = timestamp.split(".")[0]  # Remove milliseconds
+                        formatted_time = dt.replace("T", " ")
                     except:
                         formatted_time = timestamp
-                memory = entry.get('memory', '')
+                memory = entry.get("memory", "")
                 if timestamp and memory:
                     # Store tuple of (timestamp, formatted_string) for sorting
                     memories.append((timestamp, f"[{formatted_time}] {memory}"))
-            
+
             # Sort by timestamp in descending order (newest first)
             memories.sort(key=lambda x: x[0], reverse=True)
-            
+
             # Extract only the formatted strings
             memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
             logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
             return memories_str
         except Exception as e:
             logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
-            return ""
+            return ""

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -3,7 +3,9 @@ import time
 import json
 import os
 import yaml
-from core.utils.util import get_project_dir
+from config.config_loader import get_project_dir
+from config.manage_api_client import save_mem_local_short
+
 
 short_term_memory_prompt = """
 # 时空记忆编织者
@@ -71,11 +73,23 @@ short_term_memory_prompt = """
 ```
 """
 
+short_term_memory_prompt_only_content = """
+你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
+1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
+2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
+3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
+4、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
+5、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
+6、只需要返回总结摘要，严格控制在1800字内
+7、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
+"""
+
+
 def extract_json_data(json_code):
     start = json_code.find("```json")
     # 从start开始找到下一个```结束
-    end = json_code.find("```", start+1)
-    #print("start:", start, "end:", end)
+    end = json_code.find("```", start + 1)
+    # print("start:", start, "end:", end)
     if start == -1 or end == -1:
         try:
             jsonData = json.loads(json_code)
@@ -83,74 +97,89 @@ def extract_json_data(json_code):
         except Exception as e:
             print("Error:", e)
         return ""
-    jsonData = json_code[start+7:end]
+    jsonData = json_code[start + 7 : end]
     return jsonData
 
+
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory):
         super().__init__(config)
         self.short_momery = ""
-        self.memory_path = get_project_dir() + 'data/.memory.yaml'
-        self.load_memory()
+        self.save_to_file = True
+        self.memory_path = get_project_dir() + "data/.memory.yaml"
+        self.load_memory(summary_memory)
+
+    def init_memory(
+        self, role_id, llm, summary_memory=None, save_to_file=True, **kwargs
+    ):
+        super().init_memory(role_id, llm, **kwargs)
+        self.save_to_file = save_to_file
+        self.load_memory(summary_memory)
+
+    def load_memory(self, summary_memory):
+        # api获取到总结记忆后直接返回
+        if summary_memory or not self.save_to_file:
+            self.short_momery = summary_memory
+            return
 
-    def init_memory(self, role_id, llm):
-        super().init_memory(role_id, llm)
-        self.load_memory()
-    
-    def load_memory(self):
         all_memory = {}
         if os.path.exists(self.memory_path):
-            with open(self.memory_path, 'r', encoding='utf-8') as f:
+            with open(self.memory_path, "r", encoding="utf-8") as f:
                 all_memory = yaml.safe_load(f) or {}
         if self.role_id in all_memory:
             self.short_momery = all_memory[self.role_id]
-    
+
     def save_memory_to_file(self):
         all_memory = {}
         if os.path.exists(self.memory_path):
-              with open(self.memory_path, 'r', encoding='utf-8') as f:
-                  all_memory = yaml.safe_load(f) or {}
+            with open(self.memory_path, "r", encoding="utf-8") as f:
+                all_memory = yaml.safe_load(f) or {}
         all_memory[self.role_id] = self.short_momery
-        with open(self.memory_path, 'w', encoding='utf-8') as f:
+        with open(self.memory_path, "w", encoding="utf-8") as f:
             yaml.dump(all_memory, f, allow_unicode=True)
-        
+
     async def save_memory(self, msgs):
         if self.llm is None:
             logger.bind(tag=TAG).error("LLM is not set for memory provider")
             return None
-        
+
         if len(msgs) < 2:
             return None
-        
+
         msgStr = ""
         for msg in msgs:
             if msg.role == "user":
                 msgStr += f"User: {msg.content}\n"
-            elif msg.role== "assistant":
+            elif msg.role == "assistant":
                 msgStr += f"Assistant: {msg.content}\n"
-        if len(self.short_momery) > 0:
-            msgStr+="历史记忆：\n"
-            msgStr+=self.short_momery
-        
-        #当前时间
+        if self.short_momery and len(self.short_momery) > 0:
+            msgStr += "历史记忆：\n"
+            msgStr += self.short_momery
+
+        # 当前时间
         time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
         msgStr += f"当前时间：{time_str}"
 
-        result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
- 
-        json_str = extract_json_data(result)
-        try:
-            json_data = json.loads(json_str) # 检查json格式是否正确
-            self.short_momery = json_str
-        except Exception as e:
-            print("Error:", e)
-        
-        self.save_memory_to_file()
+        if self.save_to_file:
+            result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
+            json_str = extract_json_data(result)
+            try:
+                json.loads(json_str)  # 检查json格式是否正确
+                self.short_momery = json_str
+                self.save_memory_to_file()
+            except Exception as e:
+                print("Error:", e)
+        else:
+            result = self.llm.response_no_stream(
+                short_term_memory_prompt_only_content, msgStr
+            )
+            save_mem_local_short(self.role_id, result)
         logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
 
         return self.short_momery
-    
-    async def query_memory(self, query: str)-> str:
-        return self.short_momery
+
+    async def query_memory(self, query: str) -> str:
+        return self.short_momery

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -1,18 +1,20 @@
-'''
+"""
 不使用记忆，可以选择此模块
-'''
+"""
+
 from ..base import MemoryProviderBase, logger
 
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory=None):
         super().__init__(config)
-      
+
     async def save_memory(self, msgs):
         logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
         return None
 
-    async def query_memory(self, query: str)-> str:
+    async def query_memory(self, query: str) -> str:
         logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.")
-        return ""
+        return ""

main/xiaozhi-server/core/providers/vad/base.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class VADProviderBase(ABC):
+    @abstractmethod
+    def is_vad(self, conn, data) -> bool:
+        """检测音频数据中的语音活动"""
+        pass

main/xiaozhi-server/core/providers/vad/silero.py

@@ -0,0 +1,71 @@
+import time
+import numpy as np
+import torch
+import opuslib_next
+from config.logger import setup_logging
+from core.providers.vad.base import VADProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class VADProvider(VADProviderBase):
+    def __init__(self, config):
+        logger.bind(tag=TAG).info("SileroVAD", config)
+        self.model, self.utils = torch.hub.load(
+            repo_or_dir=config["model_dir"],
+            source="local",
+            model="silero_vad",
+            force_reload=False,
+        )
+        (get_speech_timestamps, _, _, _, _) = self.utils
+
+        self.decoder = opuslib_next.Decoder(16000, 1)
+
+        # 处理空字符串的情况
+        threshold = config.get("threshold", "0.5")
+        min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
+
+        self.vad_threshold = float(threshold) if threshold else 0.5
+        self.silence_threshold_ms = (
+            int(min_silence_duration_ms) if min_silence_duration_ms else 1000
+        )
+
+    def is_vad(self, conn, opus_packet):
+        try:
+            pcm_frame = self.decoder.decode(opus_packet, 960)
+            conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
+
+            # 处理缓冲区中的完整帧（每次处理512采样点）
+            client_have_voice = False
+            while len(conn.client_audio_buffer) >= 512 * 2:
+                # 提取前512个采样点（1024字节）
+                chunk = conn.client_audio_buffer[: 512 * 2]
+                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :]
+
+                # 转换为模型需要的张量格式
+                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+                audio_float32 = audio_int16.astype(np.float32) / 32768.0
+                audio_tensor = torch.from_numpy(audio_float32)
+
+                # 检测语音活动
+                with torch.no_grad():
+                    speech_prob = self.model(audio_tensor, 16000).item()
+                client_have_voice = speech_prob >= self.vad_threshold
+
+                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
+                if conn.client_have_voice and not client_have_voice:
+                    stop_duration = (
+                        time.time() * 1000 - conn.client_have_voice_last_time
+                    )
+                    if stop_duration >= self.silence_threshold_ms:
+                        conn.client_voice_stop = True
+                if client_have_voice:
+                    conn.client_have_voice = True
+                    conn.client_have_voice_last_time = time.time() * 1000
+
+            return client_have_voice
+        except opuslib_next.OpusError as e:
+            logger.bind(tag=TAG).info(f"解码错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")