merge main

main/xiaozhi-server/core/auth.py

@@ -0,0 +1,54 @@
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AuthenticationError(Exception):
+    """认证异常"""
+    pass
+
+
+class AuthMiddleware:
+    def __init__(self, config):
+        self.config = config
+        self.auth_config = config["server"].get("auth", {})
+        # 构建token查找表
+        self.tokens = {
+            item["token"]: item["name"]
+            for item in self.auth_config.get("tokens", [])
+        }
+        # 设备白名单
+        self.allowed_devices = set(
+            self.auth_config.get("allowed_devices", [])
+        )
+
+    async def authenticate(self, headers):
+        """验证连接请求"""
+        # 检查是否启用认证
+        if not self.auth_config.get("enabled", False):
+            return True
+
+        # 检查设备是否在白名单中
+        device_id = headers.get("device-id", "")
+
+        if self.allowed_devices and device_id in self.allowed_devices:
+            return True
+
+        # 验证Authorization header
+        auth_header = headers.get("authorization", "")
+        if not auth_header.startswith("Bearer "):
+            logger.bind(tag=TAG).error("Missing or invalid Authorization header")
+            raise AuthenticationError("Missing or invalid Authorization header")
+
+        token = auth_header.split(" ")[1]
+        if token not in self.tokens:
+            logger.bind(tag=TAG).error(f"Invalid token: {token}")
+            raise AuthenticationError("Invalid token")
+
+        logger.bind(tag=TAG).info(f"Authentication successful - Device: {device_id}, Token: {self.tokens[token]}")
+        return True
+
+    def get_token_name(self, token):
+        """获取token对应的设备名称"""
+        return self.tokens.get(token)

main/xiaozhi-server/core/connection.py

@@ -0,0 +1,576 @@
+import os
+import json
+import uuid
+import time
+import queue
+import asyncio
+import traceback
+from config.logger import setup_logging
+import threading
+import websockets
+from typing import Dict, Any
+from core.utils.dialogue import Message, Dialogue
+from core.handle.textHandle import handleTextMessage
+from core.utils.util import get_string_no_punctuation_or_emoji
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+from core.handle.sendAudioHandle import sendAudioMessage, sendAudioMessageStream
+from core.handle.receiveAudioHandle import handleAudioMessage
+from core.handle.intentHandler import Action, get_functions, handle_llm_function_call
+from config.private_config import PrivateConfig
+from core.auth import AuthMiddleware, AuthenticationError
+from core.utils.auth_code_gen import AuthCodeGenerator
+
+TAG = __name__
+
+
+class TTSException(RuntimeError):
+    pass
+
+
+class ConnectionHandler:
+    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent):
+        self.config = config
+        self.logger = setup_logging()
+        self.auth = AuthMiddleware(config)
+
+        self.tts_stream = self.config.get("TTS_SET", {}).get("TTS_STREAM", False)
+
+        self.websocket = None
+        self.headers = None
+        self.session_id = None
+        self.prompt = None
+        self.welcome_msg = None
+
+        # 客户端状态相关
+        self.client_abort = False
+        self.client_listen_mode = "auto"
+
+        # 线程任务相关
+        self.loop = asyncio.get_event_loop()
+        self.stop_event = threading.Event()
+        self.tts_queue = queue.Queue()
+        self.tts_queue_stream = queue.Queue()
+        self.audio_play_queue = queue.Queue()
+        max_workers = self.config.get("TTS_SET", {}).get("MAX_WORKERS", 10)
+        self.executor = ThreadPoolExecutor(max_workers=max_workers)
+
+        # 依赖的组件
+        self.vad = _vad
+        self.asr = _asr
+        self.llm = _llm
+        self.tts = _tts
+        self.memory = _memory
+        self.intent = _intent
+
+        # vad相关变量
+        self.client_audio_buffer = bytes()
+        self.client_have_voice = False
+        self.client_have_voice_last_time = 0.0
+        self.client_no_voice_last_time = 0.0
+        self.client_voice_stop = False
+
+        # asr相关变量
+        self.asr_audio = []
+        self.asr_server_receive = True
+
+        # llm相关变量
+        self.llm_finish_task = False
+        self.dialogue = Dialogue()
+
+        # tts相关变量
+        self.tts_first_text_index = -1
+        self.tts_last_text_index = -1
+        self.tts_duration = 0
+
+        # iot相关变量
+        self.iot_descriptors = {}
+
+        self.cmd_exit = self.config["CMD_exit"]
+        self.max_cmd_length = 0
+        for cmd in self.cmd_exit:
+            if len(cmd) > self.max_cmd_length:
+                self.max_cmd_length = len(cmd)
+
+        self.private_config = None
+        self.auth_code_gen = AuthCodeGenerator.get_instance()
+        self.is_device_verified = False  # 添加设备验证状态标志
+        self.music_handler = _music
+        self.close_after_chat = False  # 是否在聊天结束后关闭连接
+        self.use_function_call_mode = False
+        if self.config["selected_module"]["Intent"] == 'function_call':
+            self.use_function_call_mode = True
+
+        self.logger.bind(tag=TAG).info(f"use_function_call_mode:{self.use_function_call_mode}")
+
+    async def handle_connection(self, ws):
+        try:
+            # 获取并验证headers
+            self.headers = dict(ws.request.headers)
+            # 获取客户端ip地址
+            client_ip = ws.remote_address[0]
+            self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
+
+            # 进行认证
+            await self.auth.authenticate(self.headers)
+
+            device_id = self.headers.get("device-id", None)
+            self.memory.init_memory(device_id, self.llm)
+            self.intent.set_llm(self.llm)
+
+            # Load private configuration if device_id is provided
+            bUsePrivateConfig = self.config.get("use_private_config", False)
+            self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
+            if bUsePrivateConfig and device_id:
+                try:
+                    self.private_config = PrivateConfig(device_id, self.config, self.auth_code_gen)
+                    await self.private_config.load_or_create()
+                    # 判断是否已经绑定
+                    owner = self.private_config.get_owner()
+                    self.is_device_verified = owner is not None
+
+                    if self.is_device_verified:
+                        await self.private_config.update_last_chat_time()
+
+                    llm, tts = self.private_config.create_private_instances()
+                    if all([llm, tts]):
+                        self.llm = llm
+                        self.tts = tts
+                        self.logger.bind(tag=TAG).info(f"Loaded private config and instances for device {device_id}")
+                    else:
+                        self.logger.bind(tag=TAG).error(f"Failed to create instances for device {device_id}")
+                        self.private_config = None
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"Error initializing private config: {e}")
+                    self.private_config = None
+                    raise
+
+            # 认证通过,继续处理
+            self.websocket = ws
+            self.session_id = str(uuid.uuid4())
+
+            self.welcome_msg = self.config["xiaozhi"]
+            self.welcome_msg["session_id"] = self.session_id
+            await self.websocket.send(json.dumps(self.welcome_msg))
+
+            await self.loop.run_in_executor(None, self._initialize_components)
+
+            # tts 消化线程
+            tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
+            tts_priority.start()
+
+            # 音频播放 消化线程
+            audio_play_priority = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
+            audio_play_priority.start()
+
+            try:
+                async for message in self.websocket:
+                    await self._route_message(message)
+            except websockets.exceptions.ConnectionClosed:
+                self.logger.bind(tag=TAG).info("客户端断开连接")
+                await self.close()
+
+        except AuthenticationError as e:
+            self.logger.bind(tag=TAG).error(f"Authentication failed: {str(e)}")
+            await ws.close()
+            return
+        except Exception as e:
+            stack_trace = traceback.format_exc()
+            self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
+            await ws.close()
+            return
+        finally:
+            await self.memory.save_memory(self.dialogue.dialogue)
+
+    async def _route_message(self, message):
+        """消息路由"""
+        if isinstance(message, str):
+            await handleTextMessage(self, message)
+        elif isinstance(message, bytes):
+            await handleAudioMessage(self, message)
+
+    def _initialize_components(self):
+        self.prompt = self.config["prompt"]
+        if self.private_config:
+            self.prompt = self.private_config.private_config.get("prompt", self.prompt)
+        # 赋予LLM时间观念
+        if "{date_time}" in self.prompt:
+            date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
+            self.prompt = self.prompt.replace("{date_time}", date_time)
+        self.dialogue.put(Message(role="system", content=self.prompt))
+
+    async def _check_and_broadcast_auth_code(self):
+        """检查设备绑定状态并广播认证码"""
+        if not self.private_config.get_owner():
+            auth_code = self.private_config.get_auth_code()
+            if auth_code:
+                # 发送验证码语音提示
+                text = f"请在后台输入验证码：{' '.join(auth_code)}"
+                self.recode_first_last_text(text)
+                future = self.executor.submit(self.speak_and_play, text)
+                self.tts_queue.put(future)
+            return False
+        return True
+
+    def isNeedAuth(self):
+        bUsePrivateConfig = self.config.get("use_private_config", False)
+        if not bUsePrivateConfig:
+            # 如果不使用私有配置，就不需要验证
+            return False
+        return not self.is_device_verified
+
+    def chat(self, query):
+        if self.isNeedAuth():
+            self.llm_finish_task = True
+            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
+            future.result()
+            return True
+
+        self.dialogue.put(Message(role="user", content=query))
+
+        response_message = []
+        processed_chars = 0  # 跟踪已处理的字符位置
+        try:
+            start_time = time.time()
+            # 使用带记忆的对话
+            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
+            memory_str = future.result()
+
+            self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
+            llm_responses = self.llm.response(
+                self.session_id,
+                self.dialogue.get_llm_dialogue_with_memory(memory_str)
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
+            return None
+
+        self.llm_finish_task = False
+        text_index = 0
+        for content in llm_responses:
+            response_message.append(content)
+            if self.client_abort:
+                break
+
+            end_time = time.time()
+            self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+
+            # 合并当前全部文本并处理未分割部分
+            full_text = "".join(response_message)
+            current_text = full_text[processed_chars:]  # 从未处理的位置开始
+
+            # 查找最后一个有效标点
+            punctuations = ("。", "？", "！", "；", "：")
+            last_punct_pos = -1
+            for punct in punctuations:
+                pos = current_text.rfind(punct)
+                if pos > last_punct_pos:
+                    last_punct_pos = pos
+
+            # 找到分割点则处理
+            if last_punct_pos != -1:
+                segment_text_raw = current_text[:last_punct_pos + 1]
+                segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                if segment_text:
+                    # 强制设置空字符，测试TTS出错返回语音的健壮性
+                    # if text_index % 2 == 0:
+                    #     segment_text = " "
+                    text_index += 1
+                    self.recode_first_last_text(segment_text, text_index)
+                    if self.tts_stream:
+                        stream_queue = queue.Queue()
+                        self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue)
+                        self.tts_queue_stream.put({
+                            "text": segment_text,
+                            "chunk_queque": stream_queue,
+                            "text_index": text_index
+                        })
+                    else:
+                        future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                        self.tts_queue.put(future)
+                    processed_chars += len(segment_text_raw)  # 更新已处理字符位置
+
+        # 处理最后剩余的文本
+        full_text = "".join(response_message)
+        remaining_text = full_text[processed_chars:]
+        if remaining_text:
+            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                text_index += 1
+                self.recode_first_last_text(segment_text, text_index)
+                if self.tts_stream:
+                    stream_queue = queue.Queue()
+                    self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue, text_index)
+                    self.tts_queue_stream.put({
+                        "text": segment_text,
+                        "chunk_queque": stream_queue,
+                        "text_index": text_index
+                    })
+                else:
+                    future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                    self.tts_queue.put(future)
+
+        self.llm_finish_task = True
+        self.dialogue.put(Message(role="assistant", content="".join(response_message)))
+        self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
+        return True
+
+    def chat_with_function_calling(self, query):
+        self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
+        """Chat with function calling for intent detection using streaming"""
+        if self.isNeedAuth():
+            self.llm_finish_task = True
+            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
+            future.result()
+            return True
+
+        self.dialogue.put(Message(role="user", content=query))
+
+        # Define intent functions
+        functions = get_functions()
+
+        response_message = []
+        processed_chars = 0  # 跟踪已处理的字符位置
+        function_call_data = None  # 存储function call数据
+
+        try:
+            start_time = time.time()
+
+            # 使用带记忆的对话
+            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
+            memory_str = future.result()
+
+            # self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
+
+            # 使用支持functions的streaming接口
+            llm_responses = self.llm.response_with_functions(
+                self.session_id,
+                self.dialogue.get_llm_dialogue_with_memory(memory_str),
+                functions=functions
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
+            return None
+
+        self.llm_finish_task = False
+        text_index = 0
+
+        # 处理流式响应
+        for response in llm_responses:
+            if response["type"] == "content":
+                content = response["content"]
+                response_message.append(content)
+
+                if self.client_abort:
+                    break
+
+                end_time = time.time()
+                self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+
+                # 处理文本分段和TTS逻辑
+                # 合并当前全部文本并处理未分割部分
+                full_text = "".join(response_message)
+                current_text = full_text[processed_chars:]  # 从未处理的位置开始
+
+                # 查找最后一个有效标点
+                punctuations = ("。", "？", "！", "；", "：")
+                last_punct_pos = -1
+                for punct in punctuations:
+                    pos = current_text.rfind(punct)
+                    if pos > last_punct_pos:
+                        last_punct_pos = pos
+
+                # 找到分割点则处理
+                if last_punct_pos != -1:
+                    segment_text_raw = current_text[:last_punct_pos + 1]
+                    segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                    if segment_text:
+                        text_index += 1
+                        self.recode_first_last_text(segment_text, text_index)
+                        future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                        self.tts_queue.put(future)
+                        processed_chars += len(segment_text_raw)  # 更新已处理字符位置
+
+            elif response["type"] == "function_call":
+                # Extract function call data
+                function_call_data = {
+                    "name": response["function_call"]["function"]["name"],
+                    "arguments": response["function_call"]["function"]["arguments"]
+                }
+                self.logger.bind(tag=TAG).info(f"Function call detected: {function_call_data}")
+
+        # 处理最后剩余的文本
+        full_text = "".join(response_message)
+        remaining_text = full_text[processed_chars:]
+        if remaining_text:
+            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                text_index += 1
+                self.recode_first_last_text(segment_text, text_index)
+                future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                self.tts_queue.put(future)
+
+        # 存储对话内容
+        self.dialogue.put(Message(role="assistant", content="".join(response_message)))
+
+        # 处理function call
+        if function_call_data:
+            result = handle_llm_function_call(self, function_call_data)
+            if result.action == Action.RESPONSE:
+                text = result.response
+                text_index += 1
+                self.recode_first_last_text(text, text_index)
+                future = self.executor.submit(self.speak_and_play, text, text_index)
+                self.tts_queue.put(future)
+
+        self.llm_finish_task = True
+        self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
+
+        return True
+
+    def _tts_priority_thread(self):
+        if self.tts_stream:
+            self._tts_priority_thread_stream()
+        else:
+            self._tts_priority_thread_no_stream()
+
+    def _tts_priority_thread_no_stream(self):
+        while not self.stop_event.is_set():
+            text = None
+            try:
+                future = self.tts_queue.get()
+                if future is None:
+                    continue
+                text = None
+                opus_datas, text_index, tts_file = [], 0, None
+                try:
+                    self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
+                    tts_file, text, text_index = future.result(timeout=10)
+                    if text is None or len(text) <= 0:
+                        self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
+                    elif tts_file is None:
+                        self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
+                    else:
+                        self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
+                        if os.path.exists(tts_file):
+                            opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
+                        else:
+                            self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
+                except TimeoutError:
+                    self.logger.bind(tag=TAG).error("TTS超时")
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
+                if not self.client_abort:
+                    # 如果没有中途打断就发送语音
+                    self.audio_play_queue.put((opus_datas, text, text_index))
+                if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
+                    os.remove(tts_file)
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
+                self.clearSpeakStatus()
+                asyncio.run_coroutine_threadsafe(
+                    self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
+                    self.loop
+                )
+                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
+
+    def _tts_priority_thread_stream(self):
+        while not self.stop_event.is_set():
+            text = None
+            try:
+                tts_stream_queue_msg = self.tts_queue_stream.get()
+                try:
+                    text = tts_stream_queue_msg["text"]
+                    chunk_queque = tts_stream_queue_msg["chunk_queque"]
+                    text_index = tts_stream_queue_msg["text_index"]
+                except TimeoutError:
+                    self.logger.error("TTS 任务超时")
+                    continue
+                except Exception as e:
+                    self.logger.error(f"TTS 任务出错: {e}")
+                    continue
+                if not self.client_abort:
+                    # 如果没有中途打断就发送语音
+                    self.audio_play_queue.put((chunk_queque, text, text_index))
+            except Exception as e:
+                self.clearSpeakStatus()
+                asyncio.run_coroutine_threadsafe(
+                    self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
+                    self.loop
+                )
+                self.logger.error(f"tts_priority priority_thread: {text}{e}")
+
+    def _audio_play_priority_thread(self):
+        while not self.stop_event.is_set():
+            text = None
+            try:
+                if self.tts_stream:
+                    chunk_queque, text, text_index = self.audio_play_queue.get()
+                    future = asyncio.run_coroutine_threadsafe(
+                        sendAudioMessageStream(self, chunk_queque, text, text_index),
+                        self.loop)
+                    future.result()
+                else:
+                    opus_datas, text, text_index = self.audio_play_queue.get()
+                    future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
+                                                              self.loop)
+                    future.result()
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
+
+    def speak_and_play(self, text, text_index=0):
+        if text is None or len(text) <= 0:
+            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
+            return None, text, text_index
+        tts_file = self.tts.to_tts(text)
+        if tts_file is None:
+            self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
+            return None, text, text_index
+        self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
+        return tts_file, text, text_index
+
+    def speak_and_play_stream(self, text, queue: queue.Queue, text_index=0):
+        if text is None or len(text) <= 0:
+            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
+            return None, text
+        self.tts.to_tts_stream(text, queue, text_index)
+
+    def clearSpeakStatus(self):
+        self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
+        self.asr_server_receive = True
+        self.tts_last_text_index = -1
+        self.tts_first_text_index = -1
+        self.tts_duration = 0
+
+    def recode_first_last_text(self, text, text_index=0):
+        if self.tts_first_text_index == -1:
+            self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
+            self.tts_first_text_index = text_index
+        self.tts_last_text_index = text_index
+
+    async def close(self):
+        """资源清理方法"""
+
+        # 清理其他资源
+        self.stop_event.set()
+        self.executor.shutdown(wait=False)
+        if self.websocket:
+            await self.websocket.close()
+        self.logger.bind(tag=TAG).info("连接资源已释放")
+
+    def reset_vad_states(self):
+        self.client_audio_buffer = bytes()
+        self.client_have_voice = False
+        self.client_have_voice_last_time = 0
+        self.client_voice_stop = False
+        self.logger.bind(tag=TAG).debug("VAD states reset.")
+
+    def chat_and_close(self, text):
+        """Chat with the user and then close the connection"""
+        try:
+            # Use the existing chat method
+            self.chat(text)
+
+            # After chat is complete, close the connection
+            self.close_after_chat = True
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Chat and close error: {str(e)}")

main/xiaozhi-server/core/handle/abortHandle.py

@@ -0,0 +1,16 @@
+import json
+import queue
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleAbortMessage(conn):
+    logger.bind(tag=TAG).info("Abort message received")
+    # 设置成打断状态，会自动打断llm、tts任务
+    conn.client_abort = True
+    # 打断客户端说话状态
+    await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
+    conn.clearSpeakStatus()
+    logger.bind(tag=TAG).info("Abort message received-end")

main/xiaozhi-server/core/handle/helloHandle.py

@@ -0,0 +1,8 @@
+import json
+from config.logger import setup_logging
+
+logger = setup_logging()
+
+
+async def handleHelloMessage(conn):
+    await conn.websocket.send(json.dumps(conn.welcome_msg))

main/xiaozhi-server/core/handle/intentHandler.py

@@ -0,0 +1,170 @@
+from config.logger import setup_logging
+import json
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils.dialogue import Message
+from config.functionCallConfig import FunctionCallConfig
+import asyncio
+from enum import Enum
+
+TAG = __name__
+logger = setup_logging()
+
+
+class Action(Enum):
+    NOTFOUND = (0, "没有找到函数")
+    NONE = (1, "啥也不干")
+    RESPONSE = (2, "直接回复")
+    REQLLM = (3, "调用函数后再请求llm生成回复")
+
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+
+
+class ActionResponse:
+    def __init__(self, action: Action, result, response):
+        self.action = action  # 动作类型
+        self.result = result  # 动作产生的结果
+        self.response = response  # 直接回复的内容
+
+
+def get_functions():
+    """获取功能调用配置"""
+    return FunctionCallConfig
+
+
+def handle_llm_function_call(conn, function_call_data):
+    try:
+        function_name = function_call_data["name"]
+
+        if function_name == "handle_exit_intent":
+            # 处理退出意图
+            try:
+                say_goodbye = json.loads(function_call_data["arguments"]).get("say_goodbye", "再见")
+                conn.close_after_chat = True
+                logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
+                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
+
+        elif function_name == "play_music":
+            # 处理音乐播放意图
+            try:
+                song_name = "random"
+                arguments = function_call_data["arguments"]
+                if arguments is not None and len(arguments) > 0:
+                    args = json.loads(arguments)
+                    song_name = args.get("song_name", "random")
+                music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
+
+                # 执行音乐播放命令
+                future = asyncio.run_coroutine_threadsafe(
+                    conn.music_handler.handle_music_command(conn, music_intent),
+                    conn.loop
+                )
+                future.result()
+                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+        else:
+            return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+
+    return None
+
+
+async def handle_user_intent(conn, text):
+    """
+    Handle user intent before starting chat
+    
+    Args:
+        conn: Connection object
+        text: User's text input
+    
+    Returns:
+        bool: True if intent was handled, False if should proceed to chat
+    """
+    # 检查是否有明确的退出命令
+    if await check_direct_exit(conn, text):
+        return True
+
+    if conn.use_function_call_mode:
+        # 使用支持function calling的聊天方法,不再进行意图分析
+        return False
+
+    logger.bind(tag=TAG).info(f"分析用户意图: {text}")
+
+    # 使用LLM进行意图分析
+    intent = await analyze_intent_with_llm(conn, text)
+
+    if not intent:
+        return False
+
+    # 处理各种意图
+    return await process_intent_result(conn, intent, text)
+
+
+async def check_direct_exit(conn, text):
+    """检查是否有明确的退出命令"""
+    cmd_exit = conn.cmd_exit
+    for cmd in cmd_exit:
+        if text == cmd:
+            logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
+            await conn.close()
+            return True
+    return False
+
+
+async def analyze_intent_with_llm(conn, text):
+    """使用LLM分析用户意图"""
+    if not hasattr(conn, 'intent') or not conn.intent:
+        logger.bind(tag=TAG).warning("意图识别服务未初始化")
+        return None
+
+    # 创建对话历史记录
+    dialogue = conn.dialogue
+    dialogue.put(Message(role="user", content=text))
+
+    try:
+        intent_result = await conn.intent.detect_intent(dialogue.dialogue)
+        logger.bind(tag=TAG).info(f"意图识别结果: {intent_result}")
+
+        # 尝试解析JSON结果
+        try:
+            intent_data = json.loads(intent_result)
+            if "intent" in intent_data:
+                return intent_data["intent"]
+        except json.JSONDecodeError:
+            # 如果不是JSON格式，尝试直接获取意图文本
+            return intent_result.strip()
+
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
+
+    return None
+
+
+async def process_intent_result(conn, intent, original_text):
+    """处理意图识别结果"""
+    # 处理退出意图
+    if "结束聊天" in intent:
+        logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
+
+        # 如果正在播放音乐，可以关了 TODO
+
+        # 如果是明确的离别意图，发送告别语并关闭连接
+        await send_stt_message(conn, original_text)
+        conn.executor.submit(conn.chat_and_close, original_text)
+        return True
+
+    # 处理播放音乐意图
+    if "播放音乐" in intent:
+        logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
+        await conn.music_handler.handle_music_command(conn, intent)
+        return True
+
+    # 其他意图处理可以在这里扩展
+
+    # 默认返回False，表示继续常规聊天流程
+    return False

main/xiaozhi-server/core/handle/iotHandle.py

@@ -0,0 +1,193 @@
+import json
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IotDescriptor:
+    """
+    A class to represent an IoT descriptor.
+    Attributes:
+    ----------
+    name : str
+        The name of the IoT descriptor.
+    description : str
+        A brief description of the IoT descriptor.
+    properties : dict
+        A dictionary containing properties of the IoT descriptor.
+    methods : dict
+        A dictionary containing methods of the IoT descriptor.
+    -------
+    """
+
+    def __init__(self, name, description, properties, methods):
+        self.name = name
+        self.description = description
+        self.properties = []
+        self.methods = []
+
+        # 根据描述创建属性
+        for key, value in properties.items():
+            # "volume":{"description":"当前音量 值","type":"number"}
+            """
+            等价于
+            {
+                'name': 名字,
+                'description': 描述,
+                'value': 0
+            }
+            """
+            # setattr(self, key, {}) # 创建一个空字典, 名字是属性名
+            property_item = globals()[key] = {}  # 创建一个空字典, 名字是属性名
+            property_item['name'] = key
+            property_item["description"] = value["description"]
+            if value["type"] == "number":
+                property_item["value"] = 0
+            elif value["type"] == "boolean":
+                property_item["value"] = False
+            else:
+                property_item["value"] = ""
+            self.properties.append(property_item)
+
+        # 根据描述创建方法
+        for key, value in methods.items():
+            # "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
+            """
+            等价于
+            SetVolume = {
+                `description`: 描述,
+                `volume`: {
+                    `description`: 描述,
+                    `value`: 0
+                }
+            }
+            """
+            # setattr(self, key, {}) # 创建一个空字典, 名字是方法名
+            method = globals()[key] = {}  # 创建一个空字典, 名字是方法名
+            method["description"] = value["description"]
+            method['name'] = key
+            for k, v in value["parameters"].items():
+                # 不同的参数解析
+                method[k] = {}
+                method[k]["description"] = v["description"]
+                if v["type"] == "number":
+                    method[k]["value"] = 0
+                elif v["type"] == "boolean":
+                    method[k]["value"] = False
+                else:
+                    method[k]["value"] = ""
+
+            self.methods.append(method)
+
+
+async def handleIotDescriptors(conn, descriptors):
+    """
+    处理物联网描述
+    示例: [{
+        "name":"Speaker",
+        "description":"当前 AI 机器人的扬声器",
+        "properties":{
+            "volume":{"description":"当前音量 值","type":"number"}  可以有boolean, number, string三种类型
+        },
+        "methods":{
+            "SetVolume":{
+                "description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
+            }
+        }
+    }]
+    descriptors: 描述列表
+    """
+    for descriptor in descriptors:
+        iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
+                                       descriptor["methods"])
+        conn.iot_descriptors[descriptor["name"]] = iot_descriptor
+
+    # 暂时从配置文件中设置音量，后期通过意图识别控制音量
+    default_iot_volume = 100
+    if "iot" in conn.config:
+        default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
+    logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
+    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
+async def handleIotStatus(conn, states):
+    """
+    处理物联网状态
+    示例: [{
+        "name":"Speaker",
+        "state":{
+            "volume":100
+        }
+    }]
+    states: 状态列表
+    """
+    for state in states:
+        for key, value in conn.iot_descriptors.items():
+            if key == state["name"]:
+                for property_item in value.properties:
+                    # properties为字典列表, 记录各种属性
+                    for k, v in state["state"].items():
+                        # state为字典, 记录各种属性的值, 是需要记录的信息
+                        if property_item["name"] == k:
+                            # 检查一下属性是不是相同的
+                            if type(v) != type(property_item["value"]):
+                                logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
+                                break
+                            else:
+                                property_item["value"] = v
+                                logger.bind(tag=TAG).info(f"物联网状态更新: {key} , {property_item['name']} = {v}")
+                            break
+                break
+
+async def get_iot_status(conn, name, property_name):
+    """
+    获取物联网状态
+    name: 设备名称 "Speaker"
+    property_name: 属性名称 "volume"
+    返回值: 属性值, 实际的属性有int, bool和str三种类型
+    """
+    for key, value in conn.iot_descriptors.items():
+        if key == name:
+            for property_item in value.properties:
+                if property_item["name"] == property_name:
+                    return property_item["value"]
+    return None
+
+async def send_iot_conn(conn, name, method_name, parameters):
+    """
+    发送物联网指令
+    name: 设备名称 "Speaker"
+    method: 方法 "SetVolume"
+    parameters: 参数, 是一个字典 {"volume": 100}
+    发送示例:
+    {
+        "type": "iot",
+        "commands": [
+            {
+                "name" :  "Speaker",
+                "method": "SetVolume",
+                "parameters": {
+                    "volume": 100
+                    }
+            }
+        ]
+    }
+    """
+
+    for key, value in conn.iot_descriptors.items():
+        if key == name:
+            # 找到了设备
+            for method in value.methods:
+                # 找到了方法
+                if method["name"] == method_name:
+                    await conn.websocket.send(json.dumps({
+                        "type": "iot",
+                        "commands": [
+                            {
+                                "name": name,
+                                "method": method_name,
+                                "parameters": parameters
+                            }
+                        ]
+                    }))
+                    return
+    logger.bind(tag=TAG).error(f"未找到方法{method_name}")

main/xiaozhi-server/core/handle/musicHandler.py

@@ -0,0 +1,139 @@
+from config.logger import setup_logging
+import os
+import random
+import difflib
+import re
+import traceback
+from pathlib import Path
+import time
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils import p3
+
+TAG = __name__
+logger = setup_logging()
+
+
+def _extract_song_name(text):
+    """从用户输入中提取歌名"""
+    for keyword in ["播放音乐"]:
+        if keyword in text:
+            parts = text.split(keyword)
+            if len(parts) > 1:
+                return parts[1].strip()
+    return None
+
+
+def _find_best_match(potential_song, music_files):
+    """查找最匹配的歌曲"""
+    best_match = None
+    highest_ratio = 0
+
+    for music_file in music_files:
+        song_name = os.path.splitext(music_file)[0]
+        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
+        if ratio > highest_ratio and ratio > 0.4:
+            highest_ratio = ratio
+            best_match = music_file
+    return best_match
+
+
+class MusicManager:
+    def __init__(self, music_dir, music_ext):
+        self.music_dir = Path(music_dir)
+        self.music_ext = music_ext
+
+    def get_music_files(self):
+        music_files = []
+        for file in self.music_dir.rglob("*"):
+            # 判断是否是文件
+            if file.is_file():
+                # 获取文件扩展名
+                ext = file.suffix.lower()
+                # 判断扩展名是否在列表中
+                if ext in self.music_ext:
+                    # music_files.append(str(file.resolve()))  # 添加绝对路径
+                    # 添加相对路径
+                    music_files.append(str(file.relative_to(self.music_dir)))
+        return music_files
+
+
+class MusicHandler:
+    def __init__(self, config):
+        self.config = config
+
+        if "music" in self.config:
+            self.music_config = self.config["music"]
+            self.music_dir = os.path.abspath(
+                self.music_config.get("music_dir", "./music")  # 默认路径修改
+            )
+            self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
+            self.refresh_time = self.music_config.get("refresh_time", 60)
+        else:
+            self.music_dir = os.path.abspath("./music")
+            self.music_ext = (".mp3", ".wav", ".p3")
+            self.refresh_time = 60
+
+        # 获取音乐文件列表
+        self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
+        self.scan_time = time.time()
+        logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
+
+    async def handle_music_command(self, conn, text):
+        """处理音乐播放指令"""
+        clean_text = re.sub(r'[^\w\s]', '', text).strip()
+        logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
+
+        # 尝试匹配具体歌名
+        if os.path.exists(self.music_dir):
+            if time.time() - self.scan_time > self.refresh_time:
+                # 刷新音乐文件列表
+                self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
+                self.scan_time = time.time()
+                logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
+
+            potential_song = _extract_song_name(clean_text)
+            if potential_song:
+                best_match = _find_best_match(potential_song, self.music_files)
+                if best_match:
+                    logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
+                    await self.play_local_music(conn, specific_file=best_match)
+                    return True
+        # 检查是否是通用播放音乐命令
+        await self.play_local_music(conn)
+        return True
+
+    async def play_local_music(self, conn, specific_file=None):
+        """播放本地音乐文件"""
+        try:
+            if not os.path.exists(self.music_dir):
+                logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
+                return
+
+            # 确保路径正确性
+            if specific_file:
+                selected_music = specific_file
+                music_path = os.path.join(self.music_dir, specific_file)
+            else:
+                if not self.music_files:
+                    logger.bind(tag=TAG).error("未找到MP3音乐文件")
+                    return
+                selected_music = random.choice(self.music_files)
+                music_path = os.path.join(self.music_dir, selected_music)
+
+            if not os.path.exists(music_path):
+                logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
+                return
+            text = f"正在播放{selected_music}"
+            await send_stt_message(conn, text)
+            conn.tts_first_text_index = 0
+            conn.tts_last_text_index = 0
+            conn.llm_finish_task = True
+            if music_path.endswith(".p3"):
+                opus_packets, duration = p3.decode_opus_from_file(music_path)
+            else:
+                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
+            conn.audio_play_queue.put((opus_packets, selected_music, 0))
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
+            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -0,0 +1,74 @@
+from config.logger import setup_logging
+import time
+from core.utils.util import remove_punctuation_and_length
+from core.handle.sendAudioHandle import send_stt_message
+from core.handle.intentHandler import handle_user_intent
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleAudioMessage(conn, audio):
+    if not conn.asr_server_receive:
+        logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
+        return
+    if conn.client_listen_mode == "auto":
+        have_voice = conn.vad.is_vad(conn, audio)
+    else:
+        have_voice = conn.client_have_voice
+
+    # 如果本次没有声音，本段也没声音，就把声音丢弃了
+    if have_voice == False and conn.client_have_voice == False:
+        await no_voice_close_connect(conn)
+        conn.asr_audio.clear()
+        return
+    conn.client_no_voice_last_time = 0.0
+    conn.asr_audio.append(audio)
+    # 如果本段有声音，且已经停止了
+    if conn.client_voice_stop:
+        conn.client_abort = False
+        conn.asr_server_receive = False
+        # 音频太短了，无法识别
+        if len(conn.asr_audio) < 3:
+            conn.asr_server_receive = True
+        else:
+            text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
+            logger.bind(tag=TAG).info(f"识别文本: {text}")
+            text_len, _ = remove_punctuation_and_length(text)
+            if text_len > 0:
+                await startToChat(conn, text)
+            else:
+                conn.asr_server_receive = True
+        conn.asr_audio.clear()
+        conn.reset_vad_states()
+
+
+async def startToChat(conn, text):
+    # 首先进行意图分析
+    intent_handled = await handle_user_intent(conn, text)
+    
+    if intent_handled:
+        # 如果意图已被处理，不再进行聊天
+        conn.asr_server_receive = True
+        return
+    
+    # 意图未被处理，继续常规聊天流程
+    await send_stt_message(conn, text)
+    if conn.use_function_call_mode:
+        # 使用支持function calling的聊天方法
+        conn.executor.submit(conn.chat_with_function_calling, text)
+    else:
+        conn.executor.submit(conn.chat, text)
+
+
+async def no_voice_close_connect(conn):
+    if conn.client_no_voice_last_time == 0.0:
+        conn.client_no_voice_last_time = time.time() * 1000
+    else:
+        no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
+        close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
+        if no_voice_time > 1000 * close_connection_no_voice_time:
+            conn.client_abort = False
+            conn.asr_server_receive = False
+            prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
+            await startToChat(conn, prompt)

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -0,0 +1,136 @@
+import traceback
+
+from config.logger import setup_logging
+import json
+import asyncio
+import time
+from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
+
+TAG = __name__
+logger = setup_logging()
+
+async def sendAudioMessageStream(conn, audios_queue, text, text_index=0):
+    # 发送句子开始消息
+    if text_index == conn.tts_first_text_index:
+        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+    await send_tts_message(conn, "sentence_start", text)
+
+    # 初始化流控参数
+    frame_duration = 60  # 毫秒
+    start_time = time.time()  # 使用高精度计时器
+    play_position = 0  # 已播放的时长（毫秒）
+    time_out_stop = False
+    while True:
+        try:
+            start_get_queue = time.time()
+            # 尝试获取数据，如果没有数据，则等待一小段时间再试
+            audio_data_chunke = None
+            try:
+                audio_data_chunke = audios_queue.get(timeout=5)  # 设置超时为1秒
+            except Exception as e:
+                # 如果超时，继续等待
+                logger.bind(tag=TAG).error(f"获取队列超时～{e}")
+
+            audio_opus_datas = audio_data_chunke.get('data') if audio_data_chunke else None
+            duration = audio_data_chunke.get('duration') if audio_data_chunke else 0
+
+            if audio_data_chunke:
+                start_time = time.time()
+            # 检查是否超过 5 秒没有数据
+            if time.time() - start_time > 15:
+                logger.bind(tag=TAG).error("超过15秒没有数据，退出。")
+                break
+
+            if audio_data_chunke and audio_data_chunke.get("end", True):
+                break
+
+            if audio_opus_datas:
+                queue_duration = time.time() - start_get_queue
+                last_duration = conn.tts_duration - queue_duration
+                if last_duration <= 0:
+                    last_duration = 0
+                conn.tts_duration = duration + last_duration
+                for opus_packet in audio_opus_datas:
+                    await conn.websocket.send(opus_packet)
+                start_time = time.time()  # 更新获取数据的时间
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发生错误: {e}")
+            traceback.print_exc()  # 打印错误堆栈
+    await send_tts_message(conn, "sentence_end", text)
+
+    print(f'{text_index}-{conn.tts_last_text_index}')
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        if conn.tts_duration and conn.tts_duration > 0:
+            await asyncio.sleep(conn.tts_duration)
+        await send_tts_message(conn, 'stop', None)
+        if await isLLMWantToFinish(text):
+            await conn.close()
+
+
+
+async def sendAudioMessage(conn, audios, text, text_index=0):
+    # 发送句子开始消息
+    if text_index == conn.tts_first_text_index:
+        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+    await send_tts_message(conn, "sentence_start", text)
+
+    # 初始化流控参数
+    frame_duration = 60  # 毫秒
+    start_time = time.perf_counter()  # 使用高精度计时器
+    play_position = 0  # 已播放的时长（毫秒）
+
+    for opus_packet in audios:
+        if conn.client_abort:
+            return
+
+        # 计算当前包的预期发送时间
+        expected_time = start_time + (play_position / 1000)
+        current_time = time.perf_counter()
+
+        # 等待直到预期时间
+        delay = expected_time - current_time
+        if delay > 0:
+            await asyncio.sleep(delay)
+
+        # 发送音频包
+        await conn.websocket.send(opus_packet)
+        play_position += frame_duration  # 更新播放位置
+    await send_tts_message(conn, "sentence_end", text)
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        await send_tts_message(conn, 'stop', None)
+        if conn.close_after_chat:
+            await conn.close()
+
+async def send_tts_message(conn, state, text=None):
+    """发送 TTS 状态消息"""
+    message = {
+        "type": "tts",
+        "state": state,
+        "session_id": conn.session_id
+    }
+    if text is not None:
+        message["text"] = text
+
+    await conn.websocket.send(json.dumps(message))
+    if state == "stop":
+        conn.clearSpeakStatus()
+
+
+async def send_stt_message(conn, text):
+    """发送 STT 状态消息"""
+    stt_text = get_string_no_punctuation_or_emoji(text)
+    await conn.websocket.send(json.dumps({
+        "type": "stt",
+        "text": stt_text,
+        "session_id": conn.session_id}
+    ))
+    await conn.websocket.send(
+        json.dumps({
+            "type": "llm",
+            "text": "😊",
+            "emotion": "happy",
+            "session_id": conn.session_id}
+        ))
+    await send_tts_message(conn, "start")

main/xiaozhi-server/core/handle/textHandle.py

@@ -0,0 +1,46 @@
+from config.logger import setup_logging
+import json
+from core.handle.abortHandle import handleAbortMessage
+from core.handle.helloHandle import handleHelloMessage
+from core.handle.receiveAudioHandle import startToChat
+from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleTextMessage(conn, message):
+    """处理文本消息"""
+    logger.bind(tag=TAG).info(f"收到文本消息：{message}")
+    try:
+        msg_json = json.loads(message)
+        if isinstance(msg_json, int):
+            await conn.websocket.send(message)
+            return
+        if msg_json["type"] == "hello":
+            await handleHelloMessage(conn)
+        elif msg_json["type"] == "abort":
+            await handleAbortMessage(conn)
+        elif msg_json["type"] == "listen":
+            if "mode" in msg_json:
+                conn.client_listen_mode = msg_json["mode"]
+                logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
+            if msg_json["state"] == "start":
+                conn.client_have_voice = True
+                conn.client_voice_stop = False
+            elif msg_json["state"] == "stop":
+                conn.client_have_voice = True
+                conn.client_voice_stop = True
+            elif msg_json["state"] == "detect":
+                conn.asr_server_receive = False
+                conn.client_have_voice = False
+                conn.asr_audio.clear()
+                if "text" in msg_json:
+                    await startToChat(conn, msg_json["text"])
+        elif msg_json["type"] == "iot":
+            if "descriptors" in msg_json:
+                await handleIotDescriptors(conn, msg_json["descriptors"])
+            if "states" in msg_json:
+                await handleIotStatus(conn, msg_json["states"])  
+    except json.JSONDecodeError:
+        await conn.websocket.send(message)

main/xiaozhi-server/core/providers/asr/base.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, List
+
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProviderBase(ABC):
+    @abstractmethod
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """解码Opus数据并保存为WAV文件"""
+        pass
+
+    @abstractmethod
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        pass

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -0,0 +1,286 @@
+import time
+import io
+import wave
+import os
+from typing import Optional, Tuple, List
+import uuid
+import websockets
+import json
+import gzip
+
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+CLIENT_FULL_REQUEST = 0b0001
+CLIENT_AUDIO_ONLY_REQUEST = 0b0010
+
+NO_SEQUENCE = 0b0000
+NEG_SEQUENCE = 0b0010
+
+SERVER_FULL_RESPONSE = 0b1001
+SERVER_ACK = 0b1011
+SERVER_ERROR_RESPONSE = 0b1111
+
+NO_SERIALIZATION = 0b0000
+JSON = 0b0001
+THRIFT = 0b0011
+CUSTOM_TYPE = 0b1111
+NO_COMPRESSION = 0b0000
+GZIP = 0b0001
+CUSTOM_COMPRESSION = 0b1111
+
+
+def parse_response(res):
+    """
+    protocol_version(4 bits), header_size(4 bits),
+    message_type(4 bits), message_type_specific_flags(4 bits)
+    serialization_method(4 bits) message_compression(4 bits)
+    reserved （8bits) 保留字段
+    header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
+    payload 类似与http 请求体
+    """
+    protocol_version = res[0] >> 4
+    header_size = res[0] & 0x0f
+    message_type = res[1] >> 4
+    message_type_specific_flags = res[1] & 0x0f
+    serialization_method = res[2] >> 4
+    message_compression = res[2] & 0x0f
+    reserved = res[3]
+    header_extensions = res[4:header_size * 4]
+    payload = res[header_size * 4:]
+    result = {}
+    payload_msg = None
+    payload_size = 0
+    if message_type == SERVER_FULL_RESPONSE:
+        payload_size = int.from_bytes(payload[:4], "big", signed=True)
+        payload_msg = payload[4:]
+    elif message_type == SERVER_ACK:
+        seq = int.from_bytes(payload[:4], "big", signed=True)
+        result['seq'] = seq
+        if len(payload) >= 8:
+            payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+            payload_msg = payload[8:]
+    elif message_type == SERVER_ERROR_RESPONSE:
+        code = int.from_bytes(payload[:4], "big", signed=False)
+        result['code'] = code
+        payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+        payload_msg = payload[8:]
+    if payload_msg is None:
+        return result
+    if message_compression == GZIP:
+        payload_msg = gzip.decompress(payload_msg)
+    if serialization_method == JSON:
+        payload_msg = json.loads(str(payload_msg, "utf-8"))
+    elif serialization_method != NO_SERIALIZATION:
+        payload_msg = str(payload_msg, "utf-8")
+    result['payload_msg'] = payload_msg
+    result['payload_size'] = payload_size
+    return result
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.appid = config.get("appid")
+        self.cluster = config.get("cluster")
+        self.access_token = config.get("access_token")
+        self.output_dir = config.get("output_dir")
+
+        self.host = "openspeech.bytedance.com"
+        self.ws_url = f"wss://{self.host}/api/v2/asr"
+        self.success_code = 1000
+        self.seg_duration = 15000
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    @staticmethod
+    def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
+        """Generate protocol header."""
+        header = bytearray()
+        header_size = 1
+        header.append((0b0001 << 4) | header_size)  # Protocol version
+        header.append((message_type << 4) | message_type_specific_flags)
+        header.append((0b0001 << 4) | 0b0001)  # JSON serialization & GZIP compression
+        header.append(0x00)  # reserved
+        return header
+
+    def _construct_request(self, reqid) -> dict:
+        """Construct the request payload."""
+        return {
+            "app": {
+                "appid": f"{self.appid}",
+                "cluster": self.cluster,
+                "token": self.access_token,
+            },
+            "user": {
+                "uid": str(uuid.uuid4()),
+            },
+            "request": {
+                "reqid": reqid,
+                "show_utterances": False,
+                "sequence": 1
+            },
+            "audio": {
+                "format": "wav",
+                "rate": 16000,
+                "language": "zh-CN",
+                "bits": 16,
+                "channel": 1,
+                "codec": "raw",
+            },
+        }
+
+    async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
+        """Send request to Volcano ASR service."""
+        try:
+            auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
+            async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
+                # Prepare request data
+                request_params = self._construct_request(str(uuid.uuid4()))
+                print(request_params)
+                payload_bytes = str.encode(json.dumps(request_params))
+                payload_bytes = gzip.compress(payload_bytes)
+                full_client_request = self._generate_header()
+                full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                full_client_request.extend(payload_bytes)  # payload
+
+                # Send header and metadata
+                # full_client_request
+                await websocket.send(full_client_request)
+                res = await websocket.recv()
+                result = parse_response(res)
+                if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+                for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
+                    if last:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST,
+                            message_type_specific_flags=NEG_SEQUENCE
+                        )
+                    else:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST
+                        )
+                    payload_bytes = gzip.compress(chunk)
+                    audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                    audio_only_request.extend(payload_bytes)  # payload
+                    # Send audio data
+                    await websocket.send(audio_only_request)
+
+                # Receive response
+                response = await websocket.recv()
+                result = parse_response(response)
+
+                if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
+                    if len(result['payload_msg']['result']) > 0:
+                        return result['payload_msg']['result'][0]["text"]
+                    return None
+                else:
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
+            return None
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data
+
+    @staticmethod
+    def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
+        with io.BytesIO(data) as _f:
+            wave_fp = wave.open(_f, 'rb')
+            nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
+            wave_bytes = wave_fp.readframes(nframes)
+        return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
+
+    @staticmethod
+    def slice_data(data: bytes, chunk_size: int) -> (list, bool):
+        """
+        slice data
+        :param data: wav data
+        :param chunk_size: the segment size in one request
+        :return: segment data, last flag
+        """
+        data_len = len(data)
+        offset = 0
+        while offset + chunk_size < data_len:
+            yield data[offset: offset + chunk_size], False
+            offset += chunk_size
+        else:
+            yield data[offset: data_len], True
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        try:
+            # 合并所有opus数据包
+            pcm_data = self.decode_opus(opus_data, session_id)
+            combined_pcm_data = b''.join(pcm_data)
+
+            wav_buffer = io.BytesIO()
+
+            with wave.open(wav_buffer, "wb") as wav_file:
+                wav_file.setnchannels(1)  # 设置声道数
+                wav_file.setsampwidth(2)  # 设置采样宽度
+                wav_file.setframerate(16000)  # 设置采样率
+                wav_file.writeframes(combined_pcm_data)  # 写入 PCM 数据
+
+            # 获取封装后的 WAV 数据
+            wav_data = wav_buffer.getvalue()
+            nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
+            size_per_sec = nchannels * sampwidth * framerate
+            segment_size = int(size_per_sec * self.seg_duration / 1000)
+
+            # 语音识别
+            start_time = time.time()
+            text = await self._send_request(wav_data, segment_size)
+            if text:
+                logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+                return text, None
+            return "", None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -0,0 +1,110 @@
+import time
+import wave
+import os
+import sys
+import io
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+import uuid
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+from funasr import AutoModel
+from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+TAG = __name__
+logger = setup_logging()
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")  # 修正配置键名
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+        with CaptureOutput():
+            self.model = AutoModel(
+                model=self.model_dir,
+                vad_kwargs={"max_single_segment_time": 30000},
+                disable_update=True,
+                hub="hf"
+                # device="cuda:0",  # 启用GPU加速
+            )
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        file_path = None
+        try:
+            # 保存音频文件
+            start_time = time.time()
+            file_path = self.save_audio_to_file(opus_data, session_id)
+            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+
+            # 语音识别
+            start_time = time.time()
+            result = self.model.generate(
+                input=file_path,
+                cache={},
+                language="auto",
+                use_itn=True,
+                batch_size_s=60,
+            )
+            text = rich_transcription_postprocess(result[0]["text"])
+            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+
+            return text, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None
+
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/intent/base.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class IntentProviderBase(ABC):
+    def __init__(self, config):
+        self.config = config
+        self.intent_options = config.get("intent_options", {
+            "continue_chat": "继续聊天",
+            "end_chat": "结束聊天",
+            "play_music": "播放音乐"
+        })
+
+    def set_llm(self, llm):
+        self.llm = llm
+        logger.bind(tag=TAG).debug("Set LLM for intent provider")
+        
+    @abstractmethod
+    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+        """
+        检测用户最后一句话的意图
+        Args:
+            dialogue_history: 对话历史记录列表，每条记录包含role和content
+        Returns:
+            返回识别出的意图，格式为:
+            - "继续聊天"
+            - "结束聊天" 
+            - "播放音乐 歌名" 或 "随机播放音乐"
+        """
+        pass

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -0,0 +1,61 @@
+from typing import List, Dict
+from ..base import IntentProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+
+class IntentProvider(IntentProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.llm = None
+        self.promot = self.get_intent_system_prompt()
+
+    def get_intent_system_prompt(self) -> str:
+        """
+        根据配置的意图选项动态生成系统提示词
+        Returns:
+            格式化后的系统提示词
+        """
+        intent_list = []
+        for key, value in self.intent_options.items():
+            if key == "play_music":
+                intent_list.append(f"{value} [歌名]")
+            else:
+                intent_list.append(value)
+                
+        prompt = (
+            "你是一个意图识别助手。你需要根据和用户的对话记录，重点分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
+            f"{', '.join(intent_list)}\n"
+            "如果是唱歌、听歌、播放音乐，请指定歌名，格式为'播放音乐 [识别出的歌名]'。\n"
+            "如果听不出具体歌名，可以返回'随机播放音乐'。\n"
+            "只需要返回意图结果的json，不要解释。"
+            "返回格式如下：\n"
+            "{intent: '用户意图'}"
+        )
+        return prompt
+    
+    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+        if not self.llm:
+            raise ValueError("LLM provider not set")
+
+        # 构建用户最后一句话的提示
+        msgStr = ""
+        for msg in dialogue_history:
+            if msg.role == "user":
+                msgStr += f"User: {msg.content}\n"
+            elif msg.role== "assistant":
+                msgStr += f"Assistant: {msg.content}\n"
+
+        user_prompt = f"请分析用户的意图：\n{msgStr}"
+        
+        # 使用LLM进行意图识别
+        intent = self.llm.response_no_stream(
+            system_prompt=self.promot,
+            user_prompt=user_prompt
+        )
+  
+        logger.bind(tag=TAG).info(f"Detected intent: {intent}")
+        return intent.strip()

main/xiaozhi-server/core/providers/intent/nointent/nointent.py

@@ -0,0 +1,18 @@
+from ..base import IntentProviderBase
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class IntentProvider(IntentProviderBase):
+    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+        """
+        默认的意图识别实现，始终返回继续聊天
+        Args:
+            dialogue_history: 对话历史记录列表
+        Returns:
+            固定返回"继续聊天"
+        """
+        logger.bind(tag=TAG).debug("Using NoIntentProvider, always returning continue chat")
+        return self.intent_options["continue_chat"]

main/xiaozhi-server/core/providers/llm/base.py

@@ -0,0 +1,38 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProviderBase(ABC):
+    @abstractmethod
+    def response(self, session_id, dialogue):
+        """LLM response generator"""
+        pass
+
+    def response_no_stream(self, system_prompt, user_prompt):
+        try:
+            # 构造对话格式
+            dialogue = [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt}
+            ]
+            result = ""
+            for part in self.response("", dialogue):
+                result += part
+            return result
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
+            return "【LLM服务响应异常】"
+    
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        """
+        Default implementation for function calling (streaming)
+        This should be overridden by providers that support function calls
+        
+        Returns: generator that yields either text tokens or a special function call token
+        """
+        # For providers that don't support functions, just return regular response
+        for token in self.response(session_id, dialogue):
+            yield {"type": "content", "content": token}

main/xiaozhi-server/core/providers/llm/coze/coze.py

@@ -0,0 +1,37 @@
+from config.logger import setup_logging
+import requests
+import json
+import re
+from core.providers.llm.base import LLMProviderBase
+import os
+# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
+from cozepy import COZE_CN_BASE_URL
+from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.personal_access_token = config.get("personal_access_token")
+        self.bot_id = config.get("bot_id")
+        self.user_id = config.get("user_id")
+
+    def response(self, session_id, dialogue):
+        coze_api_token = self.personal_access_token
+        coze_api_base = COZE_CN_BASE_URL
+
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+        coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
+
+        for event in coze.chat.stream(
+            bot_id=self.bot_id,
+            user_id=self.user_id,
+            additional_messages=[
+                Message.build_user_question_text(last_msg["content"]),
+            ],
+        ):
+            if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
+                print(event.message.content, end="", flush=True)
+                yield event.message.content

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -0,0 +1,39 @@
+import json
+from config.logger import setup_logging
+import requests
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
+
+    def response(self, session_id, dialogue):
+        try:
+            # 取最后一条用户消息
+            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+            # 发起流式请求
+            with requests.post(
+                    f"{self.base_url}/chat-messages",
+                    headers={"Authorization": f"Bearer {self.api_key}"},
+                    json={
+                        "query": last_msg["content"],
+                        "response_mode": "streaming",
+                        "user": session_id,
+                        "inputs": {}
+                    },
+                    stream=True
+            ) as r:
+                for line in r.iter_lines():
+                    if line.startswith(b'data: '):
+                        event = json.loads(line[6:])
+                        if event.get('answer'):
+                            yield event['answer']
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+            yield "【服务响应异常】"

main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py

@@ -0,0 +1,65 @@
+import json
+from config.logger import setup_logging
+import requests
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.base_url = config.get("base_url")
+        self.detail = config.get("detail", False)
+        self.variables = config.get("variables", {})
+
+    def response(self, session_id, dialogue):
+        try:
+            # 取最后一条用户消息
+            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+            # 发起流式请求
+            with requests.post(
+                    f"{self.base_url}/chat/completions",
+                    headers={"Authorization": f"Bearer {self.api_key}"},
+                    json={
+                        "stream": True,
+                        "chatId": session_id,
+                        "detail": self.detail,
+                        "variables": self.variables,
+                        "messages": [
+                            {
+                                "role": "user",
+                                "content": last_msg["content"]
+                            }
+                        ]
+                    },
+                    stream=True
+            ) as r:
+                for line in r.iter_lines():
+                    if line:
+                        try:
+                            if line.startswith(b'data: '):
+                                if line[6:].decode('utf-8') == '[DONE]':
+                                    break
+
+                                data = json.loads(line[6:])
+                                if 'choices' in data and len(data['choices']) > 0:
+                                    delta = data['choices'][0].get('delta', {})
+                                    if delta and 'content' in delta and delta['content'] is not None:
+                                        content = delta['content']
+                                        if '<think>' in content:
+                                            continue
+                                        if '</think>' in content:
+                                            continue
+                                        yield content
+
+                        except json.JSONDecodeError as e:
+                            continue
+                        except Exception as e:
+                            continue
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+            yield "【服务响应异常】"

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -0,0 +1,80 @@
+import google.generativeai as genai
+from core.utils.util import check_model_key
+from core.providers.llm.base import LLMProviderBase
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        """初始化Gemini LLM Provider"""
+        self.model_name = config.get("model_name", "gemini-1.5-pro")
+        self.api_key = config.get("api_key")
+
+        have_key = check_model_key("LLM", self.api_key)
+
+        if not have_key:
+            return
+
+        try:
+            # 初始化Gemini客户端
+            genai.configure(api_key=self.api_key)
+            self.model = genai.GenerativeModel(self.model_name)
+
+            # 设置生成参数
+            self.generation_config = {
+                "temperature": 0.7,
+                "top_p": 0.9,
+                "top_k": 40,
+                "max_output_tokens": 2048,
+            }
+            self.chat = None
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
+            self.model = None
+
+    def response(self, session_id, dialogue):
+        """生成Gemini对话响应"""
+        if not self.model:
+            yield "【Gemini服务未正确初始化】"
+            return
+
+        try:
+            # 处理对话历史
+            chat_history = []
+            for msg in dialogue[:-1]:  # 历史对话
+                role = "model" if msg["role"] == "assistant" else "user"
+                content = msg["content"].strip()
+                if content:
+                    chat_history.append({
+                        "role": role,
+                        "parts": [content]
+                    })
+
+            # 获取当前消息
+            current_msg = dialogue[-1]["content"]
+
+            # 创建新的聊天会话
+            chat = self.model.start_chat(history=chat_history)
+
+            # 发送消息并获取流式响应
+            response = chat.send_message(
+                current_msg,
+                stream=True,
+                generation_config=self.generation_config
+            )
+
+            # 处理流式响应
+            for chunk in response:
+                if hasattr(chunk, 'text') and chunk.text:
+                    yield chunk.text
+
+        except Exception as e:
+            error_msg = str(e)
+            logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
+
+            # 针对不同错误返回友好提示
+            if "Rate limit" in error_msg:
+                yield "【Gemini服务请求太频繁,请稍后再试】"
+            elif "Invalid API key" in error_msg:
+                yield "【Gemini API key无效】"
+            else:
+                yield f"【Gemini服务响应异常: {error_msg}】"

main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py

@@ -0,0 +1,62 @@
+import requests
+from requests.exceptions import RequestException
+from config.logger import setup_logging
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.agent_id = config.get("agent_id")  # 对应 agent_id
+        self.api_key = config.get("api_key")
+        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
+        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
+
+    def response(self, session_id, dialogue):
+        print(dialogue)
+        try:
+            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
+
+            # 提取最后一个 role 为 'user' 的 content
+            input_text = None
+            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
+                # 逆序遍历，找到最后一个 role 为 'user' 的消息
+                for message in reversed(dialogue):
+                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
+                        input_text = message.get("content", "")
+                        break  # 找到后立即退出循环
+
+            # 构造请求数据
+            payload = {
+                "text": input_text,
+                "agent_id": self.agent_id,
+                "conversation_id": session_id  # 使用 session_id 作为 conversation_id
+            }
+            # 设置请求头
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json"
+            }
+
+            # 发起 POST 请求
+            response = requests.post(self.api_url, json=payload, headers=headers)
+
+            # 检查请求是否成功
+            response.raise_for_status()
+
+            # 解析返回数据
+            data = response.json()
+            speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
+
+            # 返回生成的内容
+            if speech:
+                yield speech
+            else:
+                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
+
+        except RequestException as e:
+            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -0,0 +1,81 @@
+from config.logger import setup_logging
+from openai import OpenAI
+import json
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.base_url = config.get("base_url", "http://localhost:11434")
+        # Initialize OpenAI client with Ollama base URL
+        #如果没有v1，增加v1
+        if not self.base_url.endswith("/v1"):
+            self.base_url = f"{self.base_url}/v1"
+            
+        self.client = OpenAI(
+            base_url=self.base_url,
+            api_key="ollama"  # Ollama doesn't need an API key but OpenAI client requires one
+        )
+
+    def response(self, session_id, dialogue):
+        try:
+            responses = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True
+            )
+            
+            for chunk in responses:
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else ''
+                    if content:
+                        yield content
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
+            yield "【Ollama服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        try:
+            stream = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True,
+                tools=functions,
+            )
+            
+            current_function_call = None
+            current_content = ""
+            
+            for chunk in stream:
+                delta = chunk.choices[0].delta
+                
+                if delta.content:
+                    current_content += delta.content
+                    yield {"type": "content", "content": delta.content}
+                
+                if delta.tool_calls:
+                    tool_call = delta.tool_calls[0]
+                    # Handle the function call data using proper attribute access
+                    if not current_function_call:
+                        current_function_call = {
+                            "function": {
+                                "name": tool_call.function.name,
+                                "arguments": tool_call.function.arguments
+                            }
+                        }
+            
+            if current_function_call:
+                logger.bind(tag=TAG).debug(f"ollama Function call detected: {current_function_call}")
+                yield {"type": "function_call", "function_call": current_function_call}
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
+            yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -0,0 +1,83 @@
+import openai
+from core.utils.util import check_model_key
+from core.providers.llm.base import LLMProviderBase
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.api_key = config.get("api_key")
+        if 'base_url' in config:
+            self.base_url = config.get("base_url")
+        else:
+            self.base_url = config.get("url")
+        check_model_key("LLM", self.api_key)
+        self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
+
+    def response(self, session_id, dialogue):
+        try:
+            responses = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True
+            )
+            
+            is_active = True
+            for chunk in responses:
+                try:
+                    # 检查是否存在有效的choice且content不为空
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else ''
+                except IndexError:
+                    content = ''
+                if content:
+                    # 处理标签跨多个chunk的情况
+                    if '<think>' in content:
+                        is_active = False
+                        content = content.split('<think>')[0]
+                    if '</think>' in content:
+                        is_active = True
+                        content = content.split('</think>')[-1]
+                    if is_active:
+                        yield content
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        try:
+            stream = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True,
+                tools=functions,
+            )
+            
+            current_function_call = None
+            current_content = ""
+            
+            for chunk in stream:
+                delta = chunk.choices[0].delta
+                
+                if delta.content:
+                    current_content += delta.content
+                    yield {"type": "content", "content": delta.content}
+                
+                if delta.tool_calls:
+                    tool_call = delta.tool_calls[0]
+                    # Handle the function call data using proper attribute access
+                    if not current_function_call:
+                        current_function_call = {
+                            "function": {
+                                "name": tool_call.function.name,
+                                "arguments": tool_call.function.arguments
+                            }
+                        }
+            
+            if current_function_call:
+                logger.bind(tag=TAG).debug(f"openai Function call detected: {current_function_call}")
+                yield {"type": "function_call", "function_call": current_function_call}
+                
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
+            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}

main/xiaozhi-server/core/providers/memory/base.py

@@ -0,0 +1,25 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class MemoryProviderBase(ABC):
+    def __init__(self, config):
+        self.config = config
+        self.role_id = None
+        self.llm = None
+
+    @abstractmethod
+    async def save_memory(self, msgs):
+        """Save a new memory for specific role and return memory ID"""
+        print("this is base func", msgs)
+
+    @abstractmethod
+    async def query_memory(self, query: str) -> str:
+        """Query memories for specific role based on similarity"""
+        return "please implement query method"
+
+    def init_memory(self, role_id, llm):
+        self.role_id = role_id    
+        self.llm = llm

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -0,0 +1,83 @@
+import traceback
+
+from ..base import MemoryProviderBase, logger
+from mem0 import MemoryClient
+from core.utils.util import check_model_key
+
+TAG = __name__
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.api_key = config.get("api_key", "")
+        self.api_version = config.get("api_version", "v1.1")
+        have_key = check_model_key("Mem0ai", self.api_key)
+        if not have_key :
+            self.use_mem0 = False
+            return
+        else:
+            self.use_mem0 = True
+        try:
+            self.client = MemoryClient(api_key=self.api_key)
+            logger.bind(tag=TAG).info("成功连接到 Mem0ai 服务")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"连接到 Mem0ai 服务时发生错误: {str(e)}")
+            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
+            self.use_mem0 = False
+
+    async def save_memory(self, msgs):
+        if not self.use_mem0:
+            return None
+        if len(msgs) < 2:
+            return None
+        
+        try:
+            # Format the content as a message list for mem0
+            messages = [
+                {"role": message.role, "content": message.content}
+                for message in msgs if message.role != "system"
+            ]
+            result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
+            logger.bind(tag=TAG).debug(f"Save memory result: {result}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
+            return None
+
+    async def query_memory(self, query: str)-> str:
+        if not self.use_mem0:
+            return ""
+        try:
+            results = self.client.search(
+                query,
+                user_id=self.role_id,
+                output_format=self.api_version
+            )
+            if not results or 'results' not in results:
+                return ""
+                
+            # Format each memory entry with its update time up to minutes
+            memories = []
+            for entry in results['results']:
+                timestamp = entry.get('updated_at', '')
+                if timestamp:
+                    try:
+                        # Parse and reformat the timestamp
+                        dt = timestamp.split('.')[0]  # Remove milliseconds
+                        formatted_time = dt.replace('T', ' ')
+                    except:
+                        formatted_time = timestamp
+                memory = entry.get('memory', '')
+                if timestamp and memory:
+                    # Store tuple of (timestamp, formatted_string) for sorting
+                    memories.append((timestamp, f"[{formatted_time}] {memory}"))
+            
+            # Sort by timestamp in descending order (newest first)
+            memories.sort(key=lambda x: x[0], reverse=True)
+            
+            # Extract only the formatted strings
+            memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
+            logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
+            return memories_str
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
+            return ""

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -0,0 +1,156 @@
+from ..base import MemoryProviderBase, logger
+import time
+import json
+import os
+import yaml
+from core.utils.util import get_project_dir
+
+short_term_memory_prompt = """
+# 时空记忆编织者
+
+## 核心使命
+构建可生长的动态记忆网络，在有限空间内保留关键信息的同时，智能维护信息演变轨迹
+根据对话记录，总结user的重要信息，以便在未来的对话中提供更个性化的服务
+
+## 记忆法则
+### 1. 三维度记忆评估（每次更新必执行）
+| 维度       | 评估标准                  | 权重分 |
+|------------|---------------------------|--------|
+| 时效性     | 信息新鲜度（按对话轮次） | 40%    |
+| 情感强度   | 含💖标记/重复提及次数     | 35%    |
+| 关联密度   | 与其他信息的连接数量      | 25%    |
+
+### 2. 动态更新机制
+**名字变更处理示例：**
+原始记忆："曾用名": ["张三"], "现用名": "张三丰"
+触发条件：当检测到「我叫X」「称呼我Y」等命名信号时
+操作流程：
+1. 将旧名移入"曾用名"列表
+2. 记录命名时间轴："2024-02-15 14:32:启用张三丰"
+3. 在记忆立方追加：「从张三到张三丰的身份蜕变」
+
+### 3. 空间优化策略
+- **信息压缩术**：用符号体系提升密度
+  - ✅"张三丰[北/软工/🐱]"
+  - ❌"北京软件工程师，养猫"
+- **淘汰预警**：当总字数≥900时触发
+  1. 删除权重分<60且3轮未提及的信息
+  2. 合并相似条目（保留时间戳最近的）
+
+## 记忆结构
+输出格式必须为可解析的json字符串，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
+```json
+{
+  "时空档案": {
+    "身份图谱": {
+      "现用名": "",
+      "特征标记": [] 
+    },
+    "记忆立方": [
+      {
+        "事件": "入职新公司",
+        "时间戳": "2024-03-20",
+        "情感值": 0.9,
+        "关联项": ["下午茶"],
+        "保鲜期": 30 
+      }
+    ]
+  },
+  "关系网络": {
+    "高频话题": {"职场": 12},
+    "暗线联系": [""]
+  },
+  "待响应": {
+    "紧急事项": ["需立即处理的任务"], 
+    "潜在关怀": ["可主动提供的帮助"]
+  },
+  "高光语录": [
+    "最打动人心的瞬间，强烈的情感表达，user的原话"
+  ]
+}
+```
+"""
+
+def extract_json_data(json_code):
+    start = json_code.find("```json")
+    # 从start开始找到下一个```结束
+    end = json_code.find("```", start+1)
+    #print("start:", start, "end:", end)
+    if start == -1 or end == -1:
+        try:
+            jsonData = json.loads(json_code)
+            return json_code
+        except Exception as e:
+            print("Error:", e)
+        return ""
+    jsonData = json_code[start+7:end]
+    return jsonData
+
+TAG = __name__
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.short_momery = ""
+        self.memory_path = get_project_dir() + 'data/.memory.yaml'
+        self.load_memory()
+
+    def init_memory(self, role_id, llm):
+        super().init_memory(role_id, llm)
+        self.load_memory()
+    
+    def load_memory(self):
+        all_memory = {}
+        if os.path.exists(self.memory_path):
+            with open(self.memory_path, 'r', encoding='utf-8') as f:
+                all_memory = yaml.safe_load(f) or {}
+        if self.role_id in all_memory:
+            self.short_momery = all_memory[self.role_id]
+    
+    def save_memory_to_file(self):
+        all_memory = {}
+        if os.path.exists(self.memory_path):
+              with open(self.memory_path, 'r', encoding='utf-8') as f:
+                  all_memory = yaml.safe_load(f) or {}
+        all_memory[self.role_id] = self.short_momery
+        with open(self.memory_path, 'w', encoding='utf-8') as f:
+            yaml.dump(all_memory, f, allow_unicode=True)
+        
+    async def save_memory(self, msgs):
+        if self.llm is None:
+            logger.bind(tag=TAG).error("LLM is not set for memory provider")
+            return None
+        
+        if len(msgs) < 2:
+            return None
+        
+        msgStr = ""
+        for msg in msgs:
+            if msg.role == "user":
+                msgStr += f"User: {msg.content}\n"
+            elif msg.role== "assistant":
+                msgStr += f"Assistant: {msg.content}\n"
+        if len(self.short_momery) > 0:
+            msgStr+="历史记忆：\n"
+            msgStr+=self.short_momery
+        
+        #当前时间
+        time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        msgStr += f"当前时间：{time_str}"
+
+        result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
+ 
+        json_str = extract_json_data(result)
+        try:
+            json_data = json.loads(json_str) # 检查json格式是否正确
+            self.short_momery = json_str
+        except Exception as e:
+            print("Error:", e)
+        
+        self.save_memory_to_file()
+        logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
+
+        return self.short_momery
+    
+    async def query_memory(self, query: str)-> str:
+        return self.short_momery

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -0,0 +1,18 @@
+'''
+不使用记忆，可以选择此模块
+'''
+from ..base import MemoryProviderBase, logger
+
+TAG = __name__
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+      
+    async def save_memory(self, msgs):
+        logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
+        return None
+
+    async def query_memory(self, query: str)-> str:
+        logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.")
+        return ""

main/xiaozhi-server/core/providers/tts/aliyun.py

@@ -0,0 +1,132 @@
+import os
+import uuid
+import json
+import hmac
+import hashlib
+import base64
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+import http.client
+import urllib.parse
+import time
+import uuid
+from urllib import parse
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {'AccessKeyId': access_key_id,
+                      'Action': 'CreateToken',
+                      'Format': 'JSON',
+                      'RegionId': 'cn-shanghai',
+                      'SignatureMethod': 'HMAC-SHA1',
+                      'SignatureNonce': str(uuid.uuid1()),
+                      'SignatureVersion': '1.0',
+                      'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                      'Version': '2019-02-28'}
+        # 构造规范化的请求字符串
+        query_string = AccessToken._encode_dict(parameters)
+        print('规范化的请求字符串: %s' % query_string)
+        # 构造待签名字符串
+        string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string)
+        print('待签名的字符串: %s' % string_to_sign)
+        # 计算签名
+        secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'),
+                                   bytes(string_to_sign, encoding='utf-8'),
+                                   hashlib.sha1).digest()
+        signature = base64.b64encode(secreted_string)
+        print('签名: %s' % signature)
+        # 进行URL编码
+        signature = AccessToken._encode_text(signature)
+        print('URL编码后的签名: %s' % signature)
+        # 调用服务
+        full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string)
+        print('url: %s' % full_url)
+        # 提交HTTP GET请求
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            key = 'Token'
+            if key in root_obj:
+                token = root_obj[key]['Id']
+                expire_time = root_obj[key]['ExpireTime']
+                return token, expire_time
+        print(response.text)
+        return None, None
+
+
+class TTSProvider(TTSProviderBase):
+
+
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        
+        # 新增空值判断逻辑
+        access_key_id = config.get("access_key_id")
+        access_key_secret = config.get("access_key_secret")
+        if access_key_id and access_key_secret:
+            # 使用密钥对生成临时token
+            token, expire_time = AccessToken.create_token(access_key_id, access_key_secret)
+        else:
+            # 直接使用预生成的长期token
+            token = config.get("token")
+            expire_time = None
+        
+        print('token: %s, expire time(s): %s' % (token, expire_time))
+
+        self.appkey = config.get("appkey")
+        self.token = token
+        self.format = config.get("format", "wav")
+        self.sample_rate = config.get("sample_rate", 16000)
+        self.voice = config.get("voice", "xiaoyun")
+        self.volume = config.get("volume", 50)
+        self.speech_rate = config.get("speech_rate", 0)
+        self.pitch_rate = config.get("pitch_rate", 0)
+
+        self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
+        self.api_url = f"https://{self.host}/stream/v1/tts"
+        self.header = {
+            "Content-Type": "application/json"
+        }
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "appkey": self.appkey,
+            "token": self.token,
+            "text": text,
+            "format": self.format,
+            "sample_rate": self.sample_rate,
+            "voice": self.voice,
+            "volume": self.volume,
+            "speech_rate": self.speech_rate,
+            "pitch_rate": self.pitch_rate
+        }
+
+        print(self.api_url, json.dumps(request_json, ensure_ascii=False))
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
+            if resp.headers['Content-Type'].startswith('audio/'):
+                with open(output_file, 'wb') as f:
+                    f.write(resp.content)
+                return output_file
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/base.py

@@ -0,0 +1,123 @@
+import asyncio
+from config.logger import setup_logging
+import os
+import numpy as np
+import opuslib_next
+from pydub import AudioSegment
+from abc import ABC, abstractmethod
+import queue
+
+TAG = __name__
+logger = setup_logging()
+
+
+class TTSProviderBase(ABC):
+    def __init__(self, config, delete_audio_file):
+        self.delete_audio_file = delete_audio_file
+        self.output_file = config.get("output_file")
+
+    @abstractmethod
+    def generate_filename(self):
+        pass
+
+    def to_tts(self, text):
+        tmp_file = self.generate_filename()
+        try:
+            max_repeat_time = 5
+            while not os.path.exists(tmp_file) and max_repeat_time > 0:
+                asyncio.run(self.text_to_speak(text, tmp_file))
+                if not os.path.exists(tmp_file):
+                    max_repeat_time = max_repeat_time - 1
+                    logger.bind(tag=TAG).error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次")
+
+            if max_repeat_time > 0:
+                logger.bind(tag=TAG).info(f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次")
+
+            return tmp_file
+        except Exception as e:
+            logger.bind(tag=TAG).info(f": {e}")
+            return None
+
+    def to_tts_stream(self, text, queue: queue.Queue, text_index=0):
+        try:
+            asyncio.run(self.text_to_speak_stream(text, queue, text_index))
+        except Exception as e:
+            logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
+            return None
+
+    @abstractmethod
+    async def text_to_speak(self, text, output_file):
+        pass
+
+    async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
+        raise Exception("该TTS还没有实现stream模式")
+
+    def wav_to_opus_data(self, wav_file_path):
+        # 使用pydub加载PCM文件
+        # 获取文件后缀名
+        file_type = os.path.splitext(wav_file_path)[1]
+        if file_type:
+            file_type = file_type.lstrip('.')
+        audio = AudioSegment.from_file(wav_file_path, format=file_type)
+
+        duration = len(audio) / 1000.0
+
+        # 转换为单声道和16kHz采样率（确保与编码器匹配）
+        audio = audio.set_channels(1).set_frame_rate(16000)
+
+        # 获取原始PCM数据（16位小端）
+        raw_data = audio.raw_data
+
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+        opus_datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i:i + frame_size * 2]
+
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                chunk += b'\x00' * (frame_size * 2 - len(chunk))
+
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+
+            # 编码Opus数据
+            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
+            opus_datas.append(opus_data)
+
+        return opus_datas, duration
+
+    def wav_to_opus_data_audio_raw(self, raw_data):
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+        opus_datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i:i + frame_size * 2]
+
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                # logger.bind(tag=TAG).info("开始补0")
+                chunk += b'\x00' * (frame_size * 2 - len(chunk))
+
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+
+            # 编码Opus数据
+            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
+            opus_datas.append(opus_data)
+
+        return opus_datas

main/xiaozhi-server/core/providers/tts/cozecn.py

@@ -0,0 +1,38 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.model = config.get("model")
+        self.access_token = config.get("access_token")
+        self.voice = config.get("voice")
+        self.response_format = config.get("response_format")
+
+        self.host = "api.coze.cn"
+        self.api_url = f"https://{self.host}/v1/audio/speech"
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "input": text,
+            "voice_id": self.voice,
+            "response_format": self.response_format,
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
+        data = response.content
+        file_to_save = open(output_file, "wb")
+        file_to_save.write(data)

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -0,0 +1,62 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.appid = config.get("appid")
+        self.access_token = config.get("access_token")
+        self.cluster = config.get("cluster")
+        self.voice = config.get("voice")
+        self.api_url = config.get("api_url")
+        self.authorization = config.get("authorization")
+        self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
+        check_model_key("TTS", self.access_token)
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "app": {
+                "appid": f"{self.appid}",
+                "token": "access_token",
+                "cluster": self.cluster
+            },
+            "user": {
+                "uid": "1"
+            },
+            "audio": {
+                "voice_type": self.voice,
+                "encoding": "wav",
+                "speed_ratio": 1.0,
+                "volume_ratio": 1.0,
+                "pitch_ratio": 1.0,
+            },
+            "request": {
+                "reqid": str(uuid.uuid4()),
+                "text": text,
+                "text_type": "plain",
+                "operation": "query",
+                "with_frontend": 1,
+                "frontend_type": "unitTson"
+            }
+        }
+
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            if "data" in resp.json():
+                data = resp.json()["data"]
+                file_to_save = open(output_file, "wb")
+                file_to_save.write(base64.b64decode(data))
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/edge.py

@@ -0,0 +1,18 @@
+import os
+import uuid
+import edge_tts
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.voice = config.get("voice")
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        communicate = edge_tts.Communicate(text, voice=self.voice)  # Use your preferred voice
+        await communicate.save(output_file)

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -0,0 +1,268 @@
+import base64
+import os
+import traceback
+import uuid
+import queue
+import io
+
+import numpy as np
+import requests
+import ormsgpack
+from pathlib import Path
+
+import torch
+import torchaudio
+from pydantic import BaseModel, Field, conint, model_validator
+from pydub import AudioSegment
+from typing_extensions import Annotated
+from datetime import datetime
+from typing import Literal
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ServeReferenceAudio(BaseModel):
+    audio: bytes
+    text: str
+
+    @model_validator(mode="before")
+    def decode_audio(cls, values):
+        audio = values.get("audio")
+        if (
+                isinstance(audio, str) and len(audio) > 255
+        ):  # Check if audio is a string (Base64)
+            try:
+                values["audio"] = base64.b64decode(audio)
+            except Exception as e:
+                # If the audio is not a valid base64 string, we will just ignore it and let the server handle it
+                pass
+        return values
+
+    def __repr__(self) -> str:
+        return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
+
+
+class ServeTTSRequest(BaseModel):
+    text: str
+    chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
+    # Audio format
+    format: Literal["wav", "pcm", "mp3"] = "wav"
+    # References audios for in-context learning
+    references: list[ServeReferenceAudio] = []
+    # Reference id
+    # For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
+    # Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
+    reference_id: str | None = None
+    seed: int | None = None
+    use_memory_cache: Literal["on", "off"] = "off"
+    # Normalize text for en & zh, this increase stability for numbers
+    normalize: bool = True
+    # not usually used below
+    streaming: bool = False
+    max_new_tokens: int = 1024
+    top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+    repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
+    temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+
+    class Config:
+        # Allow arbitrary types for pytorch related types
+        arbitrary_types_allowed = True
+
+
+def audio_to_bytes(file_path):
+    if not file_path or not Path(file_path).exists():
+        return None
+    with open(file_path, "rb") as wav_file:
+        wav = wav_file.read()
+    return wav
+
+
+def read_ref_text(ref_text):
+    path = Path(ref_text)
+    if path.exists() and path.is_file():
+        with path.open("r", encoding="utf-8") as file:
+            return file.read()
+    return ref_text
+
+
+class TTSProvider(TTSProviderBase):
+
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        self.reference_id = config.get("reference_id")
+        self.reference_audio = config.get("reference_audio", [])
+        self.reference_text = config.get("reference_text", [])
+        self.format = config.get("format", "wav")
+        self.channels = config.get("channels", 1)
+        self.rate = config.get("rate", 44100)
+        self.api_key = config.get("api_key", "YOUR_API_KEY")
+        have_key = check_model_key("FishSpeech TTS", self.api_key)
+        if not have_key:
+            return
+        self.normalize = config.get("normalize", True)
+        self.max_new_tokens = config.get("max_new_tokens", 1024)
+        self.chunk_length = config.get("chunk_length", 200)
+        self.top_p = config.get("top_p", 0.7)
+        self.repetition_penalty = config.get("repetition_penalty", 1.2)
+        self.temperature = config.get("temperature", 0.7)
+        self.streaming = config.get("streaming", False)
+        self.use_memory_cache = config.get("use_memory_cache", "on")
+        self.seed = config.get("seed")
+        self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        # Prepare reference data
+        byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+        ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+
+        data = {
+            "text": text,
+            "references": [
+                ServeReferenceAudio(
+                    audio=audio if audio else b"", text=text
+                )
+                for text, audio in zip(ref_texts, byte_audios)
+            ],
+            "reference_id": self.reference_id,
+            "normalize": self.normalize,
+            "format": self.format,
+            "max_new_tokens": self.max_new_tokens,
+            "chunk_length": self.chunk_length,
+            "top_p": self.top_p,
+            "repetition_penalty": self.repetition_penalty,
+            "temperature": self.temperature,
+            "streaming": self.streaming,
+            "use_memory_cache": self.use_memory_cache,
+            "seed": self.seed,
+        }
+
+        pydantic_data = ServeTTSRequest(**data)
+
+        response = requests.post(
+            self.api_url,
+            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/msgpack",
+            },
+        )
+
+        if response.status_code == 200:
+            audio_content = response.content
+
+            with open(output_file, "wb") as audio_file:
+                audio_file.write(audio_content)
+
+
+
+        else:
+            print(f"Request failed with status code {response.status_code}")
+            print(response.json())
+
+    def _get_audio_from_tts(self, data_bytes):
+        tts_speech = torch.from_numpy(np.array(np.frombuffer(data_bytes, dtype=np.int16))).unsqueeze(dim=0)
+        with io.BytesIO() as bf:
+            torchaudio.save(bf, tts_speech, 44100, format="wav")
+            audio = AudioSegment.from_file(bf, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        return audio
+
+    async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
+        try:
+            # Prepare reference data
+            byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+            ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+
+            data = {
+                "text": text,
+                "references": [
+                    ServeReferenceAudio(
+                        audio=audio if audio else b"", text=text
+                    )
+                    for text, audio in zip(ref_texts, byte_audios)
+                ],
+                "reference_id": self.reference_id,
+                "normalize": self.normalize,
+                "format": self.format,
+                "max_new_tokens": self.max_new_tokens,
+                "chunk_length": self.chunk_length,
+                "top_p": self.top_p,
+                "repetition_penalty": self.repetition_penalty,
+                "temperature": self.temperature,
+                "streaming": self.streaming,
+                "use_memory_cache": self.use_memory_cache,
+                "seed": self.seed,
+            }
+
+            pydantic_data = ServeTTSRequest(**data)
+            audio_buff = None
+            chunk_total = b''
+            last_raw = b''
+            audio_raw = b''
+            print("请求tts")
+            with requests.post(
+                    self.api_url,
+                    data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+                    headers={
+                        "Authorization": f"Bearer {self.api_key}",
+                        "Content-Type": "application/msgpack",
+                    },
+            ) as response:
+                if response.status_code == 200:
+                    for chunk in response.iter_content():
+                        # 拼接当前块和上一块数据
+                        chunk_total += chunk
+                        # 最后一个是静音，说明是一个完整的音频
+                        if len(chunk_total) % 2 == 0 and chunk_total[-2:] == b'\x00\x00':
+                            audio = self._get_audio_from_tts(chunk_total)
+                            audio_raw = audio_raw + audio.raw_data
+                            #长度凑够2贞开始发送，60ms*4=240ms
+                            if len(audio_raw) >= 7680:
+                                duration = 60 * len(audio_raw) // 1920
+                                if (len(audio_raw) % 1920) > 0:
+                                    duration += 60
+                                duration = duration / 1000.0
+                                logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
+                                opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
+                                queue.put({
+                                    "data": opus_datas,
+                                    "duration": duration,
+                                    "end": False,
+                                    "text_index": text_index
+                                })
+                                audio_raw = b''
+                            chunk_total = b''
+                    if len(chunk_total) > 0:
+                        audio = self._get_audio_from_tts(chunk_total)
+                        audio_raw = audio_raw + audio.raw_data
+                        duration = 60 * len(audio_raw) // 1920
+                        if (len(audio_raw) % 1920) > 0:
+                            duration += 60
+                        duration = duration / 1000.0
+                        # 把 audio 转成 opus
+                        logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
+                        opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
+                        queue.put({
+                            "data": opus_datas,
+                            "duration": duration,
+                            "end": False
+                        })
+
+                else:
+                    print('请求失败:', response.status_code, response.text)
+                queue.put({
+                    "data": None,
+                    "end": True
+                })
+        except Exception as e:
+            logger.bind(tag=TAG).error("tts发生错误")
+            traceback.print_exc()
+            raise e

main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py

@@ -0,0 +1,67 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.text_lang = config.get("text_lang", "zh")
+        self.ref_audio_path = config.get("ref_audio_path")
+        self.prompt_text = config.get("prompt_text")
+        self.prompt_lang = config.get("prompt_lang", "zh")
+        self.top_k = config.get("top_k", 5)
+        self.top_p = config.get("top_p", 1)
+        self.temperature = config.get("temperature", 1)
+        self.text_split_method = config.get("text_split_method", "cut0")
+        self.batch_size = config.get("batch_size", 1)
+        self.batch_threshold = config.get("batch_threshold", 0.75)
+        self.split_bucket = config.get("split_bucket", True)
+        self.return_fragment = config.get("return_fragment", False)
+        self.speed_factor = config.get("speed_factor", 1.0)
+        self.streaming_mode = config.get("streaming_mode", False)
+        self.seed = config.get("seed", -1)
+        self.parallel_infer = config.get("parallel_infer", True)
+        self.repetition_penalty = config.get("repetition_penalty", 1.35)
+        self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "text": text,
+            "text_lang": self.text_lang,
+            "ref_audio_path": self.ref_audio_path,
+            "aux_ref_audio_paths": self.aux_ref_audio_paths,
+            "prompt_text": self.prompt_text,
+            "prompt_lang": self.prompt_lang,
+            "top_k": self.top_k,
+            "top_p": self.top_p,
+            "temperature": self.temperature,
+            "text_split_method": self.text_split_method,
+            "batch_size": self.batch_size,
+            "batch_threshold": self.batch_threshold,
+            "split_bucket": self.split_bucket,
+            "return_fragment": self.return_fragment,
+            "speed_factor": self.speed_factor,
+            "streaming_mode": self.streaming_mode,
+            "seed": self.seed,
+            "parallel_infer": self.parallel_infer,
+            "repetition_penalty": self.repetition_penalty
+        }
+
+        resp = requests.post(self.url, json=request_json)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -0,0 +1,52 @@
+import os
+import uuid
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.text_lang = config.get("text_lang", "audo")
+        self.ref_audio_path = config.get("ref_audio_path")
+        self.prompt_lang = config.get("prompt_lang")
+        self.prompt_text = config.get("prompt_text")
+        self.top_k = config.get("top_k", 5)
+        self.top_p = config.get("top_p", 1)
+        self.temperature = config.get("temperature", 1)
+        self.sample_steps = config.get("sample_steps", 16)
+        self.media_type = config.get("media_type", "wav")
+        self.streaming_mode = config.get("streaming_mode", False)
+        self.threshold = config.get("threshold", 30)
+
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_params = {
+            "text": text,
+            "text_lang": self.text_lang,
+            "ref_audio_path": self.ref_audio_path,
+            "prompt_lang": self.prompt_lang,
+            "prompt_text": self.prompt_text,
+            "top_k": self.top_k,
+            "top_p": self.top_p,
+            "temperature": self.temperature,
+            "sample_steps": self.sample_steps,
+            "media_type": self.media_type,
+            "streaming_mode": self.streaming_mode,
+            "threshold": self.threshold,
+        }
+
+        resp = requests.get(self.url, params=request_params)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}")

main/xiaozhi-server/core/providers/tts/minimax.py

@@ -0,0 +1,77 @@
+import os
+import uuid
+import json
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.group_id = config.get("group_id")
+        self.api_key = config.get("api_key")
+        self.model = config.get("model")
+        self.voice_id = config.get("voice_id")
+
+        default_voice_setting = {
+            "voice_id": "female-shaonv",
+            "speed": 1,
+            "vol": 1,
+            "pitch": 0,
+            "emotion": "happy"
+        }
+        default_pronunciation_dict = {
+            "tone": [
+                "处理/(chu3)(li3)", "危险/dangerous"
+            ]
+        }
+        defult_audio_setting = {
+            "sample_rate": 32000,
+            "bitrate": 128000,
+            "format": "mp3",
+            "channel": 1
+        }
+        self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})}
+        self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})}
+        self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
+        self.timber_weights = config.get("timber_weights", [])
+
+        if self.voice_id:
+            self.voice_setting["voice_id"] = self.voice_id
+
+        self.host = "api.minimax.chat"
+        self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
+        self.header = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}"
+        }
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "text": text,
+            "stream": False,
+            "voice_setting": self.voice_setting,
+            "pronunciation_dict": self.pronunciation_dict,
+            "audio_setting": self.audio_setting,
+        }
+
+        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
+            request_json["timber_weights"] = self.timber_weights
+            request_json["voice_setting"]["voice_id"] = ""
+
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            # 检查返回请求数据的status_code是否为0
+            if resp.json()["base_resp"]["status_code"] == 0:
+                data = resp.json()['data']['audio']
+                file_to_save = open(output_file, "wb")
+                file_to_save.write(bytes.fromhex(data))
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/openai.py

@@ -0,0 +1,40 @@
+import os
+import uuid
+import requests
+from datetime import datetime
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.api_key = config.get("api_key")
+        self.api_url = config.get("api_url", "https://api.openai.com/v1/audio/speech")
+        self.model = config.get("model", "tts-1")
+        self.voice = config.get("voice", "alloy")
+        self.response_format = "wav"
+        self.speed = config.get("speed", 1.0)
+        self.output_file = config.get("output_file", "tmp/")
+        check_model_key("TTS", self.api_key)
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        data = {
+            "model": self.model,
+            "input": text,
+            "voice": self.voice,
+            "response_format": "wav",
+            "speed": self.speed
+        }
+        response = requests.post(self.api_url, json=data, headers=headers)
+        if response.status_code == 200:
+            with open(output_file, "wb") as audio_file:
+                audio_file.write(response.content)
+        else:
+            raise Exception(f"OpenAI TTS请求失败: {response.status_code} - {response.text}")

main/xiaozhi-server/core/providers/tts/siliconflow.py

@@ -0,0 +1,39 @@
+import os
+import uuid
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.model = config.get("model")
+        self.access_token = config.get("access_token")
+        self.voice = config.get("voice")
+        self.response_format = config.get("response_format")
+        self.sample_rate = config.get("sample_rate")
+        self.speed = config.get("speed")
+        self.gain = config.get("gain")
+
+        self.host = "api.siliconflow.cn"
+        self.api_url = f"https://{self.host}/v1/audio/speech"
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "input": text,
+            "voice": self.voice,
+            "response_format": self.response_format,
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
+        data = response.content
+        file_to_save = open(output_file, "wb")
+        file_to_save.write(data)

main/xiaozhi-server/core/providers/tts/ttson.py

@@ -0,0 +1,64 @@
+import os
+import uuid
+import json
+import requests
+import shutil
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url", "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=")
+        self.voice_id = config.get("voice_id", 1695)
+        self.token = config.get("token")
+        self.to_lang = config.get("to_lang")
+        self.volume_change_dB = config.get("volume_change_dB", 0)
+        self.speed_factor = config.get("speed_factor", 1)
+        self.stream = config.get("stream", False)
+        self.output_file = config.get("output_file")
+        self.pitch_factor = config.get("pitch_factor", 0)
+        self.format = config.get("format", "mp3")
+        self.emotion = config.get("emotion", 1)
+        self.header = {
+            "Content-Type": "application/json"
+        }
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        url = f'{self.url}{self.token}'
+        result = "firefly"
+        payload = json.dumps({
+            "to_lang": self.to_lang,
+            "text": text,
+            "emotion": self.emotion,
+            "format": self.format,
+            "volume_change_dB": self.volume_change_dB,
+            "voice_id": self.voice_id,
+            "pitch_factor": self.pitch_factor,
+            "speed_factor": self.speed_factor,
+            "token": self.token
+        })
+
+        resp = requests.request("POST", url, data=payload)
+        if resp.status_code != 200:
+            return None
+        resp_json = resp.json()
+        try:
+            result = resp_json['url'] + ':' + str(
+                resp_json[
+                    'port']) + '/flashsummary/retrieveFileData?stream=True&token=' + self.token + '&voice_audio_path=' + \
+                     resp_json['voice_path']
+        except Exception as e:
+            print("error:", e)
+
+        audio_content = requests.get(result)
+        with open(output_file, "wb") as f:
+            f.write(audio_content.content)
+            return True
+        voice_path = resp_json.get("voice_path")
+        des_path = output_file
+        shutil.move(voice_path, des_path)

main/xiaozhi-server/core/utils/asr.py

@@ -0,0 +1,24 @@
+import importlib
+import logging
+import os
+import sys
+import time
+import wave
+import uuid
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, List
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+def create_instance(class_name: str, *args, **kwargs) -> ASRProviderBase:
+    """工厂方法创建ASR实例"""
+    if os.path.exists(os.path.join('core', 'providers', 'asr', f'{class_name}.py')):
+        lib_name = f'core.providers.asr.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].ASRProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的ASR类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/auth_code_gen.py

@@ -0,0 +1,97 @@
+import random
+import threading
+import time
+from typing import Set
+
+class AuthCodeGenerator:
+    _instance = None
+    _instance_lock = threading.Lock()
+    
+    def __new__(cls):
+        if not cls._instance:
+            with cls._instance_lock:
+                if not cls._instance:
+                    cls._instance = super(AuthCodeGenerator, cls).__new__(cls)
+                    # 初始化随机种子
+                    random.seed(time.time())
+        return cls._instance
+
+    def __init__(self):
+        # 确保 __init__ 只被调用一次
+        if not hasattr(self, '_initialized'):
+            self._used_codes: Set[str] = set()
+            self._code_timestamps = {}
+            self._lock = threading.Lock()
+            self._code_timeout = 3 * 24 * 60 * 60
+            self._initialized = True
+    
+    @classmethod
+    def get_instance(cls):
+        """获取AuthCodeGenerator的单例实例"""
+        return cls()
+
+    def generate_code(self) -> str:
+        """
+        生成6位数字认证码，确保不重复
+        返回: 6位数字字符串
+        """
+        with self._lock:
+            self._clean_expired_codes()  # 清理过期code
+            while True:
+                # 使用时间戳和已用码数量作为种子，确保每次生成不同的随机数
+                seed = int(time.time() * 1000) + len(self._used_codes)
+                random.seed(seed)
+                
+                # 生成6位随机数字
+                code = ''.join(str(random.randint(0, 9)) for _ in range(6))
+                
+                # 检查是否已存在
+                if code not in self._used_codes:
+                    self._used_codes.add(code)
+                    self._code_timestamps[code] = time.time()
+                    return code
+    
+    def remove_code(self, code: str) -> bool:
+        """
+        删除已使用的认证码
+        参数:
+            code: 要删除的认证码
+        返回:
+            bool: 删除成功返回True，码不存在返回False
+        """
+        print('remove_code', code)
+        with self._lock:
+            if code in self._used_codes:
+                self._used_codes.remove(code)
+                if code in self._code_timestamps:
+                    del self._code_timestamps[code]
+                return True
+            return False
+    
+    def is_code_used(self, code: str) -> bool:
+        """
+        检查认证码是否已被使用
+        参数:
+            code: 要检查的认证码
+        返回:
+            bool: 如果码存在返回True，否则返回False
+        """
+        with self._lock:
+            return code in self._used_codes
+    
+    def clear_codes(self):
+        """清空所有已使用的认证码"""
+        with self._lock:
+            self._used_codes.clear()
+            self._code_timestamps.clear()
+    
+    def _clean_expired_codes(self):
+        """清理过期的认证码"""
+        current_time = time.time()
+        expired_codes = [
+            code for code, timestamp in self._code_timestamps.items()
+            if (current_time - timestamp) > self._code_timeout
+        ]
+        for code in expired_codes:
+            self._used_codes.remove(code)
+            del self._code_timestamps[code]

main/xiaozhi-server/core/utils/dialogue.py

@@ -0,0 +1,53 @@
+import uuid
+from typing import List, Dict
+from datetime import datetime
+
+
+class Message:
+    def __init__(self, role: str, content: str = None, uniq_id: str = None):
+        self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
+        self.role = role
+        self.content = content
+
+
+class Dialogue:
+    def __init__(self):
+        self.dialogue: List[Message] = []
+        # 获取当前时间
+        self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+    def put(self, message: Message):
+        self.dialogue.append(message)
+
+    def get_llm_dialogue(self) -> List[Dict[str, str]]:
+        dialogue = []
+        for m in self.dialogue:
+            dialogue.append({"role": m.role, "content": m.content})
+        return dialogue
+
+    def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
+        if memory_str is None or len(memory_str) == 0:
+            return self.get_llm_dialogue()
+        
+        # 构建带记忆的对话
+        dialogue = []
+        
+        # 添加系统提示和记忆
+        system_message = next(
+            (msg for msg in self.dialogue if msg.role == "system"), None
+        )
+
+
+        if system_message:
+            enhanced_system_prompt = (
+                f"{system_message.content}\n\n"
+                f"相关记忆：\n{memory_str}"
+            )
+            dialogue.append({"role": "system", "content": enhanced_system_prompt})
+
+        # 添加用户和助手的对话
+        for msg in self.dialogue:
+            if msg.role != "system":  # 跳过原始的系统消息
+                dialogue.append({"role": msg.role, "content": msg.content})
+
+        return dialogue

main/xiaozhi-server/core/utils/intent.py

@@ -0,0 +1,17 @@
+import os
+import sys
+from config.logger import setup_logging
+import importlib
+
+logger = setup_logging()
+
+
+def create_instance(class_name, *args, **kwargs):
+    # 创建intent实例
+    if os.path.exists(os.path.join('core', 'providers', 'intent', class_name, f'{class_name}.py')):
+        lib_name = f'core.providers.intent.{class_name}.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].IntentProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的intent类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/llm.py

@@ -0,0 +1,23 @@
+import os
+import sys
+
+# 添加项目根目录到Python路径
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.abspath(os.path.join(current_dir, "..", ".."))
+sys.path.insert(0, project_root)
+
+from config.logger import setup_logging
+import importlib
+
+logger = setup_logging()
+
+
+def create_instance(class_name, *args, **kwargs):
+    # 创建LLM实例
+    if os.path.exists(os.path.join('core', 'providers', 'llm', class_name, f'{class_name}.py')):
+        lib_name = f'core.providers.llm.{class_name}.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].LLMProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的LLM类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/lock_manager.py

@@ -0,0 +1,39 @@
+import asyncio
+from typing import Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class FileLockManager:
+    _instance = None
+    _locks: Dict[str, asyncio.Lock] = {}
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(FileLockManager, cls).__new__(cls)
+        return cls._instance
+
+    @classmethod
+    def get_lock(cls, file_path: str) -> asyncio.Lock:
+        """获取指定文件的锁"""
+        if file_path not in cls._locks:
+            cls._locks[file_path] = asyncio.Lock()
+        return cls._locks[file_path]
+
+    @classmethod
+    async def acquire_lock(cls, file_path: str):
+        """获取锁"""
+        lock = cls.get_lock(file_path)
+        await lock.acquire()
+        logger.bind(tag=TAG).debug(f"Acquired lock for {file_path}")
+
+    @classmethod
+    def release_lock(cls, file_path: str):
+        """释放锁"""
+        if file_path in cls._locks:
+            try:
+                cls._locks[file_path].release()
+                logger.bind(tag=TAG).debug(f"Released lock for {file_path}")
+            except RuntimeError as e:
+                logger.bind(tag=TAG).warning(f"Failed to release lock for {file_path}: {e}")

main/xiaozhi-server/core/utils/memory.py

@@ -0,0 +1,17 @@
+import os
+import sys
+import importlib
+from config.logger import setup_logging
+from core.utils.util import read_config, get_project_dir
+
+logger = setup_logging()
+
+def create_instance(class_name, *args, **kwargs):
+    if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')):
+        lib_name = f'core.providers.memory.{class_name}.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].MemoryProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的记忆服务类型: {class_name}")
+

main/xiaozhi-server/core/utils/p3.py

@@ -0,0 +1,33 @@
+import struct
+
+def decode_opus_from_file(input_file):
+    """
+    从p3文件中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。
+    """
+    opus_datas = []
+    total_frames = 0
+    sample_rate = 16000  # 文件采样率
+    frame_duration_ms = 60  # 帧时长
+    frame_size = int(sample_rate * frame_duration_ms / 1000)
+
+    with open(input_file, 'rb') as f:
+        while True:
+            # 读取头部（4字节）：[1字节类型，1字节保留，2字节长度]
+            header = f.read(4)
+            if not header:
+                break
+
+            # 解包头部信息
+            _, _, data_len = struct.unpack('>BBH', header)
+
+            # 根据头部指定的长度读取 Opus 数据
+            opus_data = f.read(data_len)
+            if len(opus_data) != data_len:
+                raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the file.")
+
+            opus_datas.append(opus_data)
+            total_frames += 1
+
+    # 计算总时长
+    total_duration = (total_frames * frame_duration_ms) / 1000.0
+    return opus_datas, total_duration

main/xiaozhi-server/core/utils/tts.py

@@ -0,0 +1,17 @@
+import os
+import sys
+from config.logger import setup_logging
+import importlib
+
+logger = setup_logging()
+
+
+def create_instance(class_name, *args, **kwargs):
+    # 创建TTS实例
+    if os.path.exists(os.path.join('core', 'providers', 'tts', f'{class_name}.py')):
+        lib_name = f'core.providers.tts.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].TTSProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/util.py

@@ -0,0 +1,121 @@
+import os
+import json
+import yaml
+import socket
+import subprocess
+import logging
+
+
+def get_project_dir():
+    """获取项目根目录"""
+    return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + '/'
+
+
+def get_local_ip():
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        # Connect to Google's DNS servers
+        s.connect(("8.8.8.8", 80))
+        local_ip = s.getsockname()[0]
+        s.close()
+        return local_ip
+    except Exception as e:
+        return "127.0.0.1"
+
+
+def read_config(config_path):
+    with open(config_path, "r", encoding="utf-8") as file:
+        config = yaml.safe_load(file)
+    return config
+
+
+def write_json_file(file_path, data):
+    """将数据写入 JSON 文件"""
+    with open(file_path, 'w', encoding='utf-8') as file:
+        json.dump(data, file, ensure_ascii=False, indent=4)
+
+
+def is_punctuation_or_emoji(char):
+    """检查字符是否为空格、指定标点或表情符号"""
+    # 定义需要去除的中英文标点（包括全角/半角）
+    punctuation_set = {
+        '，', ',',  # 中文逗号 + 英文逗号
+        '。', '.',  # 中文句号 + 英文句号
+        '！', '!',  # 中文感叹号 + 英文感叹号
+        '-', '－',  # 英文连字符 + 中文全角横线
+        '、'  # 中文顿号
+    }
+    if char.isspace() or char in punctuation_set:
+        return True
+    # 检查表情符号（保留原有逻辑）
+    code_point = ord(char)
+    emoji_ranges = [
+        (0x1F600, 0x1F64F), (0x1F300, 0x1F5FF),
+        (0x1F680, 0x1F6FF), (0x1F900, 0x1F9FF),
+        (0x1FA70, 0x1FAFF), (0x2600, 0x26FF),
+        (0x2700, 0x27BF)
+    ]
+    return any(start <= code_point <= end for start, end in emoji_ranges)
+
+
+def get_string_no_punctuation_or_emoji(s):
+    """去除字符串首尾的空格、标点符号和表情符号"""
+    chars = list(s)
+    # 处理开头的字符
+    start = 0
+    while start < len(chars) and is_punctuation_or_emoji(chars[start]):
+        start += 1
+    # 处理结尾的字符
+    end = len(chars) - 1
+    while end >= start and is_punctuation_or_emoji(chars[end]):
+        end -= 1
+    return ''.join(chars[start:end + 1])
+
+
+def remove_punctuation_and_length(text):
+    # 全角符号和半角符号的Unicode范围
+    full_width_punctuations = '！＂＃＄％＆＇（）＊＋，－。／：；＜＝＞？＠［＼］＾＿｀｛｜｝～'
+    half_width_punctuations = r'!"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~'
+    space = ' '  # 半角空格
+    full_width_space = '　'  # 全角空格
+
+    # 去除全角和半角符号以及空格
+    result = ''.join([char for char in text if
+                      char not in full_width_punctuations and char not in half_width_punctuations and char not in space and char not in full_width_space])
+
+    if result == "Yeah":
+        return 0, ""
+    return len(result), result
+
+def check_model_key(modelType, modelKey):
+    if "你" in modelKey:
+        logging.error("你还没配置" + modelType + "的密钥，请在配置文件中配置密钥，否则无法正常工作")
+        return False
+    return True
+
+
+def check_ffmpeg_installed():
+    ffmpeg_installed = False
+    try:
+        # 执行ffmpeg -version命令，并捕获输出
+        result = subprocess.run(
+            ['ffmpeg', '-version'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=True  # 如果返回码非零则抛出异常
+        )
+        # 检查输出中是否包含版本信息（可选）
+        output = result.stdout + result.stderr
+        if 'ffmpeg version' in output.lower():
+            ffmpeg_installed = True
+        return False
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        # 命令执行失败或未找到
+        ffmpeg_installed = False
+    if not ffmpeg_installed:
+        error_msg = "您的电脑还没正确安装ffmpeg\n"
+        error_msg += "\n建议您：\n"
+        error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
+        error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
+        raise ValueError(error_msg)

main/xiaozhi-server/core/utils/vad.py

@@ -0,0 +1,77 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+import opuslib_next
+import time
+import numpy as np
+import torch
+
+TAG = __name__
+logger = setup_logging()
+
+class VAD(ABC):
+    @abstractmethod
+    def is_vad(self, conn, data):
+        """检测音频数据中的语音活动"""
+        pass
+
+
+class SileroVAD(VAD):
+    def __init__(self, config):
+        logger.bind(tag=TAG).info("SileroVAD", config)
+        self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
+                                                source='local',
+                                                model='silero_vad',
+                                                force_reload=False)
+        (get_speech_timestamps, _, _, _, _) = self.utils
+
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.vad_threshold = config.get("threshold")
+        self.silence_threshold_ms = config.get("min_silence_duration_ms")
+
+    def is_vad(self, conn, opus_packet):
+        try:
+            pcm_frame = self.decoder.decode(opus_packet, 960)
+            conn.client_audio_buffer += pcm_frame  # 将新数据加入缓冲区
+
+            # 处理缓冲区中的完整帧（每次处理512采样点）
+            client_have_voice = False
+            while len(conn.client_audio_buffer) >= 512 * 2:
+                # 提取前512个采样点（1024字节）
+                chunk = conn.client_audio_buffer[:512 * 2]
+                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
+
+                # 转换为模型需要的张量格式
+                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+                audio_float32 = audio_int16.astype(np.float32) / 32768.0
+                audio_tensor = torch.from_numpy(audio_float32)
+
+                # 检测语音活动
+                speech_prob = self.model(audio_tensor, 16000).item()
+                client_have_voice = speech_prob >= self.vad_threshold
+
+                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
+                if conn.client_have_voice and not client_have_voice:
+                    stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
+                    if stop_duration >= self.silence_threshold_ms:
+                        conn.client_voice_stop = True
+                if client_have_voice:
+                    conn.client_have_voice = True
+                    conn.client_have_voice_last_time = time.time() * 1000
+
+            return client_have_voice
+        except opuslib_next.OpusError as e:
+            logger.bind(tag=TAG).info(f"解码错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
+
+
+def create_instance(class_name, *args, **kwargs) -> VAD:
+    # 获取类对象
+    cls_map = {
+        "SileroVAD": SileroVAD,
+        # 可扩展其他SileroVAD实现
+    }
+
+    if cls := cls_map.get(class_name):
+        return cls(*args, **kwargs)
+    raise ValueError(f"不支持的SileroVAD类型: {class_name}")

main/xiaozhi-server/core/websocket_server.py

@@ -0,0 +1,86 @@
+import asyncio
+import websockets
+from config.logger import setup_logging
+from core.connection import ConnectionHandler
+from core.handle.musicHandler import MusicHandler
+from core.utils.util import get_local_ip
+from core.utils import asr, vad, llm, tts, memory, intent
+
+TAG = __name__
+
+
+class WebSocketServer:
+    def __init__(self, config: dict):
+        self.config = config
+        self.logger = setup_logging()
+        self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent = self._create_processing_instances()
+        self.active_connections = set()  # 添加全局连接记录
+
+    def _create_processing_instances(self):
+        memory_cls_name = self.config["selected_module"].get("Memory", "nomem") # 默认使用nomem
+        has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
+        memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
+
+        """创建处理模块实例"""
+        return (
+            vad.create_instance(
+                self.config["selected_module"]["VAD"],
+                self.config["VAD"][self.config["selected_module"]["VAD"]]
+            ),
+            asr.create_instance(
+                self.config["selected_module"]["ASR"]
+                if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
+                else
+                self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
+                self.config["ASR"][self.config["selected_module"]["ASR"]],
+                self.config["delete_audio"]
+            ),
+            llm.create_instance(
+                self.config["selected_module"]["LLM"]
+                if not 'type' in self.config["LLM"][self.config["selected_module"]["LLM"]]
+                else
+                self.config["LLM"][self.config["selected_module"]["LLM"]]['type'],
+                self.config["LLM"][self.config["selected_module"]["LLM"]],
+            ),
+            tts.create_instance(
+                self.config["selected_module"]["TTS"]
+                if not 'type' in self.config["TTS"][self.config["selected_module"]["TTS"]]
+                else
+                self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
+                self.config["TTS"][self.config["selected_module"]["TTS"]],
+                self.config["delete_audio"]
+            ),
+            MusicHandler(self.config),
+            memory.create_instance(memory_cls_name, memory_cfg),
+            intent.create_instance(
+                self.config["selected_module"]["Intent"]
+                if not 'type' in self.config["Intent"][self.config["selected_module"]["Intent"]]
+                else
+                self.config["Intent"][self.config["selected_module"]["Intent"]]["type"],
+                self.config["Intent"][self.config["selected_module"]["Intent"]]
+            ),
+        )
+
+    async def start(self):
+        server_config = self.config["server"]
+        host = server_config["ip"]
+        port = server_config["port"]
+
+        self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
+        self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")
+        async with websockets.serve(
+                self._handle_connection,
+                host,
+                port
+        ):
+            await asyncio.Future()
+
+    async def _handle_connection(self, websocket):
+        """处理新连接，每次创建独立的ConnectionHandler"""
+        # 创建ConnectionHandler时传入当前server实例
+        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent)
+        self.active_connections.add(handler)
+        try:
+            await handler.handle_connection(websocket)
+        finally:
+            self.active_connections.discard(handler)