重新划分目录 (#214)

* 🦄 refactor(web): 修改zhikongtaiweb到web

* 🦄 refactor: 重写前端

路由守护尚未写完

* 🦄 refactor: 标准化路由

* update:前端重写，保留后端

* update:添加前端代码

* update:pip转成poetry启动

* update:增加mem0ai包依赖

* update:文档增加mem0ai的描述

* feat: play online mp3 (#181)

Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com>

* 修改前端代码

* update:调整项目目录

* update:优化

* update:配置文件去除8002端口

* update:增加开发说明

* update:更新开发协议

---------

Co-authored-by: kalicyh <34980061+kaliCYH@users.noreply.github.com>
Co-authored-by: hrz <1710360675@qq.com>
Co-authored-by: freshlife001 <talent@mises.site>
Co-authored-by: CGD <3030332422@qq.com>

core/auth.py

@@ -1,54 +0,0 @@
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-
-class AuthenticationError(Exception):
-    """认证异常"""
-    pass
-
-
-class AuthMiddleware:
-    def __init__(self, config):
-        self.config = config
-        self.auth_config = config["server"].get("auth", {})
-        # 构建token查找表
-        self.tokens = {
-            item["token"]: item["name"]
-            for item in self.auth_config.get("tokens", [])
-        }
-        # 设备白名单
-        self.allowed_devices = set(
-            self.auth_config.get("allowed_devices", [])
-        )
-
-    async def authenticate(self, headers):
-        """验证连接请求"""
-        # 检查是否启用认证
-        if not self.auth_config.get("enabled", False):
-            return True
-
-        # 检查设备是否在白名单中
-        device_id = headers.get("device-id", "")
-
-        if self.allowed_devices and device_id in self.allowed_devices:
-            return True
-
-        # 验证Authorization header
-        auth_header = headers.get("authorization", "")
-        if not auth_header.startswith("Bearer "):
-            logger.bind(tag=TAG).error("Missing or invalid Authorization header")
-            raise AuthenticationError("Missing or invalid Authorization header")
-
-        token = auth_header.split(" ")[1]
-        if token not in self.tokens:
-            logger.bind(tag=TAG).error(f"Invalid token: {token}")
-            raise AuthenticationError("Invalid token")
-
-        logger.bind(tag=TAG).info(f"Authentication successful - Device: {device_id}, Token: {self.tokens[token]}")
-        return True
-
-    def get_token_name(self, token):
-        """获取token对应的设备名称"""
-        return self.tokens.get(token)

core/connection.py

@@ -1,374 +0,0 @@
-import os
-import json
-import uuid
-import time
-import queue
-import asyncio
-import traceback
-from config.logger import setup_logging
-import threading
-import websockets
-from typing import Dict, Any
-from core.utils.dialogue import Message, Dialogue
-from core.handle.textHandle import handleTextMessage
-from core.utils.util import get_string_no_punctuation_or_emoji
-from concurrent.futures import ThreadPoolExecutor, TimeoutError
-from core.handle.sendAudioHandle import sendAudioMessage
-from core.handle.receiveAudioHandle import handleAudioMessage
-from config.private_config import PrivateConfig
-from core.auth import AuthMiddleware, AuthenticationError
-from core.utils.auth_code_gen import AuthCodeGenerator
-
-TAG = __name__
-
-
-class TTSException(RuntimeError):
-    pass
-
-
-class ConnectionHandler:
-    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory):
-        self.config = config
-        self.logger = setup_logging()
-        self.auth = AuthMiddleware(config)
-
-        self.websocket = None
-        self.headers = None
-        self.session_id = None
-        self.prompt = None
-        self.welcome_msg = None
-
-        # 客户端状态相关
-        self.client_abort = False
-        self.client_listen_mode = "auto"
-
-        # 线程任务相关
-        self.loop = asyncio.get_event_loop()
-        self.stop_event = threading.Event()
-        self.tts_queue = queue.Queue()
-        self.audio_play_queue = queue.Queue()
-        self.executor = ThreadPoolExecutor(max_workers=10)
-
-        # 依赖的组件
-        self.vad = _vad
-        self.asr = _asr
-        self.llm = _llm
-        self.tts = _tts
-        self.memory = _memory
-
-        # vad相关变量
-        self.client_audio_buffer = bytes()
-        self.client_have_voice = False
-        self.client_have_voice_last_time = 0.0
-        self.client_no_voice_last_time = 0.0
-        self.client_voice_stop = False
-
-        # asr相关变量
-        self.asr_audio = []
-        self.asr_server_receive = True
-
-        # llm相关变量
-        self.llm_finish_task = False
-        self.dialogue = Dialogue()
-
-        # tts相关变量
-        self.tts_first_text_index = -1
-        self.tts_last_text_index = -1
-
-        # iot相关变量
-        self.iot_descriptors = {}
-
-        self.cmd_exit = self.config["CMD_exit"]
-        self.max_cmd_length = 0
-        for cmd in self.cmd_exit:
-            if len(cmd) > self.max_cmd_length:
-                self.max_cmd_length = len(cmd)
-
-        self.private_config = None
-        self.auth_code_gen = AuthCodeGenerator.get_instance()
-        self.is_device_verified = False  # 添加设备验证状态标志
-        self.music_handler = _music
-
-    async def handle_connection(self, ws):
-        try:
-            # 获取并验证headers
-            self.headers = dict(ws.request.headers)
-            # 获取客户端ip地址
-            client_ip = ws.remote_address[0]
-            self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
-
-            # 进行认证
-            await self.auth.authenticate(self.headers)
-
-            device_id = self.headers.get("device-id", None)
-            self.memory.set_role_id(device_id)
-
-            # Load private configuration if device_id is provided
-            bUsePrivateConfig = self.config.get("use_private_config", False)
-            self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
-            if bUsePrivateConfig and device_id:
-                try:
-                    self.private_config = PrivateConfig(device_id, self.config, self.auth_code_gen)
-                    await self.private_config.load_or_create()
-                    # 判断是否已经绑定
-                    owner = self.private_config.get_owner()
-                    self.is_device_verified = owner is not None
-
-                    if self.is_device_verified:
-                        await self.private_config.update_last_chat_time()
-
-                    llm, tts = self.private_config.create_private_instances()
-                    if all([llm, tts]):
-                        self.llm = llm
-                        self.tts = tts
-                        self.logger.bind(tag=TAG).info(f"Loaded private config and instances for device {device_id}")
-                    else:
-                        self.logger.bind(tag=TAG).error(f"Failed to create instances for device {device_id}")
-                        self.private_config = None
-                except Exception as e:
-                    self.logger.bind(tag=TAG).error(f"Error initializing private config: {e}")
-                    self.private_config = None
-                    raise
-
-            # 认证通过,继续处理
-            self.websocket = ws
-            self.session_id = str(uuid.uuid4())
-
-            self.welcome_msg = self.config["xiaozhi"]
-            self.welcome_msg["session_id"] = self.session_id
-            await self.websocket.send(json.dumps(self.welcome_msg))
-
-            await self.loop.run_in_executor(None, self._initialize_components)
-
-            # tts 消化线程
-            tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
-            tts_priority.start()
-
-            # 音频播放 消化线程
-            audio_play_priority = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
-            audio_play_priority.start()
-
-            try:
-                async for message in self.websocket:
-                    await self._route_message(message)
-            except websockets.exceptions.ConnectionClosed:
-                self.logger.bind(tag=TAG).info("客户端断开连接")
-                await self.close()
-
-        except AuthenticationError as e:
-            self.logger.bind(tag=TAG).error(f"Authentication failed: {str(e)}")
-            await ws.close()
-            return
-        except Exception as e:
-            stack_trace = traceback.format_exc()
-            self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
-            await ws.close()
-            return
-        finally:
-            await self.memory.save_memory(self.dialogue.dialogue)
-
-    async def _route_message(self, message):
-        """消息路由"""
-        if isinstance(message, str):
-            await handleTextMessage(self, message)
-        elif isinstance(message, bytes):
-            await handleAudioMessage(self, message)
-
-    def _initialize_components(self):
-        self.prompt = self.config["prompt"]
-        if self.private_config:
-            self.prompt = self.private_config.private_config.get("prompt", self.prompt)
-        # 赋予LLM时间观念
-        if "{date_time}" in self.prompt:
-            date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
-            self.prompt = self.prompt.replace("{date_time}", date_time)
-        self.dialogue.put(Message(role="system", content=self.prompt))
-
-    async def _check_and_broadcast_auth_code(self):
-        """检查设备绑定状态并广播认证码"""
-        if not self.private_config.get_owner():
-            auth_code = self.private_config.get_auth_code()
-            if auth_code:
-                # 发送验证码语音提示
-                text = f"请在后台输入验证码：{' '.join(auth_code)}"
-                self.recode_first_last_text(text)
-                future = self.executor.submit(self.speak_and_play, text)
-                self.tts_queue.put(future)
-            return False
-        return True
-
-    def isNeedAuth(self):
-        bUsePrivateConfig = self.config.get("use_private_config", False)
-        if not bUsePrivateConfig:
-            # 如果不使用私有配置，就不需要验证
-            return False
-        return not self.is_device_verified
-
-
-    def chat(self, query):
-        if self.isNeedAuth():
-            self.llm_finish_task = True
-            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
-            future.result()
-            return True
-
-        self.dialogue.put(Message(role="user", content=query))
-        response_message = []
-        processed_chars = 0  # 跟踪已处理的字符位置
-        try:
-            start_time = time.time()
-            # 使用带记忆的对话
-            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
-            memory_str = future.result()
-            
-            self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
-            llm_responses = self.llm.response(
-                self.session_id, 
-                self.dialogue.get_llm_dialogue_with_memory(memory_str)
-            )
-        except Exception as e:
-            self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
-            return None
-
-        self.llm_finish_task = False
-        text_index = 0
-        for content in llm_responses:
-            response_message.append(content)
-            if self.client_abort:
-                break
-
-            end_time = time.time()
-            self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
-
-            # 合并当前全部文本并处理未分割部分
-            full_text = "".join(response_message)
-            current_text = full_text[processed_chars:]  # 从未处理的位置开始
-
-            # 查找最后一个有效标点
-            punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
-            last_punct_pos = -1
-            for punct in punctuations:
-                pos = current_text.rfind(punct)
-                if pos > last_punct_pos:
-                    last_punct_pos = pos
-
-            # 找到分割点则处理
-            if last_punct_pos != -1:
-                segment_text_raw = current_text[:last_punct_pos + 1]
-                segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
-                if segment_text:
-                    # 强制设置空字符，测试TTS出错返回语音的健壮性
-                    # if text_index % 2 == 0:
-                    #     segment_text = " "
-                    text_index += 1
-                    self.recode_first_last_text(segment_text, text_index)
-                    future = self.executor.submit(self.speak_and_play, segment_text, text_index)
-                    self.tts_queue.put(future)
-                    processed_chars += len(segment_text_raw)  # 更新已处理字符位置
-
-        # 处理最后剩余的文本
-        full_text = "".join(response_message)
-        remaining_text = full_text[processed_chars:]
-        if remaining_text:
-            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
-            if segment_text:
-                text_index += 1
-                self.recode_first_last_text(segment_text, text_index)
-                future = self.executor.submit(self.speak_and_play, segment_text, text_index)
-                self.tts_queue.put(future)
-
-        self.llm_finish_task = True
-        self.dialogue.put(Message(role="assistant", content="".join(response_message)))
-        self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
-        return True
-
-    def _tts_priority_thread(self):
-        while not self.stop_event.is_set():
-            text = None
-            try:
-                future = self.tts_queue.get()
-                if future is None:
-                    continue
-                text = None
-                opus_datas, text_index, tts_file = [], 0, None
-                try:
-                    self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
-                    tts_file, text, text_index = future.result(timeout=10)
-                    if text is None or len(text) <= 0:
-                        self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
-                    elif tts_file is None:
-                        self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
-                    else:
-                        self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
-                        if os.path.exists(tts_file):
-                            opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
-                        else:
-                            self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
-                except TimeoutError:
-                    self.logger.bind(tag=TAG).error("TTS超时")
-                except Exception as e:
-                    self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
-                if not self.client_abort:
-                    # 如果没有中途打断就发送语音
-                    self.audio_play_queue.put((opus_datas, text, text_index))
-                if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
-                    os.remove(tts_file)
-            except Exception as e:
-                self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
-                self.clearSpeakStatus()
-                asyncio.run_coroutine_threadsafe(
-                    self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
-                    self.loop
-                )
-                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
-
-    def _audio_play_priority_thread(self):
-        while not self.stop_event.is_set():
-            text = None
-            try:
-                opus_datas, text, text_index = self.audio_play_queue.get()
-                future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
-                                                          self.loop)
-                future.result()
-            except Exception as e:
-                self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
-
-    def speak_and_play(self, text, text_index=0):
-        if text is None or len(text) <= 0:
-            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
-            return None, text, text_index
-        tts_file = self.tts.to_tts(text)
-        if tts_file is None:
-            self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
-            return None, text, text_index
-        self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
-        return tts_file, text, text_index
-
-    def clearSpeakStatus(self):
-        self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
-        self.asr_server_receive = True
-        self.tts_last_text_index = -1
-        self.tts_first_text_index = -1
-
-    def recode_first_last_text(self, text, text_index=0):
-        if self.tts_first_text_index == -1:
-            self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
-            self.tts_first_text_index = text_index
-        self.tts_last_text_index = text_index
-
-    async def close(self):
-        """资源清理方法"""
-
-        # 清理其他资源
-        self.stop_event.set()
-        self.executor.shutdown(wait=False)
-        if self.websocket:
-            await self.websocket.close()
-        self.logger.bind(tag=TAG).info("连接资源已释放")
-
-    def reset_vad_states(self):
-        self.client_audio_buffer = bytes()
-        self.client_have_voice = False
-        self.client_have_voice_last_time = 0
-        self.client_voice_stop = False
-        self.logger.bind(tag=TAG).debug("VAD states reset.")

core/handle/abortHandle.py

@@ -1,16 +0,0 @@
-import json
-import queue
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-
-async def handleAbortMessage(conn):
-    logger.bind(tag=TAG).info("Abort message received")
-    # 设置成打断状态，会自动打断llm、tts任务
-    conn.client_abort = True
-    # 打断客户端说话状态
-    await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
-    conn.clearSpeakStatus()
-    logger.bind(tag=TAG).info("Abort message received-end")

core/handle/helloHandle.py

@@ -1,8 +0,0 @@
-import json
-from config.logger import setup_logging
-
-logger = setup_logging()
-
-
-async def handleHelloMessage(conn):
-    await conn.websocket.send(json.dumps(conn.welcome_msg))

core/handle/iotHandle.py

@@ -1,152 +0,0 @@
-import json
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-
-class IotDescriptor:
-    """
-    A class to represent an IoT descriptor.
-    Attributes:
-    ----------
-    name : str
-        The name of the IoT descriptor.
-    description : str
-        A brief description of the IoT descriptor.
-    properties : dict
-        A dictionary containing properties of the IoT descriptor.
-    methods : dict
-        A dictionary containing methods of the IoT descriptor.
-    -------
-    """
-
-    def __init__(self, name, description, properties, methods):
-        self.name = name
-        self.description = description
-        self.properties = []
-        self.methods = []
-
-        # 根据描述创建属性
-        for key, value in properties.items():
-            # "volume":{"description":"当前音量 值","type":"number"}
-            """
-            等价于
-            {
-                'name': 名字,
-                'description': 描述,
-                'value': 0
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是属性名
-            property_item = globals()[key] = {}  # 创建一个空字典, 名字是属性名
-            property_item['name'] = key
-            property_item["description"] = value["description"]
-            if value["type"] == "number":
-                property_item["value"] = 0
-            elif value["type"] == "boolean":
-                property_item["value"] = False
-            else:
-                property_item["value"] = ""
-            self.properties.append(property_item)
-
-        # 根据描述创建方法
-        for key, value in methods.items():
-            # "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
-            """
-            等价于
-            SetVolume = {
-                `description`: 描述,
-                `volume`: {
-                    `description`: 描述,
-                    `value`: 0
-                }
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是方法名
-            method = globals()[key] = {}  # 创建一个空字典, 名字是方法名
-            method["description"] = value["description"]
-            method['name'] = key
-            for k, v in value["parameters"].items():
-                # 不同的参数解析
-                method[k] = {}
-                method[k]["description"] = v["description"]
-                if v["type"] == "number":
-                    method[k]["value"] = 0
-                elif v["type"] == "boolean":
-                    method[k]["value"] = False
-                else:
-                    method[k]["value"] = ""
-
-            self.methods.append(method)
-
-
-async def handleIotDescriptors(conn, descriptors):
-    """
-    处理物联网描述
-    示例: [{
-        "name":"Speaker",
-        "description":"当前 AI 机器人的扬声器",
-        "properties":{
-            "volume":{"description":"当前音量 值","type":"number"}  可以有boolean, number, string三种类型
-        },
-        "methods":{
-            "SetVolume":{
-                "description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
-            }
-        }
-    }]
-    descriptors: 描述列表
-    """
-    for descriptor in descriptors:
-        iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
-                                       descriptor["methods"])
-        conn.iot_descriptors[descriptor["name"]] = iot_descriptor
-
-    # 暂时从配置文件中设置音量，后期通过意图识别控制音量
-    default_iot_volume = 100
-    if "iot" in conn.config:
-        default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
-    logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
-    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
-
-
-async def send_iot_conn(conn, name, method_name, parameters):
-    """
-    发送物联网指令
-    name: 设备名称 "Speaker"
-    method: 方法 "SetVolume"
-    parameters: 参数, 是一个字典 {"volume": 100}
-    发送示例:
-    {
-        "type": "iot",
-        "commands": [
-            {
-                "name" :  "Speaker",
-                "method": "SetVolume",
-                "parameters": {
-                    "volume": 100
-                    }
-            }
-        ]
-    }
-    """
-
-    for key, value in conn.iot_descriptors.items():
-        if key == name:
-            # 找到了设备
-            for method in value.methods:
-                # 找到了方法
-                if method["name"] == method_name:
-                    await conn.websocket.send(json.dumps({
-                        "type": "iot",
-                        "commands": [
-                            {
-                                "name": name,
-                                "method": method_name,
-                                "parameters": parameters
-                            }
-                        ]
-                    }))
-                    return
-    logger.bind(tag=TAG).error(f"未找到方法{method_name}")

core/handle/musicHandler.py

@@ -1,153 +0,0 @@
-from config.logger import setup_logging
-import os
-import random
-import difflib
-import re
-import traceback
-from pathlib import Path
-import time
-from core.handle.sendAudioHandle import send_stt_message
-from core.utils import p3
-
-TAG = __name__
-logger = setup_logging()
-
-
-def _extract_song_name(text):
-    """从用户输入中提取歌名"""
-    for keyword in ["听", "播放", "放", "唱"]:
-        if keyword in text:
-            parts = text.split(keyword)
-            if len(parts) > 1:
-                return parts[1].strip()
-    return None
-
-
-def _find_best_match(potential_song, music_files):
-    """查找最匹配的歌曲"""
-    best_match = None
-    highest_ratio = 0
-
-    for music_file in music_files:
-        song_name = os.path.splitext(music_file)[0]
-        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
-        if ratio > highest_ratio and ratio > 0.4:
-            highest_ratio = ratio
-            best_match = music_file
-    return best_match
-
-class MusicManager:
-    def __init__(self, music_dir, music_ext):
-        self.music_dir = Path(music_dir)
-        self.music_ext = music_ext
-
-    def get_music_files(self):
-        music_files = []
-        for file in self.music_dir.rglob("*"):
-            # 判断是否是文件
-            if file.is_file():
-                # 获取文件扩展名
-                ext = file.suffix.lower()
-                # 判断扩展名是否在列表中
-                if ext in self.music_ext:
-                    # music_files.append(str(file.resolve()))  # 添加绝对路径
-                    # 添加相对路径
-                    music_files.append(str(file.relative_to(self.music_dir)))
-        return music_files
-
-class MusicHandler:
-    def __init__(self, config):
-        self.config = config
-        self.music_related_keywords = []
-
-        if "music" in self.config:
-            self.music_config = self.config["music"]
-            self.music_dir = os.path.abspath(
-                self.music_config.get("music_dir", "./music")  # 默认路径修改
-            )
-            self.music_related_keywords = self.music_config.get("music_commands", [])
-            self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
-            self.refresh_time = self.music_config.get("refresh_time", 60)
-        else:
-            self.music_dir = os.path.abspath("./music")
-            self.music_related_keywords = ["来一首歌", "唱一首歌", "播放音乐", "来点音乐", "背景音乐", "放首歌",
-                                           "播放歌曲", "来点背景音乐", "我想听歌", "我要听歌", "放点音乐"]
-            self.music_ext = (".mp3", ".wav", ".p3")
-            self.refresh_time = 60
-
-        # 获取音乐文件列表
-        self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-        self.scan_time = time.time()
-        logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
-
-    async def handle_music_command(self, conn, text):
-        """处理音乐播放指令"""
-        clean_text = re.sub(r'[^\w\s]', '', text).strip()
-        logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
-
-        # 尝试匹配具体歌名
-        if os.path.exists(self.music_dir):
-            if time.time() - self.scan_time > self.refresh_time:
-                # 刷新音乐文件列表
-                self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-                self.scan_time = time.time()
-                logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
-
-            potential_song = _extract_song_name(clean_text)
-            if potential_song:
-                best_match = _find_best_match(potential_song, self.music_files)
-                if best_match:
-                    logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
-                    await self.play_local_music(conn, specific_file=best_match)
-                    return True
-
-        # 检查是否是通用播放音乐命令
-        if any(cmd in clean_text for cmd in self.music_related_keywords):
-            await self.play_local_music(conn)
-            return True
-
-        return False
-
-    async def play_local_music(self, conn, specific_file=None):
-        """播放本地音乐文件"""
-        try:
-            if not os.path.exists(self.music_dir):
-                logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
-                return
-
-            # 确保路径正确性
-            if specific_file:
-                music_path = os.path.join(self.music_dir, specific_file)
-                if not os.path.exists(music_path):
-                    logger.bind(tag=TAG).error(f"指定的音乐文件不存在: {music_path}")
-                    return
-                selected_music = specific_file
-            else:
-                if time.time() - self.scan_time > self.refresh_time:
-                    # 刷新音乐文件列表
-                    self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-                    self.scan_time = time.time()
-                    logger.bind(tag=TAG).debug(f"刷新的音乐文件列表: {self.music_files}")
-
-                if not self.music_files:
-                    logger.bind(tag=TAG).error("未找到MP3音乐文件")
-                    return
-                selected_music = random.choice(self.music_files)
-                music_path = os.path.join(self.music_dir, selected_music)
-                if not os.path.exists(music_path):
-                    logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
-                    return
-            text = f"正在播放{selected_music}"
-            await send_stt_message(conn, text)
-            conn.tts_first_text_index = 0
-            conn.tts_last_text_index = 0
-            conn.llm_finish_task = True
-            if music_path.endswith(".p3"):
-                opus_packets, duration = p3.decode_opus_from_file(music_path)
-            else:
-                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
-            conn.audio_play_queue.put((opus_packets, selected_music, 0))
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
-            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

core/handle/receiveAudioHandle.py

@@ -1,77 +0,0 @@
-from config.logger import setup_logging
-import time
-from core.utils.util import remove_punctuation_and_length
-from core.handle.sendAudioHandle import send_stt_message
-
-TAG = __name__
-logger = setup_logging()
-
-
-async def handleAudioMessage(conn, audio):
-    if not conn.asr_server_receive:
-        logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
-        return
-    if conn.client_listen_mode == "auto":
-        have_voice = conn.vad.is_vad(conn, audio)
-    else:
-        have_voice = conn.client_have_voice
-
-    # 如果本次没有声音，本段也没声音，就把声音丢弃了
-    if have_voice == False and conn.client_have_voice == False:
-        await no_voice_close_connect(conn)
-        conn.asr_audio.clear()
-        return
-    conn.client_no_voice_last_time = 0.0
-    conn.asr_audio.append(audio)
-    # 如果本段有声音，且已经停止了
-    if conn.client_voice_stop:
-        conn.client_abort = False
-        conn.asr_server_receive = False
-        # 音频太短了，无法识别
-        if len(conn.asr_audio) < 3:
-            conn.asr_server_receive = True
-        else:
-            text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
-            logger.bind(tag=TAG).info(f"识别文本: {text}")
-            text_len, text_without_punctuation = remove_punctuation_and_length(text)
-            if await conn.music_handler.handle_music_command(conn, text_without_punctuation):
-                conn.asr_server_receive = True
-                conn.asr_audio.clear()
-                return
-            if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation):
-                return
-            if text_len > 0:
-                await startToChat(conn, text)
-            else:
-                conn.asr_server_receive = True
-        conn.asr_audio.clear()
-        conn.reset_vad_states()
-
-
-async def handleCMDMessage(conn, text):
-    cmd_exit = conn.cmd_exit
-    for cmd in cmd_exit:
-        if text == cmd:
-            logger.bind(tag=TAG).info("识别到明确的退出命令".format(text))
-            await conn.close()
-            return True
-    return False
-
-
-async def startToChat(conn, text):
-    # 异步发送 stt 信息
-    await send_stt_message(conn, text)
-    conn.executor.submit(conn.chat, text)
-
-
-async def no_voice_close_connect(conn):
-    if conn.client_no_voice_last_time == 0.0:
-        conn.client_no_voice_last_time = time.time() * 1000
-    else:
-        no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
-        close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
-        if no_voice_time > 1000 * close_connection_no_voice_time:
-            conn.client_abort = False
-            conn.asr_server_receive = False
-            prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
-            await startToChat(conn, prompt)

core/handle/sendAudioHandle.py

@@ -1,83 +0,0 @@
-from config.logger import setup_logging
-import json
-import asyncio
-import time
-from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
-
-TAG = __name__
-logger = setup_logging()
-
-
-async def isLLMWantToFinish(last_text):
-    _, last_text_without_punctuation = remove_punctuation_and_length(last_text)
-    if "再见" in last_text_without_punctuation or "拜拜" in last_text_without_punctuation:
-        return True
-    return False
-
-
-async def sendAudioMessage(conn, audios, text, text_index=0):
-    # 发送句子开始消息
-    if text_index == conn.tts_first_text_index:
-        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
-    await send_tts_message(conn, "sentence_start", text)
-
-    # 初始化流控参数
-    frame_duration = 60  # 毫秒
-    start_time = time.perf_counter()  # 使用高精度计时器
-    play_position = 0  # 已播放的时长（毫秒）
-
-    for opus_packet in audios:
-        if conn.client_abort:
-            return
-
-        # 计算当前包的预期发送时间
-        expected_time = start_time + (play_position / 1000)
-        current_time = time.perf_counter()
-
-        # 等待直到预期时间
-        delay = expected_time - current_time
-        if delay > 0:
-            await asyncio.sleep(delay)
-
-        # 发送音频包
-        await conn.websocket.send(opus_packet)
-        play_position += frame_duration  # 更新播放位置
-    await send_tts_message(conn, "sentence_end", text)
-    # 发送结束消息（如果是最后一个文本）
-    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
-        await send_tts_message(conn, 'stop', None)
-        if await isLLMWantToFinish(text):
-            await conn.close()
-
-
-async def send_tts_message(conn, state, text=None):
-    """发送 TTS 状态消息"""
-    message = {
-        "type": "tts",
-        "state": state,
-        "session_id": conn.session_id
-    }
-    if text is not None:
-        message["text"] = text
-
-    await conn.websocket.send(json.dumps(message))
-    if state == "stop":
-        conn.clearSpeakStatus()
-
-
-async def send_stt_message(conn, text):
-    """发送 STT 状态消息"""
-    stt_text = get_string_no_punctuation_or_emoji(text)
-    await conn.websocket.send(json.dumps({
-        "type": "stt",
-        "text": stt_text,
-        "session_id": conn.session_id}
-    ))
-    await conn.websocket.send(
-        json.dumps({
-            "type": "llm",
-            "text": "😊",
-            "emotion": "happy",
-            "session_id": conn.session_id}
-        ))
-    await send_tts_message(conn, "start")

core/handle/textHandle.py

@@ -1,44 +0,0 @@
-from config.logger import setup_logging
-import json
-from core.handle.abortHandle import handleAbortMessage
-from core.handle.helloHandle import handleHelloMessage
-from core.handle.receiveAudioHandle import startToChat
-from core.handle.iotHandle import handleIotDescriptors
-
-TAG = __name__
-logger = setup_logging()
-
-
-async def handleTextMessage(conn, message):
-    """处理文本消息"""
-    logger.bind(tag=TAG).info(f"收到文本消息：{message}")
-    try:
-        msg_json = json.loads(message)
-        if isinstance(msg_json, int):
-            await conn.websocket.send(message)
-            return
-        if msg_json["type"] == "hello":
-            await handleHelloMessage(conn)
-        elif msg_json["type"] == "abort":
-            await handleAbortMessage(conn)
-        elif msg_json["type"] == "listen":
-            if "mode" in msg_json:
-                conn.client_listen_mode = msg_json["mode"]
-                logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
-            if msg_json["state"] == "start":
-                conn.client_have_voice = True
-                conn.client_voice_stop = False
-            elif msg_json["state"] == "stop":
-                conn.client_have_voice = True
-                conn.client_voice_stop = True
-            elif msg_json["state"] == "detect":
-                conn.asr_server_receive = False
-                conn.client_have_voice = False
-                conn.asr_audio.clear()
-                if "text" in msg_json:
-                    await startToChat(conn, msg_json["text"])
-        elif msg_json["type"] == "iot":
-            if "descriptors" in msg_json:
-                await handleIotDescriptors(conn, msg_json["descriptors"])
-    except json.JSONDecodeError:
-        await conn.websocket.send(message)

core/providers/asr/base.py

@@ -1,19 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Optional, Tuple, List
-
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-
-class ASRProviderBase(ABC):
-    @abstractmethod
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """解码Opus数据并保存为WAV文件"""
-        pass
-
-    @abstractmethod
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
-        """将语音数据转换为文本"""
-        pass

core/providers/asr/doubao.py

@@ -1,286 +0,0 @@
-import time
-import io
-import wave
-import os
-from typing import Optional, Tuple, List
-import uuid
-import websockets
-import json
-import gzip
-
-import opuslib_next
-from core.providers.asr.base import ASRProviderBase
-
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-CLIENT_FULL_REQUEST = 0b0001
-CLIENT_AUDIO_ONLY_REQUEST = 0b0010
-
-NO_SEQUENCE = 0b0000
-NEG_SEQUENCE = 0b0010
-
-SERVER_FULL_RESPONSE = 0b1001
-SERVER_ACK = 0b1011
-SERVER_ERROR_RESPONSE = 0b1111
-
-NO_SERIALIZATION = 0b0000
-JSON = 0b0001
-THRIFT = 0b0011
-CUSTOM_TYPE = 0b1111
-NO_COMPRESSION = 0b0000
-GZIP = 0b0001
-CUSTOM_COMPRESSION = 0b1111
-
-
-def parse_response(res):
-    """
-    protocol_version(4 bits), header_size(4 bits),
-    message_type(4 bits), message_type_specific_flags(4 bits)
-    serialization_method(4 bits) message_compression(4 bits)
-    reserved （8bits) 保留字段
-    header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
-    payload 类似与http 请求体
-    """
-    protocol_version = res[0] >> 4
-    header_size = res[0] & 0x0f
-    message_type = res[1] >> 4
-    message_type_specific_flags = res[1] & 0x0f
-    serialization_method = res[2] >> 4
-    message_compression = res[2] & 0x0f
-    reserved = res[3]
-    header_extensions = res[4:header_size * 4]
-    payload = res[header_size * 4:]
-    result = {}
-    payload_msg = None
-    payload_size = 0
-    if message_type == SERVER_FULL_RESPONSE:
-        payload_size = int.from_bytes(payload[:4], "big", signed=True)
-        payload_msg = payload[4:]
-    elif message_type == SERVER_ACK:
-        seq = int.from_bytes(payload[:4], "big", signed=True)
-        result['seq'] = seq
-        if len(payload) >= 8:
-            payload_size = int.from_bytes(payload[4:8], "big", signed=False)
-            payload_msg = payload[8:]
-    elif message_type == SERVER_ERROR_RESPONSE:
-        code = int.from_bytes(payload[:4], "big", signed=False)
-        result['code'] = code
-        payload_size = int.from_bytes(payload[4:8], "big", signed=False)
-        payload_msg = payload[8:]
-    if payload_msg is None:
-        return result
-    if message_compression == GZIP:
-        payload_msg = gzip.decompress(payload_msg)
-    if serialization_method == JSON:
-        payload_msg = json.loads(str(payload_msg, "utf-8"))
-    elif serialization_method != NO_SERIALIZATION:
-        payload_msg = str(payload_msg, "utf-8")
-    result['payload_msg'] = payload_msg
-    result['payload_size'] = payload_size
-    return result
-
-
-class ASRProvider(ASRProviderBase):
-    def __init__(self, config: dict, delete_audio_file: bool):
-        self.appid = config.get("appid")
-        self.cluster = config.get("cluster")
-        self.access_token = config.get("access_token")
-        self.output_dir = config.get("output_dir")
-
-        self.host = "openspeech.bytedance.com"
-        self.ws_url = f"wss://{self.host}/api/v2/asr"
-        self.success_code = 1000
-        self.seg_duration = 15000
-
-        # 确保输出目录存在
-        os.makedirs(self.output_dir, exist_ok=True)
-
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
-        file_path = os.path.join(self.output_dir, file_name)
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        with wave.open(file_path, "wb") as wf:
-            wf.setnchannels(1)
-            wf.setsampwidth(2)  # 2 bytes = 16-bit
-            wf.setframerate(16000)
-            wf.writeframes(b"".join(pcm_data))
-
-        return file_path
-
-    @staticmethod
-    def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
-        """Generate protocol header."""
-        header = bytearray()
-        header_size = 1
-        header.append((0b0001 << 4) | header_size)  # Protocol version
-        header.append((message_type << 4) | message_type_specific_flags)
-        header.append((0b0001 << 4) | 0b0001)  # JSON serialization & GZIP compression
-        header.append(0x00)  # reserved
-        return header
-
-    def _construct_request(self, reqid) -> dict:
-        """Construct the request payload."""
-        return {
-            "app": {
-                "appid": f"{self.appid}",
-                "cluster": self.cluster,
-                "token": self.access_token,
-            },
-            "user": {
-                "uid": str(uuid.uuid4()),
-            },
-            "request": {
-                "reqid": reqid,
-                "show_utterances": False,
-                "sequence": 1
-            },
-            "audio": {
-                "format": "wav",
-                "rate": 16000,
-                "language": "zh-CN",
-                "bits": 16,
-                "channel": 1,
-                "codec": "raw",
-            },
-        }
-
-    async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
-        """Send request to Volcano ASR service."""
-        try:
-            auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
-            async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
-                # Prepare request data
-                request_params = self._construct_request(str(uuid.uuid4()))
-                print(request_params)
-                payload_bytes = str.encode(json.dumps(request_params))
-                payload_bytes = gzip.compress(payload_bytes)
-                full_client_request = self._generate_header()
-                full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
-                full_client_request.extend(payload_bytes)  # payload
-
-                # Send header and metadata
-                # full_client_request
-                await websocket.send(full_client_request)
-                res = await websocket.recv()
-                result = parse_response(res)
-                if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
-                    logger.bind(tag=TAG).error(f"ASR error: {result}")
-                    return None
-
-                for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
-                    if last:
-                        audio_only_request = self._generate_header(
-                            message_type=CLIENT_AUDIO_ONLY_REQUEST,
-                            message_type_specific_flags=NEG_SEQUENCE
-                        )
-                    else:
-                        audio_only_request = self._generate_header(
-                            message_type=CLIENT_AUDIO_ONLY_REQUEST
-                        )
-                    payload_bytes = gzip.compress(chunk)
-                    audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
-                    audio_only_request.extend(payload_bytes)  # payload
-                    # Send audio data
-                    await websocket.send(audio_only_request)
-
-                # Receive response
-                response = await websocket.recv()
-                result = parse_response(response)
-
-                if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
-                    if len(result['payload_msg']['result']) > 0:
-                        return result['payload_msg']['result'][0]["text"]
-                    return None
-                else:
-                    logger.bind(tag=TAG).error(f"ASR error: {result}")
-                    return None
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
-            return None
-
-    @staticmethod
-    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
-
-    @staticmethod
-    def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
-        with io.BytesIO(data) as _f:
-            wave_fp = wave.open(_f, 'rb')
-            nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
-            wave_bytes = wave_fp.readframes(nframes)
-        return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
-
-    @staticmethod
-    def slice_data(data: bytes, chunk_size: int) -> (list, bool):
-        """
-        slice data
-        :param data: wav data
-        :param chunk_size: the segment size in one request
-        :return: segment data, last flag
-        """
-        data_len = len(data)
-        offset = 0
-        while offset + chunk_size < data_len:
-            yield data[offset: offset + chunk_size], False
-            offset += chunk_size
-        else:
-            yield data[offset: data_len], True
-
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
-        """将语音数据转换为文本"""
-        try:
-            # 合并所有opus数据包
-            pcm_data = self.decode_opus(opus_data, session_id)
-            combined_pcm_data = b''.join(pcm_data)
-
-            wav_buffer = io.BytesIO()
-
-            with wave.open(wav_buffer, "wb") as wav_file:
-                wav_file.setnchannels(1)  # 设置声道数
-                wav_file.setsampwidth(2)  # 设置采样宽度
-                wav_file.setframerate(16000)  # 设置采样率
-                wav_file.writeframes(combined_pcm_data)  # 写入 PCM 数据
-
-            # 获取封装后的 WAV 数据
-            wav_data = wav_buffer.getvalue()
-            nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
-            size_per_sec = nchannels * sampwidth * framerate
-            segment_size = int(size_per_sec * self.seg_duration / 1000)
-
-            # 语音识别
-            start_time = time.time()
-            text = await self._send_request(wav_data, segment_size)
-            if text:
-                logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
-                return text, None
-            return "", None
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None

core/providers/asr/fun_local.py

@@ -1,110 +0,0 @@
-import time
-import wave
-import os
-import sys
-import io
-from config.logger import setup_logging
-from typing import Optional, Tuple, List
-import uuid
-import opuslib_next
-from core.providers.asr.base import ASRProviderBase
-
-from funasr import AutoModel
-from funasr.utils.postprocess_utils import rich_transcription_postprocess
-
-TAG = __name__
-logger = setup_logging()
-
-
-# 捕获标准输出
-class CaptureOutput:
-    def __enter__(self):
-        self._output = io.StringIO()
-        self._original_stdout = sys.stdout
-        sys.stdout = self._output
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        sys.stdout = self._original_stdout
-        self.output = self._output.getvalue()
-        self._output.close()
-
-        # 将捕获到的内容通过 logger 输出
-        if self.output:
-            logger.bind(tag=TAG).info(self.output.strip())
-
-
-class ASRProvider(ASRProviderBase):
-    def __init__(self, config: dict, delete_audio_file: bool):
-        self.model_dir = config.get("model_dir")
-        self.output_dir = config.get("output_dir")  # 修正配置键名
-        self.delete_audio_file = delete_audio_file
-
-        # 确保输出目录存在
-        os.makedirs(self.output_dir, exist_ok=True)
-        with CaptureOutput():
-            self.model = AutoModel(
-                model=self.model_dir,
-                vad_kwargs={"max_single_segment_time": 30000},
-                disable_update=True,
-                hub="hf"
-                # device="cuda:0",  # 启用GPU加速
-            )
-
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
-        file_path = os.path.join(self.output_dir, file_name)
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        with wave.open(file_path, "wb") as wf:
-            wf.setnchannels(1)
-            wf.setsampwidth(2)  # 2 bytes = 16-bit
-            wf.setframerate(16000)
-            wf.writeframes(b"".join(pcm_data))
-
-        return file_path
-
-    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
-        """语音转文本主处理逻辑"""
-        file_path = None
-        try:
-            # 保存音频文件
-            start_time = time.time()
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
-
-            # 语音识别
-            start_time = time.time()
-            result = self.model.generate(
-                input=file_path,
-                cache={},
-                language="auto",
-                use_itn=True,
-                batch_size_s=60,
-            )
-            text = rich_transcription_postprocess(result[0]["text"])
-            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
-
-            return text, file_path
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", None
-
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

core/providers/llm/base.py

@@ -1,8 +0,0 @@
-from abc import ABC, abstractmethod
-
-
-class LLMProviderBase(ABC):
-    @abstractmethod
-    def response(self, session_id, dialogue):
-        """LLM response generator"""
-        pass

core/providers/llm/coze/coze.py

@@ -1,37 +0,0 @@
-from config.logger import setup_logging
-import requests
-import json
-import re
-from core.providers.llm.base import LLMProviderBase
-import os
-# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
-from cozepy import COZE_CN_BASE_URL
-from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa
-
-TAG = __name__
-logger = setup_logging()
-
-class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        self.personal_access_token = config.get("personal_access_token")
-        self.bot_id = config.get("bot_id")
-        self.user_id = config.get("user_id")
-
-    def response(self, session_id, dialogue):
-        coze_api_token = self.personal_access_token
-        coze_api_base = COZE_CN_BASE_URL
-
-        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-
-        coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
-
-        for event in coze.chat.stream(
-            bot_id=self.bot_id,
-            user_id=self.user_id,
-            additional_messages=[
-                Message.build_user_question_text(last_msg["content"]),
-            ],
-        ):
-            if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
-                print(event.message.content, end="", flush=True)
-                yield event.message.content

core/providers/llm/dify/dify.py

@@ -1,39 +0,0 @@
-import json
-from config.logger import setup_logging
-import requests
-from core.providers.llm.base import LLMProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        self.api_key = config["api_key"]
-        self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
-
-    def response(self, session_id, dialogue):
-        try:
-            # 取最后一条用户消息
-            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-
-            # 发起流式请求
-            with requests.post(
-                    f"{self.base_url}/chat-messages",
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json={
-                        "query": last_msg["content"],
-                        "response_mode": "streaming",
-                        "user": session_id,
-                        "inputs": {}
-                    },
-                    stream=True
-            ) as r:
-                for line in r.iter_lines():
-                    if line.startswith(b'data: '):
-                        event = json.loads(line[6:])
-                        if event.get('answer'):
-                            yield event['answer']
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
-            yield "【服务响应异常】"

core/providers/llm/gemini/gemini.py

@@ -1,81 +0,0 @@
-from config.logger import setup_logging
-import google.generativeai as genai
-from core.providers.llm.base import LLMProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        """初始化Gemini LLM Provider"""
-        self.model_name = config.get("model_name", "gemini-1.5-pro") 
-        self.api_key = config.get("api_key")
-        
-        if not self.api_key or "你" in self.api_key:
-            logger.bind(tag=TAG).error("你还没配置Gemini LLM的密钥，请在配置文件中配置密钥，否则无法正常工作")
-            return
-            
-        try:
-            # 初始化Gemini客户端
-            genai.configure(api_key=self.api_key)
-            self.model = genai.GenerativeModel(self.model_name)
-            
-            # 设置生成参数
-            self.generation_config = {
-                "temperature": 0.7,
-                "top_p": 0.9,
-                "top_k": 40,
-                "max_output_tokens": 2048,
-            }
-            self.chat = None
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
-            self.model = None
-
-    def response(self, session_id, dialogue):
-        """生成Gemini对话响应"""
-        if not self.model:
-            yield "【Gemini服务未正确初始化】"
-            return
-
-        try:
-            # 处理对话历史
-            chat_history = []
-            for msg in dialogue[:-1]:  # 历史对话
-                role = "model" if msg["role"] == "assistant" else "user"
-                content = msg["content"].strip()
-                if content:
-                    chat_history.append({
-                        "role": role,
-                        "parts": [content]
-                    })
-
-            # 获取当前消息
-            current_msg = dialogue[-1]["content"]
-
-            # 创建新的聊天会话
-            chat = self.model.start_chat(history=chat_history)
-            
-            # 发送消息并获取流式响应
-            response = chat.send_message(
-                current_msg,
-                stream=True,
-                generation_config=self.generation_config
-            )
-
-            # 处理流式响应
-            for chunk in response:
-                if hasattr(chunk, 'text') and chunk.text:
-                    yield chunk.text
-
-        except Exception as e:
-            error_msg = str(e)
-            logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
-            
-            # 针对不同错误返回友好提示
-            if "Rate limit" in error_msg:
-                yield "【Gemini服务请求太频繁,请稍后再试】"
-            elif "Invalid API key" in error_msg:
-                yield "【Gemini API key无效】"
-            else:
-                yield f"【Gemini服务响应异常: {error_msg}】"

core/providers/llm/homeassistant/homeassistant.py

@@ -1,62 +0,0 @@
-import requests
-from requests.exceptions import RequestException
-from config.logger import setup_logging
-from core.providers.llm.base import LLMProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-
-class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        self.agent_id = config.get("agent_id")  # 对应 agent_id
-        self.api_key = config.get("api_key")
-        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
-        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
-
-    def response(self, session_id, dialogue):
-        print(dialogue)
-        try:
-            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
-
-            # 提取最后一个 role 为 'user' 的 content
-            input_text = None
-            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
-                # 逆序遍历，找到最后一个 role 为 'user' 的消息
-                for message in reversed(dialogue):
-                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
-                        input_text = message.get("content", "")
-                        break  # 找到后立即退出循环
-
-            # 构造请求数据
-            payload = {
-                "text": input_text,
-                "agent_id": self.agent_id,
-                "conversation_id": session_id  # 使用 session_id 作为 conversation_id
-            }
-            # 设置请求头
-            headers = {
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json"
-            }
-
-            # 发起 POST 请求
-            response = requests.post(self.api_url, json=payload, headers=headers)
-
-            # 检查请求是否成功
-            response.raise_for_status()
-
-            # 解析返回数据
-            data = response.json()
-            speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
-
-            # 返回生成的内容
-            if speech:
-                yield speech
-            else:
-                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
-
-        except RequestException as e:
-            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")

core/providers/llm/ollama/ollama.py

@@ -1,46 +0,0 @@
-from config.logger import setup_logging
-import requests, json
-from core.providers.llm.base import LLMProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-
-class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-
-        self.model_name = config.get("model_name")
-        self.base_url = config.get("base_url", "http://localhost:11434")
-
-    def response(self, session_id, dialogue):        
-        try:
-            # Convert dialogue format to Ollama format
-            prompt = ""
-            for msg in dialogue:
-                if msg["role"] == "system":
-                    prompt += f"System: {msg['content']}\n"
-                elif msg["role"] == "user":
-                    prompt += f"User: {msg['content']}\n"
-                elif msg["role"] == "assistant":
-                    prompt += f"Assistant: {msg['content']}\n"
-
-            # Make request to Ollama API
-            response = requests.post(
-                f"{self.base_url}/api/generate",
-                json={
-                    "model": self.model_name,
-                    "prompt": prompt,
-                    "stream": True
-                },
-                stream=True
-            )
-
-            for line in response.iter_lines():
-                if line:
-                    json_response = json.loads(line)
-                    if "response" in json_response:
-                        yield json_response["response"]
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
-            yield "【Ollama服务响应异常】"

core/providers/llm/openai/openai.py

@@ -1,49 +0,0 @@
-from config.logger import setup_logging
-import openai
-from core.providers.llm.base import LLMProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-
-class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
-        self.model_name = config.get("model_name")
-        self.api_key = config.get("api_key")
-        if 'base_url' in config:
-            self.base_url = config.get("base_url")
-        else:
-            self.base_url = config.get("url")
-        if "你" in self.api_key:
-            logger.bind(tag=TAG).error("你还没配置LLM的密钥，请在配置文件中配置密钥，否则无法正常工作")
-        self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
-
-    def response(self, session_id, dialogue):
-        try:
-            responses = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True
-            )
-            
-            is_active = True
-            for chunk in responses:
-                try:
-                    # 检查是否存在有效的choice且content不为空
-                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
-                    content = delta.content if hasattr(delta, 'content') else ''
-                except IndexError:
-                    content = ''
-                if content:
-                    # 处理标签跨多个chunk的情况
-                    if '<think>' in content:
-                        is_active = False
-                        content = content.split('<think>')[0]
-                    if '</think>' in content:
-                        is_active = True
-                        content = content.split('</think>')[-1]
-                    if is_active:
-                        yield content
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in response generation: {e}")

core/providers/memory/base.py

@@ -1,23 +0,0 @@
-from abc import ABC, abstractmethod
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-class MemoryProviderBase(ABC):
-    def __init__(self, config):
-        self.config = config
-        self.role_id = None
-
-    @abstractmethod
-    async def save_memory(self, msgs):
-        """Save a new memory for specific role and return memory ID"""
-        print("this is base func", msgs)
-
-    @abstractmethod
-    async def query_memory(self, query: str) -> str:
-        """Query memories for specific role based on similarity"""
-        return "please implement query method"
-
-    def set_role_id(self, role_id: str):
-        self.role_id = role_id

core/providers/memory/mem0ai/mem0ai.py

@@ -1,74 +0,0 @@
-from ..base import MemoryProviderBase, logger
-from mem0 import MemoryClient
-
-TAG = __name__
-
-class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
-        super().__init__(config)
-        self.api_key = config.get("api_key", "")
-        self.api_version = config.get("api_version", "v1.1")
-        if len(self.api_key) == 0 or "你" in self.api_key:
-            logger.bind(tag=TAG).error("你还没配置Mem0ai的密钥，请在配置文件中配置密钥，否则无法提供记忆服务")
-            self.use_mem0 = False
-            return
-        else:
-            self.use_mem0 = True
-        self.client = MemoryClient(api_key=self.api_key)
-
-    async def save_memory(self, msgs):
-        if not self.use_mem0:
-            return None
-        if len(msgs) < 2:
-            return None
-        
-        try:
-            # Format the content as a message list for mem0
-            messages = [
-                {"role": message.role, "content": message.content}
-                for message in msgs if message.role != "system"
-            ]
-            result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
-            logger.bind(tag=TAG).debug(f"Save memory result: {result}")
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
-            return None
-
-    async def query_memory(self, query: str)-> str:
-        if not self.use_mem0:
-            return ""
-        try:
-            results = self.client.search(
-                query,
-                user_id=self.role_id,
-                output_format=self.api_version
-            )
-            if not results or 'results' not in results:
-                return ""
-                
-            # Format each memory entry with its update time up to minutes
-            memories = []
-            for entry in results['results']:
-                timestamp = entry.get('updated_at', '')
-                if timestamp:
-                    try:
-                        # Parse and reformat the timestamp
-                        dt = timestamp.split('.')[0]  # Remove milliseconds
-                        formatted_time = dt.replace('T', ' ')
-                    except:
-                        formatted_time = timestamp
-                memory = entry.get('memory', '')
-                if timestamp and memory:
-                    # Store tuple of (timestamp, formatted_string) for sorting
-                    memories.append((timestamp, f"[{formatted_time}] {memory}"))
-            
-            # Sort by timestamp in descending order (newest first)
-            memories.sort(key=lambda x: x[0], reverse=True)
-            
-            # Extract only the formatted strings
-            memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
-            logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
-            return memories_str
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
-            return ""

core/providers/tts/aliyun.py

@@ -1,57 +0,0 @@
-import os
-import uuid
-import json
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-import http.client
-import urllib.parse
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.appkey = config.get("appkey")
-        self.token = config.get("token")
-        self.format = config.get("format", "wav")
-        self.sample_rate = config.get("sample_rate", 16000)
-        self.voice = config.get("voice", "xiaoyun")
-        self.volume = config.get("volume", 50)
-        self.speech_rate = config.get("speech_rate", 0)
-        self.pitch_rate = config.get("pitch_rate", 0)
-
-        self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
-        self.api_url = f"https://{self.host}/stream/v1/tts"
-        self.header = {
-            "Content-Type": "application/json"
-        }
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "appkey": self.appkey,
-            "token": self.token,
-            "text": text,
-            "format": self.format,
-            "sample_rate": self.sample_rate,
-            "voice": self.voice,
-            "volume": self.volume,
-            "speech_rate": self.speech_rate,
-            "pitch_rate": self.pitch_rate
-        }
-
-        print(self.api_url, json.dumps(request_json, ensure_ascii=False))
-        try:
-            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
-            # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
-            if resp.headers['Content-Type'].startswith('audio/'):
-                with open(output_file, 'wb') as f:
-                    f.write(resp.content)
-                return output_file
-            else:
-                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

core/providers/tts/base.py

@@ -1,84 +0,0 @@
-import asyncio
-from config.logger import setup_logging
-import os
-import numpy as np
-import opuslib_next
-from pydub import AudioSegment
-from abc import ABC, abstractmethod
-
-TAG = __name__
-logger = setup_logging()
-
-
-class TTSProviderBase(ABC):
-    def __init__(self, config, delete_audio_file):
-        self.delete_audio_file = delete_audio_file
-        self.output_file = config.get("output_file")
-
-    @abstractmethod
-    def generate_filename(self):
-        pass
-
-    def to_tts(self, text):
-        tmp_file = self.generate_filename()
-        try:
-            max_repeat_time = 5
-            while not os.path.exists(tmp_file) and max_repeat_time > 0:
-                asyncio.run(self.text_to_speak(text, tmp_file))
-                if not os.path.exists(tmp_file):
-                    max_repeat_time = max_repeat_time - 1
-                    logger.bind(tag=TAG).error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次")
-
-            if max_repeat_time > 0:
-                logger.bind(tag=TAG).info(f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次")
-
-            return tmp_file
-        except Exception as e:
-            logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
-            return None
-
-    @abstractmethod
-    async def text_to_speak(self, text, output_file):
-        pass
-
-    def wav_to_opus_data(self, wav_file_path):
-        # 使用pydub加载PCM文件
-        # 获取文件后缀名
-        file_type = os.path.splitext(wav_file_path)[1]
-        if file_type:
-            file_type = file_type.lstrip('.')
-        audio = AudioSegment.from_file(wav_file_path, format=file_type)
-
-        duration = len(audio) / 1000.0
-
-        # 转换为单声道和16kHz采样率（确保与编码器匹配）
-        audio = audio.set_channels(1).set_frame_rate(16000)
-
-        # 获取原始PCM数据（16位小端）
-        raw_data = audio.raw_data
-
-        # 初始化Opus编码器
-        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
-
-        # 编码参数
-        frame_duration = 60  # 60ms per frame
-        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
-
-        opus_datas = []
-        # 按帧处理所有音频数据（包括最后一帧可能补零）
-        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
-            # 获取当前帧的二进制数据
-            chunk = raw_data[i:i + frame_size * 2]
-
-            # 如果最后一帧不足，补零
-            if len(chunk) < frame_size * 2:
-                chunk += b'\x00' * (frame_size * 2 - len(chunk))
-
-            # 转换为numpy数组处理
-            np_frame = np.frombuffer(chunk, dtype=np.int16)
-
-            # 编码Opus数据
-            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
-            opus_datas.append(opus_data)
-
-        return opus_datas, duration

core/providers/tts/cozecn.py

@@ -1,38 +0,0 @@
-import os
-import uuid
-import json
-import base64
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.model = config.get("model")
-        self.access_token = config.get("access_token")
-        self.voice = config.get("voice")
-        self.response_format = config.get("response_format")
-
-        self.host = "api.coze.cn"
-        self.api_url = f"https://{self.host}/v1/audio/speech"
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "input": text,
-            "voice_id": self.voice,
-            "response_format": self.response_format,
-        }
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-            "Content-Type": "application/json"
-        }
-        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
-        data = response.content
-        file_to_save = open(output_file, "wb")
-        file_to_save.write(data)

core/providers/tts/doubao.py

@@ -1,60 +0,0 @@
-import os
-import uuid
-import json
-import base64
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.appid = config.get("appid")
-        self.access_token = config.get("access_token")
-        self.cluster = config.get("cluster")
-        self.voice = config.get("voice")
-        self.api_url = config.get("api_url")
-        self.authorization = config.get("authorization")
-        self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "app": {
-                "appid": f"{self.appid}",
-                "token": "access_token",
-                "cluster": self.cluster
-            },
-            "user": {
-                "uid": "1"
-            },
-            "audio": {
-                "voice_type": self.voice,
-                "encoding": "wav",
-                "speed_ratio": 1.0,
-                "volume_ratio": 1.0,
-                "pitch_ratio": 1.0,
-            },
-            "request": {
-                "reqid": str(uuid.uuid4()),
-                "text": text,
-                "text_type": "plain",
-                "operation": "query",
-                "with_frontend": 1,
-                "frontend_type": "unitTson"
-            }
-        }
-
-        try:
-            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
-            if "data" in resp.json():
-                data = resp.json()["data"]
-                file_to_save = open(output_file, "wb")
-                file_to_save.write(base64.b64decode(data))
-            else:
-                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

core/providers/tts/edge.py

@@ -1,18 +0,0 @@
-import os
-import uuid
-import edge_tts
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.voice = config.get("voice")
-
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        communicate = edge_tts.Communicate(text, voice=self.voice)  # Use your preferred voice
-        await communicate.save(output_file)

core/providers/tts/fishspeech.py

@@ -1,158 +0,0 @@
-
-import base64
-import os
-import uuid
-import requests
-import ormsgpack
-from pathlib import Path
-from pydantic import BaseModel, Field, conint, model_validator
-from typing_extensions import Annotated
-from datetime import datetime
-from typing import Literal
-from core.providers.tts.base import TTSProviderBase
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-
-class ServeReferenceAudio(BaseModel):
-    audio: bytes
-    text: str
-
-    @model_validator(mode="before")
-    def decode_audio(cls, values):
-        audio = values.get("audio")
-        if (
-            isinstance(audio, str) and len(audio) > 255
-        ):  # Check if audio is a string (Base64)
-            try:
-                values["audio"] = base64.b64decode(audio)
-            except Exception as e:
-                # If the audio is not a valid base64 string, we will just ignore it and let the server handle it
-                pass
-        return values
-
-    def __repr__(self) -> str:
-        return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
-
-class ServeTTSRequest(BaseModel):
-    text: str
-    chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
-    # Audio format
-    format: Literal["wav", "pcm", "mp3"] = "wav"
-    # References audios for in-context learning
-    references: list[ServeReferenceAudio] = []
-    # Reference id
-    # For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
-    # Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
-    reference_id: str | None = None
-    seed: int | None = None
-    use_memory_cache: Literal["on", "off"] = "off"
-    # Normalize text for en & zh, this increase stability for numbers
-    normalize: bool = True
-    # not usually used below
-    streaming: bool = False
-    max_new_tokens: int = 1024
-    top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
-    repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
-    temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
-
-    class Config:
-        # Allow arbitrary types for pytorch related types
-        arbitrary_types_allowed = True
-
-
-def audio_to_bytes(file_path):
-    if not file_path or not Path(file_path).exists():
-        return None
-    with open(file_path, "rb") as wav_file:
-        wav = wav_file.read()
-    return wav
-
-def read_ref_text(ref_text):
-    path = Path(ref_text)
-    if path.exists() and path.is_file():
-        with path.open("r", encoding="utf-8") as file:
-            return file.read()
-    return ref_text
-
-class TTSProvider(TTSProviderBase):
-
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-
-        self.reference_id = config.get("reference_id")
-        self.reference_audio = config.get("reference_audio",[]) 
-        self.reference_text = config.get("reference_text",[])
-        self.format =  config.get("format","wav")
-        self.channels = config.get("channels",1)
-        self.rate = config.get("rate",44100)
-        self.api_key = config.get("api_key","YOUR_API_KEY")
-        if "你" in self.api_key:
-            logger.bind(tag=TAG).error("你还没配置FishSpeech TTS的密钥，请在配置文件中配置密钥，否则无法正常工作")
-            return
-        self.normalize = config.get("normalize",True)
-        self.max_new_tokens = config.get("max_new_tokens",1024)
-        self.chunk_length = config.get("chunk_length",200)
-        self.top_p = config.get("top_p",0.7)
-        self.repetition_penalty = config.get("repetition_penalty",1.2)
-        self.temperature = config.get("temperature",0.7)
-        self.streaming = config.get("streaming",False)
-        self.use_memory_cache = config.get("use_memory_cache","on")
-        self.seed = config.get("seed")
-        self.api_url =   config.get("api_url","http://127.0.0.1:8080/v1/tts")
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        # Prepare reference data
-        byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
-        ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
-
-        data = {
-            "text": text,
-            "references": [
-                ServeReferenceAudio(
-                    audio=audio if audio else b"", text=text
-                )
-                for text, audio in zip(ref_texts, byte_audios)
-            ],
-            "reference_id": self.reference_id,
-            "normalize": self.normalize,
-            "format": self.format,
-            "max_new_tokens": self.max_new_tokens,
-            "chunk_length": self.chunk_length,
-            "top_p": self.top_p,
-            "repetition_penalty": self.repetition_penalty,
-            "temperature": self.temperature,
-            "streaming": self.streaming,
-            "use_memory_cache": self.use_memory_cache,
-            "seed": self.seed,
-        }
-
-        pydantic_data = ServeTTSRequest(**data)
-
-        response = requests.post(
-            self.api_url,
-            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
-            headers={
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/msgpack",
-            },
-        )
-
-        if response.status_code == 200:
-            audio_content = response.content
-
-            with open(output_file, "wb") as audio_file:
-                audio_file.write(audio_content)
-                
-
-
-        else:
-            print(f"Request failed with status code {response.status_code}")
-            print(response.json())
-
-

core/providers/tts/gpt_sovits_v2.py

@@ -1,67 +0,0 @@
-import os
-import uuid
-import json
-import base64
-import requests
-from config.logger import setup_logging
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-TAG = __name__
-logger = setup_logging()
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.url = config.get("url")
-        self.text_lang = config.get("text_lang", "zh")
-        self.ref_audio_path = config.get("ref_audio_path")
-        self.prompt_text = config.get("prompt_text")
-        self.prompt_lang = config.get("prompt_lang", "zh")
-        self.top_k = config.get("top_k", 5)
-        self.top_p = config.get("top_p", 1)
-        self.temperature = config.get("temperature", 1)
-        self.text_split_method = config.get("text_split_method", "cut0")
-        self.batch_size = config.get("batch_size", 1)
-        self.batch_threshold = config.get("batch_threshold", 0.75)
-        self.split_bucket = config.get("split_bucket", True)
-        self.return_fragment = config.get("return_fragment", False)
-        self.speed_factor = config.get("speed_factor", 1.0)
-        self.streaming_mode = config.get("streaming_mode", False)
-        self.seed = config.get("seed", -1)
-        self.parallel_infer = config.get("parallel_infer", True)
-        self.repetition_penalty = config.get("repetition_penalty", 1.35)
-        self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "text": text,
-            "text_lang": self.text_lang,
-            "ref_audio_path": self.ref_audio_path,
-            "aux_ref_audio_paths": self.aux_ref_audio_paths,
-            "prompt_text": self.prompt_text,
-            "prompt_lang": self.prompt_lang,
-            "top_k": self.top_k,
-            "top_p": self.top_p,
-            "temperature": self.temperature,
-            "text_split_method": self.text_split_method,
-            "batch_size": self.batch_size,
-            "batch_threshold": self.batch_threshold,
-            "split_bucket": self.split_bucket,
-            "return_fragment": self.return_fragment,
-            "speed_factor": self.speed_factor,
-            "streaming_mode": self.streaming_mode,
-            "seed": self.seed,
-            "parallel_infer": self.parallel_infer,
-            "repetition_penalty": self.repetition_penalty
-        }
-
-        resp = requests.post(self.url, json=request_json)
-        if resp.status_code == 200:
-            with open(output_file, "wb") as file:
-                file.write(resp.content)
-        else:
-            logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")

core/providers/tts/minimax.py

@@ -1,77 +0,0 @@
-import os
-import uuid
-import json
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.group_id = config.get("group_id")
-        self.api_key = config.get("api_key")
-        self.model = config.get("model")
-        self.voice_id = config.get("voice_id")
-
-        default_voice_setting = {
-            "voice_id": "female-shaonv",
-            "speed": 1,
-            "vol": 1,
-            "pitch": 0,
-            "emotion": "happy"
-        }
-        default_pronunciation_dict = {
-            "tone": [
-                "处理/(chu3)(li3)", "危险/dangerous"
-            ]
-        }
-        defult_audio_setting = {
-            "sample_rate": 32000,
-            "bitrate": 128000,
-            "format": "mp3",
-            "channel": 1
-        }
-        self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})}
-        self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})}
-        self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
-        self.timber_weights = config.get("timber_weights", [])
-
-        if self.voice_id:
-            self.voice_setting["voice_id"] = self.voice_id
-
-        self.host = "api.minimax.chat"
-        self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
-        self.header = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {self.api_key}"
-        }
-
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "text": text,
-            "stream": False,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting,
-        }
-
-        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
-
-        try:
-            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
-            # 检查返回请求数据的status_code是否为0
-            if resp.json()["base_resp"]["status_code"] == 0:
-                data = resp.json()['data']['audio']
-                file_to_save = open(output_file, "wb")
-                file_to_save.write(bytes.fromhex(data))
-            else:
-                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

core/providers/tts/siliconflow.py

@@ -1,39 +0,0 @@
-import os
-import uuid
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.model = config.get("model")
-        self.access_token = config.get("access_token")
-        self.voice = config.get("voice")
-        self.response_format = config.get("response_format")
-        self.sample_rate = config.get("sample_rate")
-        self.speed = config.get("speed")
-        self.gain = config.get("gain")
-
-        self.host = "api.siliconflow.cn"
-        self.api_url = f"https://{self.host}/v1/audio/speech"
-
-    def generate_filename(self, extension=".wav"):
-        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "input": text,
-            "voice": self.voice,
-            "response_format": self.response_format,
-        }
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-            "Content-Type": "application/json"
-        }
-        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
-        data = response.content
-        file_to_save = open(output_file, "wb")
-        file_to_save.write(data)

core/utils/asr.py

@@ -1,24 +0,0 @@
-import importlib
-import logging
-import os
-import sys
-import time
-import wave
-import uuid
-from abc import ABC, abstractmethod
-from typing import Optional, Tuple, List
-from core.providers.asr.base import ASRProviderBase
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-def create_instance(class_name: str, *args, **kwargs) -> ASRProviderBase:
-    """工厂方法创建ASR实例"""
-    if os.path.exists(os.path.join('core', 'providers', 'asr', f'{class_name}.py')):
-        lib_name = f'core.providers.asr.{class_name}'
-        if lib_name not in sys.modules:
-            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
-        return sys.modules[lib_name].ASRProvider(*args, **kwargs)
-
-    raise ValueError(f"不支持的ASR类型: {class_name}，请检查该配置的type是否设置正确")

core/utils/auth_code_gen.py

@@ -1,97 +0,0 @@
-import random
-import threading
-import time
-from typing import Set
-
-class AuthCodeGenerator:
-    _instance = None
-    _instance_lock = threading.Lock()
-    
-    def __new__(cls):
-        if not cls._instance:
-            with cls._instance_lock:
-                if not cls._instance:
-                    cls._instance = super(AuthCodeGenerator, cls).__new__(cls)
-                    # 初始化随机种子
-                    random.seed(time.time())
-        return cls._instance
-
-    def __init__(self):
-        # 确保 __init__ 只被调用一次
-        if not hasattr(self, '_initialized'):
-            self._used_codes: Set[str] = set()
-            self._code_timestamps = {}
-            self._lock = threading.Lock()
-            self._code_timeout = 3 * 24 * 60 * 60
-            self._initialized = True
-    
-    @classmethod
-    def get_instance(cls):
-        """获取AuthCodeGenerator的单例实例"""
-        return cls()
-
-    def generate_code(self) -> str:
-        """
-        生成6位数字认证码，确保不重复
-        返回: 6位数字字符串
-        """
-        with self._lock:
-            self._clean_expired_codes()  # 清理过期code
-            while True:
-                # 使用时间戳和已用码数量作为种子，确保每次生成不同的随机数
-                seed = int(time.time() * 1000) + len(self._used_codes)
-                random.seed(seed)
-                
-                # 生成6位随机数字
-                code = ''.join(str(random.randint(0, 9)) for _ in range(6))
-                
-                # 检查是否已存在
-                if code not in self._used_codes:
-                    self._used_codes.add(code)
-                    self._code_timestamps[code] = time.time()
-                    return code
-    
-    def remove_code(self, code: str) -> bool:
-        """
-        删除已使用的认证码
-        参数:
-            code: 要删除的认证码
-        返回:
-            bool: 删除成功返回True，码不存在返回False
-        """
-        print('remove_code', code)
-        with self._lock:
-            if code in self._used_codes:
-                self._used_codes.remove(code)
-                if code in self._code_timestamps:
-                    del self._code_timestamps[code]
-                return True
-            return False
-    
-    def is_code_used(self, code: str) -> bool:
-        """
-        检查认证码是否已被使用
-        参数:
-            code: 要检查的认证码
-        返回:
-            bool: 如果码存在返回True，否则返回False
-        """
-        with self._lock:
-            return code in self._used_codes
-    
-    def clear_codes(self):
-        """清空所有已使用的认证码"""
-        with self._lock:
-            self._used_codes.clear()
-            self._code_timestamps.clear()
-    
-    def _clean_expired_codes(self):
-        """清理过期的认证码"""
-        current_time = time.time()
-        expired_codes = [
-            code for code, timestamp in self._code_timestamps.items()
-            if (current_time - timestamp) > self._code_timeout
-        ]
-        for code in expired_codes:
-            self._used_codes.remove(code)
-            del self._code_timestamps[code]

core/utils/dialogue.py

@@ -1,50 +0,0 @@
-import uuid
-from typing import List, Dict
-from datetime import datetime
-
-
-class Message:
-    def __init__(self, role: str, content: str = None, uniq_id: str = None):
-        self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
-        self.role = role
-        self.content = content
-
-
-class Dialogue:
-    def __init__(self):
-        self.dialogue: List[Message] = []
-        # 获取当前时间
-        self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-
-    def put(self, message: Message):
-        self.dialogue.append(message)
-
-    def get_llm_dialogue(self) -> List[Dict[str, str]]:
-        dialogue = []
-        for m in self.dialogue:
-            dialogue.append({"role": m.role, "content": m.content})
-        return dialogue
-
-    def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
-        # 构建带记忆的对话
-        dialogue = []
-        
-        # 添加系统提示和记忆
-        system_message = next(
-            (msg for msg in self.dialogue if msg.role == "system"), None
-        )
-
-
-        if system_message:
-            enhanced_system_prompt = (
-                f"{system_message.content}\n\n"
-                f"相关记忆：\n{memory_str}"
-            )
-            dialogue.append({"role": "system", "content": enhanced_system_prompt})
-
-        # 添加用户和助手的对话
-        for msg in self.dialogue:
-            if msg.role != "system":  # 跳过原始的系统消息
-                dialogue.append({"role": msg.role, "content": msg.content})
-
-        return dialogue

core/utils/llm.py

@@ -1,23 +0,0 @@
-import os
-import sys
-
-# 添加项目根目录到Python路径
-current_dir = os.path.dirname(os.path.abspath(__file__))
-project_root = os.path.abspath(os.path.join(current_dir, "..", ".."))
-sys.path.insert(0, project_root)
-
-from config.logger import setup_logging
-import importlib
-
-logger = setup_logging()
-
-
-def create_instance(class_name, *args, **kwargs):
-    # 创建LLM实例
-    if os.path.exists(os.path.join('core', 'providers', 'llm', class_name, f'{class_name}.py')):
-        lib_name = f'core.providers.llm.{class_name}.{class_name}'
-        if lib_name not in sys.modules:
-            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
-        return sys.modules[lib_name].LLMProvider(*args, **kwargs)
-
-    raise ValueError(f"不支持的LLM类型: {class_name}，请检查该配置的type是否设置正确")

core/utils/lock_manager.py

@@ -1,39 +0,0 @@
-import asyncio
-from typing import Dict
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
-
-class FileLockManager:
-    _instance = None
-    _locks: Dict[str, asyncio.Lock] = {}
-
-    def __new__(cls):
-        if cls._instance is None:
-            cls._instance = super(FileLockManager, cls).__new__(cls)
-        return cls._instance
-
-    @classmethod
-    def get_lock(cls, file_path: str) -> asyncio.Lock:
-        """获取指定文件的锁"""
-        if file_path not in cls._locks:
-            cls._locks[file_path] = asyncio.Lock()
-        return cls._locks[file_path]
-
-    @classmethod
-    async def acquire_lock(cls, file_path: str):
-        """获取锁"""
-        lock = cls.get_lock(file_path)
-        await lock.acquire()
-        logger.bind(tag=TAG).debug(f"Acquired lock for {file_path}")
-
-    @classmethod
-    def release_lock(cls, file_path: str):
-        """释放锁"""
-        if file_path in cls._locks:
-            try:
-                cls._locks[file_path].release()
-                logger.bind(tag=TAG).debug(f"Released lock for {file_path}")
-            except RuntimeError as e:
-                logger.bind(tag=TAG).warning(f"Failed to release lock for {file_path}: {e}")

core/utils/memory.py

@@ -1,17 +0,0 @@
-import os
-import sys
-import importlib
-from config.logger import setup_logging
-from core.utils.util import read_config, get_project_dir
-
-logger = setup_logging()
-
-def create_instance(class_name, *args, **kwargs):
-    if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')):
-        lib_name = f'core.providers.memory.{class_name}.{class_name}'
-        if lib_name not in sys.modules:
-            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
-        return sys.modules[lib_name].MemoryProvider(*args, **kwargs)
-
-    raise ValueError(f"不支持的记忆服务类型: {class_name}")
-

core/utils/p3.py

@@ -1,33 +0,0 @@
-import struct
-
-def decode_opus_from_file(input_file):
-    """
-    从p3文件中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。
-    """
-    opus_datas = []
-    total_frames = 0
-    sample_rate = 16000  # 文件采样率
-    frame_duration_ms = 60  # 帧时长
-    frame_size = int(sample_rate * frame_duration_ms / 1000)
-
-    with open(input_file, 'rb') as f:
-        while True:
-            # 读取头部（4字节）：[1字节类型，1字节保留，2字节长度]
-            header = f.read(4)
-            if not header:
-                break
-
-            # 解包头部信息
-            _, _, data_len = struct.unpack('>BBH', header)
-
-            # 根据头部指定的长度读取 Opus 数据
-            opus_data = f.read(data_len)
-            if len(opus_data) != data_len:
-                raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the file.")
-
-            opus_datas.append(opus_data)
-            total_frames += 1
-
-    # 计算总时长
-    total_duration = (total_frames * frame_duration_ms) / 1000.0
-    return opus_datas, total_duration

core/utils/tts.py

@@ -1,17 +0,0 @@
-import os
-import sys
-from config.logger import setup_logging
-import importlib
-
-logger = setup_logging()
-
-
-def create_instance(class_name, *args, **kwargs):
-    # 创建TTS实例
-    if os.path.exists(os.path.join('core', 'providers', 'tts', f'{class_name}.py')):
-        lib_name = f'core.providers.tts.{class_name}'
-        if lib_name not in sys.modules:
-            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
-        return sys.modules[lib_name].TTSProvider(*args, **kwargs)
-
-    raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")

core/utils/util.py

@@ -1,143 +0,0 @@
-import os
-import re
-import json
-import yaml
-import socket
-import subprocess
-
-
-def get_project_dir():
-    """获取项目根目录"""
-    return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + '/'
-
-
-def get_local_ip():
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        # Connect to Google's DNS servers
-        s.connect(("8.8.8.8", 80))
-        local_ip = s.getsockname()[0]
-        s.close()
-        return local_ip
-    except Exception as e:
-        return "127.0.0.1"
-
-
-def read_config(config_path):
-    with open(config_path, "r", encoding="utf-8") as file:
-        config = yaml.safe_load(file)
-    return config
-
-
-def write_json_file(file_path, data):
-    """将数据写入 JSON 文件"""
-    with open(file_path, 'w', encoding='utf-8') as file:
-        json.dump(data, file, ensure_ascii=False, indent=4)
-
-
-def is_punctuation_or_emoji(char):
-    """检查字符是否为空格、指定标点或表情符号"""
-    # 定义需要去除的中英文标点（包括全角/半角）
-    punctuation_set = {
-        '，', ',',  # 中文逗号 + 英文逗号
-        '。', '.',  # 中文句号 + 英文句号
-        '！', '!',  # 中文感叹号 + 英文感叹号
-        '-', '－',  # 英文连字符 + 中文全角横线
-        '、'  # 中文顿号
-    }
-    if char.isspace() or char in punctuation_set:
-        return True
-    # 检查表情符号（保留原有逻辑）
-    code_point = ord(char)
-    emoji_ranges = [
-        (0x1F600, 0x1F64F), (0x1F300, 0x1F5FF),
-        (0x1F680, 0x1F6FF), (0x1F900, 0x1F9FF),
-        (0x1FA70, 0x1FAFF), (0x2600, 0x26FF),
-        (0x2700, 0x27BF)
-    ]
-    return any(start <= code_point <= end for start, end in emoji_ranges)
-
-
-def get_string_no_punctuation_or_emoji(s):
-    """去除字符串首尾的空格、标点符号和表情符号"""
-    chars = list(s)
-    # 处理开头的字符
-    start = 0
-    while start < len(chars) and is_punctuation_or_emoji(chars[start]):
-        start += 1
-    # 处理结尾的字符
-    end = len(chars) - 1
-    while end >= start and is_punctuation_or_emoji(chars[end]):
-        end -= 1
-    return ''.join(chars[start:end + 1])
-
-
-def remove_punctuation_and_length(text):
-    # 全角符号和半角符号的Unicode范围
-    full_width_punctuations = '！＂＃＄％＆＇（）＊＋，－。／：；＜＝＞？＠［＼］＾＿｀｛｜｝～'
-    half_width_punctuations = '!"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~'
-    space = ' '  # 半角空格
-    full_width_space = '　'  # 全角空格
-
-    # 去除全角和半角符号以及空格
-    result = ''.join([char for char in text if
-                      char not in full_width_punctuations and char not in half_width_punctuations and char not in space and char not in full_width_space])
-
-    if result == "Yeah":
-        return 0, ""
-    return len(result), result
-
-
-def check_password(password):
-    """
-    检查密码是否满足以下条件：
-    1. 密码长度大于八位。
-    2. 密码包含英文和数字。
-    3. 密码不能包含“xiaozhi”字符。
-
-    :param password: 要检查的密码
-    :return: 如果密码满足条件，则返回True；否则返回False。
-    """
-    # 检查密码长度
-    if len(password) < 8:
-        return False
-
-    # 检查是否包含英文字符和数字
-    if not re.search(r'[A-Za-z]', password) or not re.search(r'[0-9]', password):
-        return False
-
-    # 检查是否包含“xiaozhi”字符
-    if "xiaozhi" in password:
-        return False
-
-    if "1234" in password:
-        return False
-
-    # 如果满足所有条件，则返回True
-    return True
-
-def check_ffmpeg_installed():
-    ffmpeg_installed = False
-    try:
-        # 执行ffmpeg -version命令，并捕获输出
-        result = subprocess.run(
-            ['ffmpeg', '-version'],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            check=True  # 如果返回码非零则抛出异常
-        )
-        # 检查输出中是否包含版本信息（可选）
-        output = result.stdout + result.stderr
-        if 'ffmpeg version' in output.lower():
-            ffmpeg_installed = True
-        return False
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        # 命令执行失败或未找到
-        ffmpeg_installed = False
-    if not ffmpeg_installed:
-        error_msg = "您的电脑还没正确安装ffmpeg\n"
-        error_msg += "\n建议您：\n"
-        error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
-        error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
-        raise ValueError(error_msg)

core/utils/vad.py

@@ -1,77 +0,0 @@
-from abc import ABC, abstractmethod
-from config.logger import setup_logging
-import opuslib_next
-import time
-import numpy as np
-import torch
-
-TAG = __name__
-logger = setup_logging()
-
-class VAD(ABC):
-    @abstractmethod
-    def is_vad(self, conn, data):
-        """检测音频数据中的语音活动"""
-        pass
-
-
-class SileroVAD(VAD):
-    def __init__(self, config):
-        logger.bind(tag=TAG).info("SileroVAD", config)
-        self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
-                                                source='local',
-                                                model='silero_vad',
-                                                force_reload=False)
-        (get_speech_timestamps, _, _, _, _) = self.utils
-
-        self.decoder = opuslib_next.Decoder(16000, 1)
-        self.vad_threshold = config.get("threshold")
-        self.silence_threshold_ms = config.get("min_silence_duration_ms")
-
-    def is_vad(self, conn, opus_packet):
-        try:
-            pcm_frame = self.decoder.decode(opus_packet, 960)
-            conn.client_audio_buffer += pcm_frame  # 将新数据加入缓冲区
-
-            # 处理缓冲区中的完整帧（每次处理512采样点）
-            client_have_voice = False
-            while len(conn.client_audio_buffer) >= 512 * 2:
-                # 提取前512个采样点（1024字节）
-                chunk = conn.client_audio_buffer[:512 * 2]
-                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
-
-                # 转换为模型需要的张量格式
-                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
-                audio_float32 = audio_int16.astype(np.float32) / 32768.0
-                audio_tensor = torch.from_numpy(audio_float32)
-
-                # 检测语音活动
-                speech_prob = self.model(audio_tensor, 16000).item()
-                client_have_voice = speech_prob >= self.vad_threshold
-
-                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
-                if conn.client_have_voice and not client_have_voice:
-                    stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
-                    if stop_duration >= self.silence_threshold_ms:
-                        conn.client_voice_stop = True
-                if client_have_voice:
-                    conn.client_have_voice = True
-                    conn.client_have_voice_last_time = time.time() * 1000
-
-            return client_have_voice
-        except opuslib_next.OpusError as e:
-            logger.bind(tag=TAG).info(f"解码错误: {e}")
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
-
-
-def create_instance(class_name, *args, **kwargs) -> VAD:
-    # 获取类对象
-    cls_map = {
-        "SileroVAD": SileroVAD,
-        # 可扩展其他SileroVAD实现
-    }
-
-    if cls := cls_map.get(class_name):
-        return cls(*args, **kwargs)
-    raise ValueError(f"不支持的SileroVAD类型: {class_name}")

core/websocket_server.py

@@ -1,79 +0,0 @@
-import asyncio
-import websockets
-from config.logger import setup_logging
-from core.connection import ConnectionHandler
-from core.handle.musicHandler import MusicHandler
-from core.utils.util import get_local_ip
-from core.utils import asr, vad, llm, tts, memory
-
-TAG = __name__
-
-
-class WebSocketServer:
-    def __init__(self, config: dict):
-        self.config = config
-        self.logger = setup_logging()
-        self._vad, self._asr, self._llm, self._tts, self._music, self._memory = self._create_processing_instances()
-        self.active_connections = set()  # 添加全局连接记录
-
-    def _create_processing_instances(self):
-        memory_cls_name = self.config["selected_module"].get("Memory", "mem0ai") # 默认使用mem0ai
-        has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
-        memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
-
-        """创建处理模块实例"""
-        return (
-            vad.create_instance(
-                self.config["selected_module"]["VAD"],
-                self.config["VAD"][self.config["selected_module"]["VAD"]]
-            ),
-            asr.create_instance(
-                self.config["selected_module"]["ASR"]
-                if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
-                else
-                self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
-                self.config["ASR"][self.config["selected_module"]["ASR"]],
-                self.config["delete_audio"]
-            ),
-            llm.create_instance(
-                self.config["selected_module"]["LLM"]
-                if not 'type' in self.config["LLM"][self.config["selected_module"]["LLM"]]
-                else
-                self.config["LLM"][self.config["selected_module"]["LLM"]]['type'],
-                self.config["LLM"][self.config["selected_module"]["LLM"]],
-            ),
-            tts.create_instance(
-                self.config["selected_module"]["TTS"]
-                if not 'type' in self.config["TTS"][self.config["selected_module"]["TTS"]]
-                else
-                self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
-                self.config["TTS"][self.config["selected_module"]["TTS"]],
-                self.config["delete_audio"]
-            ),
-            MusicHandler(self.config),
-            memory.create_instance(memory_cls_name, memory_cfg),
-        )
-
-    async def start(self):
-        server_config = self.config["server"]
-        host = server_config["ip"]
-        port = server_config["port"]
-
-        self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
-        self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")
-        async with websockets.serve(
-                self._handle_connection,
-                host,
-                port
-        ):
-            await asyncio.Future()
-
-    async def _handle_connection(self, websocket):
-        """处理新连接，每次创建独立的ConnectionHandler"""
-        # 创建ConnectionHandler时传入当前server实例
-        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory)
-        self.active_connections.add(handler)
-        try:
-            await handler.handle_connection(websocket)
-        finally:
-            self.active_connections.discard(handler)