update:合并非tts代码

This commit is contained in:
hrz
2025-05-21 13:18:12 +08:00
parent ede2bc6a4e
commit 851365fb58
65 changed files with 5423 additions and 1943 deletions
+144
View File
@@ -0,0 +1,144 @@
import os
import argparse
import yaml
from collections.abc import Mapping
from config.manage_api_client import init_service, get_server_config, get_agent_models
# 添加全局配置缓存
_config_cache = None
def get_project_dir():
"""获取项目根目录"""
return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/"
def read_config(config_path):
with open(config_path, "r", encoding="utf-8") as file:
config = yaml.safe_load(file)
return config
def load_config():
"""加载配置文件"""
global _config_cache
if _config_cache is not None:
return _config_cache
default_config_path = get_project_dir() + "config.yaml"
custom_config_path = get_project_dir() + "data/.config.yaml"
# 加载默认配置
default_config = read_config(default_config_path)
custom_config = read_config(custom_config_path)
if custom_config.get("manager-api", {}).get("url"):
config = get_config_from_api(custom_config)
else:
# 合并配置
config = merge_configs(default_config, custom_config)
# 初始化目录
ensure_directories(config)
_config_cache = config
return config
def get_config_from_api(config):
"""从Java API获取配置"""
# 初始化API客户端
init_service(config)
# 获取服务器配置
config_data = get_server_config()
if config_data is None:
raise Exception("Failed to fetch server config from API")
config_data["read_config_from_api"] = True
config_data["manager-api"] = {
"url": config["manager-api"].get("url", ""),
"secret": config["manager-api"].get("secret", ""),
}
if config.get("server"):
config_data["server"] = {
"ip": config["server"].get("ip", ""),
"port": config["server"].get("port", ""),
}
return config_data
def get_private_config_from_api(config, device_id, client_id):
"""从Java API获取私有配置"""
return get_agent_models(device_id, client_id, config["selected_module"])
def ensure_directories(config):
"""确保所有配置路径存在"""
dirs_to_create = set()
project_dir = get_project_dir() # 获取项目根目录
# 日志文件目录
log_dir = config.get("log", {}).get("log_dir", "tmp")
dirs_to_create.add(os.path.join(project_dir, log_dir))
# ASR/TTS模块输出目录
for module in ["ASR", "TTS"]:
if config.get(module) is None:
continue
for provider in config.get(module, {}).values():
output_dir = provider.get("output_dir", "")
if output_dir:
dirs_to_create.add(output_dir)
# 根据selected_module创建模型目录
selected_modules = config.get("selected_module", {})
for module_type in ["ASR", "LLM", "TTS"]:
selected_provider = selected_modules.get(module_type)
if not selected_provider:
continue
if config.get(module) is None:
continue
if config.get(selected_provider) is None:
continue
provider_config = config.get(module_type, {}).get(selected_provider, {})
output_dir = provider_config.get("output_dir")
if output_dir:
full_model_dir = os.path.join(project_dir, output_dir)
dirs_to_create.add(full_model_dir)
# 统一创建目录(保留原data目录创建)
for dir_path in dirs_to_create:
try:
os.makedirs(dir_path, exist_ok=True)
except PermissionError:
print(f"警告:无法创建目录 {dir_path},请检查写入权限")
def merge_configs(default_config, custom_config):
"""
递归合并配置,custom_config优先级更高
Args:
default_config: 默认配置
custom_config: 用户自定义配置
Returns:
合并后的配置
"""
if not isinstance(default_config, Mapping) or not isinstance(
custom_config, Mapping
):
return custom_config
merged = dict(default_config)
for key, value in custom_config.items():
if (
key in merged
and isinstance(merged[key], Mapping)
and isinstance(value, Mapping)
):
merged[key] = merge_configs(merged[key], value)
else:
merged[key] = value
return merged
+43 -9
View File
@@ -1,12 +1,45 @@
import os
import sys
from loguru import logger
from config.settings import load_config
from config.config_loader import load_config
from config.settings import check_config_file
SERVER_VERSION = "0.2.1"
SERVER_VERSION = "0.4.4"
def get_module_abbreviation(module_name, module_dict):
"""获取模块名称的缩写,如果为空则返回00
如果名称中包含下划线,则返回下划线后面的前两个字符
"""
module_value = module_dict.get(module_name, "")
if not module_value:
return "00"
if "_" in module_value:
parts = module_value.split("_")
return parts[-1][:2] if parts[-1] else "00"
return module_value[:2]
def build_module_string(selected_module):
"""构建模块字符串"""
return (
get_module_abbreviation("VAD", selected_module)
+ get_module_abbreviation("ASR", selected_module)
+ get_module_abbreviation("LLM", selected_module)
+ get_module_abbreviation("TTS", selected_module)
+ get_module_abbreviation("Memory", selected_module)
+ get_module_abbreviation("Intent", selected_module)
)
def formatter(record):
"""为没有 tag 的日志添加默认值"""
record["extra"].setdefault("tag", record["name"])
return record["message"]
def setup_logging():
check_config_file()
"""从配置文件中读取日志配置,并设置日志输出格式和级别"""
config = load_config()
log_config = config["log"]
@@ -18,11 +51,7 @@ def setup_logging():
"log_format_file",
"{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}",
)
selected_module = config.get("selected_module")
selected_module_str = "".join(
[value[0] + value[1] for key, value in selected_module.items()]
)
selected_module_str = build_module_string(config.get("selected_module", {}))
log_format = log_format.replace("{version}", SERVER_VERSION)
log_format = log_format.replace("{selected_module}", selected_module_str)
@@ -41,9 +70,14 @@ def setup_logging():
logger.remove()
# 输出到控制台
logger.add(sys.stdout, format=log_format, level=log_level)
logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter)
# 输出到文件
logger.add(os.path.join(log_dir, log_file), format=log_format_file, level=log_level)
logger.add(
os.path.join(log_dir, log_file),
format=log_format_file,
level=log_level,
filter=formatter,
)
return logger
@@ -0,0 +1,192 @@
import os
import time
import base64
from typing import Optional, Dict
import httpx
TAG = __name__
class DeviceNotFoundException(Exception):
pass
class DeviceBindException(Exception):
def __init__(self, bind_code):
self.bind_code = bind_code
super().__init__(f"设备绑定异常,绑定码: {bind_code}")
class ManageApiClient:
_instance = None
_client = None
_secret = None
def __new__(cls, config):
"""单例模式确保全局唯一实例,并支持传入配置参数"""
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._init_client(config)
return cls._instance
@classmethod
def _init_client(cls, config):
"""初始化持久化连接池"""
cls.config = config.get("manager-api")
if not cls.config:
raise Exception("manager-api配置错误")
if not cls.config.get("url") or not cls.config.get("secret"):
raise Exception("manager-api的url或secret配置错误")
if "" in cls.config.get("secret"):
raise Exception("请先配置manager-api的secret")
cls._secret = cls.config.get("secret")
cls.max_retries = cls.config.get("max_retries", 6) # 最大重试次数
cls.retry_delay = cls.config.get("retry_delay", 10) # 初始重试延迟(秒)
# NOTE(goody): 2025/4/16 http相关资源统一管理,后续可以增加线程池或者超时
# 后续也可以统一配置apiToken之类的走通用的Auth
cls._client = httpx.Client(
base_url=cls.config.get("url"),
headers={
"User-Agent": f"PythonClient/2.0 (PID:{os.getpid()})",
"Accept": "application/json",
"Authorization": "Bearer " + cls._secret,
},
timeout=cls.config.get("timeout", 30), # 默认超时时间30秒
)
@classmethod
def _request(cls, method: str, endpoint: str, **kwargs) -> Dict:
"""发送单次HTTP请求并处理响应"""
endpoint = endpoint.lstrip("/")
response = cls._client.request(method, endpoint, **kwargs)
response.raise_for_status()
result = response.json()
# 处理API返回的业务错误
if result.get("code") == 10041:
raise DeviceNotFoundException(result.get("msg"))
elif result.get("code") == 10042:
raise DeviceBindException(result.get("msg"))
elif result.get("code") != 0:
raise Exception(f"API返回错误: {result.get('msg', '未知错误')}")
# 返回成功数据
return result.get("data") if result.get("code") == 0 else None
@classmethod
def _should_retry(cls, exception: Exception) -> bool:
"""判断异常是否应该重试"""
# 网络连接相关错误
if isinstance(
exception, (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError)
):
return True
# HTTP状态码错误
if isinstance(exception, httpx.HTTPStatusError):
status_code = exception.response.status_code
return status_code in [408, 429, 500, 502, 503, 504]
return False
@classmethod
def _execute_request(cls, method: str, endpoint: str, **kwargs) -> Dict:
"""带重试机制的请求执行器"""
retry_count = 0
while retry_count <= cls.max_retries:
try:
# 执行请求
return cls._request(method, endpoint, **kwargs)
except Exception as e:
# 判断是否应该重试
if retry_count < cls.max_retries and cls._should_retry(e):
retry_count += 1
print(
f"{method} {endpoint} 请求失败,将在 {cls.retry_delay:.1f} 秒后进行第 {retry_count} 次重试"
)
time.sleep(cls.retry_delay)
continue
else:
# 不重试,直接抛出异常
raise
@classmethod
def safe_close(cls):
"""安全关闭连接池"""
if cls._client:
cls._client.close()
cls._instance = None
def get_server_config() -> Optional[Dict]:
"""获取服务器基础配置"""
return ManageApiClient._instance._execute_request("POST", "/config/server-base")
def get_agent_models(
mac_address: str, client_id: str, selected_module: Dict
) -> Optional[Dict]:
"""获取代理模型配置"""
return ManageApiClient._instance._execute_request(
"POST",
"/config/agent-models",
json={
"macAddress": mac_address,
"clientId": client_id,
"selectedModule": selected_module,
},
)
def save_mem_local_short(mac_address: str, short_momery: str) -> Optional[Dict]:
try:
return ManageApiClient._instance._execute_request(
"PUT",
f"/agent/saveMemory/" + mac_address,
json={
"summaryMemory": short_momery,
},
)
except Exception as e:
print(f"存储短期记忆到服务器失败: {e}")
return None
def report(
mac_address: str, session_id: str, chat_type: int, content: str, audio
) -> Optional[Dict]:
"""带熔断的业务方法示例"""
if not content or not ManageApiClient._instance:
return None
try:
return ManageApiClient._instance._execute_request(
"POST",
f"/agent/chat-history/report",
json={
"macAddress": mac_address,
"sessionId": session_id,
"chatType": chat_type,
"content": content,
"audioBase64": (
base64.b64encode(audio).decode("utf-8") if audio else None
),
},
)
except Exception as e:
print(f"TTS上报失败: {e}")
return None
def init_service(config):
ManageApiClient(config)
def manage_api_http_safe_close():
ManageApiClient.safe_close()
@@ -1,241 +0,0 @@
import os
import time
import yaml
from config.logger import setup_logging
from typing import Dict, Any, Optional
from copy import deepcopy
from core.utils.util import get_project_dir
from core.utils import llm, tts
from core.utils.lock_manager import FileLockManager
TAG = __name__
class PrivateConfig:
def __init__(self, device_id: str, default_config: Dict[str, Any], auth_code_gen=None):
self.device_id = device_id
self.default_config = default_config
self.config_path = get_project_dir() + 'data/.private_config.yaml'
self.logger = setup_logging()
self.private_config = {}
self.auth_code_gen = auth_code_gen
self.lock_manager = FileLockManager()
async def load_or_create(self):
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
if os.path.exists(self.config_path):
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
else:
all_configs = {}
if self.device_id not in all_configs:
# Get selected module names
selected_modules = self.default_config['selected_module']
selected_tts = selected_modules['TTS']
selected_llm = selected_modules['LLM']
selected_asr = selected_modules['ASR']
selected_vad = selected_modules['VAD']
# 生成认证码
auth_code = None
if self.auth_code_gen:
auth_code = self.auth_code_gen.generate_code()
# Initialize device config with only necessary configurations
device_config = {
'selected_module': deepcopy(selected_modules),
'prompt': self.default_config['prompt'],
'LLM': {
selected_llm: deepcopy(self.default_config['LLM'][selected_llm])
},
'TTS': {
selected_tts: deepcopy(self.default_config['TTS'][selected_tts])
},
'ASR': {
selected_asr: deepcopy(self.default_config['ASR'][selected_asr])
},
'VAD': {
selected_vad: deepcopy(self.default_config['VAD'][selected_vad])
},
'auth_code': auth_code # 添加认证码字段
}
all_configs[self.device_id] = device_config
# Save updated configs
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
self.private_config = all_configs[self.device_id]
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error handling private config: {e}")
self.private_config = {}
async def update_config(self, selected_modules: Dict[str, str], prompt: str, nickname: str) -> bool:
"""更新设备配置
Args:
selected_modules: 选择的模块配置,格式如 {'LLM': 'AliLLM', 'TTS': 'EdgeTTS',...}
prompt: 提示词配置
Returns:
bool: 更新是否成功
"""
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
# Read main config to get full module configurations
main_config = self.default_config
# Create new device config
device_config = {
'selected_module': selected_modules,
'prompt': prompt,
'nickname': nickname,
}
if self.private_config.get('last_chat_time'):
device_config['last_chat_time'] = self.private_config['last_chat_time']
if self.private_config.get('owner'):
device_config['owner'] = self.private_config['owner']
# Copy full module configurations from main config
for module_type, selected_name in selected_modules.items():
if selected_name and selected_name in main_config.get(module_type, {}):
device_config[module_type] = {
selected_name: main_config[module_type][selected_name]
}
# Read all configs
if os.path.exists(self.config_path):
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
else:
all_configs = {}
# Update device config
all_configs[self.device_id] = device_config
self.private_config = device_config
# Save back to file
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
return True
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error updating config: {e}")
return False
async def delete_config(self) -> bool:
"""删除设备配置
Returns:
bool: 删除是否成功
"""
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
# 读取所有配置
if os.path.exists(self.config_path):
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
else:
return False
# 删除设备配置
if self.device_id in all_configs:
del all_configs[self.device_id]
# 保存更新后的配置
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
self.private_config = {}
return True
return False
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error deleting config: {e}")
return False
def create_private_instances(self):
# 判断存在私有配置,并且self.device_id在私有配置中
if not self.private_config:
self.logger.bind(tag=TAG).error("Private config not found for device_id: {}", self.device_id)
return None, None
"""创建私有处理模块实例"""
config = self.private_config
selected_modules = config['selected_module']
return (
llm.create_instance(
selected_modules["LLM"]
if not 'type' in config["LLM"][selected_modules["LLM"]]
else
config["LLM"][selected_modules["LLM"]]['type'],
config["LLM"][selected_modules["LLM"]],
),
tts.create_instance(
selected_modules["TTS"]
if not 'type' in config["TTS"][selected_modules["TTS"]]
else
config["TTS"][selected_modules["TTS"]]["type"],
config["TTS"][selected_modules["TTS"]],
self.default_config.get("delete_audio", True) # Using default_config for global settings
)
)
async def update_last_chat_time(self, timestamp=None):
"""更新设备最近一次的聊天时间
Args:
timestamp: 指定的时间戳,不传则使用当前时间
"""
if not self.private_config:
self.logger.bind(tag=TAG).error("Private config not found")
return False
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
if timestamp is None:
timestamp = int(time.time())
self.private_config['last_chat_time'] = timestamp
# 读取所有配置
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
# 更新当前设备配置
all_configs[self.device_id] = self.private_config
# 保存回文件
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
return True
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error updating last chat time: {e}")
return False
def get_auth_code(self) -> str:
"""获取设备的认证码
Returns:
str: 认证码,如果没有返回空字符串
"""
return self.private_config.get('auth_code', '')
def get_owner(self) -> Optional[str]:
"""获取设备当前所有者"""
return self.private_config.get('owner')
+25 -119
View File
@@ -1,127 +1,33 @@
import os
import argparse
from ruamel.yaml import YAML
from collections.abc import Mapping
from core.utils.util import read_config, get_project_dir
from config.config_loader import read_config, get_project_dir, load_config
default_config_file = "config.yaml"
def ensure_directories(config):
"""确保所有配置路径存在"""
dirs_to_create = set()
project_dir = get_project_dir() # 获取项目根目录
# 日志文件目录
log_dir = config.get('log', {}).get('log_dir', 'tmp')
dirs_to_create.add(os.path.join(project_dir, log_dir))
# ASR/TTS模块输出目录
for module in ['ASR', 'TTS']:
for provider in config.get(module, {}).values():
output_dir = provider.get('output_dir', '')
if output_dir:
dirs_to_create.add(output_dir)
# 根据selected_module创建模型目录
selected_modules = config.get('selected_module', {})
for module_type in ['ASR', 'LLM', 'TTS']:
selected_provider = selected_modules.get(module_type)
if not selected_provider:
continue
provider_config = config.get(module_type, {}).get(selected_provider, {})
output_dir = provider_config.get('output_dir')
if output_dir:
full_model_dir = os.path.join(project_dir, output_dir)
dirs_to_create.add(full_model_dir)
# 统一创建目录(保留原data目录创建)
for dir_path in dirs_to_create:
try:
os.makedirs(dir_path, exist_ok=True)
except PermissionError:
print(f"警告:无法创建目录 {dir_path},请检查写入权限")
def get_config_file():
global default_config_file
"""获取配置文件路径,优先使用私有配置文件(若存在)。
Returns:
str: 配置文件路径(相对路径或默认路径)
"""
config_file = default_config_file
if os.path.exists(get_project_dir() + "data/." + default_config_file):
config_file = "data/." + default_config_file
return config_file
def load_config():
"""加载配置文件"""
parser = argparse.ArgumentParser(description="Server configuration")
config_file = get_config_file()
parser.add_argument("--config_path", type=str, default=config_file)
args = parser.parse_args()
config = read_config(args.config_path)
# 初始化目录
ensure_directories(config)
return config
def update_config(config):
yaml = YAML()
yaml.preserve_quotes = True
"""将配置保存到YAML文件"""
with open(get_config_file(), 'w') as f:
yaml.dump(config, f)
def find_missing_keys(new_config, old_config, parent_key=''):
"""
递归查找缺失的配置项
返回格式:[缺失配置路径]
"""
missing_keys = []
if not isinstance(new_config, Mapping):
return missing_keys
for key, value in new_config.items():
# 构建当前配置路径
full_path = f"{parent_key}.{key}" if parent_key else key
# 检查键是否存在
if key not in old_config:
missing_keys.append(full_path)
continue
# 递归检查嵌套字典
if isinstance(value, Mapping):
sub_missing = find_missing_keys(
value,
old_config[key],
parent_key=full_path
)
missing_keys.extend(sub_missing)
return missing_keys
config_file_valid = False
def check_config_file():
old_config_file = get_config_file()
global default_config_file
if not 'data' in old_config_file:
global config_file_valid
if config_file_valid:
return
old_config = read_config(get_project_dir() + old_config_file)
new_config = read_config(get_project_dir() + default_config_file)
# 查找缺失的配置项
missing_keys = find_missing_keys(new_config, old_config)
"""
简化的配置检查,仅提示用户配置文件的使用情况
"""
custom_config_file = get_project_dir() + "data/." + default_config_file
if not os.path.exists(custom_config_file):
raise FileNotFoundError(
"找不到data/.config.yaml文件,请按教程确认该配置文件是否存在"
)
if missing_keys:
error_msg = "您的配置文件太旧了,缺少了:\n"
error_msg += "\n".join(f"- {key}" for key in missing_keys)
error_msg += "\n建议您:\n"
error_msg += "1、备份data/.config.yaml文件\n"
error_msg += "2、将根目录的config.yaml文件复制到data下,重命名为.config.yaml\n"
error_msg += "3、将密钥逐个复制到新的配置文件中\n"
raise ValueError(error_msg)
# 检查是否从API读取配置
config = load_config()
if config.get("read_config_from_api", False):
print("从API读取配置")
old_config_origin = read_config(custom_config_file)
if old_config_origin.get("selected_module") is not None:
error_msg = "您的配置文件好像既包含智控台的配置又包含本地配置:\n"
error_msg += "\n建议您:\n"
error_msg += "1、将根目录的config_from_api.yaml文件复制到data下,重命名为.config.yaml\n"
error_msg += "2、按教程配置好接口地址和密钥\n"
raise ValueError(error_msg)
config_file_valid = True