mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-22 23:23:55 +08:00
Merge branch 'agent-plugin' into main
This commit is contained in:
hrz
@@ -227,7 +227,7 @@ public interface Constant {
|
||||
/**
|
||||
* 版本号
|
||||
*/
|
||||
public static final String VERSION = "0.5.1";
|
||||
public static final String VERSION = "0.5.4";
|
||||
|
||||
/**
|
||||
* 无效固件URL
|
||||
|
||||
@@ -27,6 +27,9 @@ public class AgentDTO {
|
||||
@Schema(description = "大语言模型名称", example = "llm_model_01")
|
||||
private String llmModelName;
|
||||
|
||||
@Schema(description = "视觉模型名称", example = "vllm_model_01")
|
||||
private String vllmModelName;
|
||||
|
||||
@Schema(description = "记忆模型ID", example = "mem_model_01")
|
||||
private String memModelId;
|
||||
|
||||
|
||||
@@ -33,7 +33,10 @@ public class AgentUpdateDTO implements Serializable {
|
||||
@Schema(description = "大语言模型标识", example = "llm_model_02", nullable = true)
|
||||
private String llmModelId;
|
||||
|
||||
@Schema(description = "语音合成模型标识", example = "tts_model_02", nullable = true)
|
||||
@Schema(description = "VLLM模型标识", example = "vllm_model_02", required = false)
|
||||
private String vllmModelId;
|
||||
|
||||
@Schema(description = "语音合成模型标识", example = "tts_model_02", required = false)
|
||||
private String ttsModelId;
|
||||
|
||||
@Schema(description = "音色标识", example = "voice_02", nullable = true)
|
||||
|
||||
@@ -36,6 +36,9 @@ public class AgentEntity {
|
||||
@Schema(description = "大语言模型标识")
|
||||
private String llmModelId;
|
||||
|
||||
@Schema(description = "VLLM模型标识")
|
||||
private String vllmModelId;
|
||||
|
||||
@Schema(description = "语音合成模型标识")
|
||||
private String ttsModelId;
|
||||
|
||||
|
||||
@@ -49,6 +49,11 @@ public class AgentTemplateEntity implements Serializable {
|
||||
*/
|
||||
private String llmModelId;
|
||||
|
||||
/**
|
||||
* VLLM模型标识
|
||||
*/
|
||||
private String vllmModelId;
|
||||
|
||||
/**
|
||||
* 语音合成模型标识
|
||||
*/
|
||||
|
||||
+3
@@ -124,6 +124,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
|
||||
// 获取 LLM 模型名称
|
||||
dto.setLlmModelName(modelConfigService.getModelNameById(agent.getLlmModelId()));
|
||||
|
||||
// 获取 VLLM 模型名称
|
||||
dto.setVllmModelName(modelConfigService.getModelNameById(agent.getVllmModelId()));
|
||||
|
||||
// 获取记忆模型名称
|
||||
dto.setMemModelId(agent.getMemModelId());
|
||||
|
||||
|
||||
+5
-2
@@ -72,6 +72,7 @@ public class ConfigServiceImpl implements ConfigService {
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
result,
|
||||
isCache);
|
||||
|
||||
@@ -153,6 +154,7 @@ public class ConfigServiceImpl implements ConfigService {
|
||||
agent.getVadModelId(),
|
||||
agent.getAsrModelId(),
|
||||
agent.getLlmModelId(),
|
||||
agent.getVllmModelId(),
|
||||
agent.getTtsModelId(),
|
||||
agent.getMemModelId(),
|
||||
agent.getIntentModelId(),
|
||||
@@ -254,6 +256,7 @@ public class ConfigServiceImpl implements ConfigService {
|
||||
String vadModelId,
|
||||
String asrModelId,
|
||||
String llmModelId,
|
||||
String vllmModelId,
|
||||
String ttsModelId,
|
||||
String memModelId,
|
||||
String intentModelId,
|
||||
@@ -261,8 +264,8 @@ public class ConfigServiceImpl implements ConfigService {
|
||||
boolean isCache) {
|
||||
Map<String, String> selectedModule = new HashMap<>();
|
||||
|
||||
String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM" };
|
||||
String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId };
|
||||
String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM" };
|
||||
String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId };
|
||||
String intentLLMModelId = null;
|
||||
String memLocalShortLLMModelId = null;
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
-- VLLM模型供应器
|
||||
delete from `ai_model_provider` where id = 'SYSTEM_VLLM_openai';
|
||||
INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
|
||||
('SYSTEM_VLLM_openai', 'VLLM', 'openai', 'OpenAI接口', '[{"key":"base_url","label":"基础URL","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"api_key","label":"API密钥","type":"string"}]', 9, 1, NOW(), 1, NOW());
|
||||
|
||||
-- VLLM模型配置
|
||||
delete from `ai_model_config` where id = 'VLLM_ChatGLMVLLM';
|
||||
INSERT INTO `ai_model_config` VALUES ('VLLM_ChatGLMVLLM', 'VLLM', 'ChatGLMVLLM', '智谱视觉AI', 1, 1, '{\"type\": \"openai\", \"model_name\": \"glm-4v-flash\", \"base_url\": \"https://open.bigmodel.cn/api/paas/v4/\", \"api_key\": \"你的api_key\"}', NULL, NULL, 1, NULL, NULL, NULL, NULL);
|
||||
|
||||
-- 更新文档
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://bigmodel.cn/usercenter/proj-mgmt/apikeys',
|
||||
`remark` = '智谱视觉AI配置说明：
|
||||
1. 访问 https://bigmodel.cn/usercenter/proj-mgmt/apikeys
|
||||
2. 注册并获取API密钥
|
||||
3. 填入配置文件中' WHERE `id` = 'VLLM_ChatGLMVLLM';
|
||||
|
||||
|
||||
-- 添加参数
|
||||
INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (113, 'server.http_port', '8003', 'number', 1, 'http服务的端口，用于启动视觉分析接口');
|
||||
INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (114, 'server.vision_explain', 'null', 'string', 1, '视觉分析接口地址，用于下发到设备，多个用;分隔');
|
||||
|
||||
-- 智能体表增加VLLM模型配置
|
||||
ALTER TABLE `ai_agent`
|
||||
ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`;
|
||||
|
||||
-- 智能体模版表增加VLLM模型配置
|
||||
ALTER TABLE `ai_agent_template`
|
||||
ADD COLUMN `vllm_model_id` varchar(32) NULL DEFAULT 'VLLM_ChatGLMVLLM' COMMENT '视觉模型标识' AFTER `llm_model_id`;
|
||||
@@ -0,0 +1,45 @@
|
||||
-- VLLM模型供应器
|
||||
delete from `ai_model_provider` where id = 'SYSTEM_ASR_DoubaoStreamASR';
|
||||
INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
|
||||
('SYSTEM_ASR_DoubaoStreamASR', 'ASR', 'doubao_stream', '火山引擎语音识别(流式)', '[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"boosting_table_name","label":"热词文件名称","type":"string"},{"key":"correct_table_name","label":"替换词文件名称","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 3, 1, NOW(), 1, NOW());
|
||||
|
||||
|
||||
-- VLLM模型配置
|
||||
delete from `ai_model_config` where id = 'ASR_DoubaoStreamASR';
|
||||
INSERT INTO `ai_model_config` VALUES ('ASR_DoubaoStreamASR', 'ASR', 'DoubaoStreamASR', '豆包语音识别(流式)', 0, 1, '{\"type\": \"doubao_stream\", \"appid\": \"\", \"access_token\": \"\", \"cluster\": \"volcengine_input_common\", \"output_dir\": \"tmp/\"}', NULL, NULL, 3, NULL, NULL, NULL, NULL);
|
||||
|
||||
|
||||
-- 更新豆包ASR配置说明
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://console.volcengine.com/speech/app',
|
||||
`remark` = '豆包ASR配置说明：
|
||||
1. 豆包ASR和豆包(流式)ASR的区别是：豆包ASR是按次收费，豆包(流式)ASR是按时收费
|
||||
2. 一般来说按次收费的更便宜，但是豆包(流式)ASR使用了大模型技术，效果更好
|
||||
3. 需要在火山引擎控制台创建应用并获取appid和access_token
|
||||
4. 支持中文语音识别
|
||||
5. 需要网络连接
|
||||
6. 输出文件保存在tmp/目录
|
||||
申请步骤：
|
||||
1. 访问 https://console.volcengine.com/speech/app
|
||||
2. 创建新应用
|
||||
3. 获取appid和access_token
|
||||
4. 填入配置文件中
|
||||
如需设置热词，请参考：https://www.volcengine.com/docs/6561/155738
|
||||
' WHERE `id` = 'ASR_DoubaoASR';
|
||||
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://console.volcengine.com/speech/app',
|
||||
`remark` = '豆包ASR配置说明：
|
||||
1. 豆包ASR和豆包(流式)ASR的区别是：豆包ASR是按次收费，豆包(流式)ASR是按时收费
|
||||
2. 一般来说按次收费的更便宜，但是豆包(流式)ASR使用了大模型技术，效果更好
|
||||
3. 需要在火山引擎控制台创建应用并获取appid和access_token
|
||||
4. 支持中文语音识别
|
||||
5. 需要网络连接
|
||||
6. 输出文件保存在tmp/目录
|
||||
申请步骤：
|
||||
1. 访问 https://console.volcengine.com/speech/app
|
||||
2. 创建新应用
|
||||
3. 获取appid和access_token
|
||||
4. 填入配置文件中
|
||||
如需设置热词，请参考：https://www.volcengine.com/docs/6561/155738
|
||||
' WHERE `id` = 'ASR_DoubaoStreamASR';
|
||||
@@ -0,0 +1,14 @@
|
||||
-- VLLM模型配置
|
||||
delete from `ai_model_config` where id = 'VLLM_QwenVLVLLM';
|
||||
INSERT INTO `ai_model_config` VALUES ('VLLM_QwenVLVLLM', 'VLLM', 'QwenVLVLLM', '千问视觉模型', 0, 1, '{\"type\": \"openai\", \"model_name\": \"qwen2.5-vl-3b-instruct\", \"base_url\": \"https://dashscope.aliyuncs.com/compatible-mode/v1\", \"api_key\": \"你的api_key\"}', NULL, NULL, 2, NULL, NULL, NULL, NULL);
|
||||
|
||||
-- 更新文档
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://bailian.console.aliyun.com/?tab=api#/api/?type=model&url=https%3A%2F%2Fhelp.aliyun.com%2Fdocument_detail%2F2845564.html&renderType=iframe',
|
||||
`remark` = '千问视觉模型配置说明：
|
||||
1. 访问 https://bailian.console.aliyun.com/?tab=model#/api-key
|
||||
2. 注册并获取API密钥
|
||||
3. 填入配置文件中' WHERE `id` = 'VLLM_QwenVLVLLM';
|
||||
|
||||
-- 删除参数，这两个参数已挪至python配置文件
|
||||
delete from `sys_params` where id in (113,114);
|
||||
@@ -176,4 +176,25 @@ databaseChangeLog:
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202505292203.sql
|
||||
path: classpath:db/changelog/202505292203.sql
|
||||
- changeSet:
|
||||
id: 202506010920
|
||||
author: hrz
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202506010920.sql
|
||||
- changeSet:
|
||||
id: 202506031639
|
||||
author: hrz
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202506031639.sql
|
||||
- changeSet:
|
||||
id: 202506032232
|
||||
author: hrz
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202506032232.sql
|
||||
|
||||
Reference in New Issue
Block a user