diff --git a/Dockerfile-server-base b/Dockerfile-server-base index ab232a17..70429ab1 100644 --- a/Dockerfile-server-base +++ b/Dockerfile-server-base @@ -4,7 +4,9 @@ FROM python:3.10-slim # 安装系统依赖 RUN apt-get update && \ - apt-get install -y --no-install-recommends libopus0 ffmpeg && \ + apt-get install -y --no-install-recommends libopus0 ffmpeg locales && \ + sed -i '/zh_CN.UTF-8/s/^# //g' /etc/locale.gen && \ + locale-gen && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -14,6 +16,12 @@ RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ && \ pip config set global.timeout 120 && \ pip config set install.retries 5 +# 设置环境变量以确保正确的字符编码 +ENV LANG=zh_CN.UTF-8 \ + LC_ALL=zh_CN.UTF-8 \ + LANGUAGE=zh_CN:zh \ + PYTHONIOENCODING=utf-8 + WORKDIR /opt/xiaozhi-esp32-server # 复制requirements.txt diff --git a/docs/Deployment.md b/docs/Deployment.md index 55fef3c3..aff1287e 100644 --- a/docs/Deployment.md +++ b/docs/Deployment.md @@ -80,7 +80,7 @@ xiaozhi-server 打开命令行工具，使用`终端`或`命令行`工具 进入到你的`xiaozhi-server`，执行以下命令 ``` -docker-compose up -d +docker compose up -d ``` 执行完后，再执行以下命令，查看日志信息。 diff --git a/docs/docker-build.md b/docs/docker-build.md index 45102a7d..118ed1c9 100644 --- a/docs/docker-build.md +++ b/docs/docker-build.md @@ -17,5 +17,5 @@ docker build -t xiaozhi-esp32-server:web_latest -f ./Dockerfile-web . # 编译完成后，可以使用docker-compose启动项目 # docker-compose.yml你需要修改成自己编译的镜像版本 cd main/xiaozhi-server -docker-compose up -d +docker compose up -d ``` diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 4987afc3..4e4dc245 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -151,6 +151,11 @@ public interface Constant { */ String VOICE_CLONE_HUOSHAN_DOUBLE_STREAM = "huoshan_double_stream"; + /** + * RAG配置类型 + */ + String RAG_CONFIG_TYPE = "RAG"; + enum SysBaseParam { /** * ICP备案号 @@ -294,7 +299,7 @@ public interface Constant { /** * 版本号 */ - public static final String VERSION = "0.8.6"; + public static final String VERSION = "0.8.7"; /** * 无效固件URL diff --git a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java index 82b16ec5..b8525e63 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java +++ b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java @@ -119,10 +119,6 @@ public interface ErrorCode { int VOICEPRINT_UNREGISTER_PROCESS_ERROR = 10090; // 声纹注销处理失败 int VOICEPRINT_IDENTIFY_REQUEST_ERROR = 10091; // 声纹识别请求失败 - // 设备相关错误码 - int MAC_ADDRESS_ALREADY_EXISTS = 10161; // Mac地址已存在 - // 模型相关错误码 - int MODEL_PROVIDER_NOT_EXIST = 10162; // 供应器不存在 int LLM_NOT_EXIST = 10092; // 设置的LLM不存在 int MODEL_REFERENCED_BY_AGENT = 10093; // 该模型配置已被智能体引用，无法删除 int LLM_REFERENCED_BY_INTENT = 10094; // 该LLM模型已被意图识别配置引用，无法删除 @@ -198,4 +194,19 @@ public interface ErrorCode { int VOICE_CLONE_PREFIX = 10158; // 复刻音色前缀 int VOICE_ID_ALREADY_EXISTS = 10159; // 音色ID已存在 int VOICE_CLONE_HUOSHAN_VOICE_ID_ERROR = 10160; // 火山引擎音色ID格式错误 + + // 设备相关错误码 + int MAC_ADDRESS_ALREADY_EXISTS = 10161; // Mac地址已存在 + // 模型相关错误码 + int MODEL_PROVIDER_NOT_EXIST = 10162; // 供应器不存在 + + // 知识库相关错误码 + int Knowledge_Base_RECORD_NOT_EXISTS = 10163; // 知识库记录不存在 + int RAG_CONFIG_NOT_FOUND = 10164; // RAG配置未找到 + int RAG_CONFIG_TYPE_ERROR = 10165; // RAG配置类型错误 + int RAG_DEFAULT_CONFIG_NOT_FOUND = 10166; // 默认RAG配置未找到 + int RAG_API_ERROR = 10167; // RAG配置缺少必要参数 + int UPLOAD_FILE_ERROR = 10168; // 上传文件失败 + int NO_PERMISSION = 10169; // 没有权限 + int KNOWLEDGE_BASE_NAME_EXISTS = 10170; // 同名知识库已存在 } diff --git a/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java b/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java index a6b33b62..95add2f0 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java +++ b/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java @@ -32,13 +32,23 @@ public class FieldMetaObjectHandler implements MetaObjectHandler { // 创建者 strictInsertFill(metaObject, CREATOR, Long.class, user.getId()); - // 创建时间 - strictInsertFill(metaObject, CREATE_DATE, Date.class, date); + // 创建时间 - 支持createDate和createdAt两种字段名 + if (metaObject.hasSetter(CREATE_DATE)) { + strictInsertFill(metaObject, CREATE_DATE, Date.class, date); + } + if (metaObject.hasSetter("createdAt")) { + strictInsertFill(metaObject, "createdAt", Date.class, date); + } // 更新者 strictInsertFill(metaObject, UPDATER, Long.class, user.getId()); - // 更新时间 - strictInsertFill(metaObject, UPDATE_DATE, Date.class, date); + // 更新时间 - 支持updateDate和updatedAt两种字段名 + if (metaObject.hasSetter(UPDATE_DATE)) { + strictInsertFill(metaObject, UPDATE_DATE, Date.class, date); + } + if (metaObject.hasSetter("updatedAt")) { + strictInsertFill(metaObject, "updatedAt", Date.class, date); + } // 数据标识 strictInsertFill(metaObject, DATA_OPERATION, String.class, Constant.DataOperation.INSERT.getValue()); @@ -46,10 +56,17 @@ public class FieldMetaObjectHandler implements MetaObjectHandler { @Override public void updateFill(MetaObject metaObject) { + Date date = new Date(); + // 更新者 strictUpdateFill(metaObject, UPDATER, Long.class, SecurityUser.getUserId()); - // 更新时间 - strictUpdateFill(metaObject, UPDATE_DATE, Date.class, new Date()); + // 更新时间 - 支持updateDate和updatedAt两种字段名 + if (metaObject.hasSetter(UPDATE_DATE)) { + strictUpdateFill(metaObject, UPDATE_DATE, Date.class, date); + } + if (metaObject.hasSetter("updatedAt")) { + strictUpdateFill(metaObject, "updatedAt", Date.class, date); + } // 数据标识 strictInsertFill(metaObject, DATA_OPERATION, String.class, Constant.DataOperation.UPDATE.getValue()); diff --git a/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java b/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java index d25c72b6..3a969713 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java +++ b/main/manager-api/src/main/java/xiaozhi/common/redis/RedisKeys.java @@ -152,4 +152,11 @@ public class RedisKeys { public static String getVoiceCloneAudioIdKey(String uuid) { return "voiceClone:audio:id:" + uuid; } + + /** + * 获取知识库缓存key + */ + public static String getKnowledgeBaseCacheKey(String datasetId) { + return "knowledge:base:" + datasetId; + } } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentPluginMappingServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentPluginMappingServiceImpl.java index 2a53d474..1ebb522a 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentPluginMappingServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentPluginMappingServiceImpl.java @@ -1,16 +1,25 @@ package xiaozhi.modules.agent.service.impl; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import lombok.RequiredArgsConstructor; +import xiaozhi.common.utils.JsonUtils; import xiaozhi.modules.agent.dao.AgentPluginMappingMapper; import xiaozhi.modules.agent.entity.AgentPluginMapping; import xiaozhi.modules.agent.service.AgentPluginMappingService; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; +import xiaozhi.modules.knowledge.service.KnowledgeBaseService; +import xiaozhi.modules.model.entity.ModelConfigEntity; +import xiaozhi.modules.model.service.ModelConfigService; /** * @description 针对表【ai_agent_plugin_mapping(Agent与插件的唯一映射表)】的数据库操作Service实现 @@ -21,10 +30,69 @@ import xiaozhi.modules.agent.service.AgentPluginMappingService; public class AgentPluginMappingServiceImpl extends ServiceImpl implements AgentPluginMappingService { private final AgentPluginMappingMapper agentPluginMappingMapper; + private final KnowledgeBaseService knowledgeBaseService; + private final ModelConfigService modelConfigService; @Override public List agentPluginParamsByAgentId(String agentId) { - return agentPluginMappingMapper.selectPluginsByAgentId(agentId); + List list = agentPluginMappingMapper.selectPluginsByAgentId(agentId); + Map> knowledgeBaseMap = new HashMap<>(); + Map modelConfigMap = new HashMap<>(); + for (int i = list.size() - 1; i >= 0; i--) { + AgentPluginMapping mapping = list.get(i); + if (StringUtils.isBlank(mapping.getProviderCode())) { + // 查询知识库插件参数 + KnowledgeBaseEntity knowledgeBaseEntity = knowledgeBaseService.selectById(mapping.getPluginId()); + if (knowledgeBaseEntity == null) { + list.remove(i); + continue; + } + ModelConfigEntity modelConfigEntity = modelConfigService + .getModelByIdFromCache(knowledgeBaseEntity.getRagModelId()); + if (modelConfigEntity == null) { + list.remove(i); + continue; + } + List knowledgeBaseList = knowledgeBaseMap.get(modelConfigEntity.getModelCode()); + if (knowledgeBaseList == null) { + knowledgeBaseList = new ArrayList<>(); + } + modelConfigMap.put(modelConfigEntity.getModelCode(), modelConfigEntity); + knowledgeBaseList.add(knowledgeBaseEntity); + knowledgeBaseMap.put(modelConfigEntity.getModelCode(), knowledgeBaseList); + list.remove(i); + } + } + if (knowledgeBaseMap.size() > 0) { + for (String pluginCode : knowledgeBaseMap.keySet()) { + List knowledgeBaseList = knowledgeBaseMap.get(pluginCode); + if (knowledgeBaseList == null || knowledgeBaseList.isEmpty()) { + continue; + } + + AgentPluginMapping agentPluginMapping = new AgentPluginMapping(); + agentPluginMapping.setAgentId(agentId); + agentPluginMapping.setPluginId(pluginCode); + agentPluginMapping.setProviderCode("search_from_" + pluginCode); + agentPluginMapping.setId(Long.valueOf(list.size() + 1)); + + Map paramInfo = new HashMap<>(4); + ModelConfigEntity modelConfigEntity = modelConfigMap.get(pluginCode); + paramInfo.put("base_url", modelConfigEntity.getConfigJson().getStr("base_url")); + paramInfo.put("api_key", modelConfigEntity.getConfigJson().getStr("api_key")); + paramInfo.put("dataset_ids", + knowledgeBaseList.stream().map(KnowledgeBaseEntity::getDatasetId).toList()); + + String description = "如果用户询问与【" + + String.join(",", knowledgeBaseList.stream().map(KnowledgeBaseEntity::getName).toList()) + + "】相关的问题应调用本方法，用于查询：" + String.join(",", + knowledgeBaseList.stream().map(KnowledgeBaseEntity::getDescription).toList()); + paramInfo.put("description", description); + agentPluginMapping.setParamInfo(JsonUtils.toJsonString(paramInfo)); + list.add(agentPluginMapping); + } + } + return list; } @Override diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java index 8c562edb..321b5cb2 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java @@ -45,6 +45,10 @@ public class AgentTemplateServiceImpl extends ServiceImpl wrapper = new UpdateWrapper<>(); switch (modelType) { diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index a34c73dd..7883b8c1 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -91,6 +91,7 @@ public class ConfigServiceImpl implements ConfigService { null, null, null, + null, result, isCache); @@ -195,6 +196,7 @@ public class ConfigServiceImpl implements ConfigService { agent.getTtsModelId(), agent.getMemModelId(), agent.getIntentModelId(), + null, result, true); @@ -371,12 +373,14 @@ public class ConfigServiceImpl implements ConfigService { String ttsModelId, String memModelId, String intentModelId, + String ragModelId, Map result, boolean isCache) { Map selectedModule = new HashMap<>(); - String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM" }; - String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId }; + String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM", "RAG" }; + String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId, + ragModelId }; String intentLLMModelId = null; String memLocalShortLLMModelId = null; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeBaseController.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeBaseController.java new file mode 100644 index 00000000..cddfead5 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeBaseController.java @@ -0,0 +1,159 @@ +package xiaozhi.modules.knowledge.controller; + +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; +import xiaozhi.common.page.PageData; +import xiaozhi.common.utils.Result; +import xiaozhi.modules.knowledge.dto.KnowledgeBaseDTO; +import xiaozhi.modules.knowledge.service.KnowledgeBaseService; +import xiaozhi.modules.security.user.SecurityUser; + +@AllArgsConstructor +@RestController +@RequestMapping("/datasets") +@Tag(name = "知识库管理") +public class KnowledgeBaseController { + + private final KnowledgeBaseService knowledgeBaseService; + + @GetMapping + @Operation(summary = "分页查询知识库列表") + @RequiresPermissions("sys:role:normal") + public Result> getPageList( + @RequestParam(required = false) String name, + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "10") Integer page_size) { + // 获取当前登录用户ID + Long currentUserId = SecurityUser.getUserId(); + + KnowledgeBaseDTO knowledgeBaseDTO = new KnowledgeBaseDTO(); + knowledgeBaseDTO.setName(name); + knowledgeBaseDTO.setCreator(currentUserId); // 设置创建者ID，用于权限过滤 + + PageData pageData = knowledgeBaseService.getPageList(knowledgeBaseDTO, page, page_size); + return new Result>().ok(pageData); + } + + @GetMapping("/{dataset_id}") + @Operation(summary = "根据知识库ID获取知识库详情") + @RequiresPermissions("sys:role:normal") + public Result getByDatasetId(@PathVariable("dataset_id") String datasetId) { + // 获取当前登录用户ID + Long currentUserId = SecurityUser.getUserId(); + + KnowledgeBaseDTO knowledgeBaseDTO = knowledgeBaseService.getByDatasetId(datasetId); + + // 检查权限：用户只能查看自己创建的知识库 + if (knowledgeBaseDTO.getCreator() == null || !knowledgeBaseDTO.getCreator().equals(currentUserId)) { + throw new RenException(ErrorCode.NO_PERMISSION); + } + + return new Result().ok(knowledgeBaseDTO); + } + + @PostMapping + @Operation(summary = "创建知识库") + @RequiresPermissions("sys:role:normal") + public Result save(@RequestBody @Validated KnowledgeBaseDTO knowledgeBaseDTO) { + KnowledgeBaseDTO resp = knowledgeBaseService.save(knowledgeBaseDTO); + return new Result().ok(resp); + } + + @PutMapping("/{dataset_id}") + @Operation(summary = "更新知识库") + @RequiresPermissions("sys:role:normal") + public Result update(@PathVariable("dataset_id") String datasetId, + @RequestBody @Validated KnowledgeBaseDTO knowledgeBaseDTO) { + // 获取当前登录用户ID + Long currentUserId = SecurityUser.getUserId(); + + // 先获取现有知识库信息以检查权限 + KnowledgeBaseDTO existingKnowledgeBase = knowledgeBaseService.getByDatasetId(datasetId); + + // 检查权限：用户只能更新自己创建的知识库 + if (existingKnowledgeBase.getCreator() == null || !existingKnowledgeBase.getCreator().equals(currentUserId)) { + throw new RenException(ErrorCode.NO_PERMISSION); + } + + knowledgeBaseDTO.setDatasetId(datasetId); + KnowledgeBaseDTO resp = knowledgeBaseService.update(knowledgeBaseDTO); + return new Result().ok(resp); + } + + @DeleteMapping("/{dataset_id}") + @Operation(summary = "删除单个知识库") + @Parameter(name = "dataset_id", description = "知识库ID", required = true) + @RequiresPermissions("sys:role:normal") + public Result delete(@PathVariable("dataset_id") String datasetId) { + // 获取当前登录用户ID + Long currentUserId = SecurityUser.getUserId(); + + // 先获取现有知识库信息以检查权限 + KnowledgeBaseDTO existingKnowledgeBase = knowledgeBaseService.getByDatasetId(datasetId); + + // 检查权限：用户只能删除自己创建的知识库 + if (existingKnowledgeBase.getCreator() == null || !existingKnowledgeBase.getCreator().equals(currentUserId)) { + throw new RenException(ErrorCode.NO_PERMISSION); + } + + knowledgeBaseService.deleteByDatasetId(datasetId); + return new Result<>(); + } + + @DeleteMapping("/batch") + @Operation(summary = "批量删除知识库") + @Parameter(name = "ids", description = "知识库ID列表，用逗号分隔", required = true) + @RequiresPermissions("sys:role:normal") + public Result deleteBatch(@RequestParam("ids") String ids) { + if (StringUtils.isBlank(ids)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + // 获取当前登录用户ID + Long currentUserId = SecurityUser.getUserId(); + String[] idArray = ids.split(","); + for (String datasetId : idArray) { + if (StringUtils.isNotBlank(datasetId)) { + // 先获取现有知识库信息以检查权限 + KnowledgeBaseDTO existingKnowledgeBase = knowledgeBaseService.getByDatasetId(datasetId.trim()); + + // 检查权限：用户只能删除自己创建的知识库 + if (existingKnowledgeBase.getCreator() == null + || !existingKnowledgeBase.getCreator().equals(currentUserId)) { + throw new RenException(ErrorCode.NO_PERMISSION); + } + + knowledgeBaseService.deleteByDatasetId(datasetId.trim()); + } + } + return new Result<>(); + } + + @GetMapping("/rag-models") + @Operation(summary = "获取RAG模型列表") + @RequiresPermissions("sys:role:normal") + public Result>> getRAGModels() { + List> result = knowledgeBaseService.getRAGModels(); + return new Result>>().ok(result); + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeFilesController.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeFilesController.java new file mode 100644 index 00000000..fbc62309 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeFilesController.java @@ -0,0 +1,224 @@ +package xiaozhi.modules.knowledge.controller; + +import java.util.List; +import java.util.Map; + +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; +import xiaozhi.common.page.PageData; +import xiaozhi.common.utils.Result; +import xiaozhi.modules.knowledge.dto.KnowledgeBaseDTO; +import xiaozhi.modules.knowledge.dto.KnowledgeFilesDTO; +import xiaozhi.modules.knowledge.service.KnowledgeBaseService; +import xiaozhi.modules.knowledge.service.KnowledgeFilesService; +import xiaozhi.modules.security.user.SecurityUser; + +@AllArgsConstructor +@RestController +@RequestMapping("/datasets/{dataset_id}") +@Tag(name = "知识库文档管理") +public class KnowledgeFilesController { + + private final KnowledgeFilesService knowledgeFilesService; + private final KnowledgeBaseService knowledgeBaseService; + + /** + * 验证当前用户是否有权限操作指定知识库 + * + * @param datasetId 知识库ID + */ + private void validateKnowledgeBasePermission(String datasetId) { + // 获取当前登录用户ID + Long currentUserId = SecurityUser.getUserId(); + + // 获取知识库信息 + KnowledgeBaseDTO knowledgeBase = knowledgeBaseService.getByDatasetId(datasetId); + + // 检查权限：用户只能操作自己创建的知识库 + if (knowledgeBase.getCreator() == null || !knowledgeBase.getCreator().equals(currentUserId)) { + throw new RenException(ErrorCode.NO_PERMISSION); + } + } + + @GetMapping("/documents") + @Operation(summary = "分页查询文档列表") + @RequiresPermissions("sys:role:normal") + public Result> getPageList( + @PathVariable("dataset_id") String datasetId, + @RequestParam(required = false) String name, + @RequestParam(required = false) Integer status, + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "10") Integer page_size) { + // 验证知识库权限 + validateKnowledgeBasePermission(datasetId); + + // 如果指定了状态参数，使用状态查询接口 + if (status != null) { + PageData pageData = knowledgeFilesService.getPageListByStatus(datasetId, status, page, page_size); + return new Result>().ok(pageData); + } + + // 否则使用通用查询接口 + KnowledgeFilesDTO knowledgeFilesDTO = new KnowledgeFilesDTO(); + knowledgeFilesDTO.setDatasetId(datasetId); + knowledgeFilesDTO.setName(name); + PageData pageData = knowledgeFilesService.getPageList(knowledgeFilesDTO, page, page_size); + return new Result>().ok(pageData); + } + + @GetMapping("/documents/status/{status}") + @Operation(summary = "根据状态分页查询文档列表") + @RequiresPermissions("sys:role:normal") + public Result> getPageListByStatus( + @PathVariable("dataset_id") String datasetId, + @PathVariable("status") Integer status, + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "10") Integer page_size) { + // 验证知识库权限 + validateKnowledgeBasePermission(datasetId); + + PageData pageData = knowledgeFilesService.getPageListByStatus(datasetId, status, page, page_size); + return new Result>().ok(pageData); + } + + @PostMapping("/documents") + @Operation(summary = "上传文档到知识库") + @RequiresPermissions("sys:role:normal") + public Result uploadDocument( + @PathVariable("dataset_id") String datasetId, + @RequestParam("file") MultipartFile file, + @RequestParam(required = false) String name, + @RequestParam(required = false) String chunkMethod, + @RequestParam(required = false) String metaFields, + @RequestParam(required = false) String parserConfig) { + + // 验证知识库权限 + validateKnowledgeBasePermission(datasetId); + + KnowledgeFilesDTO resp = knowledgeFilesService.uploadDocument(datasetId, file, name, + metaFields != null ? parseJsonMap(metaFields) : null, + chunkMethod, + parserConfig != null ? parseJsonMap(parserConfig) : null); + return new Result().ok(resp); + } + + @DeleteMapping("/documents/{document_id}") + @Operation(summary = "删除单个文档") + @Parameter(name = "document_id", description = "文档ID", required = true) + @RequiresPermissions("sys:role:normal") + public Result delete(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId) { + // 验证知识库权限 + validateKnowledgeBasePermission(datasetId); + + knowledgeFilesService.deleteByDocumentId(documentId, datasetId); + return new Result<>(); + } + + @PostMapping("/chunks") + @Operation(summary = "解析文档（切块）") + @RequiresPermissions("sys:role:normal") + public Result parseDocuments(@PathVariable("dataset_id") String datasetId, + @RequestBody Map> requestBody) { + // 验证知识库权限 + validateKnowledgeBasePermission(datasetId); + + List documentIds = requestBody.get("document_ids"); + if (documentIds == null || documentIds.isEmpty()) { + return new Result().error("document_ids参数不能为空"); + } + + boolean success = knowledgeFilesService.parseDocuments(datasetId, documentIds); + if (success) { + return new Result(); + } else { + return new Result().error("文档解析失败，文档可能正在处理中"); + } + } + + @GetMapping("/documents/{document_id}/chunks") + @Operation(summary = "列出指定文档的切片") + @RequiresPermissions("sys:role:normal") + public Result> listChunks(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId, + @RequestParam(required = false) String keywords, + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "1024") Integer page_size, + @RequestParam(required = false) String id) { + // 验证知识库权限 + validateKnowledgeBasePermission(datasetId); + + Map result = knowledgeFilesService.listChunks(datasetId, documentId, keywords, page, page_size, + id); + return new Result>().ok(result); + } + + /** + * 召回测试 + */ + @PostMapping("/retrieval-test") + @Operation(summary = "召回测试") + @RequiresPermissions("sys:role:normal") + public Result> retrievalTest(@PathVariable("dataset_id") String datasetId, + @RequestBody Map params) { + // 验证知识库权限 + validateKnowledgeBasePermission(datasetId); + + try { + // 提取参数 + String question = (String) params.get("question"); + if (question == null || question.trim().isEmpty()) { + return new Result>().error("问题不能为空"); + } + + List datasetIds = (List) params.get("dataset_ids"); + List documentIds = (List) params.get("document_ids"); + Integer page = (Integer) params.get("page"); + Integer pageSize = (Integer) params.get("page_size"); + Float similarityThreshold = (Float) params.get("similarity_threshold"); + Float vectorSimilarityWeight = (Float) params.get("vector_similarity_weight"); + Integer topK = (Integer) params.get("top_k"); + String rerankId = (String) params.get("rerank_id"); + Boolean keyword = (Boolean) params.get("keyword"); + Boolean highlight = (Boolean) params.get("highlight"); + List crossLanguages = (List) params.get("cross_languages"); + Map metadataCondition = (Map) params.get("metadata_condition"); + + // 如果未指定数据集ID，使用当前数据集 + if (datasetIds == null || datasetIds.isEmpty()) { + datasetIds = java.util.Arrays.asList(datasetId); + } + + Map result = knowledgeFilesService.retrievalTest( + question, datasetIds, documentIds, page, pageSize, similarityThreshold, + vectorSimilarityWeight, topK, rerankId, keyword, highlight, crossLanguages, metadataCondition); + + return new Result>().ok(result); + } catch (Exception e) { + return new Result>().error("召回测试失败: " + e.getMessage()); + } + } + /** + * 解析JSON字符串为Map对象 + */ + private Map parseJsonMap(String jsonString) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readValue(jsonString, new TypeReference>() { + }); + } catch (Exception e) { + throw new RuntimeException("解析JSON字符串失败: " + jsonString, e); + } + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dao/KnowledgeBaseDao.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dao/KnowledgeBaseDao.java new file mode 100644 index 00000000..7f94b0af --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dao/KnowledgeBaseDao.java @@ -0,0 +1,14 @@ +package xiaozhi.modules.knowledge.dao; + +import org.apache.ibatis.annotations.Mapper; + +import xiaozhi.common.dao.BaseDao; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; + +/** + * 知识库知识库 + */ +@Mapper +public interface KnowledgeBaseDao extends BaseDao { + +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeBaseDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeBaseDTO.java new file mode 100644 index 00000000..77f6d198 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeBaseDTO.java @@ -0,0 +1,49 @@ +package xiaozhi.modules.knowledge.dto; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "知识库知识库") +public class KnowledgeBaseDTO implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + @Schema(description = "唯一标识") + private String id; + + @Schema(description = "知识库ID") + private String datasetId; + + @Schema(description = "RAG模型配置ID") + private String ragModelId; + + @Schema(description = "知识库名称") + private String name; + + @Schema(description = "知识库描述") + private String description; + + @Schema(description = "状态(0:禁用 1:启用)") + private Integer status; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + private Date createdAt; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + private Date updatedAt; + + @Schema(description = "文档数量") + private Integer documentCount; +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeFilesDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeFilesDTO.java new file mode 100644 index 00000000..84774bdd --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeFilesDTO.java @@ -0,0 +1,96 @@ +package xiaozhi.modules.knowledge.dto; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; +import java.util.Map; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "知识库文档") +public class KnowledgeFilesDTO implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + @Schema(description = "唯一标识") + private String id; + + @Schema(description = "文档ID") + private String documentId; + + @Schema(description = "知识库ID") + private String datasetId; + + @Schema(description = "文档名称") + private String name; + + @Schema(description = "文档类型") + private String fileType; + + @Schema(description = "文件大小（字节）") + private Long fileSize; + + @Schema(description = "文件路径") + private String filePath; + + @Schema(description = "元数据字段") + private Map metaFields; + + @Schema(description = "分块方法") + private String chunkMethod; + + @Schema(description = "解析器配置") + private Map parserConfig; + + @Schema(description = "状态") + private Integer status; + + @Schema(description = "文档解析状态") + private String run; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + private Date createdAt; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + private Date updatedAt; + + // 文档解析状态常量定义 + private static final Integer STATUS_UNSTART = 0; + private static final Integer STATUS_RUNNING = 1; + private static final Integer STATUS_CANCEL = 2; + private static final Integer STATUS_DONE = 3; + private static final Integer STATUS_FAIL = 4; + + /** + * 获取文档解析状态码（基于run字段转换） + */ + public Integer getParseStatusCode() { + if (run == null) { + return STATUS_UNSTART; + } + + // 根据run字段的值直接映射到对应的状态码 + switch (run.toUpperCase()) { + case "RUNNING": + return STATUS_RUNNING; + case "CANCEL": + return STATUS_CANCEL; + case "DONE": + return STATUS_DONE; + case "FAIL": + return STATUS_FAIL; + case "UNSTART": + default: + return STATUS_UNSTART; + } + } + +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/entity/KnowledgeBaseEntity.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/entity/KnowledgeBaseEntity.java new file mode 100644 index 00000000..e6c897ea --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/entity/KnowledgeBaseEntity.java @@ -0,0 +1,53 @@ +package xiaozhi.modules.knowledge.entity; + +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@TableName(value = "ai_rag_dataset", autoResultMap = true) +@Schema(description = "知识库知识库表") +public class KnowledgeBaseEntity { + + @TableId(type = IdType.ASSIGN_UUID) + @Schema(description = "唯一标识") + private String id; + + @Schema(description = "知识库ID") + private String datasetId; + + @Schema(description = "RAG模型配置ID") + private String ragModelId; + + @Schema(description = "知识库名称") + private String name; + + @Schema(description = "知识库描述") + private String description; + + @Schema(description = "状态(0:禁用 1:启用)") + private Integer status; + + @Schema(description = "创建者") + @TableField(fill = FieldFill.INSERT) + private Long creator; + + @Schema(description = "创建时间") + @TableField(fill = FieldFill.INSERT) + private Date createdAt; + + @Schema(description = "更新者") + @TableField(fill = FieldFill.UPDATE) + private Long updater; + + @Schema(description = "更新时间") + @TableField(fill = FieldFill.UPDATE) + private Date updatedAt; +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeBaseService.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeBaseService.java new file mode 100644 index 00000000..2cf64cf0 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeBaseService.java @@ -0,0 +1,86 @@ +package xiaozhi.modules.knowledge.service; + +import java.util.List; +import java.util.Map; + +import xiaozhi.common.page.PageData; +import xiaozhi.common.service.BaseService; +import xiaozhi.modules.knowledge.dto.KnowledgeBaseDTO; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; + +/** + * 知识库知识库服务接口 + */ +public interface KnowledgeBaseService extends BaseService { + + /** + * 分页查询知识库列表 + * + * @param knowledgeBaseDTO 查询条件 + * @param page 页码 + * @param limit 每页数量 + * @return 分页数据 + */ + PageData getPageList(KnowledgeBaseDTO knowledgeBaseDTO, Integer page, Integer limit); + + /** + * 根据ID获取知识库详情 + * + * @param id 知识库ID + * @return 知识库详情 + */ + KnowledgeBaseDTO getById(String id); + + /** + * 新增知识库 + * + * @param knowledgeBaseDTO 知识库信息 + * @return 新增的知识库 + */ + KnowledgeBaseDTO save(KnowledgeBaseDTO knowledgeBaseDTO); + + /** + * 更新知识库 + * + * @param knowledgeBaseDTO 知识库信息 + * @return 更新的知识库 + */ + KnowledgeBaseDTO update(KnowledgeBaseDTO knowledgeBaseDTO); + + /** + * 根据知识库ID查询知识库 + * + * @param datasetId 知识库ID + * @return 知识库详情 + */ + KnowledgeBaseDTO getByDatasetId(String datasetId); + + /** + * 根据知识库ID删除知识库 + * + * @param datasetId 知识库ID + */ + void deleteByDatasetId(String datasetId); + + /** + * 获取RAG配置信息 + * + * @param ragModelId RAG模型配置ID + * @return RAG配置信息 + */ + Map getRAGConfig(String ragModelId); + + /** + * 获取默认RAG配置信息 + * + * @return 默认RAG配置信息 + */ + Map getDefaultRAGConfig(); + + /** + * 获取RAG模型列表 + * + * @return RAG模型列表 + */ + List> getRAGModels(); +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeFilesService.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeFilesService.java new file mode 100644 index 00000000..68c0de4b --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeFilesService.java @@ -0,0 +1,130 @@ +package xiaozhi.modules.knowledge.service; + +import java.util.List; +import java.util.Map; + +import org.springframework.web.multipart.MultipartFile; + +import xiaozhi.common.page.PageData; +import xiaozhi.modules.knowledge.dto.KnowledgeFilesDTO; + +/** + * 知识库文档服务接口 + */ +public interface KnowledgeFilesService { + + /** + * 分页查询文档列表 + * + * @param knowledgeFilesDTO 查询条件 + * @param page 页码 + * @param limit 每页数量 + * @return 分页数据 + */ + PageData getPageList(KnowledgeFilesDTO knowledgeFilesDTO, Integer page, Integer limit); + + /** + * 根据文档ID和知识库ID获取文档详情 + * + * @param documentId 文档ID + * @param datasetId 知识库ID + * @return 文档详情 + */ + KnowledgeFilesDTO getByDocumentId(String documentId, String datasetId); + + /** + * 上传文档到知识库 + * + * @param datasetId 知识库ID + * @param file 上传的文件 + * @param name 文档名称 + * @param metaFields 元数据字段 + * @param chunkMethod 分块方法 + * @param parserConfig 解析器配置 + * @return 上传的文档信息 + */ + KnowledgeFilesDTO uploadDocument(String datasetId, MultipartFile file, String name, + Map metaFields, String chunkMethod, + Map parserConfig); + + /** + * 根据状态分页查询文档列表 + * + * @param datasetId 知识库ID + * @param status 文档解析状态（0-未开始，1-进行中，2-已取消，3-已完成，4-失败） + * @param page 页码 + * @param limit 每页数量 + * @return 分页数据 + */ + PageData getPageListByStatus(String datasetId, Integer status, Integer page, Integer limit); + + /** + * 根据文档ID和知识库ID删除文档 + * + * @param documentId 文档ID + * @param datasetId 知识库ID + */ + void deleteByDocumentId(String documentId, String datasetId); + + /** + * 获取RAG配置信息 + * + * @param ragModelId RAG模型配置ID + * @return RAG配置信息 + */ + Map getRAGConfig(String ragModelId); + + /** + * 获取默认RAG配置信息 + * + * @return 默认RAG配置信息 + */ + Map getDefaultRAGConfig(); + + /** + * 解析文档（切块） + * + * @param datasetId 知识库ID + * @param documentIds 文档ID列表 + * @return 解析结果 + */ + boolean parseDocuments(String datasetId, List documentIds); + + /** + * 列出指定文档的切片 + * + * @param datasetId 知识库ID + * @param documentId 文档ID + * @param keywords 关键词过滤 + * @param page 页码 + * @param pageSize 每页数量 + * @param chunkId 切片ID + * @return 切片列表信息 + */ + Map listChunks(String datasetId, String documentId, String keywords, + Integer page, Integer pageSize, String chunkId); + + /** + * 召回测试 - 从指定数据集或文档中检索相关切片 + * + * @param question 用户查询或查询关键词 + * @param datasetIds 数据集ID列表 + * @param documentIds 文档ID列表 + * @param page 页码 + * @param pageSize 每页数量 + * @param similarityThreshold 最小相似度阈值 + * @param vectorSimilarityWeight 向量相似度权重 + * @param topK 参与向量余弦计算的切片数量 + * @param rerankId 重排模型ID + * @param keyword 是否启用关键词匹配 + * @param highlight 是否启用高亮显示 + * @param crossLanguages 跨语言翻译列表 + * @param metadataCondition 元数据过滤条件 + * @return 召回测试结果 + */ + Map retrievalTest(String question, List datasetIds, List documentIds, + Integer page, Integer pageSize, Float similarityThreshold, + Float vectorSimilarityWeight, Integer topK, String rerankId, + Boolean keyword, Boolean highlight, List crossLanguages, + Map metadataCondition); +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java new file mode 100644 index 00000000..56a404dc --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java @@ -0,0 +1,705 @@ +package xiaozhi.modules.knowledge.service.impl; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import xiaozhi.common.constant.Constant; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; +import xiaozhi.common.page.PageData; +import xiaozhi.common.redis.RedisKeys; +import xiaozhi.common.redis.RedisUtils; +import xiaozhi.common.service.impl.BaseServiceImpl; +import xiaozhi.common.utils.ConvertUtils; +import xiaozhi.common.utils.MessageUtils; +import xiaozhi.modules.knowledge.dao.KnowledgeBaseDao; +import xiaozhi.modules.knowledge.dto.KnowledgeBaseDTO; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; +import xiaozhi.modules.knowledge.service.KnowledgeBaseService; +import xiaozhi.modules.model.dao.ModelConfigDao; +import xiaozhi.modules.model.entity.ModelConfigEntity; +import xiaozhi.modules.model.service.ModelConfigService; +import xiaozhi.modules.security.user.SecurityUser; + +@Service +@AllArgsConstructor +@Slf4j +public class KnowledgeBaseServiceImpl extends BaseServiceImpl + implements KnowledgeBaseService { + + private final KnowledgeBaseDao knowledgeBaseDao; + private final ModelConfigService modelConfigService; + private final ModelConfigDao modelConfigDao; + private final RedisUtils redisUtils; + private RestTemplate restTemplate = new RestTemplate(); + + @Override + public KnowledgeBaseEntity selectById(Serializable datasetId) { + if (datasetId == null) { + return null; + } + + // 先从Redis获取缓存 + String key = RedisKeys.getKnowledgeBaseCacheKey(datasetId.toString()); + KnowledgeBaseEntity cachedEntity = (KnowledgeBaseEntity) redisUtils.get(key); + if (cachedEntity != null) { + return cachedEntity; + } + + // 如果缓存中没有，则从数据库获取 + KnowledgeBaseEntity entity = knowledgeBaseDao.selectById(datasetId); + if (entity == null) { + return null; + } + + // 存入Redis缓存 + redisUtils.set(key, entity); + + return entity; + } + + @Override + public PageData getPageList(KnowledgeBaseDTO knowledgeBaseDTO, Integer page, Integer limit) { + long curPage = page; + long pageSize = limit; + Page pageInfo = new Page<>(curPage, pageSize); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + + // 添加查询条件 + if (knowledgeBaseDTO != null) { + queryWrapper.like(StringUtils.isNotBlank(knowledgeBaseDTO.getName()), "name", knowledgeBaseDTO.getName()); + queryWrapper.eq(knowledgeBaseDTO.getStatus() != null, "status", knowledgeBaseDTO.getStatus()); + queryWrapper.eq("creator", knowledgeBaseDTO.getCreator()); + } + + // 添加排序规则：按创建时间降序 + queryWrapper.orderByDesc("created_at"); + + IPage knowledgeBaseEntityIPage = knowledgeBaseDao.selectPage(pageInfo, queryWrapper); + + // 获取分页数据 + PageData pageData = getPageData(knowledgeBaseEntityIPage, KnowledgeBaseDTO.class); + + // 为每个知识库获取文档数量 + if (pageData != null && pageData.getList() != null) { + for (KnowledgeBaseDTO knowledgeBase : pageData.getList()) { + try { + Integer documentCount = getDocumentCountFromRAGFlow(knowledgeBase.getDatasetId(), + knowledgeBase.getRagModelId()); + knowledgeBase.setDocumentCount(documentCount); + } catch (Exception e) { + log.warn("获取知识库 {} 的文档数量失败: {}", knowledgeBase.getDatasetId(), e.getMessage()); + knowledgeBase.setDocumentCount(0); // 设置默认值 + } + } + } + + return pageData; + } + + @Override + public KnowledgeBaseDTO getById(String id) { + if (StringUtils.isBlank(id)) { + throw new RenException(ErrorCode.IDENTIFIER_NOT_NULL); + } + + KnowledgeBaseEntity entity = knowledgeBaseDao.selectById(id); + if (entity == null) { + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public KnowledgeBaseDTO save(KnowledgeBaseDTO knowledgeBaseDTO) { + if (knowledgeBaseDTO == null) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + // 检查是否存在同名知识库 + checkDuplicateKnowledgeBaseName(knowledgeBaseDTO, null); + + String datasetId = null; + // 调用RAGFlow API创建数据集 + try { + Map ragConfig = getValidatedRAGConfig(knowledgeBaseDTO.getRagModelId()); + datasetId = createDatasetInRAGFlow( + knowledgeBaseDTO.getName(), + knowledgeBaseDTO.getDescription(), + ragConfig); + } catch (Exception e) { + // 如果RAG API调用失败，直接抛出异常，无需回滚（因为还没有插入本地数据库） + throw e; + } + + // 验证数据集ID是否已存在 + KnowledgeBaseEntity existingEntity = knowledgeBaseDao.selectOne( + new QueryWrapper().eq("dataset_id", datasetId)); + if (existingEntity != null) { + // 如果datasetId已存在，删除RAGFlow中的数据集并抛出异常 + try { + Map ragConfig = getValidatedRAGConfig(knowledgeBaseDTO.getRagModelId()); + deleteDatasetInRAGFlow(datasetId, ragConfig); + } catch (Exception deleteException) { + log.warn("删除重复datasetId的RAGFlow数据集失败: {}", deleteException.getMessage()); + } + throw new RenException(ErrorCode.DB_RECORD_EXISTS); + } + + // 创建本地实体并保存 + KnowledgeBaseEntity entity = ConvertUtils.sourceToTarget(knowledgeBaseDTO, KnowledgeBaseEntity.class); + entity.setDatasetId(datasetId); + knowledgeBaseDao.insert(entity); + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public KnowledgeBaseDTO update(KnowledgeBaseDTO knowledgeBaseDTO) { + if (knowledgeBaseDTO == null || StringUtils.isBlank(knowledgeBaseDTO.getId())) { + throw new RenException(ErrorCode.IDENTIFIER_NOT_NULL); + } + + // 检查记录是否存在 + KnowledgeBaseEntity existingEntity = knowledgeBaseDao.selectById(knowledgeBaseDTO.getId()); + if (existingEntity == null) { + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + // 检查是否存在同名知识库（排除当前记录） + checkDuplicateKnowledgeBaseName(knowledgeBaseDTO, knowledgeBaseDTO.getId()); + + // 验证数据集ID是否与其他记录冲突 + if (StringUtils.isNotBlank(knowledgeBaseDTO.getDatasetId())) { + KnowledgeBaseEntity conflictEntity = knowledgeBaseDao.selectOne( + new QueryWrapper() + .eq("dataset_id", knowledgeBaseDTO.getDatasetId()) + .ne("id", knowledgeBaseDTO.getId())); + if (conflictEntity != null) { + throw new RenException(ErrorCode.DB_RECORD_EXISTS); + } + } + + KnowledgeBaseEntity entity = ConvertUtils.sourceToTarget(knowledgeBaseDTO, KnowledgeBaseEntity.class); + knowledgeBaseDao.updateById(entity); + + // 删除缓存 + if (entity.getDatasetId() != null) { + redisUtils.delete(RedisKeys.getKnowledgeBaseCacheKey(entity.getId())); + } + + // 调用RAGFlow API更新数据集 + if (StringUtils.isNotBlank(knowledgeBaseDTO.getDatasetId())) { + try { + Map ragConfig = getValidatedRAGConfig(knowledgeBaseDTO.getRagModelId()); + updateDatasetInRAGFlow( + knowledgeBaseDTO.getDatasetId(), + knowledgeBaseDTO.getName(), + knowledgeBaseDTO.getDescription(), + ragConfig); + } catch (Exception e) { + // 如果RAG API调用失败，回滚本地数据库操作 + knowledgeBaseDao.updateById(existingEntity); + throw e; + } + } + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public KnowledgeBaseDTO getByDatasetId(String datasetId) { + if (StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + KnowledgeBaseEntity entity = knowledgeBaseDao.selectOne( + new QueryWrapper().eq("dataset_id", datasetId)); + + if (entity == null) { + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public void deleteByDatasetId(String datasetId) { + if (StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + log.info("=== 开始通过datasetId删除操作 ==="); + log.info("删除datasetId: {}", datasetId); + + KnowledgeBaseEntity entity = knowledgeBaseDao.selectOne( + new QueryWrapper().eq("dataset_id", datasetId)); + + if (entity == null) { + log.warn("记录不存在，datasetId: {}", datasetId); + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + redisUtils.delete(RedisKeys.getKnowledgeBaseCacheKey(entity.getId())); + + log.info("找到记录: ID={}, datasetId={}, ragModelId={}", + entity.getId(), entity.getDatasetId(), entity.getRagModelId()); + + // 先删除本地数据库记录 + int deleteCount = knowledgeBaseDao.deleteById(entity.getId()); + log.info("本地数据库删除结果: {}", deleteCount > 0 ? "成功" : "失败"); + + // 调用RAGFlow API删除数据集 + if (StringUtils.isNotBlank(entity.getDatasetId()) && StringUtils.isNotBlank(entity.getRagModelId())) { + try { + log.info("开始调用RAGFlow API删除数据集"); + Map ragConfig = getValidatedRAGConfig(entity.getRagModelId()); + deleteDatasetInRAGFlow(entity.getDatasetId(), ragConfig); + log.info("RAGFlow API删除调用完成"); + } catch (Exception e) { + log.warn("删除RAGFlow数据集失败: {}", e.getMessage()); + } + } else { + log.warn("datasetId或ragModelId为空，跳过RAGFlow删除"); + } + + log.info("=== 通过datasetId删除操作结束 ==="); + } + + @Override + public Map getRAGConfig(String ragModelId) { + if (StringUtils.isBlank(ragModelId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + // 从缓存获取模型配置 + ModelConfigEntity modelConfig = modelConfigService.getModelByIdFromCache(ragModelId); + if (modelConfig == null || modelConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 验证是否为RAG类型配置 + if (!Constant.RAG_CONFIG_TYPE.equals(modelConfig.getModelType().toUpperCase())) { + throw new RenException(ErrorCode.RAG_CONFIG_TYPE_ERROR); + } + + Map config = modelConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + // 返回配置信息 + return config; + } + + @Override + public Map getDefaultRAGConfig() { + // 获取默认RAG模型配置 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("model_type", Constant.RAG_CONFIG_TYPE) + .eq("is_default", 1) + .eq("is_enabled", 1); + + List modelConfigs = modelConfigDao.selectList(queryWrapper); + if (modelConfigs == null || modelConfigs.isEmpty()) { + throw new RenException(ErrorCode.RAG_DEFAULT_CONFIG_NOT_FOUND); + } + + ModelConfigEntity defaultConfig = modelConfigs.get(0); + if (defaultConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + Map config = defaultConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + return config; + } + + @Override + public List> getRAGModels() { + // 查询RAG类型的模型配置 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("model_type", Constant.RAG_CONFIG_TYPE) + .eq("is_enabled", 1) + .orderByDesc("is_default") + .orderByDesc("create_date"); + + List modelConfigs = modelConfigDao.selectList(queryWrapper); + + List> modelList = new ArrayList<>(); + for (ModelConfigEntity modelConfig : modelConfigs) { + Map modelInfo = new HashMap<>(); + modelInfo.put("id", modelConfig.getId()); + modelInfo.put("modelName", modelConfig.getModelName()); + modelList.add(modelInfo); + } + return modelList; + } + + /** + * 验证RAG配置中是否包含必要的参数 + */ + private void validateRagConfig(Map config) { + if (config == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 从配置中提取必要的参数 + String baseUrl = (String) config.get("base_url"); + String apiKey = (String) config.get("api_key"); + + // 验证base_url是否存在且非空 + if (StringUtils.isBlank(baseUrl)) { + throw new RenException(ErrorCode.RAG_API_ERROR); + } + } + + /** + * 调用RAGFlow API创建数据集 + */ + private String createDatasetInRAGFlow(String name, String description, Map ragConfig) { + String datasetId = null; + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API创建数据集, name: {}", name); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + String username = SecurityUser.getUser().getUsername(); + requestBody.put("name", username + "_" + name); + if (StringUtils.isNotBlank(description)) { + requestBody.put("description", description); + } + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送POST请求 + log.info("发送POST请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR, response.getStatusCode().toString()); + } + + // 解析响应体，提取datasetId + String responseBody = response.getBody(); + if (StringUtils.isNotBlank(responseBody)) { + try { + // 解析RAGFlow API响应，支持多种可能的字段名 + ObjectMapper objectMapper = new ObjectMapper(); + Map responseMap = objectMapper.readValue(responseBody, Map.class); + + log.debug("RAGFlow API响应解析结果: {}", responseMap); + + // 首先检查响应码 + Integer code = (Integer) responseMap.get("code"); + String message = (String) responseMap.get("message"); + if (code != null && code == 0) { + // 响应码为0表示成功，从data字段中获取datasetId + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + datasetId = (String) dataMap.get("id"); + + if (StringUtils.isBlank(datasetId)) { + // 如果id字段为空，尝试其他可能的字段名 + datasetId = (String) dataMap.get("dataset_id"); + datasetId = (String) dataMap.get("datasetId"); + } + } + } else { + // 如果响应码不为0，说明API调用失败 + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, message); + throw new RenException(ErrorCode.RAG_API_ERROR, + "RAGFlow API调用失败，响应码: " + code + ", 消息: " + message); + } + + log.info("从RAGFlow API响应中解析出datasetId: {}", datasetId); + log.debug("完整响应内容: {}", responseBody); + } catch (Exception e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析响应失败: " + e.getMessage()); + } + } + + if (StringUtils.isBlank(datasetId)) { + log.error("无法从RAGFlow API响应中获取datasetId，响应内容: {}", responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API响应中未包含datasetId"); + } + log.info("RAGFlow数据集创建成功，datasetId: {}", datasetId); + + return datasetId; + } + + /** + * 调用RAGFlow API更新数据集 + */ + private void updateDatasetInRAGFlow(String datasetId, String name, String description, + Map ragConfig) { + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API更新数据集，datasetId: {}, name: {}", datasetId, name); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets/" + datasetId; + log.debug("请求URL: {}", url); + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("dataset_id", datasetId); + String username = SecurityUser.getUser().getUsername(); + requestBody.put("name", username + "_" + name); + if (StringUtils.isNotBlank(description)) { + requestBody.put("description", description); + } + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送PUT请求 + log.info("发送PUT请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.PUT, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + log.info("RAGFlow数据集更新成功，datasetId: {}", datasetId); + + } + + /** + * 调用RAGFlow API删除数据集 + */ + private void deleteDatasetInRAGFlow(String datasetId, Map ragConfig) { + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API删除数据集，datasetId: {}", datasetId); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("ids", List.of(datasetId)); + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送DELETE请求 + log.info("发送DELETE请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.DELETE, requestEntity, + String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + log.info("RAGFlow数据集删除成功，datasetId: {}", datasetId); + + } + + /** + * 获取RAG配置并验证 + */ + private Map getValidatedRAGConfig(String ragModelId) { + Map ragConfig; + if (StringUtils.isNotBlank(ragModelId)) { + ragConfig = getRAGConfig(ragModelId); + } else { + ragConfig = getDefaultRAGConfig(); + } + + // 验证baseUrl和apiKey + validateRAGConfigParameters(ragConfig); + + return ragConfig; + } + + /** + * 验证RAG配置参数 + */ + private void validateRAGConfigParameters(Map ragConfig) { + if (ragConfig == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND, "RAG配置为空，请检查配置"); + } + + // 从配置中提取必要的参数 + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + if (StringUtils.isBlank(baseUrl)) { + throw new RenException(ErrorCode.RAG_API_ERROR, "RAG配置中base_url为空，请完善配置"); + } + + if (StringUtils.isBlank(apiKey)) { + throw new RenException(ErrorCode.RAG_API_ERROR, "RAG配置中api_key为空，请完善配置"); + } + + if (apiKey.contains("你")) { + throw new RenException(ErrorCode.RAG_API_ERROR, "RAG配置中api_key包含占位符'你'，请替换为实际的API密钥"); + } + + if (!baseUrl.startsWith("http://") && !baseUrl.startsWith("https://")) { + throw new RenException(ErrorCode.RAG_API_ERROR, "RAG配置中base_url格式不正确，必须以http://或https://开头"); + } + } + + /** + * 检查是否存在同名知识库 + * + * @param knowledgeBaseDTO 知识库DTO + * @param excludeId 排除的ID（更新时使用） + */ + private void checkDuplicateKnowledgeBaseName(KnowledgeBaseDTO knowledgeBaseDTO, String excludeId) { + if (StringUtils.isNotBlank(knowledgeBaseDTO.getName())) { + Long currentUserId = SecurityUser.getUserId(); + QueryWrapper queryWrapper = new QueryWrapper() + .eq("name", knowledgeBaseDTO.getName()) + .eq("creator", currentUserId); + + // 如果提供了排除ID，则排除该记录 + if (StringUtils.isNotBlank(excludeId)) { + queryWrapper.ne("id", excludeId); + } + + long count = knowledgeBaseDao.selectCount(queryWrapper); + if (count > 0) { + throw new RenException(ErrorCode.KNOWLEDGE_BASE_NAME_EXISTS, + MessageUtils.getMessage(ErrorCode.KNOWLEDGE_BASE_NAME_EXISTS)); + } + } + } + + /** + * 从RAGFlow API获取知识库的文档数量 + */ + private Integer getDocumentCountFromRAGFlow(String datasetId, String ragModelId) { + if (StringUtils.isBlank(datasetId) || StringUtils.isBlank(ragModelId)) { + log.warn("datasetId或ragModelId为空，无法获取文档数量"); + return 0; + } + + log.info("开始获取知识库 {} 的文档数量", datasetId); + + // 获取RAG配置 + Map ragConfig = getValidatedRAGConfig(ragModelId); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 调用RAGFlow API获取文档列表，但不返回文档详情，只获取总数 + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents?page=1&size=1"; + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API获取文档数量..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + return 0; + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + try { + ObjectMapper objectMapper = new ObjectMapper(); + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + Object totalObj = dataMap.get("total"); + if (totalObj instanceof Integer) { + Integer documentCount = (Integer) totalObj; + log.info("获取知识库 {} 的文档数量成功: {}", datasetId, documentCount); + return documentCount; + } else if (totalObj instanceof Long) { + Long documentCount = (Long) totalObj; + log.info("获取知识库 {} 的文档数量成功: {}", datasetId, documentCount); + return documentCount.intValue(); + } + } + } else { + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + } + } catch (Exception e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + } + return 0; + } + +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeFilesServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeFilesServiceImpl.java new file mode 100644 index 00000000..458ef063 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeFilesServiceImpl.java @@ -0,0 +1,1788 @@ +package xiaozhi.modules.knowledge.service.impl; + +import java.io.IOException; +import java.io.InputStream; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.io.AbstractResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.ResourceAccessException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import xiaozhi.common.constant.Constant; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; +import xiaozhi.common.page.PageData; +import xiaozhi.modules.knowledge.dto.KnowledgeFilesDTO; +import xiaozhi.modules.knowledge.service.KnowledgeFilesService; +import xiaozhi.modules.model.dao.ModelConfigDao; +import xiaozhi.modules.model.entity.ModelConfigEntity; +import xiaozhi.modules.model.service.ModelConfigService; + +@Service +@AllArgsConstructor +@Slf4j +public class KnowledgeFilesServiceImpl implements KnowledgeFilesService { + + private final ModelConfigService modelConfigService; + private final ModelConfigDao modelConfigDao; + private RestTemplate restTemplate = new RestTemplate(); + private ObjectMapper objectMapper = new ObjectMapper(); + + @Override + public PageData getPageList(KnowledgeFilesDTO knowledgeFilesDTO, Integer page, Integer limit) { + try { + + log.info("=== 开始获取文档列表 ==="); + log.info("查询条件: datasetId={}, name={}, status={}, page={}, limit={}", + knowledgeFilesDTO != null ? knowledgeFilesDTO.getDatasetId() : null, + knowledgeFilesDTO != null ? knowledgeFilesDTO.getName() : null, + knowledgeFilesDTO != null ? knowledgeFilesDTO.getStatus() : null, + page, limit); + + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 根据RAGFlow API文档，获取文档列表的接口 + String datasetId = knowledgeFilesDTO != null ? knowledgeFilesDTO.getDatasetId() : null; + if (StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId不能为空"); + } + + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents"; + + // 添加查询参数 + StringBuilder urlBuilder = new StringBuilder(url); + List params = new ArrayList<>(); + + if (knowledgeFilesDTO != null && StringUtils.isNotBlank(knowledgeFilesDTO.getName())) { + params.add("keywords=" + knowledgeFilesDTO.getName()); + } + if (page > 0) { + params.add("page=" + page); + } + if (limit > 0) { + params.add("page_size=" + limit); + } + + if (!params.isEmpty()) { + urlBuilder.append("?").append(String.join("&", params)); + } + + url = urlBuilder.toString(); + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API获取文档列表..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + return parseDocumentListResponse(dataObj, page, limit); + } else { + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API调用失败，响应码: " + code); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档列表失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档列表失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档列表失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("获取文档列表失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取文档列表失败: " + e.getMessage()); + } finally { + log.info("=== 获取文档列表操作结束 ==="); + } + } + + /** + * 解析RAGFlow API返回的文档列表响应 + */ + private PageData parseDocumentListResponse(Object dataObj, long curPage, long pageSize) { + try { + List documents = new ArrayList<>(); + long totalCount = 0; + + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + + // 获取文档列表 - 支持多种可能的字段名 + Object documentsObj = null; + + // 支持多种可能的文档列表字段名 + String[] possibleDocumentFields = { "docs", "documents", "items", "list", "data" }; + + for (String fieldName : possibleDocumentFields) { + if (dataMap.containsKey(fieldName) && dataMap.get(fieldName) instanceof List) { + documentsObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取文档列表", fieldName); + break; + } + } + + // 如果标准字段不存在，尝试自动检测 + if (documentsObj == null) { + for (Map.Entry entry : dataMap.entrySet()) { + if (entry.getValue() instanceof List) { + List list = (List) entry.getValue(); + if (!list.isEmpty() && list.get(0) instanceof Map) { + documentsObj = entry.getValue(); + log.warn("自动检测到文档列表字段: '{}'，建议检查RAGFlow API文档", entry.getKey()); + break; + } + } + } + } + + if (documentsObj instanceof List) { + List> documentList = (List>) documentsObj; + + for (Map docMap : documentList) { + KnowledgeFilesDTO dto = convertRAGDocumentToDTO(docMap); + if (dto != null) { + // 在文档列表获取时也进行状态同步检查 + syncDocumentStatusWithRAGFlow(dto); + documents.add(dto); + } + } + } + + // 解析总数 - 支持多种可能的字段名 + Object totalObj = null; + String[] possibleTotalFields = { "total", "totalCount", "total_count", "count" }; + + for (String fieldName : possibleTotalFields) { + if (dataMap.containsKey(fieldName)) { + totalObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取总数", fieldName); + break; + } + } + + if (totalObj instanceof Integer) { + totalCount = ((Integer) totalObj).longValue(); + } else if (totalObj instanceof Long) { + totalCount = (Long) totalObj; + } else if (totalObj instanceof String) { + try { + totalCount = Long.parseLong((String) totalObj); + } catch (NumberFormatException e) { + log.warn("无法解析总数字段: {}", totalObj); + } + } + } + + // 创建分页数据 + PageData pageData = new PageData<>(documents, totalCount); + + log.info("获取文档列表成功，共{}个文档，总数: {}", documents.size(), totalCount); + return pageData; + + } catch (Exception e) { + log.error("获取文档列表响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取文档列表响应失败: " + e.getMessage()); + } + } + + /** + * 同步文档状态与RAGFlow实际状态 + * 优化状态同步逻辑，确保解析中状态能够正常显示 + * 只有当文档有切片且解析时间超过30秒时，才更新为完成状态 + */ + private void syncDocumentStatusWithRAGFlow(KnowledgeFilesDTO dto) { + if (dto == null || StringUtils.isBlank(dto.getDocumentId())) { + return; + } + + String documentId = dto.getDocumentId(); + Integer currentStatus = dto.getStatus(); + + // 只有当状态明确为处理中(1)时，才进行状态同步检查 + // 避免在状态不确定或已完成的文档上重复检查 + if (currentStatus != null && currentStatus == 1) { + try { + long currentTime = System.currentTimeMillis(); + + // 调用RAGFlow API获取文档切片信息 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + String url = baseUrl + "/api/v1/documents/" + documentId + "/chunks"; + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + log.debug("检查文档切片状态，documentId: {}", documentId); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, + String.class); + + if (response.getStatusCode().is2xxSuccessful()) { + String responseBody = response.getBody(); + Map responseMap = objectMapper.readValue(responseBody, Map.class); + + Integer code = (Integer) responseMap.get("code"); + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + + // 检查是否有切片数据 + Object chunksObj = getValueFromMultipleKeys(dataMap, "chunks", "items", "list", "data"); + if (chunksObj instanceof List) { + List chunks = (List) chunksObj; + + // 如果有切片且数量大于0，说明解析已完成 + if (!chunks.isEmpty()) { + // 检查文档创建时间，确保解析过程有足够的时间显示 + Date createdAt = dto.getCreatedAt(); + long parseDuration = currentTime + - (createdAt != null ? createdAt.getTime() : currentTime); + + // 只有当解析时间超过30秒时，才更新为完成状态 + // 这样可以确保解析中状态有足够的时间显示 + if (parseDuration > 30000) { + log.info("状态同步：文档已有切片且解析时间超过30秒，更新为完成状态，documentId: {}, 切片数量: {}, 解析时长: {}ms", + documentId, chunks.size(), parseDuration); + + // 更新状态为完成(3) + dto.setStatus(3); + } else { + log.debug("文档已有切片但解析时间不足30秒，保持解析中状态，documentId: {}, 解析时长: {}ms", + documentId, parseDuration); + } + } + } + } + } + } + } catch (Exception e) { + log.debug("检查文档切片状态失败，documentId: {}, 错误: {}", documentId, e.getMessage()); + // 忽略检查失败，保持原状态 + } + } + } + + /** + * 将RAGFlow文档数据转换为KnowledgeFilesDTO + */ + private KnowledgeFilesDTO convertRAGDocumentToDTO(Map docMap) { + try { + if (docMap == null) + return null; + + KnowledgeFilesDTO dto = new KnowledgeFilesDTO(); + + // 设置基本字段 - 支持多种可能的字段名 + dto.setId(getStringValueFromMultipleKeys(docMap, "id", "document_id", "doc_id")); // 使用RAGFlow的文档ID作为本地ID + dto.setDocumentId(getStringValueFromMultipleKeys(docMap, "id", "document_id", "doc_id")); // RAGFlow文档ID + dto.setName(getStringValueFromMultipleKeys(docMap, "name", "filename", "file_name", "title")); + dto.setDatasetId(getStringValueFromMultipleKeys(docMap, "dataset_id", "dataset", "knowledge_base_id")); + + // 设置文件信息 - 支持多种可能的字段名 + dto.setFileType(getStringValueFromMultipleKeys(docMap, "file_type", "type", "format", "extension")); + + // 文件大小 - 支持多种可能的字段名 + Object sizeObj = getValueFromMultipleKeys(docMap, "size", "file_size", "size_bytes"); + if (sizeObj instanceof Integer) { + dto.setFileSize(((Integer) sizeObj).longValue()); + } else if (sizeObj instanceof Long) { + dto.setFileSize((Long) sizeObj); + } else if (sizeObj instanceof String) { + try { + dto.setFileSize(Long.parseLong((String) sizeObj)); + } catch (NumberFormatException e) { + log.warn("无法解析文件大小: {}", sizeObj); + } + } + + // 设置元数据和配置 - 支持多种可能的字段名 + Object metaFieldsObj = getValueFromMultipleKeys(docMap, "meta_fields", "metadata", "meta", "properties"); + if (metaFieldsObj instanceof Map) { + dto.setMetaFields((Map) metaFieldsObj); + } + + Object parserConfigObj = getValueFromMultipleKeys(docMap, "parser_config", "parser", "parse_config", + "config"); + if (parserConfigObj instanceof Map) { + dto.setParserConfig((Map) parserConfigObj); + } + + dto.setChunkMethod(getStringValueFromMultipleKeys(docMap, "chunk_method", "chunking", "chunk_strategy")); + + // 设置时间信息 - 支持多种可能的字段名 + Object createTimeObj = getValueFromMultipleKeys(docMap, "create_time", "created_at", "creation_time", + "created"); + if (createTimeObj instanceof Long) { + dto.setCreatedAt(new Date((Long) createTimeObj)); + } else if (createTimeObj instanceof String) { + // 尝试解析时间字符串 + try { + // 这里可以根据实际的时间格式进行调整 + dto.setCreatedAt(new Date(Long.parseLong((String) createTimeObj))); + } catch (NumberFormatException e) { + log.warn("无法解析创建时间: {}", createTimeObj); + } + } + + Object updateTimeObj = getValueFromMultipleKeys(docMap, "update_time", "updated_at", "modified_time", + "modified"); + if (updateTimeObj instanceof Long) { + dto.setUpdatedAt(new Date((Long) updateTimeObj)); + } else if (updateTimeObj instanceof String) { + try { + dto.setUpdatedAt(new Date(Long.parseLong((String) updateTimeObj))); + } catch (NumberFormatException e) { + log.warn("无法解析更新时间: {}", updateTimeObj); + } + } + + // 设置文档解析状态信息 - 直接使用RAGFlow最新状态 + String documentId = dto.getDocumentId(); + if (StringUtils.isNotBlank(documentId)) { + // 获取RAGFlow的最新状态 + Object runObj = getValueFromMultipleKeys(docMap, "run", "status", "parse_status"); + Integer ragFlowStatus = null; + if (runObj != null) { + dto.setRun(runObj.toString()); + ragFlowStatus = dto.getParseStatusCode(); + log.debug("获取RAGFlow最新状态，documentId: {}, run: {}, status: {}", + documentId, runObj, ragFlowStatus); + } + + } + + return dto; + + } catch (Exception e) { + log.error("转换RAGFlow文档数据失败: {}", e.getMessage(), e); + return null; + } + } + + /** + * 从多个可能的字段名中获取字符串值 + */ + private String getStringValueFromMultipleKeys(Map map, String... keys) { + for (String key : keys) { + Object value = map.get(key); + if (value != null) { + return value.toString(); + } + } + return null; + } + + /** + * 从多个可能的字段名中获取值 + */ + private Object getValueFromMultipleKeys(Map map, String... keys) { + for (String key : keys) { + Object value = map.get(key); + if (value != null) { + return value; + } + } + return null; + } + + @Override + public KnowledgeFilesDTO getByDocumentId(String documentId, String datasetId) { + if (StringUtils.isBlank(documentId) || StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "documentId和datasetId不能为空"); + } + + log.info("=== 开始根据documentId获取文档 ==="); + log.info("documentId: {}, datasetId: {}", documentId, datasetId); + + try { + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 修正API路径 - 根据RAGFlow API规范，获取单个文档需要datasetId + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents/" + documentId; + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API获取文档详情..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + KnowledgeFilesDTO dto = convertRAGDocumentToDTO((Map) dataObj); + if (dto != null) { + log.info("获取文档详情成功，documentId: {}", documentId); + return dto; + } + } + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } else { + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API调用失败，响应码: " + code); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, + "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + if (e.getStatusCode() == HttpStatus.NOT_FOUND) { + log.warn("文档不存在，documentId: {}, datasetId: {}", documentId, datasetId); + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString()); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString()); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("根据documentId获取文档失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取文档失败: " + e.getMessage()); + } finally { + log.info("=== 根据documentId获取文档操作结束 ==="); + } + } + + @Override + public PageData getPageListByStatus(String datasetId, Integer status, Integer page, + Integer limit) { + if (StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId不能为空"); + } + + log.info("=== 开始根据状态查询文档列表 ==="); + log.info("datasetId: {}, status: {}, page: {}, limit: {}", datasetId, status, page, limit); + + try { + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 获取文档列表 + StringBuilder urlBuilder = new StringBuilder(); + urlBuilder.append(baseUrl).append("/api/v1/datasets/").append(datasetId).append("/documents"); + + // 构建查询参数 + List params = new ArrayList<>(); + if (page != null && page > 0) { + params.add("page=" + page); + } + if (limit != null && limit > 0) { + params.add("page_size=" + limit); + } + + if (!params.isEmpty()) { + urlBuilder.append("?").append(String.join("&", params)); + } + + String url = urlBuilder.toString(); + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API获取文档列表..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + + // 解析文档列表并过滤状态 + PageData pageData = parseDocumentListResponse(dataObj, page, limit); + + if (status != null) { + // 根据状态过滤文档列表 + List filteredDocuments = pageData.getList().stream() + .filter(doc -> status.equals(doc.getStatus())) + .collect(Collectors.toList()); + + // 更新分页数据 + pageData.setList(filteredDocuments); + pageData.setTotal(filteredDocuments.size()); + } + + log.info("根据状态查询文档列表成功，datasetId: {}, 状态: {}, 文档数量: {}", + datasetId, status, pageData.getList().size()); + return pageData; + } else { + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API调用失败，响应码: " + code); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, + "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString()); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档列表失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString()); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档列表失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "获取RAGFlow文档列表失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("根据状态查询文档列表失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "查询文档列表失败: " + e.getMessage()); + } finally { + log.info("=== 根据状态查询文档列表操作结束 ==="); + } + } + + @Override + public KnowledgeFilesDTO uploadDocument(String datasetId, MultipartFile file, String name, + Map metaFields, String chunkMethod, + Map parserConfig) { + if (StringUtils.isBlank(datasetId) || file == null || file.isEmpty()) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + log.info("=== 开始文档上传操作 ==="); + log.info("上传文档到数据集: {}, 文件名: {}", datasetId, file.getOriginalFilename()); + + try { + // 在文件上传前添加详细日志 + log.info("1. 开始处理文件信息"); + String fileName = file.getOriginalFilename(); + String fileType = getFileType(fileName); + long fileSize = file.getSize(); + + log.info("文件信息 - 文件名: {}, 文件类型: {}, 文件大小: {} bytes", + fileName, fileType, fileSize); + + // 检查文件基本信息 + if (StringUtils.isBlank(fileName)) { + log.error("文件名为空"); + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "文件名不能为空"); + } + + if (fileSize == 0) { + log.error("文件大小为0"); + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "文件内容为空"); + } + + log.info("2. 开始流式上传到RAGFlow"); + // 直接调用RAGFlow API上传文档 - 使用流式上传 + String documentId = uploadDocumentToRAGFlow(datasetId, file, name, metaFields, chunkMethod, parserConfig); + + log.info("文档上传成功，documentId: {}", documentId); + + // 返回上传的文档信息 + KnowledgeFilesDTO result = new KnowledgeFilesDTO(); + result.setId(documentId); // 使用documentId作为ID + result.setDocumentId(documentId); + result.setDatasetId(datasetId); + result.setName(StringUtils.isNotBlank(name) ? name : fileName); + result.setFileType(fileType); + result.setFileSize(fileSize); + result.setStatus(1); // 上传成功，设置为处理中状态 + + return result; + + } catch (Exception e) { + log.error("文档上传失败: {}", e.getMessage()); + log.error("文档上传失败详细异常: ", e); + if (e instanceof RenException) { + throw (RenException) e; + } + throw new RenException(ErrorCode.INTERNAL_SERVER_ERROR); + } finally { + log.info("=== 文档上传操作结束 ==="); + } + } + + @Override + public void deleteByDocumentId(String documentId, String datasetId) { + if (StringUtils.isBlank(documentId) || StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "documentId和datasetId不能为空"); + } + + log.info("=== 开始根据documentId删除文档 ==="); + log.info("删除文档documentId: {}, datasetId: {}", documentId, datasetId); + + try { + // 直接调用RAGFlow API删除文档，不进行前置验证 + // 因为即使文档不存在，RAGFlow API也会返回相应的错误信息 + log.info("开始调用RAGFlow API删除文档"); + deleteDocumentInRAGFlow(documentId, datasetId); + log.info("RAGFlow API删除调用完成"); + + } catch (Exception e) { + log.error("删除文档失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "删除文档失败: " + e.getMessage()); + } finally { + log.info("=== 根据documentId删除文档操作结束 ==="); + } + } + + /** + * 获取文件类型 - 支持RAGFlow四种文档格式类型 + */ + private String getFileType(String fileName) { + if (StringUtils.isBlank(fileName)) { + log.warn("文件名为空，返回unknown类型"); + return "unknown"; + } + + try { + int lastDotIndex = fileName.lastIndexOf('.'); + if (lastDotIndex > 0 && lastDotIndex < fileName.length() - 1) { + String extension = fileName.substring(lastDotIndex + 1).toLowerCase(); + + // 文档格式类型 + String[] documentTypes = { "pdf", "doc", "docx", "txt", "md", "mdx" }; + String[] spreadsheetTypes = { "csv", "xls", "xlsx" }; + String[] presentationTypes = { "ppt", "pptx" }; + + // 检查文档类型 + for (String type : documentTypes) { + if (type.equals(extension)) { + return "document"; + } + } + + // 检查表格类型 + for (String type : spreadsheetTypes) { + if (type.equals(extension)) { + return "spreadsheet"; + } + } + // 检查幻灯片类型 + for (String type : presentationTypes) { + if (type.equals(extension)) { + return "presentation"; + } + } + // 返回原始扩展名作为文件类型 + return extension; + } + return "unknown"; + } catch (Exception e) { + log.error("获取文件类型失败: ", e); + return "unknown"; + } + } + + @Override + public Map getRAGConfig(String ragModelId) { + if (StringUtils.isBlank(ragModelId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + // 从缓存获取模型配置 + ModelConfigEntity modelConfig = modelConfigService.getModelByIdFromCache(ragModelId); + if (modelConfig == null || modelConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 验证是否为RAG类型配置 + if (!Constant.RAG_CONFIG_TYPE.equals(modelConfig.getModelType().toUpperCase())) { + throw new RenException(ErrorCode.RAG_CONFIG_TYPE_ERROR); + } + + Map config = modelConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + // 返回配置信息 + return config; + } + + @Override + public Map getDefaultRAGConfig() { + // 获取默认RAG模型配置 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("model_type", Constant.RAG_CONFIG_TYPE) + .eq("is_default", 1) + .eq("is_enabled", 1); + + List modelConfigs = modelConfigDao.selectList(queryWrapper); + if (modelConfigs == null || modelConfigs.isEmpty()) { + throw new RenException(ErrorCode.RAG_DEFAULT_CONFIG_NOT_FOUND); + } + + ModelConfigEntity defaultConfig = modelConfigs.get(0); + if (defaultConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + Map config = defaultConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + return config; + } + + /** + * 验证RAG配置中是否包含必要的参数 + */ + private void validateRagConfig(Map config) { + if (config == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 从配置中提取必要的参数 + String baseUrl = (String) config.get("base_url"); + String apiKey = (String) config.get("api_key"); + + // 验证base_url是否存在且非空 + if (StringUtils.isBlank(baseUrl)) { + throw new RenException(ErrorCode.RAG_API_ERROR); + } + } + + /** + * 调用RAGFlow API上传文档 - 流式上传版本 + */ + private String uploadDocumentToRAGFlow(String datasetId, MultipartFile file, String name, + Map metaFields, String chunkMethod, + Map parserConfig) { + try { + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API流式上传文档，datasetId: {}, 文件名: {}", datasetId, file.getOriginalFilename()); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents"; + log.debug("请求URL: {}", url); + + // 构建multipart/form-data请求 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.set("Authorization", "Bearer " + apiKey); + + // 创建多部分请求体 - 使用MultipartFileResource进行流式上传 + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", new MultipartFileResource(file, file.getOriginalFilename())); + + // 添加其他参数 + if (StringUtils.isNotBlank(name)) { + body.add("name", name); + } else { + body.add("name", file.getOriginalFilename()); + } + + if (metaFields != null && !metaFields.isEmpty()) { + try { + body.add("meta_fields", objectMapper.writeValueAsString(metaFields)); + } catch (Exception e) { + log.warn("序列化meta_fields失败: {}", e.getMessage()); + } + } + + if (StringUtils.isNotBlank(chunkMethod)) { + body.add("chunk_method", chunkMethod); + } + + if (parserConfig != null && !parserConfig.isEmpty()) { + try { + body.add("parser_config", objectMapper.writeValueAsString(parserConfig)); + } catch (Exception e) { + log.warn("序列化parser_config失败: {}", e.getMessage()); + } + } + + log.debug("multipart请求体参数数量: {}", body.size()); + log.debug("multipart请求体参数: {}", body.keySet()); + + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + + // 发送POST请求 + log.info("发送multipart/form-data POST请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + String responseBody = response.getBody(); + String documentId = null; + + if (response.getStatusCode().is2xxSuccessful()) { + try { + Map responseMap = objectMapper.readValue(responseBody, Map.class); + + log.debug("RAGFlow API响应解析结果: {}", responseMap); + + // 首先检查响应码 + Integer code = (Integer) responseMap.get("code"); + if (code != null && code == 0) { + // 响应码为0表示成功，从data字段中获取documentId + Object dataObj = responseMap.get("data"); + + // 增强的documentId提取逻辑 + documentId = extractDocumentIdFromResponse(dataObj); + + // 如果从data字段无法提取，尝试从根级别提取 + if (StringUtils.isBlank(documentId)) { + documentId = extractDocumentIdFromRoot(responseMap); + } + + log.info("文档上传成功，documentId: {}", documentId); + } else { + // 如果响应码不为0，说明API调用失败 + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API调用失败，响应码: " + code); + } + + log.info("从RAGFlow API响应中解析出documentId: {}", documentId); + log.debug("完整响应内容: {}", responseBody); + } catch (Exception e) { + log.error("解析RAGFlow API响应失败: {}, 响应内容: {}", e.getMessage(), responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow响应失败: " + e.getMessage()); + } + } else { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + if (StringUtils.isBlank(documentId)) { + log.error("无法从RAGFlow API响应中获取documentId，响应内容: {}", responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API响应中未包含documentId"); + } + log.info("RAGFlow文档上传成功，documentId: {}，文档已开始自动解析切片", documentId); + return documentId; + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, + "上传RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, + "上传RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "上传RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "上传RAGFlow文档失败: " + e.getMessage()); + } + } + + /** + * 从响应数据中提取documentId + */ + private String extractDocumentIdFromResponse(Object dataObj) { + String documentId = null; + + if (dataObj instanceof List) { + // data是一个数组，取第一个元素的id字段 + List> dataList = (List>) dataObj; + if (!dataList.isEmpty()) { + Map firstItem = dataList.get(0); + documentId = extractDocumentIdFromMap(firstItem); + } + } else if (dataObj instanceof Map) { + // data是一个对象 + Map dataMap = (Map) dataObj; + documentId = extractDocumentIdFromMap(dataMap); + } + + return documentId; + } + + /** + * 从Map中提取documentId，支持多种可能的字段名 + */ + private String extractDocumentIdFromMap(Map map) { + if (map == null) + return null; + + // 尝试多种可能的字段名 + String[] possibleFieldNames = { "id", "document_id", "documentId", "doc_id", "documentId" }; + + for (String fieldName : possibleFieldNames) { + Object value = map.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 从根级别响应中提取documentId + */ + private String extractDocumentIdFromRoot(Map responseMap) { + if (responseMap == null) + return null; + + // 尝试从根级别提取 + String[] possibleFieldNames = { "id", "document_id", "documentId", "doc_id", "documentId" }; + + for (String fieldName : possibleFieldNames) { + Object value = responseMap.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 调用RAGFlow API删除文档 + */ + private void deleteDocumentInRAGFlow(String documentId, String datasetId) { + try { + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API删除文档，documentId: {}, datasetId: {}", documentId, datasetId); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL - 根据RAGFlow API文档，使用正确的路径参数名称 + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents"; + log.debug("请求URL: {}", url); + + // 构建请求体 - 严格按照API文档格式 + Map requestBody = new HashMap<>(); + requestBody.put("ids", Arrays.asList(documentId)); // 使用Arrays.asList确保序列化正确 + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送DELETE请求 + log.info("发送DELETE请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.DELETE, requestEntity, + String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + String responseBody = response.getBody(); + + if (response.getStatusCode().is2xxSuccessful()) { + // 验证响应格式 + if (responseBody != null) { + try { + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + log.info("RAGFlow文档删除成功，documentId: {}", documentId); + return; + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 消息: {}", code, message); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API调用失败: " + message); + } + } catch (Exception e) { + log.warn("解析RAGFlow响应失败，但HTTP状态码成功，视为删除成功: {}", e.getMessage()); + log.info("RAGFlow文档删除成功，documentId: {}", documentId); + return; + } + } else { + log.info("RAGFlow文档删除成功（无响应体），documentId: {}", documentId); + return; + } + } else { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, + "删除RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, + "删除RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "删除RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "删除RAGFlow文档失败: " + e.getMessage()); + } + } + + /** + * 辅助类：将MultipartFile转换为Resource用于流式上传 + */ + private static class MultipartFileResource extends AbstractResource { + private final MultipartFile multipartFile; + private final String filename; + + public MultipartFileResource(MultipartFile multipartFile, String filename) { + this.multipartFile = multipartFile; + this.filename = filename; + } + + @Override + public String getFilename() { + return this.filename; + } + + @Override + public InputStream getInputStream() throws IOException { + return this.multipartFile.getInputStream(); + } + + @Override + public long contentLength() throws IOException { + return this.multipartFile.getSize(); + } + + @Override + public boolean exists() { + return true; + } + + @Override + public String getDescription() { + return "MultipartFile resource for " + this.filename; + } + } + + @Override + public boolean parseDocuments(String datasetId, List documentIds) { + if (StringUtils.isBlank(datasetId) || documentIds == null || documentIds.isEmpty()) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId和documentIds不能为空"); + } + + log.info("=== 开始解析文档（切块） ==="); + log.info("datasetId: {}, documentIds: {}", datasetId, documentIds); + + try { + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 根据RAGFlow API文档，解析文档的接口 + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/chunks"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("document_ids", documentIds); + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送POST请求 + log.info("发送POST请求到RAGFlow API解析文档..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + log.info("文档解析成功，datasetId: {}, documentIds: {}", datasetId, documentIds); + return true; + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 消息: {}, 响应内容: {}", code, message, responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, response.getStatusCode().toString()); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow文档失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow文档失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("解析文档失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析文档失败: " + e.getMessage()); + } finally { + log.info("=== 解析文档操作结束 ==="); + } + } + + @Override + public Map listChunks(String datasetId, String documentId, String keywords, + Integer page, Integer pageSize, String chunkId) { + if (StringUtils.isBlank(datasetId) || StringUtils.isBlank(documentId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId和documentId不能为空"); + } + + log.info("=== 开始列出切片 ==="); + log.info("datasetId: {}, documentId: {}, keywords: {}, page: {}, pageSize: {}, chunkId: {}", + datasetId, documentId, keywords, page, pageSize, chunkId); + + try { + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 根据RAGFlow API文档，列出切片的接口 + StringBuilder urlBuilder = new StringBuilder(); + urlBuilder.append(baseUrl).append("/api/v1/datasets/").append(datasetId) + .append("/documents/").append(documentId).append("/chunks"); + + // 添加查询参数 + List params = new ArrayList<>(); + + if (StringUtils.isNotBlank(keywords)) { + params.add("keywords=" + keywords); + } + + if (page != null && page > 0) { + params.add("page=" + page); + } + + if (pageSize != null && pageSize > 0) { + params.add("page_size=" + pageSize); + } + + if (StringUtils.isNotBlank(chunkId)) { + params.add("id=" + chunkId); + } + + if (!params.isEmpty()) { + urlBuilder.append("?").append(String.join("&", params)); + } + + String url = urlBuilder.toString(); + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API列出切片..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR, "列出切片失败"); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + log.info("切片列表获取成功，datasetId: {}, documentId: {}", datasetId, documentId); + + // 解析切片数据并格式化返回 + return parseChunkListResponse(responseMap); + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 消息: {}, 响应内容: {}", code, message, responseBody); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API调用失败: " + message); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "列出切片失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "列出切片失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "列出切片失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("列出切片失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "列出切片失败: " + e.getMessage()); + } finally { + log.info("=== 列出切片操作结束 ==="); + } + } + + /** + * 解析RAGFlow API返回的切片列表响应 + */ + private Map parseChunkListResponse(Map responseMap) { + Map result = new HashMap<>(); + List> chunkList = new ArrayList<>(); + long totalCount = 0; + + try { + // 首先检查是否有data字段 + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + + // 解析切片列表 - 支持多种可能的字段名 + Object chunksObj = null; + + // 支持多种可能的切片列表字段名 + String[] possibleChunkFields = { "chunks", "items", "list", "data", "docs" }; + + for (String fieldName : possibleChunkFields) { + if (dataMap.containsKey(fieldName) && dataMap.get(fieldName) instanceof List) { + chunksObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取切片列表", fieldName); + break; + } + } + + // 如果标准字段不存在，尝试自动检测 + if (chunksObj == null) { + for (Map.Entry entry : dataMap.entrySet()) { + if (entry.getValue() instanceof List) { + List list = (List) entry.getValue(); + if (!list.isEmpty() && list.get(0) instanceof Map) { + chunksObj = entry.getValue(); + log.warn("自动检测到切片列表字段: '{}'，建议检查RAGFlow API文档", entry.getKey()); + break; + } + } + } + } + + if (chunksObj instanceof List) { + List> rawChunkList = (List>) chunksObj; + + for (Map chunkMap : rawChunkList) { + Map formattedChunk = formatChunkData(chunkMap); + if (formattedChunk != null) { + chunkList.add(formattedChunk); + } + } + } + + // 解析总数 - 支持多种可能的字段名 + Object totalObj = null; + String[] possibleTotalFields = { "total", "totalCount", "total_count", "count" }; + + for (String fieldName : possibleTotalFields) { + if (dataMap.containsKey(fieldName)) { + totalObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取总数", fieldName); + break; + } + } + + if (totalObj instanceof Integer) { + totalCount = ((Integer) totalObj).longValue(); + } else if (totalObj instanceof Long) { + totalCount = (Long) totalObj; + } else if (totalObj instanceof String) { + try { + totalCount = Long.parseLong((String) totalObj); + } catch (NumberFormatException e) { + log.warn("无法解析总数字段: {}", totalObj); + } + } + + // 如果没有找到总数，使用切片列表的大小 + if (totalCount == 0 && !chunkList.isEmpty()) { + totalCount = chunkList.size(); + } + } else { + log.warn("RAGFlow API响应缺少data字段，尝试直接解析响应"); + + // 如果没有data字段，尝试直接解析响应 + Object chunksObj = null; + String[] possibleChunkFields = { "chunks", "items", "list", "data", "docs" }; + + for (String fieldName : possibleChunkFields) { + if (responseMap.containsKey(fieldName) && responseMap.get(fieldName) instanceof List) { + chunksObj = responseMap.get(fieldName); + log.debug("使用字段名'{}'获取切片列表", fieldName); + break; + } + } + + if (chunksObj instanceof List) { + List> rawChunkList = (List>) chunksObj; + + for (Map chunkMap : rawChunkList) { + Map formattedChunk = formatChunkData(chunkMap); + if (formattedChunk != null) { + chunkList.add(formattedChunk); + } + } + } + + // 解析总数 + Object totalObj = responseMap.get("total"); + if (totalObj instanceof Integer) { + totalCount = ((Integer) totalObj).longValue(); + } else if (totalObj instanceof Long) { + totalCount = (Long) totalObj; + } + + if (totalCount == 0 && !chunkList.isEmpty()) { + totalCount = chunkList.size(); + } + } + + } catch (Exception e) { + log.error("解析切片列表响应失败: {}", e.getMessage(), e); + } + + result.put("list", chunkList); + result.put("total", totalCount); + + log.debug("解析后的切片列表: {} 条记录", chunkList.size()); + return result; + } + + /** + * 格式化切片数据 + */ + private Map formatChunkData(Map chunkMap) { + if (chunkMap == null || chunkMap.isEmpty()) { + return null; + } + + Map formattedChunk = new HashMap<>(); + + try { + // 提取切片ID - 支持多种可能的字段名 + String chunkId = extractChunkId(chunkMap); + if (StringUtils.isBlank(chunkId)) { + log.warn("切片数据缺少ID字段，跳过处理: {}", chunkMap); + return null; + } + formattedChunk.put("id", chunkId); + + // 提取切片内容 - 支持多种可能的字段名 + String content = extractChunkContent(chunkMap); + formattedChunk.put("content", content != null ? content : ""); + + // 提取重要关键词 - 支持多种可能的字段名 + List importantKeywords = extractImportantKeywords(chunkMap); + formattedChunk.put("important_keywords", importantKeywords); + + // 提取问题列表 - 支持多种可能的字段名 + List questions = extractQuestions(chunkMap); + formattedChunk.put("questions", questions); + + // 提取创建时间 - 支持多种可能的字段名 + String createTime = extractCreateTime(chunkMap); + formattedChunk.put("create_time", createTime != null ? createTime : ""); + + } catch (Exception e) { + log.error("格式化切片数据失败: {}", e.getMessage(), e); + return null; + } + + return formattedChunk; + } + + /** + * 提取切片ID + */ + private String extractChunkId(Map chunkMap) { + String[] possibleIdFields = { "id", "chunk_id", "chunkId", "chunkId" }; + + for (String fieldName : possibleIdFields) { + Object value = chunkMap.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 提取切片内容 + */ + private String extractChunkContent(Map chunkMap) { + String[] possibleContentFields = { "content", "text", "chunk_content", "chunkContent" }; + + for (String fieldName : possibleContentFields) { + Object value = chunkMap.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 提取重要关键词 + */ + private List extractImportantKeywords(Map chunkMap) { + String[] possibleKeywordFields = { "important_keywords", "keywords", "importantKeywords", "key_words" }; + + for (String fieldName : possibleKeywordFields) { + Object value = chunkMap.get(fieldName); + if (value instanceof List) { + List list = (List) value; + List keywords = new ArrayList<>(); + for (Object item : list) { + if (item instanceof String && StringUtils.isNotBlank((String) item)) { + keywords.add((String) item); + } + } + return keywords; + } else if (value instanceof String && StringUtils.isNotBlank((String) value)) { + // 如果是逗号分隔的字符串，分割成列表 + String[] parts = ((String) value).split(","); + List keywords = new ArrayList<>(); + for (String part : parts) { + String trimmed = part.trim(); + if (StringUtils.isNotBlank(trimmed)) { + keywords.add(trimmed); + } + } + return keywords; + } + } + + return new ArrayList<>(); + } + + /** + * 提取问题列表 + */ + private List extractQuestions(Map chunkMap) { + String[] possibleQuestionFields = { "questions", "question_list", "questionList", "qas" }; + + for (String fieldName : possibleQuestionFields) { + Object value = chunkMap.get(fieldName); + if (value instanceof List) { + List list = (List) value; + List questions = new ArrayList<>(); + for (Object item : list) { + if (item instanceof String && StringUtils.isNotBlank((String) item)) { + questions.add((String) item); + } + } + return questions; + } else if (value instanceof String && StringUtils.isNotBlank((String) value)) { + // 如果是逗号分隔的字符串，分割成列表 + String[] parts = ((String) value).split(","); + List questions = new ArrayList<>(); + for (String part : parts) { + String trimmed = part.trim(); + if (StringUtils.isNotBlank(trimmed)) { + questions.add(trimmed); + } + } + return questions; + } + } + + return new ArrayList<>(); + } + + /** + * 提取创建时间 + */ + private String extractCreateTime(Map chunkMap) { + String[] possibleTimeFields = { "create_time", "created_at", "createTime", "timestamp" }; + + for (String fieldName : possibleTimeFields) { + Object value = chunkMap.get(fieldName); + if (value != null) { + if (value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } else if (value instanceof Long) { + // 如果是时间戳，转换为字符串 + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date((Long) value)); + } + } + } + + return null; + } + + @Override + public Map retrievalTest(String question, List datasetIds, List documentIds, + Integer page, Integer pageSize, Float similarityThreshold, + Float vectorSimilarityWeight, Integer topK, String rerankId, + Boolean keyword, Boolean highlight, List crossLanguages, + Map metadataCondition) { + + log.info("=== 开始召回测试 ==="); + log.info("问题: {}, 数据集ID: {}, 文档ID: {}, 页码: {}, 每页数量: {}", + question, datasetIds, documentIds, page, pageSize); + + try { + // 获取RAG配置 + Map ragConfig = getDefaultRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL + String url = baseUrl + "/api/v1/retrieval"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("question", question); + + if (datasetIds != null && !datasetIds.isEmpty()) { + requestBody.put("dataset_ids", datasetIds); + } + + if (documentIds != null && !documentIds.isEmpty()) { + requestBody.put("document_ids", documentIds); + } + + if (page != null && page > 0) { + requestBody.put("page", page); + } + + if (pageSize != null && pageSize > 0) { + requestBody.put("page_size", pageSize); + } + + if (similarityThreshold != null) { + requestBody.put("similarity_threshold", similarityThreshold); + } + + if (vectorSimilarityWeight != null) { + requestBody.put("vector_similarity_weight", vectorSimilarityWeight); + } + + if (topK != null && topK > 0) { + requestBody.put("top_k", topK); + } + + if (rerankId != null) { + requestBody.put("rerank_id", rerankId); + } + + if (keyword != null) { + requestBody.put("keyword", keyword); + } + + if (highlight != null) { + requestBody.put("highlight", highlight); + } + + if (crossLanguages != null && !crossLanguages.isEmpty()) { + requestBody.put("cross_languages", crossLanguages); + } + + if (metadataCondition != null) { + requestBody.put("metadata_condition", metadataCondition); + } + + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送POST请求 + log.info("发送POST请求到RAGFlow API进行召回测试..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_ERROR); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map result = (Map) dataObj; + log.info("召回测试成功，返回 {} 条切片", result.get("total")); + return result; + } else { + log.error("RAGFlow API响应格式错误，data字段不是Map类型: {}", dataObj); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API响应格式错误"); + } + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 错误信息: {}", code, message); + throw new RenException(ErrorCode.RAG_API_ERROR, "RAGFlow API调用失败: " + message); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "召回测试失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "召回测试失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "召回测试失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("召回测试失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_ERROR, "召回测试失败: " + e.getMessage()); + } finally { + log.info("=== 召回测试操作结束 ==="); + } + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelProviderServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelProviderServiceImpl.java index 69e6b75c..8b8f8672 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelProviderServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/model/service/impl/ModelProviderServiceImpl.java @@ -22,6 +22,8 @@ import xiaozhi.common.page.PageData; import xiaozhi.common.service.impl.BaseServiceImpl; import xiaozhi.common.user.UserDetail; import xiaozhi.common.utils.ConvertUtils; +import xiaozhi.modules.knowledge.dao.KnowledgeBaseDao; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; import xiaozhi.modules.model.dao.ModelProviderDao; import xiaozhi.modules.model.dto.ModelProviderDTO; import xiaozhi.modules.model.entity.ModelProviderEntity; @@ -34,13 +36,43 @@ public class ModelProviderServiceImpl extends BaseServiceImpl getPluginList() { + // 1. 获取插件列表 LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); queryWrapper.eq(ModelProviderEntity::getModelType, "Plugin"); List providerEntities = modelProviderDao.selectList(queryWrapper); - return ConvertUtils.sourceToTarget(providerEntities, ModelProviderDTO.class); + List resultList = ConvertUtils.sourceToTarget(providerEntities, ModelProviderDTO.class); + + // 2. 获取当前用户的知识库列表并追加到结果中 + UserDetail userDetail = SecurityUser.getUser(); + if (userDetail != null && userDetail.getId() != null) { + // 查询当前用户的知识库 + LambdaQueryWrapper kbQueryWrapper = new LambdaQueryWrapper<>(); + kbQueryWrapper.eq(KnowledgeBaseEntity::getCreator, userDetail.getId()); + kbQueryWrapper.eq(KnowledgeBaseEntity::getStatus, 1); // 只获取启用状态的知识库 + List knowledgeBases = knowledgeBaseDao.selectList(kbQueryWrapper); + + // 将知识库转换为ModelProviderDTO格式并添加到结果列表 + for (KnowledgeBaseEntity kb : knowledgeBases) { + ModelProviderDTO dto = new ModelProviderDTO(); + dto.setId(kb.getId()); + dto.setModelType("Rag"); + dto.setName("[知识库]" + kb.getName()); + dto.setProviderCode("ragflow"); // 假设所有RAG都使用ragflow + dto.setFields("[]"); + dto.setSort(0); + dto.setCreateDate(kb.getCreatedAt()); + dto.setUpdateDate(kb.getUpdatedAt()); + dto.setCreator(0L); + dto.setUpdater(0L); + resultList.add(dto); + } + } + + return resultList; } @Override diff --git a/main/manager-api/src/main/resources/db/changelog/202510250956.sql b/main/manager-api/src/main/resources/db/changelog/202510250956.sql new file mode 100644 index 00000000..b4129af1 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202510250956.sql @@ -0,0 +1,24 @@ +-- 添加RAG模型供应器和配置 +-- ------------------------------------------------------- + +-- 添加RAG模型供应器 +delete from `ai_model_provider` where id = 'SYSTEM_RAG_ragflow'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_RAG_ragflow', 'RAG', 'ragflow', 'RAGFlow', '[{"key": "base_url", "type": "string", "label": "服务地址"}, {"key": "api_key", "type": "string", "label": "API密钥"}]', 1, 1, NOW(), 1, NOW()); + +-- 添加RAG模型配置 +delete from `ai_model_config` where id = 'RAG_RAGFlow'; +INSERT INTO `ai_model_config` VALUES ('RAG_RAGFlow', 'RAG', 'ragflow', 'RAGFlow', 1, 1, '{"type": "ragflow", "base_url": "http://localhost", "api_key": "你的RAG密钥"}', 'https://github.com/infiniflow/ragflow/blob/main/README_zh.md', 'RAGFlow配置说明： +一、快速部署教程（docker部署） +1.$ sysctl vm.max_map_count +2.$ sysctl -w vm.max_map_count=262144 +3.$ git clone https://github.com/infiniflow/ragflow.git +4.docker compose -f docker-compose.yml up -d +5.$ docker logs -f docker-ragflow-cpu-1 +6.注冊登录后，点击右上角头像，获得RAGFlow的API KEY和API服务器地址。使用RAGFlow前请在Model Provider中添加模型和设置默认模型。 +二、如果您希望关掉注册功能 +1.停止服务 docker compose down +2. sed -i ''s/REGISTER_ENABLED=1/REGISTER_ENABLED=0/g'' .env +3.cat .env | grep -i register +4.看到REGISTER_ENABLED=0 重启服务即可。', 1, NULL, NULL, NULL, NULL); + diff --git a/main/manager-api/src/main/resources/db/changelog/202510251150.sql b/main/manager-api/src/main/resources/db/changelog/202510251150.sql new file mode 100644 index 00000000..4dea2bec --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202510251150.sql @@ -0,0 +1,19 @@ +-- 知识库表 +DROP TABLE IF EXISTS `ai_rag_dataset`; +CREATE TABLE `ai_rag_dataset` ( + `id` VARCHAR(32) NOT NULL COMMENT '唯一标识', + `dataset_id` VARCHAR(64) NOT NULL COMMENT '知识库ID', + `rag_model_id` VARCHAR(64) COMMENT 'RAG模型配置ID', + `name` VARCHAR(100) NOT NULL COMMENT '知识库名称', + `description` TEXT COMMENT '知识库描述', + `status` TINYINT(1) DEFAULT 1 COMMENT '状态：0停用 1启用', + `creator` BIGINT COMMENT '创建者', + `created_at` DATETIME COMMENT '创建时间', + `updater` BIGINT COMMENT '更新者', + `updated_at` DATETIME COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_dataset_id` (`dataset_id`), + INDEX `idx_ai_rag_dataset_status` (`status`), + INDEX `idx_ai_rag_dataset_creator` (`creator`), + INDEX `idx_ai_rag_dataset_created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='知识库表'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index b6b88bf3..326d6f57 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -402,3 +402,17 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202510191042.sql + - changeSet: + id: 202510250956 + author: rainv123 + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202510250956.sql + - changeSet: + id: 202510251150 + author: rainv123 + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202510251150.sql \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages.properties b/main/manager-api/src/main/resources/i18n/messages.properties index 77689569..d2c02815 100644 --- a/main/manager-api/src/main/resources/i18n/messages.properties +++ b/main/manager-api/src/main/resources/i18n/messages.properties @@ -168,4 +168,12 @@ 10159=\u97F3\u8272ID\u5DF2\u5B58\u5728 10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u9519\u8BEF\uFF0C\u5FC5\u987B\u4EE5S_\u5F00\u5934 10161=Mac\u5730\u5740\u5DF2\u5B58\u5728 -10162=\u6A21\u578B\u4F9B\u5E94\u5668\u4E0D\u5B58\u5728 \ No newline at end of file +10162=\u6A21\u578B\u4F9B\u5E94\u5668\u4E0D\u5B58\u5728 +10163=\u77E5\u8BC6\u5E93\u8BB0\u5F55\u4E0D\u5B58\u5728 +10164=RAG\u914D\u7F6E\u672A\u627E\u5230 +10165=RAG\u914D\u7F6E\u7C7B\u578B\u9519\u8BEF +10166=\u9ED8\u8BA4RAG\u914D\u7F6E\u672A\u627E\u5230 +10167=RAG\u63A5\u53E3\u9519\u8BEF\uFF0C{0} +10168=\u4E0A\u4F20\u6587\u4EF6\u5931\u8D25 +10169=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55 +10170=\u77E5\u8BC6\u5E93\u540D\u79F0\u91CD\u590D \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages_en_US.properties b/main/manager-api/src/main/resources/i18n/messages_en_US.properties index 04354f63..7610fd8a 100644 --- a/main/manager-api/src/main/resources/i18n/messages_en_US.properties +++ b/main/manager-api/src/main/resources/i18n/messages_en_US.properties @@ -168,4 +168,12 @@ 10159=Voice ID already exists 10160=Huoshan Engine voice ID format error, must start with S_ 10161=Mac address already exists -10162=Model provider does not exist \ No newline at end of file +10162=Model provider does not exist +10163=Knowledge base record does not exist +10164=RAG configuration not found +10165=RAG configuration type error +10166=Default RAG configuration not found +10167=RAG API interface error: {0} +10168=Upload file failed +10169=No permission to operate this knowledge base +10170=Knowledge base name already exists \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties b/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties index 6fb10371..e72f34b7 100644 --- a/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties @@ -168,4 +168,12 @@ 10159=\u97F3\u8272ID\u5DF2\u5B58\u5728 10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u9519\u8BEF\uFF0C\u5FC5\u987B\u4EE5S_\u5F00\u5934 10161=Mac\u5730\u5740\u5DF2\u5B58\u5728 -10162=\u6A21\u578B\u4F9B\u5E94\u5668\u4E0D\u5B58\u5728 \ No newline at end of file +10162=\u6A21\u578B\u4F9B\u5E94\u5668\u4E0D\u5B58\u5728 +10163=\u77E5\u8BC6\u5E93\u8BB0\u5F55\u4E0D\u5B58\u5728 +10164=RAG\u914D\u7F6E\u672A\u627E\u5230 +10165=RAG\u914D\u7F6E\u7C7B\u578B\u9519\u8BEF +10166=\u9ED8\u8BA4RAG\u914D\u7F6E\u672A\u627E\u5230 +10167=RAG\u63A5\u53E3\u9519\u8BEF\uFF0C{0} +10168=\u4E0A\u4F20\u6587\u4EF6\u5931\u8D25 +10169=\u60A8\u6CA1\u6709\u6743\u9650\u64CD\u4F5C\u8BE5\u8BB0\u5F55 +10170=\u77E5\u8BC6\u5E93\u540D\u79F0\u91CD\u590D \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties b/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties index 0f5d316c..b5d118e3 100644 --- a/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties +++ b/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties @@ -169,4 +169,11 @@ 10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u932F\u8AA4\uFF0C\u5FC5\u9808\u4EE5S_\u958B\u982D 10161=Mac\u5730\u5740\u5DF2\u5B58\u5728 10162=\u6A21\u578B\u63D0\u4F9B\u5546\u4E0D\u5B58\u5728 - +10163=\u77E5\u8B58\u5EAB\u8A18\u9304\u4E0D\u5B58\u5728 +10164=RAG\u914D\u7F6E\u672A\u627E\u5230 +10165=RAG\u914D\u7F6E\u985E\u578B\u932F\u8AA4 +10166=\u9810\u8A2DRAG\u914D\u7F6E\u672A\u627E\u5230 +10167=RAG\u63A5\u53E3\u932F\u8AA4\uFF0C{0} +10168=\u4E0A\u50B3\u6587\u4EF6\u5931\u6557 +10169=\u60A8\u6C92\u6709\u6B0A\u9650\u64CD\u4F5C\u8A72\u8A18\u9304 +10170=\u77E5\u8B58\u5EAB\u540D\u7A31\u91CD\u8907 diff --git a/main/manager-mobile/src/pages/settings/index.vue b/main/manager-mobile/src/pages/settings/index.vue index 3c699f96..8a620c78 100644 --- a/main/manager-mobile/src/pages/settings/index.vue +++ b/main/manager-mobile/src/pages/settings/index.vue @@ -235,7 +235,7 @@ function showAbout() { title: t('settings.aboutApp', { appName: import.meta.env.VITE_APP_TITLE }), content: t('settings.aboutContent', { appName: import.meta.env.VITE_APP_TITLE, - version: '0.8.6' + version: '0.8.7' }), showCancel: false, confirmText: t('common.confirm'), diff --git a/main/manager-web/src/apis/api.js b/main/manager-web/src/apis/api.js index 3f2595b4..45a810c7 100755 --- a/main/manager-web/src/apis/api.js +++ b/main/manager-web/src/apis/api.js @@ -9,6 +9,7 @@ import timbre from "./module/timbre.js" import user from './module/user.js' import voiceClone from './module/voiceClone.js' import voiceResource from './module/voiceResource.js' +import knowledgeBase from './module/knowledgeBase.js' @@ -39,5 +40,6 @@ export default { ota, dict, voiceResource, - voiceClone -} + voiceClone, + knowledgeBase + } diff --git a/main/manager-web/src/apis/module/knowledgeBase.js b/main/manager-web/src/apis/module/knowledgeBase.js new file mode 100644 index 00000000..8f2d7c0f --- /dev/null +++ b/main/manager-web/src/apis/module/knowledgeBase.js @@ -0,0 +1,318 @@ +import { getServiceUrl } from '../api'; +import RequestService from '../httpRequest'; + +/** + * 获取认证token + */ +function getAuthToken() { + return localStorage.getItem('token') || ''; +} + +/** + * 通用API请求包装器 + * @param {Object} config - 请求配置 + * @param {string} config.url - 请求URL + * @param {string} config.method - 请求方法 + * @param {Object} [config.data] - 请求数据 + * @param {Object} [config.headers] - 额外请求头 + * @param {Function} config.callback - 成功回调 + * @param {Function} [config.errorCallback] - 错误回调 + * @param {string} [config.errorMessage] - 错误消息 + * @param {Function} [config.retryFunction] - 重试函数 + */ +function makeApiRequest(config) { + const token = getAuthToken(); + const { url, method, data, headers, callback, errorCallback, errorMessage, retryFunction } = config; + + const requestBuilder = RequestService.sendRequest() + .url(url) + .method(method) + .header({ + 'Authorization': `Bearer ${token}`, + ...headers + }); + + if (data) { + requestBuilder.data(data); + } + + requestBuilder + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error(errorMessage || '操作失败', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + if (retryFunction) { + RequestService.reAjaxFun(() => { + retryFunction(); + }); + } + }).send(); +} + +/** + * 知识库管理相关API + */ +export default { + /** + * 获取知识库列表 + * @param {Object} params - 查询参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + getKnowledgeBaseList(params, callback, errorCallback) { + const queryParams = new URLSearchParams({ + page: params.page, + page_size: params.page_size, + name: params.name || '' + }).toString(); + + makeApiRequest({ + url: `${getServiceUrl()}/datasets?${queryParams}`, + method: 'GET', + callback: callback, + errorCallback: errorCallback, + errorMessage: '获取知识库列表失败', + retryFunction: () => this.getKnowledgeBaseList(params, callback, errorCallback) + }); + }, + + /** + * 创建知识库 + * @param {Object} data - 知识库数据 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + createKnowledgeBase(data, callback, errorCallback) { + console.log('createKnowledgeBase called with data:', data); + console.log('API URL:', `${getServiceUrl()}/datasets`); + + makeApiRequest({ + url: `${getServiceUrl()}/datasets`, + method: 'POST', + data: data, + headers: { 'Content-Type': 'application/json' }, + callback: (res) => { + console.log('createKnowledgeBase success response:', res); + callback(res); + }, + errorCallback: (err) => { + console.error('创建知识库失败:', err); + if (err.response) { + console.error('Error response data:', err.response.data); + console.error('Error response status:', err.response.status); + } + if (errorCallback) { + errorCallback(err); + } + }, + errorMessage: '创建知识库失败', + retryFunction: () => this.createKnowledgeBase(data, callback, errorCallback) + }); + }, + + /** + * 更新知识库 + * @param {string} datasetId - 知识库ID + * @param {Object} data - 更新数据 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + updateKnowledgeBase(datasetId, data, callback, errorCallback) { + console.log('updateKnowledgeBase called with datasetId:', datasetId, 'data:', data); + console.log('API URL:', `${getServiceUrl()}/datasets/${datasetId}`); + + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}`, + method: 'PUT', + data: data, + headers: { 'Content-Type': 'application/json' }, + callback: callback, + errorCallback: errorCallback, + errorMessage: '更新知识库失败', + retryFunction: () => this.updateKnowledgeBase(datasetId, data, callback, errorCallback) + }); + }, + + /** + * 删除单个知识库 + * @param {string} datasetId - 知识库ID + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + deleteKnowledgeBase(datasetId, callback, errorCallback) { + console.log('deleteKnowledgeBase called with datasetId:', datasetId); + console.log('API URL:', `${getServiceUrl()}/datasets/${datasetId}`); + + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}`, + method: 'DELETE', + callback: callback, + errorCallback: errorCallback, + errorMessage: '删除知识库失败', + retryFunction: () => this.deleteKnowledgeBase(datasetId, callback, errorCallback) + }); + }, + + /** + * 批量删除知识库 + * @param {string|Array} ids - 知识库ID字符串或数组 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + deleteKnowledgeBases(ids, callback, errorCallback) { + // 确保ids是正确格式的字符串 + const idsStr = Array.isArray(ids) ? ids.join(',') : ids; + + makeApiRequest({ + url: `${getServiceUrl()}/datasets/batch?ids=${idsStr}`, + method: 'DELETE', + callback: callback, + errorCallback: errorCallback, + errorMessage: '批量删除知识库失败', + retryFunction: () => this.deleteKnowledgeBases(ids, callback, errorCallback) + }); + }, + + /** + * 获取文档列表 + * @param {string} datasetId - 知识库ID + * @param {Object} params - 查询参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + getDocumentList(datasetId, params, callback, errorCallback) { + const queryParams = new URLSearchParams({ + page: params.page, + page_size: params.page_size, + name: params.name || '' + }).toString(); + + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}/documents?${queryParams}`, + method: 'GET', + callback: callback, + errorCallback: errorCallback, + errorMessage: '获取文档列表失败', + retryFunction: () => this.getDocumentList(datasetId, params, callback, errorCallback) + }); + }, + + /** + * 上传文档 + * @param {string} datasetId - 知识库ID + * @param {Object} formData - 表单数据 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + uploadDocument(datasetId, formData, callback, errorCallback) { + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}/documents`, + method: 'POST', + data: formData, + headers: { 'Content-Type': 'multipart/form-data' }, + callback: callback, + errorCallback: errorCallback, + errorMessage: '上传文档失败', + retryFunction: () => this.uploadDocument(datasetId, formData, callback, errorCallback) + }); + }, + + /** + * 解析文档 + * @param {string} datasetId - 知识库ID + * @param {string} documentId - 文档ID + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + parseDocument(datasetId, documentId, callback, errorCallback) { + const requestBody = { + document_ids: [documentId] + }; + + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}/chunks`, + method: 'POST', + data: requestBody, + headers: { 'Content-Type': 'application/json' }, + callback: callback, + errorCallback: errorCallback, + errorMessage: '解析文档失败', + retryFunction: () => this.parseDocument(datasetId, documentId, callback, errorCallback) + }); + }, + + /** + * 删除文档 + * @param {string} datasetId - 知识库ID + * @param {string} documentId - 文档ID + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + deleteDocument(datasetId, documentId, callback, errorCallback) { + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}/documents/${documentId}`, + method: 'DELETE', + callback: callback, + errorCallback: errorCallback, + errorMessage: '删除文档失败', + retryFunction: () => this.deleteDocument(datasetId, documentId, callback, errorCallback) + }); + }, + + /** + * 获取文档切片列表 + * @param {string} datasetId - 知识库ID + * @param {string} documentId - 文档ID + * @param {Object} params - 查询参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + listChunks(datasetId, documentId, params, callback, errorCallback) { + let queryParams = new URLSearchParams({ + page: params.page || 1, + page_size: params.page_size || 10 + }).toString(); + + // 添加关键词搜索参数 + if (params.keywords) { + queryParams += `&keywords=${encodeURIComponent(params.keywords)}`; + } + + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}/documents/${documentId}/chunks?${queryParams}`, + method: 'GET', + callback: callback, + errorCallback: errorCallback, + errorMessage: '获取切片列表失败', + retryFunction: () => this.listChunks(datasetId, documentId, params, callback, errorCallback) + }); + }, + + /** + * 召回测试 + * @param {string} datasetId - 知识库ID + * @param {Object} data - 召回测试参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + retrievalTest(datasetId, data, callback, errorCallback) { + makeApiRequest({ + url: `${getServiceUrl()}/datasets/${datasetId}/retrieval-test`, + method: 'POST', + data: data, + headers: { 'Content-Type': 'application/json' }, + callback: callback, + errorCallback: errorCallback, + errorMessage: '召回测试失败', + retryFunction: () => this.retrievalTest(datasetId, data, callback, errorCallback) + }); + } + +}; \ No newline at end of file diff --git a/main/manager-web/src/apis/module/model.js b/main/manager-web/src/apis/module/model.js index 5b64811a..df879919 100644 --- a/main/manager-web/src/apis/module/model.js +++ b/main/manager-web/src/apis/module/model.js @@ -337,5 +337,23 @@ export default { this.getPluginFunctionList(params, callback) }) }).send() + }, + + // 获取RAG模型列表 + getRAGModels(callback) { + RequestService.sendRequest() + .url(`${getServiceUrl()}/datasets/rag-models`) + .method('GET') + .success((res) => { + RequestService.clearRequestTime() + callback(res) + }) + .networkFail((err) => { + console.error('获取RAG模型列表失败:', err) + this.$message.error(err.msg || '获取RAG模型列表失败') + RequestService.reAjaxFun(() => { + this.getRAGModels(callback) + }) + }).send() } } diff --git a/main/manager-web/src/assets/header/knowledge_base.png b/main/manager-web/src/assets/header/knowledge_base.png new file mode 100644 index 00000000..36b91599 Binary files /dev/null and b/main/manager-web/src/assets/header/knowledge_base.png differ diff --git a/main/manager-web/src/components/HeaderBar.vue b/main/manager-web/src/components/HeaderBar.vue index 80094830..42e3a520 100644 --- a/main/manager-web/src/components/HeaderBar.vue +++ b/main/manager-web/src/components/HeaderBar.vue @@ -9,82 +9,50 @@
-
+ - + ? 'brightness(0) invert(1)' + : 'None', + }" /> {{ $t("header.smartManagement") }}
-
- +
+ {{ $t("header.voiceCloneManagement") }}
- + - + ? 'brightness(0) invert(1)' + : 'None', + }" /> {{ $t("header.voiceCloneManagement") }} - + @@ -96,82 +64,56 @@ -
- +
+ {{ $t("header.modelConfig") }}
-
- - {{ $t("header.userManagement") }} +
+ + {{ $t("header.knowledgeBase") }}
- + - + $route.path === '/ota-management' || + $route.path === '/user-management' + ? 'brightness(0) invert(1)' + : 'None', + }" /> {{ $t("header.paramDictionary") }} - + {{ $t("header.paramManagement") }} + + {{ $t("header.userManagement") }} + {{ $t("header.otaManagement") }} @@ -193,85 +135,40 @@
-
+
- - + + -
+
{{ $t("header.searchHistory") }} - + {{ $t("header.clearHistory") }}
-
+
{{ item }} - +
- + {{ userInfo.username || "加载中..." }} - + - + @@ -399,6 +296,9 @@ export default { goModelConfig() { this.$router.push("/model-config"); }, + goKnowledgeBaseManagement() { + this.$router.push("/knowledge-base-management"); + }, goVoiceCloneManagement() { this.$router.push("/voice-clone-management"); }, @@ -863,7 +763,7 @@ export default { color: #ff4949; } -.custom-search-input >>> .el-input__inner { +.custom-search-input>>>.el-input__inner { height: 18px; border-radius: 9px; background-color: #fff; @@ -933,6 +833,7 @@ export default { color: #606266; white-space: nowrap; } + /* 添加倒三角旋转样式 */ .rotate-down { transform: rotate(180deg); diff --git a/main/manager-web/src/components/KnowledgeBaseDialog.vue b/main/manager-web/src/components/KnowledgeBaseDialog.vue new file mode 100644 index 00000000..99d01341 --- /dev/null +++ b/main/manager-web/src/components/KnowledgeBaseDialog.vue @@ -0,0 +1,238 @@ + + + + + \ No newline at end of file diff --git a/main/manager-web/src/components/ProviderDialog.vue b/main/manager-web/src/components/ProviderDialog.vue index 46945832..61e5b9de 100644 --- a/main/manager-web/src/components/ProviderDialog.vue +++ b/main/manager-web/src/components/ProviderDialog.vue @@ -88,6 +88,7 @@ +