diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 4987afc3..8085f079 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -150,6 +150,16 @@ public interface Constant { * 火山引擎双声道语音克隆 */ String VOICE_CLONE_HUOSHAN_DOUBLE_STREAM = "huoshan_double_stream"; + + /** + * RAG配置类型 + */ + String RAG_CONFIG_TYPE = "RAG"; + + /** + * 默认RAG模型配置ID + */ + String DEFAULT_RAG_MODEL_ID = "RAG_DEFAULT_MODEL"; enum SysBaseParam { /** diff --git a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java index 82b16ec5..1adbb9d2 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java +++ b/main/manager-api/src/main/java/xiaozhi/common/exception/ErrorCode.java @@ -119,10 +119,6 @@ public interface ErrorCode { int VOICEPRINT_UNREGISTER_PROCESS_ERROR = 10090; // 声纹注销处理失败 int VOICEPRINT_IDENTIFY_REQUEST_ERROR = 10091; // 声纹识别请求失败 - // 设备相关错误码 - int MAC_ADDRESS_ALREADY_EXISTS = 10161; // Mac地址已存在 - // 模型相关错误码 - int MODEL_PROVIDER_NOT_EXIST = 10162; // 供应器不存在 int LLM_NOT_EXIST = 10092; // 设置的LLM不存在 int MODEL_REFERENCED_BY_AGENT = 10093; // 该模型配置已被智能体引用，无法删除 int LLM_REFERENCED_BY_INTENT = 10094; // 该LLM模型已被意图识别配置引用，无法删除 @@ -198,4 +194,28 @@ public interface ErrorCode { int VOICE_CLONE_PREFIX = 10158; // 复刻音色前缀 int VOICE_ID_ALREADY_EXISTS = 10159; // 音色ID已存在 int VOICE_CLONE_HUOSHAN_VOICE_ID_ERROR = 10160; // 火山引擎音色ID格式错误 + + // 设备相关错误码 + int MAC_ADDRESS_ALREADY_EXISTS = 10161; // Mac地址已存在 + // 模型相关错误码 + int MODEL_PROVIDER_NOT_EXIST = 10162; // 供应器不存在 + + // 知识库数据集相关错误码 + int Knowledge_Base_RECORD_NOT_EXISTS = 10163; // 知识库记录不存在 + + // RAG配置相关错误码 + int RAG_CONFIG_NOT_FOUND = 10164; // RAG配置未找到 + int RAG_CONFIG_TYPE_ERROR = 10165; // RAG配置类型错误 + int RAG_DEFAULT_CONFIG_NOT_FOUND = 10166; // 默认RAG配置未找到 + int RAG_CONFIG_MISSING_PARAMS = 10167; // RAG配置缺少必要参数 + + // RAG API调用相关错误码 + int RAG_API_CREATE_FAILED = 10168; // RAG API创建数据集失败 + int RAG_API_UPDATE_FAILED = 10169; // RAG API更新数据集失败 + int RAG_API_DELETE_FAILED = 10170; // RAG API删除数据集失败 + + int UPLOAD_FILE_ERROR = 10171; // 上传文件失败 + int RAG_API_QUERY_FAILED = 10172; // RAG API查询失败 + int RAG_API_PARSE_FAILED = 10173; // RAG API解析失败 + int RAG_API_OPERATION_FAILED = 10174; // RAG API操作失败 } diff --git a/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java b/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java index a6b33b62..3b8585cf 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java +++ b/main/manager-api/src/main/java/xiaozhi/common/handler/FieldMetaObjectHandler.java @@ -32,13 +32,23 @@ public class FieldMetaObjectHandler implements MetaObjectHandler { // 创建者 strictInsertFill(metaObject, CREATOR, Long.class, user.getId()); - // 创建时间 - strictInsertFill(metaObject, CREATE_DATE, Date.class, date); + // 创建时间 - 支持createDate和createdAt两种字段名 + if (metaObject.hasSetter(CREATE_DATE)) { + strictInsertFill(metaObject, CREATE_DATE, Date.class, date); + } + if (metaObject.hasSetter("createdAt")) { + strictInsertFill(metaObject, "createdAt", Date.class, date); + } // 更新者 strictInsertFill(metaObject, UPDATER, Long.class, user.getId()); - // 更新时间 - strictInsertFill(metaObject, UPDATE_DATE, Date.class, date); + // 更新时间 - 支持updateDate和updatedAt两种字段名 + if (metaObject.hasSetter(UPDATE_DATE)) { + strictInsertFill(metaObject, UPDATE_DATE, Date.class, date); + } + if (metaObject.hasSetter("updatedAt")) { + strictInsertFill(metaObject, "updatedAt", Date.class, date); + } // 数据标识 strictInsertFill(metaObject, DATA_OPERATION, String.class, Constant.DataOperation.INSERT.getValue()); @@ -46,10 +56,17 @@ public class FieldMetaObjectHandler implements MetaObjectHandler { @Override public void updateFill(MetaObject metaObject) { + Date date = new Date(); + // 更新者 strictUpdateFill(metaObject, UPDATER, Long.class, SecurityUser.getUserId()); - // 更新时间 - strictUpdateFill(metaObject, UPDATE_DATE, Date.class, new Date()); + // 更新时间 - 支持updateDate和updatedAt两种字段名 + if (metaObject.hasSetter(UPDATE_DATE)) { + strictUpdateFill(metaObject, UPDATE_DATE, Date.class, date); + } + if (metaObject.hasSetter("updatedAt")) { + strictUpdateFill(metaObject, "updatedAt", Date.class, date); + } // 数据标识 strictInsertFill(metaObject, DATA_OPERATION, String.class, Constant.DataOperation.UPDATE.getValue()); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java index 8c562edb..321b5cb2 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java @@ -45,6 +45,10 @@ public class AgentTemplateServiceImpl extends ServiceImpl wrapper = new UpdateWrapper<>(); switch (modelType) { diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index a34c73dd..dce101cf 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -91,6 +91,7 @@ public class ConfigServiceImpl implements ConfigService { null, null, null, + null, result, isCache); @@ -195,6 +196,7 @@ public class ConfigServiceImpl implements ConfigService { agent.getTtsModelId(), agent.getMemModelId(), agent.getIntentModelId(), + null, result, true); @@ -371,12 +373,13 @@ public class ConfigServiceImpl implements ConfigService { String ttsModelId, String memModelId, String intentModelId, + String ragModelId, Map result, boolean isCache) { Map selectedModule = new HashMap<>(); - String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM" }; - String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId }; + String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM", "VLLM","RAG" }; + String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId, vllmModelId, ragModelId }; String intentLLMModelId = null; String memLocalShortLLMModelId = null; diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeBaseController.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeBaseController.java new file mode 100644 index 00000000..13dbc3eb --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeBaseController.java @@ -0,0 +1,114 @@ +package xiaozhi.modules.knowledge.controller; + +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.apache.commons.lang3.StringUtils; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import xiaozhi.common.page.PageData; +import xiaozhi.common.utils.Result; +import xiaozhi.modules.knowledge.dto.KnowledgeBaseDTO; +import xiaozhi.modules.knowledge.service.KnowledgeBaseService; +import java.util.Map; + + +@AllArgsConstructor +@RestController +@RequestMapping("/api/v1") +@Tag(name = "知识库管理") +public class KnowledgeBaseController { + + private final KnowledgeBaseService knowledgeBaseService; + + @GetMapping("/datasets") + @Operation(summary = "分页查询知识库列表") + @RequiresPermissions("sys:role:normal") + public Result> getPageList( + @RequestParam(required = false) String name, + @RequestParam(required = false) String id, + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "10") Integer page_size, + @RequestParam(required = false) String orderby, + @RequestParam(required = false) Boolean desc) { + KnowledgeBaseDTO knowledgeBaseDTO = new KnowledgeBaseDTO(); + knowledgeBaseDTO.setName(name); + knowledgeBaseDTO.setDatasetId(id); + PageData pageData = knowledgeBaseService.getPageList(knowledgeBaseDTO, String.valueOf(page), String.valueOf(page_size)); + return new Result>().ok(pageData); + } + + @GetMapping("/datasets/{dataset_id}") + @Operation(summary = "根据知识库ID获取知识库详情") + @RequiresPermissions("sys:role:normal") + public Result getByDatasetId(@PathVariable("dataset_id") String datasetId) { + KnowledgeBaseDTO knowledgeBaseDTO = knowledgeBaseService.getByDatasetId(datasetId); + return new Result().ok(knowledgeBaseDTO); + } + + @PostMapping("/datasets") + @Operation(summary = "创建知识库") + @RequiresPermissions("sys:role:normal") + public Result save(@RequestBody @Validated KnowledgeBaseDTO knowledgeBaseDTO) { + KnowledgeBaseDTO resp = knowledgeBaseService.save(knowledgeBaseDTO); + return new Result().ok(resp); + } + + @PutMapping("/datasets/{dataset_id}") + @Operation(summary = "更新知识库") + @RequiresPermissions("sys:role:normal") + public Result update(@PathVariable("dataset_id") String datasetId, + @RequestBody @Validated KnowledgeBaseDTO knowledgeBaseDTO) { + knowledgeBaseDTO.setDatasetId(datasetId); + KnowledgeBaseDTO resp = knowledgeBaseService.update(knowledgeBaseDTO); + return new Result().ok(resp); + } + + @DeleteMapping("/datasets/{dataset_id}") + @Operation(summary = "删除单个知识库") + @Parameter(name = "dataset_id", description = "知识库ID", required = true) + @RequiresPermissions("sys:role:normal") + public Result delete(@PathVariable("dataset_id") String datasetId) { + knowledgeBaseService.deleteByDatasetId(datasetId); + return new Result<>(); + } + + @DeleteMapping("/datasets/batch") + @Operation(summary = "批量删除知识库") + @Parameter(name = "ids", description = "知识库ID列表，用逗号分隔", required = true) + @RequiresPermissions("sys:role:normal") + public Result deleteBatch(@RequestParam("ids") String ids) { + if (StringUtils.isBlank(ids)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + String[] idArray = ids.split(","); + for (String datasetId : idArray) { + if (StringUtils.isNotBlank(datasetId)) { + knowledgeBaseService.deleteByDatasetId(datasetId.trim()); + } + } + return new Result<>(); + } + + @GetMapping("/rag-config/default") + @Operation(summary = "获取默认RAG配置") + @RequiresPermissions("sys:role:normal") + public Result> getDefaultRAGConfig() { + Map config = knowledgeBaseService.getDefaultRAGConfig(); + return new Result>().ok(config); + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeFilesController.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeFilesController.java new file mode 100644 index 00000000..2152f360 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/controller/KnowledgeFilesController.java @@ -0,0 +1,222 @@ +package xiaozhi.modules.knowledge.controller; + +import java.util.List; +import java.util.Map; + +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import xiaozhi.common.page.PageData; +import xiaozhi.common.utils.Result; +import xiaozhi.modules.knowledge.dto.KnowledgeFilesDTO; +import xiaozhi.modules.knowledge.service.KnowledgeFilesService; + +@AllArgsConstructor +@RestController +@RequestMapping("/api/v1/datasets/{dataset_id}") +@Tag(name = "知识库文档管理") +public class KnowledgeFilesController { + + private final KnowledgeFilesService knowledgeFilesService; + + @GetMapping("/documents") + @Operation(summary = "分页查询文档列表") + @RequiresPermissions("sys:role:normal") + public Result> getPageList( + @PathVariable("dataset_id") String datasetId, + @RequestParam(required = false) String name, + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "10") Integer page_size) { + KnowledgeFilesDTO knowledgeFilesDTO = new KnowledgeFilesDTO(); + knowledgeFilesDTO.setDatasetId(datasetId); + knowledgeFilesDTO.setName(name); + PageData pageData = knowledgeFilesService.getPageList(knowledgeFilesDTO, page, page_size); + return new Result>().ok(pageData); + } + + @GetMapping("/documents/{document_id}") + @Operation(summary = "根据文档ID获取文档详情") + @RequiresPermissions("sys:role:normal") + public Result getByDocumentId(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId) { + KnowledgeFilesDTO knowledgeFilesDTO = knowledgeFilesService.getByDocumentId(documentId); + return new Result().ok(knowledgeFilesDTO); + } + + @PostMapping("/documents") + @Operation(summary = "上传文档到知识库") + @RequiresPermissions("sys:role:normal") + public Result uploadDocument( + @PathVariable("dataset_id") String datasetId, + @RequestParam("file") MultipartFile file, + @RequestParam(required = false) String name, + @RequestParam(required = false) String chunkMethod, + @RequestParam(required = false) String metaFields, + @RequestParam(required = false) String parserConfig) { + + KnowledgeFilesDTO resp = knowledgeFilesService.uploadDocument(datasetId, file, name, + metaFields != null ? parseJsonMap(metaFields) : null, + chunkMethod, + parserConfig != null ? parseJsonMap(parserConfig) : null); + return new Result().ok(resp); + } + + @PutMapping("/documents/{document_id}") + @Operation(summary = "更新文档配置") + @RequiresPermissions("sys:role:normal") + public Result update(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId, + @RequestBody @Validated KnowledgeFilesDTO knowledgeFilesDTO) { + knowledgeFilesDTO.setDatasetId(datasetId); + knowledgeFilesDTO.setDocumentId(documentId); + KnowledgeFilesDTO resp = knowledgeFilesService.update(knowledgeFilesDTO); + return new Result().ok(resp); + } + + @DeleteMapping("/documents/{document_id}") + @Operation(summary = "删除单个文档") + @Parameter(name = "document_id", description = "文档ID", required = true) + @RequiresPermissions("sys:role:normal") + public Result delete(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId) { + knowledgeFilesService.deleteByDocumentId(documentId, datasetId); + return new Result<>(); + } + + @DeleteMapping("/documents") + @Operation(summary = "批量删除文档") + @RequiresPermissions("sys:role:normal") + public Result deleteBatch(@PathVariable("dataset_id") String datasetId, + @RequestBody Map> requestBody) { + List ids = requestBody.get("ids"); + if (ids != null && !ids.isEmpty()) { + knowledgeFilesService.deleteBatch(ids); + } + return new Result<>(); + } + + @PostMapping("/chunks") + @Operation(summary = "批量解析文档（切块）") + @RequiresPermissions("sys:role:normal") + public Result parseDocuments(@PathVariable("dataset_id") String datasetId, + @RequestBody Map> requestBody) { + List documentIds = requestBody.get("document_ids"); + if (documentIds == null || documentIds.isEmpty()) { + return new Result().error("document_ids参数不能为空"); + } + + boolean success = knowledgeFilesService.parseDocuments(datasetId, documentIds); + if (success) { + return new Result(); + } else { + return new Result().error("文档解析失败，文档可能正在处理中"); + } + } + + @PostMapping("/documents/{document_id}/parse") + @Operation(summary = "解析单个文档（切块）") + @RequiresPermissions("sys:role:normal") + public Result parseDocument(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId) { + List documentIds = java.util.Arrays.asList(documentId); + + boolean success = knowledgeFilesService.parseDocuments(datasetId, documentIds); + if (success) { + return new Result(); + } else { + return new Result().error("文档解析失败，文档可能正在处理中"); + } + } + + @PostMapping("/documents/{document_id}/chunks") + @Operation(summary = "添加切片到指定文档") + @RequiresPermissions("sys:role:normal") + public Result> addChunk(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId, + @RequestBody Map requestBody) { + String content = (String) requestBody.get("content"); + List importantKeywords = (List) requestBody.get("important_keywords"); + List questions = (List) requestBody.get("questions"); + + Map result = knowledgeFilesService.addChunk(datasetId, documentId, content, importantKeywords, questions); + return new Result>().ok(result); + } + + @GetMapping("/documents/{document_id}/chunks") + @Operation(summary = "列出指定文档的切片") + @RequiresPermissions("sys:role:normal") + public Result> listChunks(@PathVariable("dataset_id") String datasetId, + @PathVariable("document_id") String documentId, + @RequestParam(required = false) String keywords, + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "1024") Integer page_size, + @RequestParam(required = false) String id) { + Map result = knowledgeFilesService.listChunks(datasetId, documentId, keywords, page, page_size, id); + return new Result>().ok(result); + } + + /** + * 召回测试 + */ + @PostMapping("/retrieval-test") + @Operation(summary = "召回测试") + @RequiresPermissions("sys:role:normal") + public Result> retrievalTest(@PathVariable("dataset_id") String datasetId, + @RequestBody Map params) { + try { + // 提取参数 + String question = (String) params.get("question"); + if (question == null || question.trim().isEmpty()) { + return new Result>().error("问题不能为空"); + } + + List datasetIds = (List) params.get("dataset_ids"); + List documentIds = (List) params.get("document_ids"); + Integer page = (Integer) params.get("page"); + Integer pageSize = (Integer) params.get("page_size"); + Float similarityThreshold = (Float) params.get("similarity_threshold"); + Float vectorSimilarityWeight = (Float) params.get("vector_similarity_weight"); + Integer topK = (Integer) params.get("top_k"); + String rerankId = (String) params.get("rerank_id"); + Boolean keyword = (Boolean) params.get("keyword"); + Boolean highlight = (Boolean) params.get("highlight"); + List crossLanguages = (List) params.get("cross_languages"); + Map metadataCondition = (Map) params.get("metadata_condition"); + + // 如果未指定数据集ID，使用当前数据集 + if (datasetIds == null || datasetIds.isEmpty()) { + datasetIds = java.util.Arrays.asList(datasetId); + } + + Map result = knowledgeFilesService.retrievalTest( + question, datasetIds, documentIds, page, pageSize, similarityThreshold, + vectorSimilarityWeight, topK, rerankId, keyword, highlight, crossLanguages, metadataCondition + ); + + return new Result>().ok(result); + } catch (Exception e) { + return new Result>().error("召回测试失败: " + e.getMessage()); + } + } + + /** + * 解析JSON字符串为Map对象 + */ + private Map parseJsonMap(String jsonString) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readValue(jsonString, new TypeReference>() {}); + } catch (Exception e) { + throw new RuntimeException("解析JSON字符串失败: " + jsonString, e); + } + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dao/KnowledgeBaseDao.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dao/KnowledgeBaseDao.java new file mode 100644 index 00000000..e49e5e22 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dao/KnowledgeBaseDao.java @@ -0,0 +1,14 @@ +package xiaozhi.modules.knowledge.dao; + +import org.apache.ibatis.annotations.Mapper; + +import xiaozhi.common.dao.BaseDao; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; + +/** + * 知识库知识库 + */ +@Mapper +public interface KnowledgeBaseDao extends BaseDao { + +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeBaseDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeBaseDTO.java new file mode 100644 index 00000000..77f6d198 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeBaseDTO.java @@ -0,0 +1,49 @@ +package xiaozhi.modules.knowledge.dto; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "知识库知识库") +public class KnowledgeBaseDTO implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + @Schema(description = "唯一标识") + private String id; + + @Schema(description = "知识库ID") + private String datasetId; + + @Schema(description = "RAG模型配置ID") + private String ragModelId; + + @Schema(description = "知识库名称") + private String name; + + @Schema(description = "知识库描述") + private String description; + + @Schema(description = "状态(0:禁用 1:启用)") + private Integer status; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + private Date createdAt; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + private Date updatedAt; + + @Schema(description = "文档数量") + private Integer documentCount; +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeFilesDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeFilesDTO.java new file mode 100644 index 00000000..3afdad9c --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/dto/KnowledgeFilesDTO.java @@ -0,0 +1,61 @@ +package xiaozhi.modules.knowledge.dto; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; +import java.util.Map; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "知识库文档") +public class KnowledgeFilesDTO implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + @Schema(description = "唯一标识") + private String id; + + @Schema(description = "文档ID") + private String documentId; + + @Schema(description = "知识库ID") + private String datasetId; + + @Schema(description = "文档名称") + private String name; + + @Schema(description = "文档类型") + private String fileType; + + @Schema(description = "文件大小（字节）") + private Long fileSize; + + @Schema(description = "文件路径") + private String filePath; + + @Schema(description = "元数据字段") + private Map metaFields; + + @Schema(description = "分块方法") + private String chunkMethod; + + @Schema(description = "解析器配置") + private Map parserConfig; + + @Schema(description = "状态(0:待解析 1:解析中 2:解析成功 3：解析失败)") + private Integer status; + + @Schema(description = "创建者") + private Long creator; + + @Schema(description = "创建时间") + private Date createdAt; + + @Schema(description = "更新者") + private Long updater; + + @Schema(description = "更新时间") + private Date updatedAt; +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/entity/KnowledgeBaseEntity.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/entity/KnowledgeBaseEntity.java new file mode 100644 index 00000000..e6c897ea --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/entity/KnowledgeBaseEntity.java @@ -0,0 +1,53 @@ +package xiaozhi.modules.knowledge.entity; + +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@TableName(value = "ai_rag_dataset", autoResultMap = true) +@Schema(description = "知识库知识库表") +public class KnowledgeBaseEntity { + + @TableId(type = IdType.ASSIGN_UUID) + @Schema(description = "唯一标识") + private String id; + + @Schema(description = "知识库ID") + private String datasetId; + + @Schema(description = "RAG模型配置ID") + private String ragModelId; + + @Schema(description = "知识库名称") + private String name; + + @Schema(description = "知识库描述") + private String description; + + @Schema(description = "状态(0:禁用 1:启用)") + private Integer status; + + @Schema(description = "创建者") + @TableField(fill = FieldFill.INSERT) + private Long creator; + + @Schema(description = "创建时间") + @TableField(fill = FieldFill.INSERT) + private Date createdAt; + + @Schema(description = "更新者") + @TableField(fill = FieldFill.UPDATE) + private Long updater; + + @Schema(description = "更新时间") + @TableField(fill = FieldFill.UPDATE) + private Date updatedAt; +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeBaseService.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeBaseService.java new file mode 100644 index 00000000..6a41375f --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeBaseService.java @@ -0,0 +1,85 @@ +package xiaozhi.modules.knowledge.service; + +import java.util.Map; + +import xiaozhi.common.page.PageData; +import xiaozhi.common.service.BaseService; +import xiaozhi.modules.knowledge.dto.KnowledgeBaseDTO; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; + +/** + * 知识库知识库服务接口 + */ +public interface KnowledgeBaseService extends BaseService { + + /** + * 分页查询知识库列表 + * + * @param knowledgeBaseDTO 查询条件 + * @param page 页码 + * @param limit 每页数量 + * @return 分页数据 + */ + PageData getPageList(KnowledgeBaseDTO knowledgeBaseDTO, String page, String limit); + + /** + * 根据ID获取知识库详情 + * + * @param id 知识库ID + * @return 知识库详情 + */ + KnowledgeBaseDTO getById(String id); + + /** + * 新增知识库 + * + * @param knowledgeBaseDTO 知识库信息 + * @return 新增的知识库 + */ + KnowledgeBaseDTO save(KnowledgeBaseDTO knowledgeBaseDTO); + + /** + * 更新知识库 + * + * @param knowledgeBaseDTO 知识库信息 + * @return 更新的知识库 + */ + KnowledgeBaseDTO update(KnowledgeBaseDTO knowledgeBaseDTO); + + /** + * 根据ID删除知识库 + * + * @param id 知识库ID + */ + void delete(String id); + + /** + * 根据知识库ID查询知识库 + * + * @param datasetId 知识库ID + * @return 知识库详情 + */ + KnowledgeBaseDTO getByDatasetId(String datasetId); + + /** + * 根据知识库ID删除知识库 + * + * @param datasetId 知识库ID + */ + void deleteByDatasetId(String datasetId); + + /** + * 获取RAG配置信息 + * + * @param ragModelId RAG模型配置ID + * @return RAG配置信息 + */ + Map getRAGConfig(String ragModelId); + + /** + * 获取默认RAG配置信息 + * + * @return 默认RAG配置信息 + */ + Map getDefaultRAGConfig(); +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeFilesService.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeFilesService.java new file mode 100644 index 00000000..ab51f9d6 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/KnowledgeFilesService.java @@ -0,0 +1,178 @@ +package xiaozhi.modules.knowledge.service; + +import java.util.List; +import java.util.Map; + +import org.springframework.web.multipart.MultipartFile; + +import xiaozhi.common.page.PageData; +import xiaozhi.modules.knowledge.dto.KnowledgeFilesDTO; + +/** + * 知识库文档服务接口 + */ +public interface KnowledgeFilesService { + + /** + * 分页查询文档列表 + * + * @param knowledgeFilesDTO 查询条件 + * @param page 页码 + * @param limit 每页数量 + * @return 分页数据 + */ + PageData getPageList(KnowledgeFilesDTO knowledgeFilesDTO, Integer page, Integer limit); + + /** + * 根据ID获取文档详情 + * + * @param id 文档ID + * @return 文档详情 + */ + KnowledgeFilesDTO getById(String id); + + /** + * 根据文档ID获取文档详情 + * + * @param documentId 文档ID + * @return 文档详情 + */ + KnowledgeFilesDTO getByDocumentId(String documentId); + + /** + * 根据文档ID和知识库ID获取文档详情 + * + * @param documentId 文档ID + * @param datasetId 知识库ID + * @return 文档详情 + */ + KnowledgeFilesDTO getByDocumentId(String documentId, String datasetId); + + /** + * 上传文档到知识库 + * + * @param datasetId 知识库ID + * @param file 上传的文件 + * @param name 文档名称 + * @param metaFields 元数据字段 + * @param chunkMethod 分块方法 + * @param parserConfig 解析器配置 + * @return 上传的文档信息 + */ + KnowledgeFilesDTO uploadDocument(String datasetId, MultipartFile file, String name, + Map metaFields, String chunkMethod, + Map parserConfig); + + /** + * 更新文档配置 + * + * @param knowledgeFilesDTO 文档信息 + * @return 更新的文档信息 + */ + KnowledgeFilesDTO update(KnowledgeFilesDTO knowledgeFilesDTO); + + /** + * 根据ID删除文档 + * + * @param id 文档ID + */ + void delete(String id); + + /** + * 根据文档ID删除文档 + * + * @param documentId 文档ID + */ + void deleteByDocumentId(String documentId); + + /** + * 根据文档ID和知识库ID删除文档 + * + * @param documentId 文档ID + * @param datasetId 知识库ID + */ + void deleteByDocumentId(String documentId, String datasetId); + + /** + * 批量删除文档 + * + * @param ids 文档ID列表 + */ + void deleteBatch(List ids); + + + /** + * 获取RAG配置信息 + * + * @param ragModelId RAG模型配置ID + * @return RAG配置信息 + */ + Map getRAGConfig(String ragModelId); + + /** + * 获取默认RAG配置信息 + * + * @return 默认RAG配置信息 + */ + Map getDefaultRAGConfig(); + + /** + * 解析文档（切块） + * + * @param datasetId 知识库ID + * @param documentIds 文档ID列表 + * @return 解析结果 + */ + boolean parseDocuments(String datasetId, List documentIds); + + /** + * 添加切片到指定文档 + * + * @param datasetId 知识库ID + * @param documentId 文档ID + * @param content 切片内容 + * @param importantKeywords 重要关键词列表 + * @param questions 问题列表 + * @return 添加的切片信息 + */ + Map addChunk(String datasetId, String documentId, String content, + List importantKeywords, List questions); + + /** + * 列出指定文档的切片 + * + * @param datasetId 知识库ID + * @param documentId 文档ID + * @param keywords 关键词过滤 + * @param page 页码 + * @param pageSize 每页数量 + * @param chunkId 切片ID + * @return 切片列表信息 + */ + Map listChunks(String datasetId, String documentId, String keywords, + Integer page, Integer pageSize, String chunkId); + + /** + * 召回测试 - 从指定数据集或文档中检索相关切片 + * + * @param question 用户查询或查询关键词 + * @param datasetIds 数据集ID列表 + * @param documentIds 文档ID列表 + * @param page 页码 + * @param pageSize 每页数量 + * @param similarityThreshold 最小相似度阈值 + * @param vectorSimilarityWeight 向量相似度权重 + * @param topK 参与向量余弦计算的切片数量 + * @param rerankId 重排模型ID + * @param keyword 是否启用关键词匹配 + * @param highlight 是否启用高亮显示 + * @param crossLanguages 跨语言翻译列表 + * @param metadataCondition 元数据过滤条件 + * @return 召回测试结果 + */ + Map retrievalTest(String question, List datasetIds, List documentIds, + Integer page, Integer pageSize, Float similarityThreshold, + Float vectorSimilarityWeight, Integer topK, String rerankId, + Boolean keyword, Boolean highlight, List crossLanguages, + Map metadataCondition); +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java new file mode 100644 index 00000000..39c8cc7b --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java @@ -0,0 +1,771 @@ +package xiaozhi.modules.knowledge.service.impl; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.http.*; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.ResourceAccessException; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import xiaozhi.common.constant.Constant; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; +import xiaozhi.common.page.PageData; +import xiaozhi.common.service.impl.BaseServiceImpl; +import xiaozhi.common.utils.ConvertUtils; +import xiaozhi.modules.knowledge.dao.KnowledgeBaseDao; +import xiaozhi.modules.knowledge.dto.KnowledgeBaseDTO; +import xiaozhi.modules.knowledge.entity.KnowledgeBaseEntity; +import xiaozhi.modules.knowledge.service.KnowledgeBaseService; +import xiaozhi.modules.model.dao.ModelConfigDao; +import xiaozhi.modules.model.entity.ModelConfigEntity; +import xiaozhi.modules.model.service.ModelConfigService; + +@Service +@AllArgsConstructor +@Slf4j +public class KnowledgeBaseServiceImpl extends BaseServiceImpl implements KnowledgeBaseService { + + private final KnowledgeBaseDao knowledgeBaseDao; + private final ModelConfigService modelConfigService; + private final ModelConfigDao modelConfigDao; + private RestTemplate restTemplate = new RestTemplate(); + + + + @Override + public PageData getPageList(KnowledgeBaseDTO knowledgeBaseDTO, String page, String limit) { + long curPage = Long.parseLong(page); + long pageSize = Long.parseLong(limit); + Page pageInfo = new Page<>(curPage, pageSize); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + + // 添加查询条件 + if (knowledgeBaseDTO != null) { + queryWrapper.like(StringUtils.isNotBlank(knowledgeBaseDTO.getName()), "name", knowledgeBaseDTO.getName()) + .eq(knowledgeBaseDTO.getStatus() != null, "status", knowledgeBaseDTO.getStatus()); + } + + // 添加排序规则：按创建时间降序 + queryWrapper.orderByDesc("created_at"); + + IPage knowledgeBaseEntityIPage = knowledgeBaseDao.selectPage(pageInfo, queryWrapper); + + // 同步RAGFlow API的数据集状态（可选功能，可根据需要开启） + syncRAGFlowDatasetStatus(knowledgeBaseEntityIPage.getRecords()); + + // 获取分页数据 + PageData pageData = getPageData(knowledgeBaseEntityIPage, KnowledgeBaseDTO.class); + + // 为每个知识库获取文档数量 + if (pageData != null && pageData.getList() != null) { + for (KnowledgeBaseDTO knowledgeBase : pageData.getList()) { + try { + Integer documentCount = getDocumentCountFromRAGFlow(knowledgeBase.getDatasetId(), knowledgeBase.getRagModelId()); + knowledgeBase.setDocumentCount(documentCount); + } catch (Exception e) { + log.warn("获取知识库 {} 的文档数量失败: {}", knowledgeBase.getDatasetId(), e.getMessage()); + knowledgeBase.setDocumentCount(0); // 设置默认值 + } + } + } + + return pageData; + } + + @Override + public KnowledgeBaseDTO getById(String id) { + if (StringUtils.isBlank(id)) { + throw new RenException(ErrorCode.IDENTIFIER_NOT_NULL); + } + + KnowledgeBaseEntity entity = knowledgeBaseDao.selectById(id); + if (entity == null) { + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public KnowledgeBaseDTO save(KnowledgeBaseDTO knowledgeBaseDTO) { + if (knowledgeBaseDTO == null) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + String datasetId = null; + // 调用RAGFlow API创建数据集 + try { + Map ragConfig = getValidatedRAGConfig(knowledgeBaseDTO.getRagModelId()); + datasetId = createDatasetInRAGFlow( + knowledgeBaseDTO.getName(), + knowledgeBaseDTO.getDescription(), + ragConfig + ); + } catch (Exception e) { + // 如果RAG API调用失败，直接抛出异常，无需回滚（因为还没有插入本地数据库） + throw e; + } + + // 验证数据集ID是否已存在 + KnowledgeBaseEntity existingEntity = knowledgeBaseDao.selectOne( + new QueryWrapper().eq("dataset_id", datasetId) + ); + if (existingEntity != null) { + // 如果datasetId已存在，删除RAGFlow中的数据集并抛出异常 + try { + Map ragConfig = getValidatedRAGConfig(knowledgeBaseDTO.getRagModelId()); + deleteDatasetInRAGFlow(datasetId, ragConfig); + } catch (Exception deleteException) { + log.warn("删除重复datasetId的RAGFlow数据集失败: {}", deleteException.getMessage()); + } + throw new RenException(ErrorCode.DB_RECORD_EXISTS); + } + + // 创建本地实体并保存 + KnowledgeBaseEntity entity = ConvertUtils.sourceToTarget(knowledgeBaseDTO, KnowledgeBaseEntity.class); + entity.setDatasetId(datasetId); + knowledgeBaseDao.insert(entity); + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public KnowledgeBaseDTO update(KnowledgeBaseDTO knowledgeBaseDTO) { + if (knowledgeBaseDTO == null || StringUtils.isBlank(knowledgeBaseDTO.getId())) { + throw new RenException(ErrorCode.IDENTIFIER_NOT_NULL); + } + + // 检查记录是否存在 + KnowledgeBaseEntity existingEntity = knowledgeBaseDao.selectById(knowledgeBaseDTO.getId()); + if (existingEntity == null) { + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + // 验证数据集ID是否与其他记录冲突 + if (StringUtils.isNotBlank(knowledgeBaseDTO.getDatasetId())) { + KnowledgeBaseEntity conflictEntity = knowledgeBaseDao.selectOne( + new QueryWrapper() + .eq("dataset_id", knowledgeBaseDTO.getDatasetId()) + .ne("id", knowledgeBaseDTO.getId()) + ); + if (conflictEntity != null) { + throw new RenException(ErrorCode.DB_RECORD_EXISTS); + } + } + + KnowledgeBaseEntity entity = ConvertUtils.sourceToTarget(knowledgeBaseDTO, KnowledgeBaseEntity.class); + knowledgeBaseDao.updateById(entity); + + // 调用RAGFlow API更新数据集 + if (StringUtils.isNotBlank(knowledgeBaseDTO.getDatasetId())) { + try { + Map ragConfig = getValidatedRAGConfig(knowledgeBaseDTO.getRagModelId()); + updateDatasetInRAGFlow( + knowledgeBaseDTO.getDatasetId(), + knowledgeBaseDTO.getName(), + knowledgeBaseDTO.getDescription(), + ragConfig + ); + } catch (Exception e) { + // 如果RAG API调用失败，回滚本地数据库操作 + knowledgeBaseDao.updateById(existingEntity); + throw e; + } + } + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public void delete(String id) { + if (StringUtils.isBlank(id)) { + throw new RenException(ErrorCode.IDENTIFIER_NOT_NULL); + } + + log.info("=== 开始删除操作 ==="); + log.info("删除ID: {}", id); + + KnowledgeBaseEntity entity = knowledgeBaseDao.selectById(id); + if (entity == null) { + log.warn("记录不存在，ID: {}", id); + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + log.info("找到记录: ID={}, datasetId={}, ragModelId={}", + entity.getId(), entity.getDatasetId(), entity.getRagModelId()); + + // 先调用RAGFlow API删除数据集 + boolean apiDeleteSuccess = false; + if (StringUtils.isNotBlank(entity.getDatasetId()) && StringUtils.isNotBlank(entity.getRagModelId())) { + try { + log.info("开始调用RAGFlow API删除数据集"); + Map ragConfig = getRAGConfig(entity.getRagModelId()); + validateRagConfig(ragConfig); + deleteDatasetInRAGFlow(entity.getDatasetId(), ragConfig); + log.info("RAGFlow API删除调用完成"); + apiDeleteSuccess = true; + } catch (Exception e) { + log.error("删除RAGFlow数据集失败: {}", e.getMessage()); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow数据集失败: " + e.getMessage()); + } + } else { + log.warn("datasetId或ragModelId为空，跳过RAGFlow删除"); + apiDeleteSuccess = true; // 没有RAG数据集，视为成功 + } + + // API删除成功后再删除本地记录 + if (apiDeleteSuccess) { + int deleteCount = knowledgeBaseDao.deleteById(id); + log.info("本地数据库删除结果: {}", deleteCount > 0 ? "成功" : "失败"); + } + + log.info("=== 删除操作结束 ==="); + } + + @Override + public KnowledgeBaseDTO getByDatasetId(String datasetId) { + if (StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + KnowledgeBaseEntity entity = knowledgeBaseDao.selectOne( + new QueryWrapper().eq("dataset_id", datasetId) + ); + + if (entity == null) { + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + return ConvertUtils.sourceToTarget(entity, KnowledgeBaseDTO.class); + } + + @Override + public void deleteByDatasetId(String datasetId) { + if (StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + log.info("=== 开始通过datasetId删除操作 ==="); + log.info("删除datasetId: {}", datasetId); + + KnowledgeBaseEntity entity = knowledgeBaseDao.selectOne( + new QueryWrapper().eq("dataset_id", datasetId) + ); + + if (entity == null) { + log.warn("记录不存在，datasetId: {}", datasetId); + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + + log.info("找到记录: ID={}, datasetId={}, ragModelId={}", + entity.getId(), entity.getDatasetId(), entity.getRagModelId()); + + // 先删除本地数据库记录 + int deleteCount = knowledgeBaseDao.deleteById(entity.getId()); + log.info("本地数据库删除结果: {}", deleteCount > 0 ? "成功" : "失败"); + + // 调用RAGFlow API删除数据集 + if (StringUtils.isNotBlank(entity.getDatasetId()) && StringUtils.isNotBlank(entity.getRagModelId())) { + try { + log.info("开始调用RAGFlow API删除数据集"); + Map ragConfig = getValidatedRAGConfig(entity.getRagModelId()); + deleteDatasetInRAGFlow(entity.getDatasetId(), ragConfig); + log.info("RAGFlow API删除调用完成"); + } catch (Exception e) { + log.warn("删除RAGFlow数据集失败: {}", e.getMessage()); + } + } else { + log.warn("datasetId或ragModelId为空，跳过RAGFlow删除"); + } + + log.info("=== 通过datasetId删除操作结束 ==="); + } + + @Override + public Map getRAGConfig(String ragModelId) { + if (StringUtils.isBlank(ragModelId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + // 从缓存获取模型配置 + ModelConfigEntity modelConfig = modelConfigService.getModelByIdFromCache(ragModelId); + if (modelConfig == null || modelConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 验证是否为RAG类型配置 + if (!Constant.RAG_CONFIG_TYPE.equals(modelConfig.getModelType().toUpperCase())) { + throw new RenException(ErrorCode.RAG_CONFIG_TYPE_ERROR); + } + + Map config = modelConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + // 返回配置信息 + return config; + } + + @Override + public Map getDefaultRAGConfig() { + // 获取默认RAG模型配置 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("model_type", Constant.RAG_CONFIG_TYPE) + .eq("is_default", 1) + .eq("is_enabled", 1); + + List modelConfigs = modelConfigDao.selectList(queryWrapper); + if (modelConfigs == null || modelConfigs.isEmpty()) { + throw new RenException(ErrorCode.RAG_DEFAULT_CONFIG_NOT_FOUND); + } + + ModelConfigEntity defaultConfig = modelConfigs.get(0); + if (defaultConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + Map config = defaultConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + return config; + } + + /** + * 验证RAG配置中是否包含必要的参数 + */ + private void validateRagConfig(Map config) { + if (config == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 从配置中提取必要的参数 + String baseUrl = (String) config.get("base_url"); + String apiKey = (String) config.get("api_key"); + + // 验证base_url是否存在且非空 + if (StringUtils.isBlank(baseUrl)) { + throw new RenException(ErrorCode.RAG_CONFIG_MISSING_PARAMS); + } + } + + /** + * 调用RAGFlow API创建数据集 + */ + private String createDatasetInRAGFlow(String name, String description, Map ragConfig) { + String datasetId = null; + try { + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API创建数据集, name: {}", name); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("name", name); + if (StringUtils.isNotBlank(description)) { + requestBody.put("description", description); + } + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送POST请求 + log.info("发送POST请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED); + } + + // 解析响应体，提取datasetId + String responseBody = response.getBody(); + if (StringUtils.isNotBlank(responseBody)) { + try { + // 解析RAGFlow API响应，支持多种可能的字段名 + ObjectMapper objectMapper = new ObjectMapper(); + Map responseMap = objectMapper.readValue(responseBody, Map.class); + + log.debug("RAGFlow API响应解析结果: {}", responseMap); + + // 首先检查响应码 + Integer code = (Integer) responseMap.get("code"); + if (code != null && code == 0) { + // 响应码为0表示成功，从data字段中获取datasetId + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + datasetId = (String) dataMap.get("id"); + + if (StringUtils.isBlank(datasetId)) { + // 如果id字段为空，尝试其他可能的字段名 + datasetId = (String) dataMap.get("dataset_id"); + datasetId = (String) dataMap.get("datasetId"); + } + } + } else { + // 如果响应码不为0，说明API调用失败 + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "RAGFlow API调用失败，响应码: " + code); + } + + log.info("从RAGFlow API响应中解析出datasetId: {}", datasetId); + log.debug("完整响应内容: {}", responseBody); + } catch (Exception e) { + log.error("解析RAGFlow API响应失败: {}, 响应内容: {}", e.getMessage(), responseBody); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } + } + + if (StringUtils.isBlank(datasetId)) { + log.error("无法从RAGFlow API响应中获取datasetId，响应内容: {}", responseBody); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "RAGFlow API响应中未包含datasetId"); + } + + log.info("RAGFlow数据集创建成功，datasetId: {}", datasetId); + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "创建RAGFlow数据集失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "创建RAGFlow数据集失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "创建RAGFlow数据集失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "创建RAGFlow数据集失败: " + e.getMessage()); + } + return datasetId; + } + + /** + * 调用RAGFlow API更新数据集 + */ + private void updateDatasetInRAGFlow(String datasetId, String name, String description, Map ragConfig) { + try { + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API更新数据集，datasetId: {}, name: {}", datasetId, name); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets/" + datasetId; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("dataset_id", datasetId); + requestBody.put("name", name); + if (StringUtils.isNotBlank(description)) { + requestBody.put("description", description); + } + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送PUT请求 + log.info("发送PUT请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.PUT, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED); + } + + log.info("RAGFlow数据集更新成功，datasetId: {}", datasetId); + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow数据集失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow数据集失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow数据集失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow数据集失败: " + e.getMessage()); + } + } + + /** + * 调用RAGFlow API删除数据集 + */ + private void deleteDatasetInRAGFlow(String datasetId, Map ragConfig) { + try { + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API删除数据集，datasetId: {}", datasetId); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("ids", List.of(datasetId)); + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送DELETE请求 + log.info("发送DELETE请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.DELETE, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED); + } + + log.info("RAGFlow数据集删除成功，datasetId: {}", datasetId); + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow数据集失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow数据集失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow数据集失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow数据集失败: " + e.getMessage()); + } + } + + /** + * 获取RAG配置并验证 + */ + private Map getValidatedRAGConfig(String ragModelId) { + Map ragConfig; + if (StringUtils.isNotBlank(ragModelId)) { + ragConfig = getRAGConfig(ragModelId); + } else { + ragConfig = getDefaultRAGConfig(); + } + + // 验证配置 + validateRagConfig(ragConfig); + return ragConfig; + } + + /** + * 从RAGFlow API获取知识库的文档数量 + */ + private Integer getDocumentCountFromRAGFlow(String datasetId, String ragModelId) { + if (StringUtils.isBlank(datasetId) || StringUtils.isBlank(ragModelId)) { + log.warn("datasetId或ragModelId为空，无法获取文档数量"); + return 0; + } + + try { + log.info("开始获取知识库 {} 的文档数量", datasetId); + + // 获取RAG配置 + Map ragConfig = getValidatedRAGConfig(ragModelId); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 调用RAGFlow API获取文档列表，但不返回文档详情，只获取总数 + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents?page=1&size=1"; + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API获取文档数量..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + return 0; + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + ObjectMapper objectMapper = new ObjectMapper(); + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + Object totalObj = dataMap.get("total"); + if (totalObj instanceof Integer) { + Integer documentCount = (Integer) totalObj; + log.info("获取知识库 {} 的文档数量成功: {}", datasetId, documentCount); + return documentCount; + } else if (totalObj instanceof Long) { + Long documentCount = (Long) totalObj; + log.info("获取知识库 {} 的文档数量成功: {}", datasetId, documentCount); + return documentCount.intValue(); + } + } + } else { + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + } catch (Exception e) { + log.error("获取文档数量失败: {}", e.getMessage(), e); + } + + return 0; + } + + /** + * 同步RAGFlow API的数据集状态（可选功能） + */ + private void syncRAGFlowDatasetStatus(List entities) { + if (entities == null || entities.isEmpty()) { + log.debug("没有需要同步状态的数据集"); + return; + } + + log.info("开始同步RAGFlow数据集状态，共{}个数据集", entities.size()); + + for (KnowledgeBaseEntity entity : entities) { + if (StringUtils.isNotBlank(entity.getDatasetId()) && StringUtils.isNotBlank(entity.getRagModelId())) { + try { + log.debug("开始同步数据集 {} 的状态", entity.getDatasetId()); + + Map ragConfig = getValidatedRAGConfig(entity.getRagModelId()); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 使用正确的API端点获取数据集列表，然后过滤 + String url = baseUrl + "/api/v1/datasets"; + log.debug("请求URL: {}", url); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求获取所有数据集 + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, Map.class); + + if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { + Map responseBody = response.getBody(); + // 根据响应结构判断数据集是否存在 + boolean exists = checkDatasetExists(responseBody, entity.getDatasetId()); + + if (exists) { + log.info("数据集 {} 在RAGFlow中状态正常", entity.getDatasetId()); + } else { + log.warn("数据集 {} 在RAGFlow中不存在", entity.getDatasetId()); + } + } else { + log.warn("获取数据集列表失败，状态码: {}", response.getStatusCode()); + } + + } catch (Exception e) { + log.error("同步数据集 {} 状态失败: {}", entity.getDatasetId(), e.getMessage()); + } + } + } + + log.info("RAGFlow数据集状态同步完成"); + } + + /** + * 检查数据集是否存在 + */ + private boolean checkDatasetExists(Map responseBody, String datasetId) { + try { + // 根据RAGFlow API的实际响应结构来解析 + if (responseBody.containsKey("data")) { + Object data = responseBody.get("data"); + if (data instanceof List) { + List> datasets = (List>) data; + return datasets.stream() + .anyMatch(dataset -> datasetId.equals(dataset.get("id")) || + datasetId.equals(dataset.get("dataset_id"))); + } + } + return false; + } catch (Exception e) { + log.error("检查数据集存在性失败: {}", e.getMessage()); + return false; + } + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeFilesServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeFilesServiceImpl.java new file mode 100644 index 00000000..0708e6c5 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/knowledge/service/impl/KnowledgeFilesServiceImpl.java @@ -0,0 +1,1872 @@ +package xiaozhi.modules.knowledge.service.impl; + +import java.io.IOException; +import java.util.*; +import java.io.InputStream; +import java.util.Arrays; +import java.text.SimpleDateFormat; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.http.*; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.core.io.AbstractResource; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.ResourceAccessException; +import org.springframework.web.multipart.MultipartFile; + +import xiaozhi.common.constant.Constant; +import xiaozhi.common.exception.ErrorCode; +import xiaozhi.common.exception.RenException; +import xiaozhi.common.page.PageData; +import xiaozhi.modules.knowledge.dto.KnowledgeFilesDTO; +import xiaozhi.modules.knowledge.service.KnowledgeFilesService; +import xiaozhi.modules.model.service.ModelConfigService; +import xiaozhi.modules.model.dao.ModelConfigDao; +import xiaozhi.modules.model.entity.ModelConfigEntity; + +@Service +@AllArgsConstructor +@Slf4j +public class KnowledgeFilesServiceImpl implements KnowledgeFilesService { + + private final ModelConfigService modelConfigService; + private final ModelConfigDao modelConfigDao; + private RestTemplate restTemplate = new RestTemplate(); + private ObjectMapper objectMapper = new ObjectMapper(); + + @Override + public PageData getPageList(KnowledgeFilesDTO knowledgeFilesDTO, Integer page, Integer limit) { + try { + + log.info("=== 开始获取文档列表 ==="); + log.info("查询条件: datasetId={}, name={}, status={}, page={}, limit={}", + knowledgeFilesDTO != null ? knowledgeFilesDTO.getDatasetId() : null, + knowledgeFilesDTO != null ? knowledgeFilesDTO.getName() : null, + knowledgeFilesDTO != null ? knowledgeFilesDTO.getStatus() : null, + page, limit); + + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 根据RAGFlow API文档，获取文档列表的接口 + String datasetId = knowledgeFilesDTO != null ? knowledgeFilesDTO.getDatasetId() : null; + if (StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId不能为空"); + } + + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents"; + + // 添加查询参数 + StringBuilder urlBuilder = new StringBuilder(url); + List params = new ArrayList<>(); + + if (knowledgeFilesDTO != null && StringUtils.isNotBlank(knowledgeFilesDTO.getName())) { + params.add("keywords=" + knowledgeFilesDTO.getName()); + } + if (page > 0) { + params.add("page=" + page); + } + if (limit > 0) { + params.add("page_size=" + limit); + } + + if (!params.isEmpty()) { + urlBuilder.append("?").append(String.join("&", params)); + } + + url = urlBuilder.toString(); + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API获取文档列表..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + return parseDocumentListResponse(dataObj, page, limit); + } else { + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "RAGFlow API调用失败，响应码: " + code); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取RAGFlow文档列表失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取RAGFlow文档列表失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取RAGFlow文档列表失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("获取文档列表失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取文档列表失败: " + e.getMessage()); + } finally { + log.info("=== 获取文档列表操作结束 ==="); + } + } + + /** + * 解析RAGFlow API返回的文档列表响应 + */ + private PageData parseDocumentListResponse(Object dataObj, long curPage, long pageSize) { + try { + List documents = new ArrayList<>(); + long totalCount = 0; + + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + + // 获取文档列表 - 支持多种可能的字段名 + Object documentsObj = null; + + // 支持多种可能的文档列表字段名 + String[] possibleDocumentFields = {"docs", "documents", "items", "list", "data"}; + + for (String fieldName : possibleDocumentFields) { + if (dataMap.containsKey(fieldName) && dataMap.get(fieldName) instanceof List) { + documentsObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取文档列表", fieldName); + break; + } + } + + // 如果标准字段不存在，尝试自动检测 + if (documentsObj == null) { + for (Map.Entry entry : dataMap.entrySet()) { + if (entry.getValue() instanceof List) { + List list = (List) entry.getValue(); + if (!list.isEmpty() && list.get(0) instanceof Map) { + documentsObj = entry.getValue(); + log.warn("自动检测到文档列表字段: '{}'，建议检查RAGFlow API文档", entry.getKey()); + break; + } + } + } + } + + if (documentsObj instanceof List) { + List> documentList = (List>) documentsObj; + + for (Map docMap : documentList) { + KnowledgeFilesDTO dto = convertRAGDocumentToDTO(docMap); + if (dto != null) { + documents.add(dto); + } + } + } + + // 解析总数 - 支持多种可能的字段名 + Object totalObj = null; + String[] possibleTotalFields = {"total", "totalCount", "total_count", "count"}; + + for (String fieldName : possibleTotalFields) { + if (dataMap.containsKey(fieldName)) { + totalObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取总数", fieldName); + break; + } + } + + if (totalObj instanceof Integer) { + totalCount = ((Integer) totalObj).longValue(); + } else if (totalObj instanceof Long) { + totalCount = (Long) totalObj; + } else if (totalObj instanceof String) { + try { + totalCount = Long.parseLong((String) totalObj); + } catch (NumberFormatException e) { + log.warn("无法解析总数字段: {}", totalObj); + } + } + } + + // 创建分页数据 + PageData pageData = new PageData<>(documents, totalCount); + + log.info("获取文档列表成功，共{}个文档，总数: {}", documents.size(), totalCount); + return pageData; + + } catch (Exception e) { + log.error("获取文档列表响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取文档列表响应失败: " + e.getMessage()); + } + } + + /** + * 将RAGFlow文档数据转换为KnowledgeFilesDTO + */ + private KnowledgeFilesDTO convertRAGDocumentToDTO(Map docMap) { + try { + if (docMap == null) return null; + + KnowledgeFilesDTO dto = new KnowledgeFilesDTO(); + + // 设置基本字段 - 支持多种可能的字段名 + dto.setId(getStringValueFromMultipleKeys(docMap, "id", "document_id", "doc_id")); // 使用RAGFlow的文档ID作为本地ID + dto.setDocumentId(getStringValueFromMultipleKeys(docMap, "id", "document_id", "doc_id")); // RAGFlow文档ID + dto.setName(getStringValueFromMultipleKeys(docMap, "name", "filename", "file_name", "title")); + dto.setDatasetId(getStringValueFromMultipleKeys(docMap, "dataset_id", "dataset", "knowledge_base_id")); + + // 设置文件信息 - 支持多种可能的字段名 + dto.setFileType(getStringValueFromMultipleKeys(docMap, "file_type", "type", "format", "extension")); + + // 文件大小 - 支持多种可能的字段名 + Object sizeObj = getValueFromMultipleKeys(docMap, "size", "file_size", "size_bytes"); + if (sizeObj instanceof Integer) { + dto.setFileSize(((Integer) sizeObj).longValue()); + } else if (sizeObj instanceof Long) { + dto.setFileSize((Long) sizeObj); + } else if (sizeObj instanceof String) { + try { + dto.setFileSize(Long.parseLong((String) sizeObj)); + } catch (NumberFormatException e) { + log.warn("无法解析文件大小: {}", sizeObj); + } + } + + // 设置状态 - 支持多种可能的字段名 + Object statusObj = getValueFromMultipleKeys(docMap, "status", "state", "processing_status"); + if (statusObj instanceof String) { + dto.setStatus(convertRAGStatusToLocal((String) statusObj)); + } else if (statusObj instanceof Integer) { + dto.setStatus((Integer) statusObj); + } + + // 设置元数据和配置 - 支持多种可能的字段名 + Object metaFieldsObj = getValueFromMultipleKeys(docMap, "meta_fields", "metadata", "meta", "properties"); + if (metaFieldsObj instanceof Map) { + dto.setMetaFields((Map) metaFieldsObj); + } + + Object parserConfigObj = getValueFromMultipleKeys(docMap, "parser_config", "parser", "parse_config", "config"); + if (parserConfigObj instanceof Map) { + dto.setParserConfig((Map) parserConfigObj); + } + + dto.setChunkMethod(getStringValueFromMultipleKeys(docMap, "chunk_method", "chunking", "chunk_strategy")); + + // 设置时间信息 - 支持多种可能的字段名 + Object createTimeObj = getValueFromMultipleKeys(docMap, "create_time", "created_at", "creation_time", "created"); + if (createTimeObj instanceof Long) { + dto.setCreatedAt(new Date((Long) createTimeObj)); + } else if (createTimeObj instanceof String) { + // 尝试解析时间字符串 + try { + // 这里可以根据实际的时间格式进行调整 + dto.setCreatedAt(new Date(Long.parseLong((String) createTimeObj))); + } catch (NumberFormatException e) { + log.warn("无法解析创建时间: {}", createTimeObj); + } + } + + Object updateTimeObj = getValueFromMultipleKeys(docMap, "update_time", "updated_at", "modified_time", "modified"); + if (updateTimeObj instanceof Long) { + dto.setUpdatedAt(new Date((Long) updateTimeObj)); + } else if (updateTimeObj instanceof String) { + try { + dto.setUpdatedAt(new Date(Long.parseLong((String) updateTimeObj))); + } catch (NumberFormatException e) { + log.warn("无法解析更新时间: {}", updateTimeObj); + } + } + + return dto; + + } catch (Exception e) { + log.error("转换RAGFlow文档数据失败: {}", e.getMessage(), e); + return null; + } + } + + /** + * 转换RAGFlow状态到本地状态 + */ + private Integer convertRAGStatusToLocal(String ragStatus) { + if (ragStatus == null) return 0; + + switch (ragStatus.toLowerCase()) { + case "pending": + case "processing": + return 1; // 处理中 + case "completed": + case "finished": + return 2; // 已完成 + case "failed": + case "error": + return 3; // 失败 + default: + return 0; // 未知 + } + } + + /** + * 从Map中获取字符串值 + */ + private String getStringValue(Map map, String key) { + Object value = map.get(key); + return value != null ? value.toString() : null; + } + + /** + * 从多个可能的字段名中获取字符串值 + */ + private String getStringValueFromMultipleKeys(Map map, String... keys) { + for (String key : keys) { + Object value = map.get(key); + if (value != null) { + return value.toString(); + } + } + return null; + } + + /** + * 从多个可能的字段名中获取值 + */ + private Object getValueFromMultipleKeys(Map map, String... keys) { + for (String key : keys) { + Object value = map.get(key); + if (value != null) { + return value; + } + } + return null; + } + + @Override + public KnowledgeFilesDTO getById(String id) { + if (StringUtils.isBlank(id)) { + throw new RenException(ErrorCode.IDENTIFIER_NOT_NULL); + } + + log.info("=== 开始根据ID获取文档 ==="); + log.info("文档ID: {}", id); + + try { + KnowledgeFilesDTO queryDTO = new KnowledgeFilesDTO(); + + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "请使用getByDocumentId方法，并提供datasetId"); + + } catch (Exception e) { + log.error("根据ID获取文档失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取文档失败: " + e.getMessage()); + } finally { + log.info("=== 根据ID获取文档操作结束 ==="); + } + } + + @Override + public KnowledgeFilesDTO getByDocumentId(String documentId) { + // 重载方法，保持向后兼容，但实际需要datasetId + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "请使用getByDocumentId(documentId, datasetId)方法，并提供datasetId"); + } + + @Override + public KnowledgeFilesDTO getByDocumentId(String documentId, String datasetId) { + if (StringUtils.isBlank(documentId) || StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "documentId和datasetId不能为空"); + } + + log.info("=== 开始根据documentId获取文档 ==="); + log.info("documentId: {}, datasetId: {}", documentId, datasetId); + + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 修正API路径 - 根据RAGFlow API规范，获取单个文档需要datasetId + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents/" + documentId; + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API获取文档详情..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + KnowledgeFilesDTO dto = convertRAGDocumentToDTO((Map) dataObj); + if (dto != null) { + log.info("获取文档详情成功，documentId: {}", documentId); + return dto; + } + } + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } else { + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "RAGFlow API调用失败，响应码: " + code); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + if (e.getStatusCode() == HttpStatus.NOT_FOUND) { + log.warn("文档不存在，documentId: {}, datasetId: {}", documentId, datasetId); + throw new RenException(ErrorCode.Knowledge_Base_RECORD_NOT_EXISTS); + } + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取RAGFlow文档失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取RAGFlow文档失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("根据documentId获取文档失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "获取文档失败: " + e.getMessage()); + } finally { + log.info("=== 根据documentId获取文档操作结束 ==="); + } + } + + @Override + public KnowledgeFilesDTO uploadDocument(String datasetId, MultipartFile file, String name, + Map metaFields, String chunkMethod, + Map parserConfig) { + if (StringUtils.isBlank(datasetId) || file == null || file.isEmpty()) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + log.info("=== 开始文档上传操作 ==="); + log.info("上传文档到数据集: {}, 文件名: {}", datasetId, file.getOriginalFilename()); + + try { + // 在文件上传前添加详细日志 + log.info("1. 开始处理文件信息"); + String fileName = file.getOriginalFilename(); + String fileType = getFileType(fileName); + long fileSize = file.getSize(); + + log.info("文件信息 - 文件名: {}, 文件类型: {}, 文件大小: {} bytes", + fileName, fileType, fileSize); + + // 检查文件基本信息 + if (StringUtils.isBlank(fileName)) { + log.error("文件名为空"); + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "文件名不能为空"); + } + + if (fileSize == 0) { + log.error("文件大小为0"); + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "文件内容为空"); + } + + log.info("2. 开始流式上传到RAGFlow"); + // 直接调用RAGFlow API上传文档 - 使用流式上传 + String documentId = uploadDocumentToRAGFlow(datasetId, file, name, metaFields, chunkMethod, parserConfig); + + log.info("文档上传成功，documentId: {}", documentId); + + // 返回上传的文档信息 + KnowledgeFilesDTO result = new KnowledgeFilesDTO(); + result.setId(documentId); // 使用documentId作为ID + result.setDocumentId(documentId); + result.setDatasetId(datasetId); + result.setName(StringUtils.isNotBlank(name) ? name : fileName); + result.setFileType(fileType); + result.setFileSize(fileSize); + result.setStatus(1); // 上传成功，设置为处理中状态 + + return result; + + } catch (Exception e) { + log.error("文档上传失败: {}", e.getMessage()); + log.error("文档上传失败详细异常: ", e); + if (e instanceof RenException) { + throw (RenException) e; + } + throw new RenException(ErrorCode.INTERNAL_SERVER_ERROR); + } finally { + log.info("=== 文档上传操作结束 ==="); + } + } + + @Override + public KnowledgeFilesDTO update(KnowledgeFilesDTO knowledgeFilesDTO) { + if (knowledgeFilesDTO == null || StringUtils.isBlank(knowledgeFilesDTO.getDocumentId())) { + throw new RenException(ErrorCode.IDENTIFIER_NOT_NULL); + } + + log.info("=== 开始更新文档操作 ==="); + log.info("更新文档documentId: {}", knowledgeFilesDTO.getDocumentId()); + + try { + // 调用RAGFlow API更新文档配置 + log.info("开始调用RAGFlow API更新文档配置"); + updateDocumentInRAGFlow( + knowledgeFilesDTO.getDocumentId(), + knowledgeFilesDTO.getName(), + knowledgeFilesDTO.getMetaFields(), + knowledgeFilesDTO.getChunkMethod(), + knowledgeFilesDTO.getParserConfig() + ); + log.info("RAGFlow API更新调用完成"); + + // 返回更新后的文档信息（通过查询获取最新状态） + // 需要datasetId，这里假设knowledgeFilesDTO中包含datasetId + if (StringUtils.isBlank(knowledgeFilesDTO.getDatasetId())) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "更新文档需要datasetId"); + } + return getByDocumentId(knowledgeFilesDTO.getDocumentId(), knowledgeFilesDTO.getDatasetId()); + + } catch (Exception e) { + log.error("更新文档失败: {}", e.getMessage(), e); + if (e instanceof RenException) { + throw (RenException) e; + } + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新文档失败: " + e.getMessage()); + } finally { + log.info("=== 更新文档操作结束 ==="); + } + } + + @Override + public void delete(String documentId) { + // 重载方法，保持向后兼容，但实际需要datasetId + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "请使用deleteByDocumentId(documentId, datasetId)方法，并提供datasetId"); + } + + @Override + public void deleteByDocumentId(String documentId) { + // 重载方法，保持向后兼容，但实际需要datasetId + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "请使用deleteByDocumentId(documentId, datasetId)方法，并提供datasetId"); + } + + @Override + public void deleteByDocumentId(String documentId, String datasetId) { + if (StringUtils.isBlank(documentId) || StringUtils.isBlank(datasetId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "documentId和datasetId不能为空"); + } + + log.info("=== 开始根据documentId删除文档 ==="); + log.info("删除文档documentId: {}, datasetId: {}", documentId, datasetId); + + try { + // 直接调用RAGFlow API删除文档，不进行前置验证 + // 因为即使文档不存在，RAGFlow API也会返回相应的错误信息 + log.info("开始调用RAGFlow API删除文档"); + deleteDocumentInRAGFlow(documentId, datasetId); + log.info("RAGFlow API删除调用完成"); + + } catch (Exception e) { + log.error("删除文档失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除文档失败: " + e.getMessage()); + } finally { + log.info("=== 根据documentId删除文档操作结束 ==="); + } + } + + @Override + public void deleteBatch(List ids) { + if (ids == null || ids.isEmpty()) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + log.info("=== 开始批量删除文档操作 ==="); + log.info("批量删除文档数量: {}", ids.size()); + + // 由于批量删除需要datasetId，这里无法处理 + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "批量删除需要datasetId，请使用单个删除接口"); + } + + + + /** + * 获取文件类型 - 支持RAGFlow四种文档格式类型 + */ + private String getFileType(String fileName) { + if (StringUtils.isBlank(fileName)) { + log.warn("文件名为空，返回unknown类型"); + return "unknown"; + } + + try { + int lastDotIndex = fileName.lastIndexOf('.'); + if (lastDotIndex > 0 && lastDotIndex < fileName.length() - 1) { + String extension = fileName.substring(lastDotIndex + 1).toLowerCase(); + + // 支持RAGFlow四种文档格式类型 + String[] documentTypes = {"pdf", "doc", "docx", "txt", "md", "mdx"}; + String[] spreadsheetTypes = {"csv", "xls", "xlsx"}; + String[] imageTypes = {"jpeg", "jpg", "png", "tif", "gif"}; + String[] presentationTypes = {"ppt", "pptx"}; + + // 检查文档类型 + for (String type : documentTypes) { + if (type.equals(extension)) { + return "document"; + } + } + + // 检查表格类型 + for (String type : spreadsheetTypes) { + if (type.equals(extension)) { + return "spreadsheet"; + } + } + + // 检查图片类型 + for (String type : imageTypes) { + if (type.equals(extension)) { + return "image"; + } + } + + // 检查幻灯片类型 + for (String type : presentationTypes) { + if (type.equals(extension)) { + return "presentation"; + } + } + + // 返回原始扩展名作为文件类型 + return extension; + } + return "unknown"; + } catch (Exception e) { + log.error("获取文件类型失败: ", e); + return "unknown"; + } + } + + @Override + public Map getRAGConfig(String ragModelId) { + if (StringUtils.isBlank(ragModelId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR); + } + + // 从缓存获取模型配置 + ModelConfigEntity modelConfig = modelConfigService.getModelByIdFromCache(ragModelId); + if (modelConfig == null || modelConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 验证是否为RAG类型配置 + if (!Constant.RAG_CONFIG_TYPE.equals(modelConfig.getModelType().toUpperCase())) { + throw new RenException(ErrorCode.RAG_CONFIG_TYPE_ERROR); + } + + Map config = modelConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + // 返回配置信息 + return config; + } + + @Override + public Map getDefaultRAGConfig() { + // 获取默认RAG模型配置 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("model_type", Constant.RAG_CONFIG_TYPE) + .eq("is_default", 1) + .eq("is_enabled", 1); + + List modelConfigs = modelConfigDao.selectList(queryWrapper); + if (modelConfigs == null || modelConfigs.isEmpty()) { + throw new RenException(ErrorCode.RAG_DEFAULT_CONFIG_NOT_FOUND); + } + + ModelConfigEntity defaultConfig = modelConfigs.get(0); + if (defaultConfig.getConfigJson() == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + Map config = defaultConfig.getConfigJson(); + + // 验证必要的配置参数 + validateRagConfig(config); + + return config; + } + + /** + * 验证RAG配置中是否包含必要的参数 + */ + private void validateRagConfig(Map config) { + if (config == null) { + throw new RenException(ErrorCode.RAG_CONFIG_NOT_FOUND); + } + + // 从配置中提取必要的参数 + String baseUrl = (String) config.get("base_url"); + String apiKey = (String) config.get("api_key"); + + // 验证base_url是否存在且非空 + if (StringUtils.isBlank(baseUrl)) { + throw new RenException(ErrorCode.RAG_CONFIG_MISSING_PARAMS); + } + } + + /** + * 获取默认RAG配置 + */ + private Map getRAGConfig() { + return getDefaultRAGConfig(); + } + + /** + * 调用RAGFlow API上传文档 - 流式上传版本 + */ + private String uploadDocumentToRAGFlow(String datasetId, MultipartFile file, String name, + Map metaFields, String chunkMethod, + Map parserConfig) { + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API流式上传文档，datasetId: {}, 文件名: {}", datasetId, file.getOriginalFilename()); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents"; + log.debug("请求URL: {}", url); + + // 构建multipart/form-data请求 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.set("Authorization", "Bearer " + apiKey); + + // 创建多部分请求体 - 使用MultipartFileResource进行流式上传 + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", new MultipartFileResource(file, file.getOriginalFilename())); + + // 添加其他参数 + if (StringUtils.isNotBlank(name)) { + body.add("name", name); + } else { + body.add("name", file.getOriginalFilename()); + } + + if (metaFields != null && !metaFields.isEmpty()) { + try { + body.add("meta_fields", objectMapper.writeValueAsString(metaFields)); + } catch (Exception e) { + log.warn("序列化meta_fields失败: {}", e.getMessage()); + } + } + + if (StringUtils.isNotBlank(chunkMethod)) { + body.add("chunk_method", chunkMethod); + } + + if (parserConfig != null && !parserConfig.isEmpty()) { + try { + body.add("parser_config", objectMapper.writeValueAsString(parserConfig)); + } catch (Exception e) { + log.warn("序列化parser_config失败: {}", e.getMessage()); + } + } + + log.debug("multipart请求体参数数量: {}", body.size()); + log.debug("multipart请求体参数: {}", body.keySet()); + + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + + // 发送POST请求 + log.info("发送multipart/form-data POST请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + String responseBody = response.getBody(); + String documentId = null; + + if (response.getStatusCode().is2xxSuccessful()) { + try { + Map responseMap = objectMapper.readValue(responseBody, Map.class); + + log.debug("RAGFlow API响应解析结果: {}", responseMap); + + // 首先检查响应码 + Integer code = (Integer) responseMap.get("code"); + if (code != null && code == 0) { + // 响应码为0表示成功，从data字段中获取documentId + Object dataObj = responseMap.get("data"); + + // 增强的documentId提取逻辑 + documentId = extractDocumentIdFromResponse(dataObj); + + // 如果从data字段无法提取，尝试从根级别提取 + if (StringUtils.isBlank(documentId)) { + documentId = extractDocumentIdFromRoot(responseMap); + } + + log.info("文档上传成功，documentId: {}", documentId); + } else { + // 如果响应码不为0，说明API调用失败 + log.error("RAGFlow API调用失败，响应码: {}, 响应内容: {}", code, responseBody); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "RAGFlow API调用失败，响应码: " + code); + } + + log.info("从RAGFlow API响应中解析出documentId: {}", documentId); + log.debug("完整响应内容: {}", responseBody); + } catch (Exception e) { + log.error("解析RAGFlow API响应失败: {}, 响应内容: {}", e.getMessage(), responseBody); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } + } else { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), responseBody); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED); + } + + if (StringUtils.isBlank(documentId)) { + log.error("无法从RAGFlow API响应中获取documentId，响应内容: {}", responseBody); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "RAGFlow API响应中未包含documentId"); + } + + log.info("RAGFlow文档上传成功，documentId: {}，文档已开始自动解析切片", documentId); + return documentId; + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "上传RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "上传RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "上传RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_CREATE_FAILED, "上传RAGFlow文档失败: " + e.getMessage()); + } + } + + /** + * 从响应数据中提取documentId + */ + private String extractDocumentIdFromResponse(Object dataObj) { + String documentId = null; + + if (dataObj instanceof List) { + // data是一个数组，取第一个元素的id字段 + List> dataList = (List>) dataObj; + if (!dataList.isEmpty()) { + Map firstItem = dataList.get(0); + documentId = extractDocumentIdFromMap(firstItem); + } + } else if (dataObj instanceof Map) { + // data是一个对象 + Map dataMap = (Map) dataObj; + documentId = extractDocumentIdFromMap(dataMap); + } + + return documentId; + } + + /** + * 从Map中提取documentId，支持多种可能的字段名 + */ + private String extractDocumentIdFromMap(Map map) { + if (map == null) return null; + + // 尝试多种可能的字段名 + String[] possibleFieldNames = {"id", "document_id", "documentId", "doc_id", "documentId"}; + + for (String fieldName : possibleFieldNames) { + Object value = map.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 从根级别响应中提取documentId + */ + private String extractDocumentIdFromRoot(Map responseMap) { + if (responseMap == null) return null; + + // 尝试从根级别提取 + String[] possibleFieldNames = {"id", "document_id", "documentId", "doc_id", "documentId"}; + + for (String fieldName : possibleFieldNames) { + Object value = responseMap.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 调用RAGFlow API更新文档配置 + */ + private void updateDocumentInRAGFlow(String documentId, String name, Map metaFields, + String chunkMethod, Map parserConfig) { + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API更新文档配置，documentId: {}, name: {}", documentId, name); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL - 根据RAGFlow API规范，更新文档需要datasetId，但这里假设documentId足够 + // 如果更新失败，可能需要调整URL格式 + String url = baseUrl + "/api/v1/documents/" + documentId; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("document_id", documentId); + if (StringUtils.isNotBlank(name)) { + requestBody.put("name", name); + } + if (metaFields != null && !metaFields.isEmpty()) { + requestBody.put("meta_fields", metaFields); + } + if (StringUtils.isNotBlank(chunkMethod)) { + requestBody.put("chunk_method", chunkMethod); + } + if (parserConfig != null && !parserConfig.isEmpty()) { + requestBody.put("parser_config", parserConfig); + } + + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送PUT请求 + log.info("发送PUT请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.PUT, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED); + } + + log.info("RAGFlow文档配置更新成功，documentId: {}", documentId); + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow文档配置失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow文档配置失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow文档配置失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_UPDATE_FAILED, "更新RAGFlow文档配置失败: " + e.getMessage()); + } + } + + /** + * 调用RAGFlow API删除文档 + */ + private void deleteDocumentInRAGFlow(String documentId, String datasetId) { + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + log.info("开始调用RAGFlow API删除文档，documentId: {}, datasetId: {}", documentId, datasetId); + log.debug("RAGFlow配置 - baseUrl: {}, apiKey: {}", baseUrl, StringUtils.isBlank(apiKey) ? "未配置" : "已配置"); + + // 构建请求URL - 根据RAGFlow API文档，使用正确的路径参数名称 + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents"; + log.debug("请求URL: {}", url); + + // 构建请求体 - 严格按照API文档格式 + Map requestBody = new HashMap<>(); + requestBody.put("ids", Arrays.asList(documentId)); // 使用Arrays.asList确保序列化正确 + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送DELETE请求 + log.info("发送DELETE请求到RAGFlow API..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.DELETE, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + log.debug("RAGFlow API响应内容: {}", response.getBody()); + + String responseBody = response.getBody(); + + if (response.getStatusCode().is2xxSuccessful()) { + // 验证响应格式 + if (responseBody != null) { + try { + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + log.info("RAGFlow文档删除成功，documentId: {}", documentId); + return; + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 消息: {}", code, message); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "RAGFlow API调用失败: " + message); + } + } catch (Exception e) { + log.warn("解析RAGFlow响应失败，但HTTP状态码成功，视为删除成功: {}", e.getMessage()); + log.info("RAGFlow文档删除成功，documentId: {}", documentId); + return; + } + } else { + log.info("RAGFlow文档删除成功（无响应体），documentId: {}", documentId); + return; + } + } else { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), responseBody); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED); + } + + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow文档失败: " + e.getMessage() + ", 响应: " + e.getResponseBodyAsString()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("RAGFlow API调用失败 - 未知错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_DELETE_FAILED, "删除RAGFlow文档失败: " + e.getMessage()); + } + } + + /** + * 辅助类：将MultipartFile转换为Resource用于流式上传 + */ + private static class MultipartFileResource extends AbstractResource { + private final MultipartFile multipartFile; + private final String filename; + + public MultipartFileResource(MultipartFile multipartFile, String filename) { + this.multipartFile = multipartFile; + this.filename = filename; + } + + @Override + public String getFilename() { + return this.filename; + } + + @Override + public InputStream getInputStream() throws IOException { + return this.multipartFile.getInputStream(); + } + + @Override + public long contentLength() throws IOException { + return this.multipartFile.getSize(); + } + + @Override + public boolean exists() { + return true; + } + + @Override + public String getDescription() { + return "MultipartFile resource for " + this.filename; + } + } + + @Override + public boolean parseDocuments(String datasetId, List documentIds) { + if (StringUtils.isBlank(datasetId) || documentIds == null || documentIds.isEmpty()) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId和documentIds不能为空"); + } + + log.info("=== 开始解析文档（切块） ==="); + log.info("datasetId: {}, documentIds: {}", datasetId, documentIds); + + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 根据RAGFlow API文档，解析文档的接口 + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/chunks"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("document_ids", documentIds); + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送POST请求 + log.info("发送POST请求到RAGFlow API解析文档..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_PARSE_FAILED); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + log.info("文档解析成功，datasetId: {}, documentIds: {}", datasetId, documentIds); + return true; + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 消息: {}, 响应内容: {}", code, message, responseBody); + throw new RenException(ErrorCode.RAG_API_PARSE_FAILED, "RAGFlow API调用失败: " + message); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_PARSE_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_PARSE_FAILED, "解析RAGFlow文档失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_PARSE_FAILED, "解析RAGFlow文档失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_PARSE_FAILED, "解析RAGFlow文档失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("解析文档失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_PARSE_FAILED, "解析文档失败: " + e.getMessage()); + } finally { + log.info("=== 解析文档操作结束 ==="); + } + } + + @Override + public Map addChunk(String datasetId, String documentId, String content, + List importantKeywords, List questions) { + if (StringUtils.isBlank(datasetId) || StringUtils.isBlank(documentId) || StringUtils.isBlank(content)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId、documentId和content不能为空"); + } + + log.info("=== 开始添加切片 ==="); + log.info("datasetId: {}, documentId: {}, content长度: {}", datasetId, documentId, content.length()); + + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 根据RAGFlow API文档，添加切片的接口 + String url = baseUrl + "/api/v1/datasets/" + datasetId + "/documents/" + documentId + "/chunks"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("content", content); + + if (importantKeywords != null && !importantKeywords.isEmpty()) { + requestBody.put("important_keywords", importantKeywords); + } + + if (questions != null && !questions.isEmpty()) { + requestBody.put("questions", questions); + } + + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送POST请求 + log.info("发送POST请求到RAGFlow API添加切片..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "添加切片失败"); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + log.info("切片添加成功，datasetId: {}, documentId: {}", datasetId, documentId); + return responseMap; + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 消息: {}, 响应内容: {}", code, message, responseBody); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "RAGFlow API调用失败: " + message); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "添加切片失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "添加切片失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "添加切片失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("添加切片失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "添加切片失败: " + e.getMessage()); + } finally { + log.info("=== 添加切片操作结束 ==="); + } + } + + @Override + public Map listChunks(String datasetId, String documentId, String keywords, + Integer page, Integer pageSize, String chunkId) { + if (StringUtils.isBlank(datasetId) || StringUtils.isBlank(documentId)) { + throw new RenException(ErrorCode.PARAMS_GET_ERROR, "datasetId和documentId不能为空"); + } + + log.info("=== 开始列出切片 ==="); + log.info("datasetId: {}, documentId: {}, keywords: {}, page: {}, pageSize: {}, chunkId: {}", + datasetId, documentId, keywords, page, pageSize, chunkId); + + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL - 根据RAGFlow API文档，列出切片的接口 + StringBuilder urlBuilder = new StringBuilder(); + urlBuilder.append(baseUrl).append("/api/v1/datasets/").append(datasetId) + .append("/documents/").append(documentId).append("/chunks"); + + // 添加查询参数 + List params = new ArrayList<>(); + + if (StringUtils.isNotBlank(keywords)) { + params.add("keywords=" + keywords); + } + + if (page != null && page > 0) { + params.add("page=" + page); + } + + if (pageSize != null && pageSize > 0) { + params.add("page_size=" + pageSize); + } + + if (StringUtils.isNotBlank(chunkId)) { + params.add("id=" + chunkId); + } + + if (!params.isEmpty()) { + urlBuilder.append("?").append(String.join("&", params)); + } + + String url = urlBuilder.toString(); + log.debug("请求URL: {}", url); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity requestEntity = new HttpEntity<>(headers); + + // 发送GET请求 + log.info("发送GET请求到RAGFlow API列出切片..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "列出切片失败"); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + log.info("切片列表获取成功，datasetId: {}, documentId: {}", datasetId, documentId); + + // 解析切片数据并格式化返回 + return parseChunkListResponse(responseMap); + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 消息: {}, 响应内容: {}", code, message, responseBody); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "RAGFlow API调用失败: " + message); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "列出切片失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "列出切片失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "列出切片失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("列出切片失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_OPERATION_FAILED, "列出切片失败: " + e.getMessage()); + } finally { + log.info("=== 列出切片操作结束 ==="); + } + } + + /** + * 解析RAGFlow API返回的切片列表响应 + */ + private Map parseChunkListResponse(Map responseMap) { + Map result = new HashMap<>(); + List> chunkList = new ArrayList<>(); + long totalCount = 0; + + try { + // 首先检查是否有data字段 + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map dataMap = (Map) dataObj; + + // 解析切片列表 - 支持多种可能的字段名 + Object chunksObj = null; + + // 支持多种可能的切片列表字段名 + String[] possibleChunkFields = {"chunks", "items", "list", "data", "docs"}; + + for (String fieldName : possibleChunkFields) { + if (dataMap.containsKey(fieldName) && dataMap.get(fieldName) instanceof List) { + chunksObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取切片列表", fieldName); + break; + } + } + + // 如果标准字段不存在，尝试自动检测 + if (chunksObj == null) { + for (Map.Entry entry : dataMap.entrySet()) { + if (entry.getValue() instanceof List) { + List list = (List) entry.getValue(); + if (!list.isEmpty() && list.get(0) instanceof Map) { + chunksObj = entry.getValue(); + log.warn("自动检测到切片列表字段: '{}'，建议检查RAGFlow API文档", entry.getKey()); + break; + } + } + } + } + + if (chunksObj instanceof List) { + List> rawChunkList = (List>) chunksObj; + + for (Map chunkMap : rawChunkList) { + Map formattedChunk = formatChunkData(chunkMap); + if (formattedChunk != null) { + chunkList.add(formattedChunk); + } + } + } + + // 解析总数 - 支持多种可能的字段名 + Object totalObj = null; + String[] possibleTotalFields = {"total", "totalCount", "total_count", "count"}; + + for (String fieldName : possibleTotalFields) { + if (dataMap.containsKey(fieldName)) { + totalObj = dataMap.get(fieldName); + log.debug("使用字段名'{}'获取总数", fieldName); + break; + } + } + + if (totalObj instanceof Integer) { + totalCount = ((Integer) totalObj).longValue(); + } else if (totalObj instanceof Long) { + totalCount = (Long) totalObj; + } else if (totalObj instanceof String) { + try { + totalCount = Long.parseLong((String) totalObj); + } catch (NumberFormatException e) { + log.warn("无法解析总数字段: {}", totalObj); + } + } + + // 如果没有找到总数，使用切片列表的大小 + if (totalCount == 0 && !chunkList.isEmpty()) { + totalCount = chunkList.size(); + } + } else { + log.warn("RAGFlow API响应缺少data字段，尝试直接解析响应"); + + // 如果没有data字段，尝试直接解析响应 + Object chunksObj = null; + String[] possibleChunkFields = {"chunks", "items", "list", "data", "docs"}; + + for (String fieldName : possibleChunkFields) { + if (responseMap.containsKey(fieldName) && responseMap.get(fieldName) instanceof List) { + chunksObj = responseMap.get(fieldName); + log.debug("使用字段名'{}'获取切片列表", fieldName); + break; + } + } + + if (chunksObj instanceof List) { + List> rawChunkList = (List>) chunksObj; + + for (Map chunkMap : rawChunkList) { + Map formattedChunk = formatChunkData(chunkMap); + if (formattedChunk != null) { + chunkList.add(formattedChunk); + } + } + } + + // 解析总数 + Object totalObj = responseMap.get("total"); + if (totalObj instanceof Integer) { + totalCount = ((Integer) totalObj).longValue(); + } else if (totalObj instanceof Long) { + totalCount = (Long) totalObj; + } + + if (totalCount == 0 && !chunkList.isEmpty()) { + totalCount = chunkList.size(); + } + } + + } catch (Exception e) { + log.error("解析切片列表响应失败: {}", e.getMessage(), e); + } + + result.put("list", chunkList); + result.put("total", totalCount); + + log.debug("解析后的切片列表: {} 条记录", chunkList.size()); + return result; + } + + /** + * 格式化切片数据 + */ + private Map formatChunkData(Map chunkMap) { + if (chunkMap == null || chunkMap.isEmpty()) { + return null; + } + + Map formattedChunk = new HashMap<>(); + + try { + // 提取切片ID - 支持多种可能的字段名 + String chunkId = extractChunkId(chunkMap); + if (StringUtils.isBlank(chunkId)) { + log.warn("切片数据缺少ID字段，跳过处理: {}", chunkMap); + return null; + } + formattedChunk.put("id", chunkId); + + // 提取切片内容 - 支持多种可能的字段名 + String content = extractChunkContent(chunkMap); + formattedChunk.put("content", content != null ? content : ""); + + // 提取重要关键词 - 支持多种可能的字段名 + List importantKeywords = extractImportantKeywords(chunkMap); + formattedChunk.put("important_keywords", importantKeywords); + + // 提取问题列表 - 支持多种可能的字段名 + List questions = extractQuestions(chunkMap); + formattedChunk.put("questions", questions); + + // 提取创建时间 - 支持多种可能的字段名 + String createTime = extractCreateTime(chunkMap); + formattedChunk.put("create_time", createTime != null ? createTime : ""); + + } catch (Exception e) { + log.error("格式化切片数据失败: {}", e.getMessage(), e); + return null; + } + + return formattedChunk; + } + + /** + * 提取切片ID + */ + private String extractChunkId(Map chunkMap) { + String[] possibleIdFields = {"id", "chunk_id", "chunkId", "chunkId"}; + + for (String fieldName : possibleIdFields) { + Object value = chunkMap.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 提取切片内容 + */ + private String extractChunkContent(Map chunkMap) { + String[] possibleContentFields = {"content", "text", "chunk_content", "chunkContent"}; + + for (String fieldName : possibleContentFields) { + Object value = chunkMap.get(fieldName); + if (value != null && value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } + } + + return null; + } + + /** + * 提取重要关键词 + */ + private List extractImportantKeywords(Map chunkMap) { + String[] possibleKeywordFields = {"important_keywords", "keywords", "importantKeywords", "key_words"}; + + for (String fieldName : possibleKeywordFields) { + Object value = chunkMap.get(fieldName); + if (value instanceof List) { + List list = (List) value; + List keywords = new ArrayList<>(); + for (Object item : list) { + if (item instanceof String && StringUtils.isNotBlank((String) item)) { + keywords.add((String) item); + } + } + return keywords; + } else if (value instanceof String && StringUtils.isNotBlank((String) value)) { + // 如果是逗号分隔的字符串，分割成列表 + String[] parts = ((String) value).split(","); + List keywords = new ArrayList<>(); + for (String part : parts) { + String trimmed = part.trim(); + if (StringUtils.isNotBlank(trimmed)) { + keywords.add(trimmed); + } + } + return keywords; + } + } + + return new ArrayList<>(); + } + + /** + * 提取问题列表 + */ + private List extractQuestions(Map chunkMap) { + String[] possibleQuestionFields = {"questions", "question_list", "questionList", "qas"}; + + for (String fieldName : possibleQuestionFields) { + Object value = chunkMap.get(fieldName); + if (value instanceof List) { + List list = (List) value; + List questions = new ArrayList<>(); + for (Object item : list) { + if (item instanceof String && StringUtils.isNotBlank((String) item)) { + questions.add((String) item); + } + } + return questions; + } else if (value instanceof String && StringUtils.isNotBlank((String) value)) { + // 如果是逗号分隔的字符串，分割成列表 + String[] parts = ((String) value).split(","); + List questions = new ArrayList<>(); + for (String part : parts) { + String trimmed = part.trim(); + if (StringUtils.isNotBlank(trimmed)) { + questions.add(trimmed); + } + } + return questions; + } + } + + return new ArrayList<>(); + } + + /** + * 提取创建时间 + */ + private String extractCreateTime(Map chunkMap) { + String[] possibleTimeFields = {"create_time", "created_at", "createTime", "timestamp"}; + + for (String fieldName : possibleTimeFields) { + Object value = chunkMap.get(fieldName); + if (value != null) { + if (value instanceof String && StringUtils.isNotBlank((String) value)) { + return (String) value; + } else if (value instanceof Long) { + // 如果是时间戳，转换为字符串 + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date((Long) value)); + } + } + } + + return null; + } + + @Override + public Map retrievalTest(String question, List datasetIds, List documentIds, + Integer page, Integer pageSize, Float similarityThreshold, + Float vectorSimilarityWeight, Integer topK, String rerankId, + Boolean keyword, Boolean highlight, List crossLanguages, + Map metadataCondition) { + + log.info("=== 开始召回测试 ==="); + log.info("问题: {}, 数据集ID: {}, 文档ID: {}, 页码: {}, 每页数量: {}", + question, datasetIds, documentIds, page, pageSize); + + try { + // 获取RAG配置 + Map ragConfig = getRAGConfig(); + String baseUrl = (String) ragConfig.get("base_url"); + String apiKey = (String) ragConfig.get("api_key"); + + // 构建请求URL + String url = baseUrl + "/api/v1/retrieval"; + log.debug("请求URL: {}", url); + + // 构建请求体 + Map requestBody = new HashMap<>(); + requestBody.put("question", question); + + if (datasetIds != null && !datasetIds.isEmpty()) { + requestBody.put("dataset_ids", datasetIds); + } + + if (documentIds != null && !documentIds.isEmpty()) { + requestBody.put("document_ids", documentIds); + } + + if (page != null && page > 0) { + requestBody.put("page", page); + } + + if (pageSize != null && pageSize > 0) { + requestBody.put("page_size", pageSize); + } + + if (similarityThreshold != null) { + requestBody.put("similarity_threshold", similarityThreshold); + } + + if (vectorSimilarityWeight != null) { + requestBody.put("vector_similarity_weight", vectorSimilarityWeight); + } + + if (topK != null && topK > 0) { + requestBody.put("top_k", topK); + } + + if (rerankId != null) { + requestBody.put("rerank_id", rerankId); + } + + if (keyword != null) { + requestBody.put("keyword", keyword); + } + + if (highlight != null) { + requestBody.put("highlight", highlight); + } + + if (crossLanguages != null && !crossLanguages.isEmpty()) { + requestBody.put("cross_languages", crossLanguages); + } + + if (metadataCondition != null) { + requestBody.put("metadata_condition", metadataCondition); + } + + log.debug("请求体: {}", requestBody); + + // 设置请求头 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Authorization", "Bearer " + apiKey); + + HttpEntity> requestEntity = new HttpEntity<>(requestBody, headers); + + // 发送POST请求 + log.info("发送POST请求到RAGFlow API进行召回测试..."); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class); + + log.info("RAGFlow API响应状态码: {}", response.getStatusCode()); + + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("RAGFlow API调用失败，状态码: {}, 响应内容: {}", response.getStatusCode(), response.getBody()); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED); + } + + String responseBody = response.getBody(); + log.debug("RAGFlow API响应内容: {}", responseBody); + + // 解析响应 + Map responseMap = objectMapper.readValue(responseBody, Map.class); + Integer code = (Integer) responseMap.get("code"); + + if (code != null && code == 0) { + Object dataObj = responseMap.get("data"); + if (dataObj instanceof Map) { + Map result = (Map) dataObj; + log.info("召回测试成功，返回 {} 条切片", result.get("total")); + return result; + } else { + log.error("RAGFlow API响应格式错误，data字段不是Map类型: {}", dataObj); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "RAGFlow API响应格式错误"); + } + } else { + String message = (String) responseMap.get("message"); + log.error("RAGFlow API调用失败，响应码: {}, 错误信息: {}", code, message); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "RAGFlow API调用失败: " + message); + } + + } catch (IOException e) { + log.error("解析RAGFlow API响应失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "解析RAGFlow响应失败: " + e.getMessage()); + } catch (HttpClientErrorException e) { + log.error("RAGFlow API调用失败 - HTTP错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "召回测试失败: " + e.getMessage()); + } catch (HttpServerErrorException e) { + log.error("RAGFlow API调用失败 - 服务器错误: {}, 状态码: {}, 响应内容: {}", + e.getMessage(), e.getStatusCode(), e.getResponseBodyAsString(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "召回测试失败: " + e.getMessage()); + } catch (ResourceAccessException e) { + log.error("RAGFlow API调用失败 - 网络连接错误: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "召回测试失败: 网络连接错误 - " + e.getMessage()); + } catch (Exception e) { + log.error("召回测试失败: {}", e.getMessage(), e); + throw new RenException(ErrorCode.RAG_API_QUERY_FAILED, "召回测试失败: " + e.getMessage()); + } finally { + log.info("=== 召回测试操作结束 ==="); + } + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202510250955.sql b/main/manager-api/src/main/resources/db/changelog/202510250955.sql new file mode 100644 index 00000000..0256a7a2 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202510250955.sql @@ -0,0 +1,24 @@ +-- 添加RAG模型供应器和配置 +-- ------------------------------------------------------- + +-- 添加RAG模型供应器 +delete from `ai_model_provider` where id = 'SYSTEM_RAG_ragflow'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_RAG_ragflow', 'RAG', 'ragflow', 'RAGFlow', '[{"key": "base_url", "type": "string", "label": "服务地址"}, {"key": "api_key", "type": "string", "label": "API密钥"}]', 1, 1, NOW(), 1, NOW()); + +-- 添加RAG模型配置 +delete from `ai_model_config` where id = 'RAG_RAGFlow'; +INSERT INTO `ai_model_config` VALUES ('RAG_RAGFlow', 'RAG', 'ragflow', 'RAGFlow', 1, 1, '{"type": "ragflow", "base_url": "http://localhost", "api_key": "your_api_key_here"}', 'https://github.com/infiniflow/ragflow/blob/main/README_zh.md', 'RAGFlow配置说明： +一、快速部署教程（docker部署） +1.$ sysctl vm.max_map_count +2.$ sysctl -w vm.max_map_count=262144 +3.$ git clone https://github.com/infiniflow/ragflow.git +4.docker compose -f docker-compose.yml up -d +5.$ docker logs -f docker-ragflow-cpu-1 +6.注冊登录后，点击右上角头像，获得RAGFlow的API KEY和API服务器地址。使用RAGFlow前请在Model Provider中添加模型和设置默认模型。 +二、如果您希望关掉注册功能 +1.停止服务 docker compose down +2. sed -i ''s/REGISTER_ENABLED=1/REGISTER_ENABLED=0/g'' .env +3.cat .env | grep -i register +4.看到REGISTER_ENABLED=0 重启服务即可。', 1, NULL, NULL, NULL, NULL); + diff --git a/main/manager-api/src/main/resources/db/changelog/202510251150.sql b/main/manager-api/src/main/resources/db/changelog/202510251150.sql new file mode 100644 index 00000000..4dea2bec --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202510251150.sql @@ -0,0 +1,19 @@ +-- 知识库表 +DROP TABLE IF EXISTS `ai_rag_dataset`; +CREATE TABLE `ai_rag_dataset` ( + `id` VARCHAR(32) NOT NULL COMMENT '唯一标识', + `dataset_id` VARCHAR(64) NOT NULL COMMENT '知识库ID', + `rag_model_id` VARCHAR(64) COMMENT 'RAG模型配置ID', + `name` VARCHAR(100) NOT NULL COMMENT '知识库名称', + `description` TEXT COMMENT '知识库描述', + `status` TINYINT(1) DEFAULT 1 COMMENT '状态：0停用 1启用', + `creator` BIGINT COMMENT '创建者', + `created_at` DATETIME COMMENT '创建时间', + `updater` BIGINT COMMENT '更新者', + `updated_at` DATETIME COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_dataset_id` (`dataset_id`), + INDEX `idx_ai_rag_dataset_status` (`status`), + INDEX `idx_ai_rag_dataset_creator` (`creator`), + INDEX `idx_ai_rag_dataset_created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='知识库表'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index b6b88bf3..5fd1905a 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -402,3 +402,17 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202510191042.sql + - changeSet: + id: 202510250955 + author: rainv123 + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202510250955.sql + - changeSet: + id: 202510251150 + author: rainv123 + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202510251150.sql \ No newline at end of file diff --git a/main/manager-api/src/main/resources/i18n/messages_en_US.properties b/main/manager-api/src/main/resources/i18n/messages_en_US.properties index 04354f63..6d77df3e 100644 --- a/main/manager-api/src/main/resources/i18n/messages_en_US.properties +++ b/main/manager-api/src/main/resources/i18n/messages_en_US.properties @@ -168,4 +168,17 @@ 10159=Voice ID already exists 10160=Huoshan Engine voice ID format error, must start with S_ 10161=Mac address already exists -10162=Model provider does not exist \ No newline at end of file +10162=Model provider does not exist +10163=Knowledge base record does not exist +10164=RAG configuration not found +10165=RAG configuration type error +10166=Default RAG configuration not found +10167=RAG configuration missing required parameters +10168=RAG API create dataset failed +10169=RAG API update dataset failed +10170=RAG API delete dataset failed +10171=Upload file failed +10172=RAG API query failed +10173=RAG API parse failed +10174=RAG API operation failed + diff --git a/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties b/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties index 6fb10371..a92ab1d3 100644 --- a/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/main/manager-api/src/main/resources/i18n/messages_zh_CN.properties @@ -168,4 +168,16 @@ 10159=\u97F3\u8272ID\u5DF2\u5B58\u5728 10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u9519\u8BEF\uFF0C\u5FC5\u987B\u4EE5S_\u5F00\u5934 10161=Mac\u5730\u5740\u5DF2\u5B58\u5728 -10162=\u6A21\u578B\u4F9B\u5E94\u5668\u4E0D\u5B58\u5728 \ No newline at end of file +10162=\u6A21\u578B\u4F9B\u5E94\u5668\u4E0D\u5B58\u5728 +10163=\u77E5\u8BC6\u5E93\u8BB0\u5F55\u4E0D\u5B58\u5728 +10164=RAG\u914D\u7F6E\u672A\u627E\u5230 +10165=RAG\u914D\u7F6E\u7C7B\u578B\u9519\u8BEF +10166=\u9ED8\u8BA4RAG\u914D\u7F6E\u672A\u627E\u5230 +10167=RAG\u914D\u7F6E\u7F3A\u5C11\u5FC5\u8981\u53C2\u6570 +10168=RAG API\u521B\u5EFA\u6570\u636E\u96C6\u5931\u8D25 +10169=RAG API\u66F4\u65B0\u6570\u636E\u96C6\u5931\u8D25 +10170=RAG API\u5220\u9664\u6570\u636E\u96C6\u5931\u8D25 +10171=\u4E0A\u4F20\u6587\u4EF6\u5931\u8D25 +10172=RAG API\u67E5\u8BE2\u5931\u8D25 +10173=RAG API\u89E3\u6790\u5931\u8D25 +10174=RAG API\u64CD\u4F5C\u5931\u8D25 diff --git a/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties b/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties index 0f5d316c..3d45ff3c 100644 --- a/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties +++ b/main/manager-api/src/main/resources/i18n/messages_zh_TW.properties @@ -169,4 +169,15 @@ 10160=\u706B\u5C71\u5F15\u64CE\u97F3\u8272ID\u683C\u5F0F\u932F\u8AA4\uFF0C\u5FC5\u9808\u4EE5S_\u958B\u982D 10161=Mac\u5730\u5740\u5DF2\u5B58\u5728 10162=\u6A21\u578B\u63D0\u4F9B\u5546\u4E0D\u5B58\u5728 - +10163=\u77E5\u8B58\u5EAB\u8A18\u9304\u4E0D\u5B58\u5728 +10164=RAG\u914D\u7F6E\u672A\u627E\u5230 +10165=RAG\u914D\u7F6E\u985E\u578B\u932F\u8AA4 +10166=\u9810\u8A2DRAG\u914D\u7F6E\u672A\u627E\u5230 +10167=RAG\u914D\u7F6E\u7F3A\u5C11\u5FC5\u8981\u53C3\u6578 +10168=RAG API\u5275\u5EFA\u6578\u64DA\u96C6\u5931\u6557 +10169=RAG API\u66F4\u65B0\u6578\u64DA\u96C6\u5931\u6557 +10170=RAG API\u522A\u9664\u6578\u64DA\u96C6\u5931\u6557 +10171=\u4E0A\u50B3\u6587\u4EF6\u5931\u6557 +10172=RAG API\u67E5\u8A62\u5931\u6557 +10173=RAG API\u5256\u6790\u5931\u6557 +10174=RAG API\u64CD\u4F5C\u5931\u6557 diff --git a/main/manager-api/src/main/resources/mapper/knowledge/KnowledgeBaseDao.xml b/main/manager-api/src/main/resources/mapper/knowledge/KnowledgeBaseDao.xml new file mode 100644 index 00000000..e1b6ac12 --- /dev/null +++ b/main/manager-api/src/main/resources/mapper/knowledge/KnowledgeBaseDao.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/main/manager-web/src/apis/api.js b/main/manager-web/src/apis/api.js index 3f2595b4..45a810c7 100755 --- a/main/manager-web/src/apis/api.js +++ b/main/manager-web/src/apis/api.js @@ -9,6 +9,7 @@ import timbre from "./module/timbre.js" import user from './module/user.js' import voiceClone from './module/voiceClone.js' import voiceResource from './module/voiceResource.js' +import knowledgeBase from './module/knowledgeBase.js' @@ -39,5 +40,6 @@ export default { ota, dict, voiceResource, - voiceClone -} + voiceClone, + knowledgeBase + } diff --git a/main/manager-web/src/apis/module/knowledgeBase.js b/main/manager-web/src/apis/module/knowledgeBase.js new file mode 100644 index 00000000..1e0bf728 --- /dev/null +++ b/main/manager-web/src/apis/module/knowledgeBase.js @@ -0,0 +1,440 @@ +import { getServiceUrl } from '../api'; +import RequestService from '../httpRequest'; + +/** + * 获取认证token + */ +function getAuthToken() { + return localStorage.getItem('token') || ''; +} + +/** + * 知识库管理相关API + */ +export default { + /** + * 获取知识库列表 + * @param {Object} params - 查询参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + getKnowledgeBaseList(params, callback, errorCallback) { + const token = getAuthToken(); + const queryParams = new URLSearchParams({ + page: params.page, + page_size: params.page_size, + name: params.name || '' + }).toString(); + + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets?${queryParams}`) + .method('GET') + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('获取知识库列表失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.getKnowledgeBaseList(params, callback, errorCallback); + }); + }).send(); + }, + + /** + * 创建知识库 + * @param {Object} data - 知识库数据 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + createKnowledgeBase(data, callback, errorCallback) { + console.log('createKnowledgeBase called with data:', data); + const token = getAuthToken(); + console.log('Token exists:', !!token); + console.log('API URL:', `${getServiceUrl()}/api/v1/datasets`); + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets`) + .method('POST') + .data(data) + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + console.log('createKnowledgeBase success response:', res); + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('创建知识库失败:', err); + if (err.response) { + console.error('Error response data:', err.response.data); + console.error('Error response status:', err.response.status); + } + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + console.log('Network failure, retrying...'); + const self = this; + RequestService.reAjaxFun(() => { + self.createKnowledgeBase(data, callback, errorCallback); + }); + }).send(); + }, + + /** + * 更新知识库 + * @param {string} datasetId - 知识库ID + * @param {Object} data - 更新数据 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + updateKnowledgeBase(datasetId, data, callback, errorCallback) { + console.log('updateKnowledgeBase called with datasetId:', datasetId, 'data:', data); + const token = getAuthToken(); + console.log('Token exists:', !!token); + console.log('API URL:', `${getServiceUrl()}/api/v1/datasets/${datasetId}`); + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}`) + .method('PUT') + .data(data) + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('更新知识库失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.updateKnowledgeBase(datasetId, data, callback, errorCallback); + }); + }).send(); + }, + + /** + * 删除单个知识库 + * @param {string} datasetId - 知识库ID + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + deleteKnowledgeBase(datasetId, callback, errorCallback) { + console.log('deleteKnowledgeBase called with datasetId:', datasetId); + const token = getAuthToken(); + console.log('Token exists:', !!token); + console.log('API URL:', `${getServiceUrl()}/api/v1/datasets/${datasetId}`); + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}`) + .method('DELETE') + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('删除知识库失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.deleteKnowledgeBase(datasetId, callback, errorCallback); + }); + }).send(); + }, + + /** + * 批量删除知识库 + * @param {string|Array} ids - 知识库ID字符串或数组 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + deleteKnowledgeBases(ids, callback, errorCallback) { + const token = getAuthToken(); + // 确保ids是正确格式的字符串 + const idsStr = Array.isArray(ids) ? ids.join(',') : ids; + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/batch?ids=${idsStr}`) + .method('DELETE') + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('批量删除知识库失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.deleteKnowledgeBases(ids, callback, errorCallback); + }); + }).send(); + }, + + /** + * 获取文档列表 + * @param {string} datasetId - 知识库ID + * @param {Object} params - 查询参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + getDocumentList(datasetId, params, callback, errorCallback) { + const token = getAuthToken(); + const queryParams = new URLSearchParams({ + page: params.page, + page_size: params.page_size, + name: params.name || '' + }).toString(); + + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}/documents?${queryParams}`) + .method('GET') + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('获取文档列表失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.getDocumentList(datasetId, params, callback, errorCallback); + }); + }).send(); + }, + + /** + * 上传文档 + * @param {string} datasetId - 知识库ID + * @param {Object} formData - 表单数据 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + uploadDocument(datasetId, formData, callback, errorCallback) { + const token = getAuthToken(); + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}/documents`) + .method('POST') + .data(formData) + .header({ + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'multipart/form-data' + }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('上传文档失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.uploadDocument(datasetId, formData, callback, errorCallback); + }); + }).send(); + }, + + /** + * 解析文档 + * @param {string} datasetId - 知识库ID + * @param {string} documentId - 文档ID + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + parseDocument(datasetId, documentId, callback, errorCallback) { + const token = getAuthToken(); + const requestBody = { + document_ids: [documentId] + }; + + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}/chunks`) + .method('POST') + .data(requestBody) + .header({ + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json' + }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('解析文档失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.parseDocument(datasetId, documentId, callback, errorCallback); + }); + }).send(); + }, + + /** + * 删除文档 + * @param {string} datasetId - 知识库ID + * @param {string} documentId - 文档ID + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + deleteDocument(datasetId, documentId, callback, errorCallback) { + const token = getAuthToken(); + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}/documents/${documentId}`) + .method('DELETE') + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('删除文档失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.deleteDocument(datasetId, documentId, callback, errorCallback); + }); + }).send(); + }, + + /** + * 获取文档切片列表 + * @param {string} datasetId - 知识库ID + * @param {string} documentId - 文档ID + * @param {Object} params - 查询参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + listChunks(datasetId, documentId, params, callback, errorCallback) { + const token = getAuthToken(); + const queryParams = new URLSearchParams({ + page: params.page || 1, + page_size: params.page_size || 10 + }).toString(); + + // 添加关键词搜索参数 + if (params.keywords) { + queryParams += `&keywords=${encodeURIComponent(params.keywords)}`; + } + + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}/documents/${documentId}/chunks?${queryParams}`) + .method('GET') + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('获取切片列表失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.listChunks(datasetId, documentId, params, callback, errorCallback); + }); + }).send(); + }, + + /** + * 添加切片 + * @param {string} datasetId - 知识库ID + * @param {string} documentId - 文档ID + * @param {Object} data - 切片数据 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + addChunk(datasetId, documentId, data, callback, errorCallback) { + const token = getAuthToken(); + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}/documents/${documentId}/chunks`) + .method('POST') + .data(data) + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('添加切片失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.addChunk(datasetId, documentId, data, callback, errorCallback); + }); + }).send(); + }, + + /** + * 召回测试 + * @param {string} datasetId - 知识库ID + * @param {Object} data - 召回测试参数 + * @param {Function} callback - 回调函数 + * @param {Function} errorCallback - 错误回调 + */ + retrievalTest(datasetId, data, callback, errorCallback) { + const token = getAuthToken(); + RequestService.sendRequest() + .url(`${getServiceUrl()}/api/v1/datasets/${datasetId}/retrieval-test`) + .method('POST') + .data(data) + .header({ 'Authorization': `Bearer ${token}` }) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .fail((err) => { + console.error('召回测试失败:', err); + if (errorCallback) { + errorCallback(err); + } + }) + .networkFail(() => { + const self = this; + RequestService.reAjaxFun(() => { + self.retrievalTest(datasetId, data, callback, errorCallback); + }); + }).send(); + } + +}; \ No newline at end of file diff --git a/main/manager-web/src/apis/module/model.js b/main/manager-web/src/apis/module/model.js index 5b64811a..e9e67104 100644 --- a/main/manager-web/src/apis/module/model.js +++ b/main/manager-web/src/apis/module/model.js @@ -337,5 +337,23 @@ export default { this.getPluginFunctionList(params, callback) }) }).send() + }, + + // 获取RAG模型列表 + getRAGModels(callback) { + RequestService.sendRequest() + .url(`${getServiceUrl()}/models/list?modelType=RAG&page=0&limit=100`) + .method('GET') + .success((res) => { + RequestService.clearRequestTime() + callback(res) + }) + .networkFail((err) => { + console.error('获取RAG模型列表失败:', err) + this.$message.error(err.msg || '获取RAG模型列表失败') + RequestService.reAjaxFun(() => { + this.getRAGModels(callback) + }) + }).send() } } diff --git a/main/manager-web/src/components/HeaderBar.vue b/main/manager-web/src/components/HeaderBar.vue index 80094830..30328f8a 100644 --- a/main/manager-web/src/components/HeaderBar.vue +++ b/main/manager-web/src/components/HeaderBar.vue @@ -116,19 +116,19 @@
- {{ $t("header.userManagement") }} + {{ $t("header.knowledgeBase") }}
@@ -157,7 +158,8 @@ $route.path === '/provider-management' || $route.path === '/server-side-management' || $route.path === '/agent-template-management' || - $route.path === '/ota-management' + $route.path === '/ota-management' || + $route.path === '/user-management' ? 'brightness(0) invert(1)' : 'None', }" @@ -187,6 +189,9 @@ {{ $t("header.serverSideManagement") }} + + {{ $t("header.userManagement") }} + @@ -399,6 +404,9 @@ export default { goModelConfig() { this.$router.push("/model-config"); }, + goKnowledgeBaseManagement() { + this.$router.push("/knowledge-base-management"); + }, goVoiceCloneManagement() { this.$router.push("/voice-clone-management"); }, diff --git a/main/manager-web/src/components/KnowledgeBaseDialog.vue b/main/manager-web/src/components/KnowledgeBaseDialog.vue new file mode 100644 index 00000000..d2779395 --- /dev/null +++ b/main/manager-web/src/components/KnowledgeBaseDialog.vue @@ -0,0 +1,267 @@ + + + + + \ No newline at end of file diff --git a/main/manager-web/src/components/ProviderDialog.vue b/main/manager-web/src/components/ProviderDialog.vue index 46945832..61e5b9de 100644 --- a/main/manager-web/src/components/ProviderDialog.vue +++ b/main/manager-web/src/components/ProviderDialog.vue @@ -88,6 +88,7 @@ +