diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java index acb856d6..7dba0a17 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java @@ -1,6 +1,7 @@ package com.ruoyi.platform.scheduling; import cn.hutool.core.lang.UUID; +import cn.hutool.core.map.MapUtil; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; @@ -18,6 +19,7 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -186,21 +188,36 @@ public class AppScheduleTask { JSONArray fileList = csCollectService.queryTaskResult(taskId); JSONObject taskResult = new JSONObject(); JSONArray filePaths = new JSONArray(); + JSONObject taskInfo = JSONUtil.parseObj(trainingTask.getTaskInfo()); String savePath = "/mini-model-platform-data/cs-data/" +trainingTask.getServiceId() + "/" + UUID.fastUUID(); + String saveFilePath = savePath + "/file/"; fileList.forEach(file -> { JSONObject fileObj = JSONUtil.parseObj(file); Integer objectId = (Integer) fileObj.get("objectID"); String fileName = (String) fileObj.get("path"); try { - JSONObject filePath = csCollectService.singleFileDownload(objectId, savePath,fileName ); + JSONObject filePath = csCollectService.singleFileDownload(objectId, saveFilePath,fileName ); filePaths.set(filePath); } catch (Exception e) { log.error("checkCSAppTaskResult error={}",e.getMessage()); } }); + + String zipFileName = "batch_download_" + taskId + ".zip"; + String zipFilePath = savePath + "/" + zipFileName; + HashMap upload = MapUtil.newHashMap(); + upload.put("input", (String) taskInfo.getByPath("input_file.direct_url")); + upload.put("output", saveFilePath); + csCollectService.allFileUpload(upload, savePath + "/" + zipFileName); + HashMap batch_download = MapUtil.newHashMap(); + batch_download.put("direct_url", zipFilePath); + batch_download.put("filename", zipFileName); + taskResult.set("files", filePaths); taskResult.set("result_path", savePath); taskResult.set("task_id", taskId); + taskResult.set("task_id", taskId); + taskResult.set("batch_download", batch_download); trainingTask.setStatus(Constant.Succeeded); trainingTask.setTaskResult(taskResult.toString()); trainingTaskDao.update(trainingTask); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java index 9d74ec66..33505a7c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java @@ -4,6 +4,8 @@ import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import org.springframework.web.multipart.MultipartFile; +import java.util.HashMap; + public interface CSCollectService { JSONArray getHpcAppCluster(String appCode) throws Exception; @@ -19,4 +21,6 @@ public interface CSCollectService { JSONArray queryTaskResult(String taskId) throws Exception; JSONObject singleFileDownload(Integer objectId, String savePath,String fileName) throws Exception; + + void allFileUpload(HashMap upload, String zipFilePath) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java index 460722b3..bcaa51d4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java @@ -25,6 +25,7 @@ import javax.annotation.Resource; import java.io.*; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @Service @@ -300,29 +301,40 @@ public class CSCollectServiceImpl implements CSCollectService { CloseableHttpResponse response = HttpUtils.sendGetReturnResponse(presignUrl, csAuthInfo.getToken()); InputStream content = response.getEntity().getContent(); //存入minio - return fileSaveToMinIO(content, savePath,fileName); + return fileSaveToMinIO(content, savePath, fileName); } /** - * 所有文件打包下载 + * 所有文件打包上传 * - * @param packageId - * @param userId - * @param token + * @param upload + * @param zipFilePath * @throws Exception */ - private void allFileDownload(String packageId, String userId, String token) throws Exception { - String url = csUrl + "/jsm/v2/storage/presign"; - String paramStr = "{\"userID\":137,\"info\":{\"type\":\"batchDownload\",\"params\":{\"packageID\":13627,\"zip\":true}}}"; - JSONObject params = JSONUtil.parseObj(paramStr); - params.set("userID", userId); - params.putByPath("info.params.packageID", packageId); - String res = HttpUtils.sendBodyPostWithToken(url, params, token); + @Override + public void allFileUpload(HashMap upload, String zipFilePath) throws Exception { + // 异步提交任务 + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + minioUtil.createZipFromDirectory(bucketName, upload, zipFilePath); + return "执行成功"; + } catch (Exception e) { + throw new RuntimeException("异步提交任务失败", e); + } + }); + // 处理异步结果 + future.thenAcceptAsync(result -> { + + log.info("所有文件打包上传成功，result: {}", result); + }).exceptionally(ex -> { + log.error("异步处理任务失败", ex); + return null; + }); } //将内容同步到minio中 - private JSONObject fileSaveToMinIO(InputStream inputStream, String savePath,String fileName) throws Exception { - String path = savePath+"/" +fileName; + private JSONObject fileSaveToMinIO(InputStream inputStream, String savePath, String fileName) throws Exception { + String path = savePath + "/" + fileName; ByteArrayInputStream byteArrayOutputStream = readAllBytes(inputStream); minioUtil.uploadObject(bucketName, path, byteArrayOutputStream); @@ -380,7 +392,7 @@ public class CSCollectServiceImpl implements CSCollectService { if (csAuthInfo == null) { return; } - redisService.setCacheObject(TOKEN_KEY, BeanUtil.beanToMap(csAuthInfo),10L, TimeUnit.MINUTES); + redisService.setCacheObject(TOKEN_KEY, BeanUtil.beanToMap(csAuthInfo), 10L, TimeUnit.MINUTES); } /** diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java index 2a28d900..fd51cc62 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java @@ -512,4 +512,106 @@ public class MinioUtil { return new ByteArrayInputStream(zipOutputStream.toByteArray()); } } + + /** + * 创建目录的压缩包 + */ + public void createZipFromDirectory(String bucket, Map dirMap, String zipPath) + throws Exception { + + // 创建内存中的ZIP流 + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + + // 遍历每个目录映射 + for (Map.Entry entry : dirMap.entrySet()) { + String folderName = entry.getKey(); // 压缩包内的文件夹名 + String sourcePath = entry.getValue(); // MinIO中的路径 + + // 1. 自动提取目录路径（去除文件名） + String sourceDir = extractDirectoryPath(sourcePath); + + // 2. 规范化文件夹名 + if (!folderName.endsWith("/")) { + folderName = folderName + "/"; + } + + // 3. 获取目录下的所有文件 + Iterable> items = minioClient.listObjects( + ListObjectsArgs.builder() + .bucket(bucket) + .prefix(sourceDir) + .recursive(true) + .build() + ); + + // 4. 添加文件到ZIP + for (Result result : items) { + Item item = result.get(); + if (item.isDir()) continue; + + String objectName = item.objectName(); + String relativePath = objectName.substring(sourceDir.length()); + + // 使用自定义的文件夹名 + String zipEntryName = folderName + relativePath; + + // ZIP条目 + zos.putNextEntry(new ZipEntry(zipEntryName)); + + try (InputStream is = minioClient.getObject( + GetObjectArgs.builder() + .bucket(bucket) + .object(objectName) + .build())) { + + // 使用缓冲区读写 + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + zos.write(buffer, 0, bytesRead); + } + } + + zos.closeEntry(); + } + } + + zos.finish(); + } + + // 上传压缩包到MinIO + byte[] zipBytes = baos.toByteArray(); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(zipBytes)) { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(zipPath) + .stream(bais, zipBytes.length, -1) + .contentType("application/zip") + .build() + ); + } + } + + private String extractDirectoryPath(String path) { + path = path.trim(); + + // 如果路径包含点号且不以斜杠结尾，可能是文件 + if (path.contains(".") && !path.endsWith("/")) { + int lastSlash = path.lastIndexOf("/"); + if (lastSlash > 0) { + // 返回目录部分 + return path.substring(0, lastSlash + 1); + } + } + + // 如果不是文件路径，直接返回 + if (!path.endsWith("/")) { + return path + "/"; + } + return path; + } } \ No newline at end of file