ci4s
/
ci4sManagement-cloud
0
0
Fork 7
Code Issues Pull Requests 1 Packages Projects Releases 1 Wiki Activity

超算打包下载接口联调

This commit is contained in:
chenpeng0206 2026-02-06 12:40:03 +08:00
parent d00c415387
commit 67bf49ea5c
4 changed files with 151 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java
View File

@ -1,6 +1,7 @@
package com.ruoyi.platform.scheduling;
import cn.hutool.core.lang.UUID;
import cn.hutool.core.map.MapUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
@ -18,6 +19,7 @@ import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
@ -186,21 +188,36 @@ public class AppScheduleTask {
JSONArray fileList = csCollectService.queryTaskResult(taskId);
JSONObject taskResult = new JSONObject();
JSONArray filePaths = new JSONArray();
JSONObject taskInfo = JSONUtil.parseObj(trainingTask.getTaskInfo());
String savePath = "/mini-model-platform-data/cs-data/" +trainingTask.getServiceId() + "/" + UUID.fastUUID();
String saveFilePath = savePath + "/file/";
fileList.forEach(file -> {
JSONObject fileObj = JSONUtil.parseObj(file);
Integer objectId = (Integer) fileObj.get("objectID");
String fileName = (String) fileObj.get("path");
try {
JSONObject filePath = csCollectService.singleFileDownload(objectId, savePath,fileName );
JSONObject filePath = csCollectService.singleFileDownload(objectId, saveFilePath,fileName );
filePaths.set(filePath);
} catch (Exception e) {
log.error("checkCSAppTaskResult error={}",e.getMessage());
}
});
String zipFileName = "batch_download_" + taskId + ".zip";
String zipFilePath = savePath + "/" + zipFileName;
HashMap<String, String> upload = MapUtil.newHashMap();
upload.put("input", (String) taskInfo.getByPath("input_file.direct_url"));
upload.put("output", saveFilePath);
csCollectService.allFileUpload(upload, savePath + "/" + zipFileName);
HashMap<String, String> batch_download = MapUtil.newHashMap();
batch_download.put("direct_url", zipFilePath);
batch_download.put("filename", zipFileName);
taskResult.set("files", filePaths);
taskResult.set("result_path", savePath);
taskResult.set("task_id", taskId);
taskResult.set("task_id", taskId);
taskResult.set("batch_download", batch_download);
trainingTask.setStatus(Constant.Succeeded);
trainingTask.setTaskResult(taskResult.toString());
trainingTaskDao.update(trainingTask);

4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java
View File

@ -4,6 +4,8 @@ import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import org.springframework.web.multipart.MultipartFile;
import java.util.HashMap;
public interface CSCollectService {
JSONArray getHpcAppCluster(String appCode) throws Exception;
@ -19,4 +21,6 @@ public interface CSCollectService {
JSONArray queryTaskResult(String taskId) throws Exception;
JSONObject singleFileDownload(Integer objectId, String savePath,String fileName) throws Exception;
void allFileUpload(HashMap<String, String> upload, String zipFilePath) throws Exception;
}

42
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java
View File

@ -25,6 +25,7 @@ import javax.annotation.Resource;
import java.io.*;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@Service
@ -300,29 +301,40 @@ public class CSCollectServiceImpl implements CSCollectService {
CloseableHttpResponse response = HttpUtils.sendGetReturnResponse(presignUrl, csAuthInfo.getToken());
InputStream content = response.getEntity().getContent();
//存入minio
return fileSaveToMinIO(content, savePath,fileName);
return fileSaveToMinIO(content, savePath, fileName);
}
/**
* 所有文件打包下载
* 所有文件打包上传
*
* @param packageId
* @param userId
* @param token
* @param upload
* @param zipFilePath
* @throws Exception
*/
private void allFileDownload(String packageId, String userId, String token) throws Exception {
String url = csUrl + "/jsm/v2/storage/presign";
String paramStr = "{\"userID\":137,\"info\":{\"type\":\"batchDownload\",\"params\":{\"packageID\":13627,\"zip\":true}}}";
JSONObject params = JSONUtil.parseObj(paramStr);
params.set("userID", userId);
params.putByPath("info.params.packageID", packageId);
String res = HttpUtils.sendBodyPostWithToken(url, params, token);
@Override
public void allFileUpload(HashMap<String, String> upload, String zipFilePath) throws Exception {
// 异步提交任务
CompletableFuture<String> future = CompletableFuture.supplyAsync(() -> {
try {
minioUtil.createZipFromDirectory(bucketName, upload, zipFilePath);
return "执行成功";
} catch (Exception e) {
throw new RuntimeException("异步提交任务失败", e);
}
});
// 处理异步结果
future.thenAcceptAsync(result -> {
log.info("所有文件打包上传成功，result: {}", result);
}).exceptionally(ex -> {
log.error("异步处理任务失败", ex);
return null;
});
}
//将内容同步到minio中
private JSONObject fileSaveToMinIO(InputStream inputStream, String savePath,String fileName) throws Exception {
String path = savePath+"/" +fileName;
private JSONObject fileSaveToMinIO(InputStream inputStream, String savePath, String fileName) throws Exception {
String path = savePath + "/" + fileName;
ByteArrayInputStream byteArrayOutputStream = readAllBytes(inputStream);
minioUtil.uploadObject(bucketName, path, byteArrayOutputStream);
@ -380,7 +392,7 @@ public class CSCollectServiceImpl implements CSCollectService {
if (csAuthInfo == null) {
return;
}
redisService.setCacheObject(TOKEN_KEY, BeanUtil.beanToMap(csAuthInfo),10L, TimeUnit.MINUTES);
redisService.setCacheObject(TOKEN_KEY, BeanUtil.beanToMap(csAuthInfo), 10L, TimeUnit.MINUTES);
}
/**

102
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java
View File

@ -512,4 +512,106 @@ public class MinioUtil {
return new ByteArrayInputStream(zipOutputStream.toByteArray());
}
}
/**
* 创建目录的压缩包
*/
public void createZipFromDirectory(String bucket, Map<String,String> dirMap, String zipPath)
throws Exception {
// 创建内存中的ZIP流
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
// 遍历每个目录映射
for (Map.Entry<String, String> entry : dirMap.entrySet()) {
String folderName = entry.getKey(); // 压缩包内的文件夹名
String sourcePath = entry.getValue(); // MinIO中的路径
// 1. 自动提取目录路径（去除文件名）
String sourceDir = extractDirectoryPath(sourcePath);
// 2. 规范化文件夹名
if (!folderName.endsWith("/")) {
folderName = folderName + "/";
}
// 3. 获取目录下的所有文件
Iterable<Result<Item>> items = minioClient.listObjects(
ListObjectsArgs.builder()
.bucket(bucket)
.prefix(sourceDir)
.recursive(true)
.build()
);
// 4. 添加文件到ZIP
for (Result<Item> result : items) {
Item item = result.get();
if (item.isDir()) continue;
String objectName = item.objectName();
String relativePath = objectName.substring(sourceDir.length());
// 使用自定义的文件夹名
String zipEntryName = folderName + relativePath;
// ZIP条目
zos.putNextEntry(new ZipEntry(zipEntryName));
try (InputStream is = minioClient.getObject(
GetObjectArgs.builder()
.bucket(bucket)
.object(objectName)
.build())) {
// 使用缓冲区读写
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = is.read(buffer)) != -1) {
zos.write(buffer, 0, bytesRead);
}
}
zos.closeEntry();
}
}
zos.finish();
}
// 上传压缩包到MinIO
byte[] zipBytes = baos.toByteArray();
try (ByteArrayInputStream bais = new ByteArrayInputStream(zipBytes)) {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(bucket)
.object(zipPath)
.stream(bais, zipBytes.length, -1)
.contentType("application/zip")
.build()
);
}
}
private String extractDirectoryPath(String path) {
path = path.trim();
// 如果路径包含点号且不以斜杠结尾，可能是文件
if (path.contains(".") && !path.endsWith("/")) {
int lastSlash = path.lastIndexOf("/");
if (lastSlash > 0) {
// 返回目录部分
return path.substring(0, lastSlash + 1);
}
}
// 如果不是文件路径，直接返回
if (!path.endsWith("/")) {
return path + "/";
}
return path;
}
}