feat: 小模型上传文件改为分片上传
This commit is contained in:
zhaowei
parent
840dc7e286
commit
d63d306178
|
|
@ -93,7 +93,8 @@
|
|||
"react-dom": "^18.2.0",
|
||||
"react-draggable": "^4.4.6",
|
||||
"react-helmet-async": "^1.3.0",
|
||||
"react-highlight": "^0.15.0"
|
||||
"react-highlight": "^0.15.0",
|
||||
"spark-md5": "~3.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ant-design/pro-cli": "^3.1.0",
|
||||
|
|
@ -119,6 +120,7 @@
|
|||
"@types/react-dom": "^18.0.11",
|
||||
"@types/react-helmet": "^6.1.5",
|
||||
"@types/react-highlight": "^0.12.5",
|
||||
"@types/spark-md5": "~3.0.5",
|
||||
"@umijs/lint": "^4.0.66",
|
||||
"@umijs/max": "^4.0.66",
|
||||
"cross-env": "^7.0.3",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import KFIcon from '@/components/KFIcon';
|
|||
import KFModal from '@/components/KFModal';
|
||||
import { DataSource, ResourceType, resourceConfig } from '@/pages/Dataset/config';
|
||||
import { to } from '@/utils/promise';
|
||||
import { getFileListFromEvent, removeUploadedFile, validateUploadFiles } from '@/utils/ui';
|
||||
import { getFileListFromEvent, validateUploadFiles } from '@/utils/ui';
|
||||
import {
|
||||
Button,
|
||||
Form,
|
||||
|
|
@ -17,6 +17,7 @@ import {
|
|||
import { omit } from 'lodash';
|
||||
import { useEffect, useState } from 'react';
|
||||
import styles from '../AddDatasetModal/index.less';
|
||||
import useSplitUpload from './useSplitUpload';
|
||||
|
||||
interface AddVersionModalProps extends Omit<ModalProps, 'onOk'> {
|
||||
resourceType: ResourceType;
|
||||
|
|
@ -41,6 +42,7 @@ function AddVersionModal({
|
|||
const [uuid] = useState(Date.now());
|
||||
const config = resourceConfig[resourceType];
|
||||
const [form] = Form.useForm();
|
||||
const [sliptUploadRequest, cancelUpload] = useSplitUpload();
|
||||
|
||||
useEffect(() => {
|
||||
const getNextVersion = async () => {
|
||||
|
|
@ -67,7 +69,8 @@ function AddVersionModal({
|
|||
defaultFileList: [],
|
||||
beforeUpload: config.beforeUpload,
|
||||
accept: config.uploadAccept,
|
||||
onRemove: removeUploadedFile,
|
||||
customRequest: sliptUploadRequest,
|
||||
onRemove: cancelUpload,
|
||||
};
|
||||
|
||||
// 上传请求
|
||||
|
|
@ -85,11 +88,11 @@ function AddVersionModal({
|
|||
const fileList: UploadFile[] = formData['fileList'] ?? [];
|
||||
if (validateUploadFiles(fileList)) {
|
||||
const version_vos = fileList.map((item) => {
|
||||
const data = item.response?.data?.[0] ?? {};
|
||||
const data = item.response?.data ?? {};
|
||||
return {
|
||||
file_name: data.fileName,
|
||||
file_size: data.fileSize,
|
||||
url: data.url,
|
||||
file_name: data.filename,
|
||||
file_size: data.totalSize,
|
||||
url: data.location,
|
||||
};
|
||||
});
|
||||
const params = {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,393 @@
|
|||
import { getUploadChunkReq, mergeChunkReq, uploadChunkReq } from '@/services/dataset';
|
||||
import { generateUUID } from '@/utils';
|
||||
import { to } from '@/utils/promise';
|
||||
import { type UploadFile, type UploadProps } from 'antd';
|
||||
import PQueue from 'p-queue';
|
||||
import { useRef } from 'react';
|
||||
import SparkMD5 from 'spark-md5';
|
||||
|
||||
type OnProgress = ({ percent }: { percent: number }) => void;
|
||||
type FileUploadChunkQueueValue = {
|
||||
stop: () => void;
|
||||
cancelFlag: boolean;
|
||||
};
|
||||
|
||||
const useSplitUpload = () => {
|
||||
// 使用 useRef 来持久化存储队列对象，避免重新渲染时丢失
|
||||
const fileUploadChunkQueue = useRef<Record<string, FileUploadChunkQueueValue>>({});
|
||||
// 分片大小
|
||||
const DEFAULT_SIZE = 10 * 1024 * 1024;
|
||||
// 重试次数
|
||||
const RETRY_COUNT = 2;
|
||||
// 并发数
|
||||
const CONCURRENCY_COUNT = 5;
|
||||
|
||||
const cancelUpload = (file: UploadFile) => {
|
||||
// 停止特定文件的队列
|
||||
const fileUid = file.uid;
|
||||
if (fileUploadChunkQueue.current[fileUid]) {
|
||||
fileUploadChunkQueue.current[fileUid].cancelFlag = true;
|
||||
fileUploadChunkQueue.current[fileUid].stop();
|
||||
console.log(`已停止文件 ${file.name} 的上传`);
|
||||
|
||||
// 从队列中移除
|
||||
delete fileUploadChunkQueue.current[fileUid];
|
||||
return true;
|
||||
} else {
|
||||
console.warn(`未找到文件 ${file.name} 的上传队列`);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 取消所有文件的上传
|
||||
*/
|
||||
const cancelAllUploads = () => {
|
||||
// 停止所有上传队列
|
||||
Object.keys(fileUploadChunkQueue.current).forEach((fileUid) => {
|
||||
fileUploadChunkQueue.current[fileUid].cancelFlag = true;
|
||||
fileUploadChunkQueue.current[fileUid].stop();
|
||||
console.log(`已停止文件 ${fileUid} 的上传`);
|
||||
});
|
||||
|
||||
// 清空所有队列
|
||||
fileUploadChunkQueue.current = {};
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* 获取指定文件的取消状态
|
||||
*/
|
||||
const getCancelFlag = (fileUid: string): boolean => {
|
||||
return fileUploadChunkQueue.current[fileUid]?.cancelFlag || false;
|
||||
};
|
||||
|
||||
/**
|
||||
* 重置指定文件的取消状态
|
||||
*/
|
||||
const resetCancelFlag = (fileUid: string) => {
|
||||
if (fileUploadChunkQueue.current[fileUid]) {
|
||||
fileUploadChunkQueue.current[fileUid].cancelFlag = false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 获取一个上传任务，没有则初始化一个
|
||||
*/
|
||||
const getTaskInfo = async (file: any, identifier: string) => {
|
||||
const res = await getUploadChunkReq({
|
||||
chunkNumber: 1,
|
||||
chunkSize: DEFAULT_SIZE,
|
||||
currentChunkSize: Math.min(DEFAULT_SIZE, file.size),
|
||||
totalSize: file.size,
|
||||
identifier: identifier,
|
||||
filename: file.name,
|
||||
relativePath: file.name,
|
||||
totalChunks: Math.ceil(file.size / DEFAULT_SIZE),
|
||||
});
|
||||
return res;
|
||||
};
|
||||
|
||||
/**
|
||||
* 上传逻辑处理，如果文件已经上传完成（完成分块合并操作），则不会进入到此方法中
|
||||
*/
|
||||
const handleUpload = async (
|
||||
file: any,
|
||||
taskRecord: any,
|
||||
chunkNum: number,
|
||||
identifier: string,
|
||||
onProgress: OnProgress,
|
||||
) => {
|
||||
let uploadedSize = 0; // 已上传的大小
|
||||
const totalSize = file.size || 0; // 文件总大小
|
||||
const failedChunks = new Set<number>(); // 存储失败任务
|
||||
const partMsgList: any = []; // 分片文件集
|
||||
const { uploaded_chunks } = taskRecord;
|
||||
const uploadedChunk = uploaded_chunks || [];
|
||||
|
||||
// 创建队列实例，设置并发数为5
|
||||
const queue = new PQueue({
|
||||
concurrency: CONCURRENCY_COUNT,
|
||||
});
|
||||
|
||||
const uploadChunk = async (partNumber: number) => {
|
||||
// 检查该文件的取消标志
|
||||
if (getCancelFlag(file.uid)) {
|
||||
throw new Error('上传已取消');
|
||||
}
|
||||
|
||||
const start = Number(DEFAULT_SIZE) * (partNumber - 1);
|
||||
const end = Math.min(start + Number(DEFAULT_SIZE), totalSize);
|
||||
const blob = file.slice(start, end);
|
||||
const currentChunkSize = end - start;
|
||||
|
||||
const uploadData = new FormData();
|
||||
uploadData.append('chunkNumber', `${partNumber}`);
|
||||
uploadData.append('chunkSize', String(DEFAULT_SIZE));
|
||||
uploadData.append('currentChunkSize', String(currentChunkSize));
|
||||
uploadData.append('filename', file.name);
|
||||
uploadData.append('relativePath', file.name);
|
||||
uploadData.append('identifier', identifier);
|
||||
uploadData.append('totalChunks', String(chunkNum));
|
||||
uploadData.append('totalSize', totalSize);
|
||||
uploadData.append('upfile', new File([blob], `${partNumber}`));
|
||||
|
||||
const [res, error] = await to(uploadChunkReq(uploadData));
|
||||
if (error || !res) {
|
||||
failedChunks.add(partNumber);
|
||||
console.log(`分片${partNumber}上传失败`, error);
|
||||
throw new Error(`分片${partNumber}上传失败`);
|
||||
}
|
||||
|
||||
partMsgList.push({
|
||||
chunkIndex: partNumber,
|
||||
chunkSize: blob.size,
|
||||
etag: res.data,
|
||||
});
|
||||
|
||||
failedChunks.delete(partNumber);
|
||||
return blob.size;
|
||||
};
|
||||
|
||||
/**
|
||||
* 更新上传进度
|
||||
* @param increment 为已上传的进度增加的字节量
|
||||
*/
|
||||
const updateProcess = (increment: number) => {
|
||||
uploadedSize += increment;
|
||||
const percent = Math.min(100, (uploadedSize / totalSize) * 100);
|
||||
onProgress({ percent: Number(percent.toFixed(2)) });
|
||||
};
|
||||
|
||||
// 处理已上传的分片
|
||||
uploadedChunk.forEach((partNumber: number) => {
|
||||
const chunkSize =
|
||||
partNumber === chunkNum
|
||||
? totalSize % Number(DEFAULT_SIZE) || Number(DEFAULT_SIZE)
|
||||
: Number(DEFAULT_SIZE);
|
||||
updateProcess(chunkSize);
|
||||
});
|
||||
|
||||
// 获取待上传的分片
|
||||
const pendingChunks = Array.from({ length: chunkNum }, (_, i) => i + 1).filter(
|
||||
(partNumber) => !uploadedChunk.includes(partNumber),
|
||||
);
|
||||
|
||||
if (pendingChunks.length === 0) return [];
|
||||
|
||||
// 设置取消功能 - 更新队列中的 stop 方法
|
||||
fileUploadChunkQueue.current[file.uid] = {
|
||||
...fileUploadChunkQueue.current[file.uid],
|
||||
stop: () => {
|
||||
// 只设置当前文件的取消标志
|
||||
fileUploadChunkQueue.current[file.uid].cancelFlag = true;
|
||||
queue.clear(); // 清空未开始的任务
|
||||
queue.pause(); // 暂停队列
|
||||
console.log(`文件 ${file.uid} 的上传已被取消`);
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
// 第一轮上传
|
||||
const firstRoundTasks = pendingChunks.map((partNumber) => async () => {
|
||||
// 每次任务开始前检查该文件的取消状态
|
||||
if (getCancelFlag(file.uid)) {
|
||||
throw new Error('上传已取消');
|
||||
}
|
||||
|
||||
try {
|
||||
const chunkSize = await uploadChunk(partNumber);
|
||||
updateProcess(chunkSize);
|
||||
} catch (error) {
|
||||
// 如果是取消错误，直接抛出
|
||||
if ((error as Error).message === '上传已取消') {
|
||||
throw error;
|
||||
}
|
||||
console.warn(`分片${partNumber}上传失败:`, (error as Error).message);
|
||||
}
|
||||
});
|
||||
|
||||
await queue.addAll(firstRoundTasks);
|
||||
|
||||
// 检查该文件是否被取消
|
||||
if (getCancelFlag(file.uid)) {
|
||||
throw new Error('上传已取消');
|
||||
}
|
||||
|
||||
// 检查失败率
|
||||
const failureRate = failedChunks.size / pendingChunks.length;
|
||||
if (failureRate > 0.2) {
|
||||
throw new Error(`上传失败率过高 (${Math.round(failureRate * 100)}%)，请检查网络连接`);
|
||||
}
|
||||
|
||||
// 重试失败的分片（最多重试2次）
|
||||
if (failedChunks.size > 0) {
|
||||
console.log(`开始重试 ${failedChunks.size} 个失败分片`);
|
||||
|
||||
for (let retry = 1; retry <= RETRY_COUNT && failedChunks.size > 0; retry++) {
|
||||
// 每次重试前检查取消状态
|
||||
if (getCancelFlag(file.uid)) {
|
||||
throw new Error('上传已取消');
|
||||
}
|
||||
|
||||
const retryTasks = Array.from(failedChunks).map((partNumber) => async () => {
|
||||
// 每个任务开始前检查取消状态
|
||||
if (getCancelFlag(file.uid)) {
|
||||
throw new Error('上传已取消');
|
||||
}
|
||||
|
||||
try {
|
||||
const chunkSize = await uploadChunk(partNumber);
|
||||
updateProcess(chunkSize);
|
||||
console.log(`分片${partNumber}第${retry}次重试成功`);
|
||||
} catch (error) {
|
||||
// 如果是取消错误，直接抛出
|
||||
if ((error as Error).message === '上传已取消') {
|
||||
throw error;
|
||||
}
|
||||
console.warn(`分片${partNumber}第${retry}次重试失败`);
|
||||
}
|
||||
});
|
||||
|
||||
await queue.addAll(retryTasks);
|
||||
}
|
||||
}
|
||||
|
||||
// 最终检查取消状态
|
||||
if (getCancelFlag(file.uid)) {
|
||||
throw new Error('上传已取消');
|
||||
}
|
||||
|
||||
return Array.from(failedChunks);
|
||||
} finally {
|
||||
// 上传完成或取消后，清理队列引用
|
||||
delete fileUploadChunkQueue.current[file.uid];
|
||||
}
|
||||
};
|
||||
|
||||
const handleHttpRequest: UploadProps['customRequest'] = async ({
|
||||
onProgress,
|
||||
onError,
|
||||
onSuccess,
|
||||
file: uploadFile,
|
||||
}) => {
|
||||
const file = uploadFile as UploadFile;
|
||||
const md5 = await computeMD5(file);
|
||||
|
||||
// 开始新上传前初始化该文件的取消状态
|
||||
if (!fileUploadChunkQueue.current[file.uid]) {
|
||||
fileUploadChunkQueue.current[file.uid] = {
|
||||
stop: () => {}, // 先设置一个空函数，后面会覆盖
|
||||
cancelFlag: false,
|
||||
};
|
||||
} else {
|
||||
resetCancelFlag(file.uid);
|
||||
}
|
||||
|
||||
const [taskRes, taskError] = await to(getTaskInfo(file, md5));
|
||||
if (taskError || !taskRes) {
|
||||
onError?.(new Error('获取分片上传任务失败'));
|
||||
return;
|
||||
}
|
||||
|
||||
const task = taskRes.data;
|
||||
const { skip_upload: skipUpload } = task;
|
||||
const chunkNum = Math.ceil((file.size ?? 0) / DEFAULT_SIZE);
|
||||
|
||||
// 文件已经上传过的，获取记录
|
||||
if (skipUpload) {
|
||||
onSuccess?.({
|
||||
...taskRes,
|
||||
data: {
|
||||
location: taskRes.data.location,
|
||||
filename: file.name,
|
||||
totalSize: file.size,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 处理取消情况
|
||||
if (getCancelFlag(file.uid)) {
|
||||
onError?.(new Error('用户已取消'));
|
||||
return;
|
||||
}
|
||||
|
||||
const [failedChuncks, uploadError] = await to(
|
||||
handleUpload(file, task, chunkNum, md5, onProgress as OnProgress),
|
||||
);
|
||||
|
||||
// 处理取消情况
|
||||
if (getCancelFlag(file.uid)) {
|
||||
onError?.(new Error('用户已取消'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (uploadError || !failedChuncks) {
|
||||
onError?.(new Error('上传文件异常'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (failedChuncks.length !== 0) {
|
||||
onError?.(new Error('部分分片上传失败'));
|
||||
return;
|
||||
}
|
||||
|
||||
const [mergeRes] = await to(
|
||||
mergeChunkReq({
|
||||
fileType: file.type,
|
||||
name: file.name,
|
||||
relativePath: file.name,
|
||||
size: file.size,
|
||||
uniqueIdentifier: md5,
|
||||
refProjectId: '123456789', // refProjectId为预留字段，可关联附件所属目标，例如所属档案，所属工程等
|
||||
}),
|
||||
);
|
||||
|
||||
if (mergeRes) {
|
||||
onSuccess?.(mergeRes);
|
||||
} else {
|
||||
onError?.(new Error('文件合并失败'));
|
||||
}
|
||||
};
|
||||
|
||||
return [handleHttpRequest, cancelUpload, cancelAllUploads] as const;
|
||||
};
|
||||
|
||||
// 计算文件的md5，考虑性能问题，只算第一个分片的md5，如果失败获取一个uuid
|
||||
function computeMD5(file: any): Promise<string> {
|
||||
let fileReader = new FileReader();
|
||||
let time = new Date().getTime();
|
||||
let blobSlice = File.prototype.slice;
|
||||
const chunkSize = 10 * 1024 * 1024;
|
||||
let chunks = Math.ceil(file.size / chunkSize);
|
||||
let spark = new SparkMD5.ArrayBuffer();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
fileReader.onload = (e) => {
|
||||
if (e.target?.result) {
|
||||
spark.append(e.target.result as ArrayBuffer);
|
||||
const md5 = spark.end();
|
||||
resolve(md5);
|
||||
console.log(
|
||||
`MD5计算完毕：${file.name} \nMD5：${md5} \n分片：${chunks} 大小:${file.size} 用时：${
|
||||
new Date().getTime() - time
|
||||
} ms`,
|
||||
);
|
||||
} else {
|
||||
resolve(generateUUID());
|
||||
}
|
||||
};
|
||||
|
||||
fileReader.onerror = function () {
|
||||
resolve(generateUUID());
|
||||
};
|
||||
|
||||
//由于计算整个文件的Md5太慢，因此采用只计算第1块文件的md5的方式
|
||||
let start = 0;
|
||||
let end = chunkSize >= file.size ? file.size : start + chunkSize;
|
||||
fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
|
||||
});
|
||||
}
|
||||
|
||||
export default useSplitUpload;
|
||||
|
|
@ -269,4 +269,45 @@ export function getModelFileUrlReq(params) {
|
|||
});
|
||||
}
|
||||
|
||||
// ----------------------------大文件上传---------------------------------
|
||||
|
||||
// 创建上传任务
|
||||
export function getUploadChunkReq(params) {
|
||||
return request(`/api/mmp/uploader/chunk`, {
|
||||
method: 'GET',
|
||||
|
||||
params,
|
||||
skipLoading: true,
|
||||
});
|
||||
}
|
||||
|
||||
// 上传分片
|
||||
export function uploadChunkReq(data) {
|
||||
return request(`/api/mmp/uploader/chunk`, {
|
||||
method: 'POST',
|
||||
data,
|
||||
skipLoading: true,
|
||||
skipErrorHandler: true,
|
||||
timeout: 600 * 1000, // 10分钟
|
||||
});
|
||||
}
|
||||
|
||||
// 合并分片
|
||||
export function mergeChunkReq(data) {
|
||||
return request(`/api/mmp/uploader/mergeFile`, {
|
||||
method: 'POST',
|
||||
|
||||
data,
|
||||
skipLoading: true,
|
||||
});
|
||||
}
|
||||
|
||||
// 获取上传文件列表
|
||||
export function getUploadFileListReq(params) {
|
||||
return request(`/api/mmp/uploader/selectFileList`, {
|
||||
method: 'GET',
|
||||
|
||||
params,
|
||||
skipLoading: true,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ export function downLoadZip(url: string, params?: any) {
|
|||
params,
|
||||
responseType: 'blob',
|
||||
getResponse: true,
|
||||
timeout: 3600 * 1000, // 1小时
|
||||
}).then((res) => {
|
||||
resolveBlob(res, MimeType.ZIP);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue