From d63d306178efbe409ee9044c19a54c53bed6cd63 Mon Sep 17 00:00:00 2001 From: zhaowei Date: Mon, 13 Oct 2025 08:26:27 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=B0=8F=E6=A8=A1=E5=9E=8B=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E6=96=87=E4=BB=B6=E6=94=B9=E4=B8=BA=E5=88=86=E7=89=87?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- react-ui/package.json | 4 +- .../components/AddVersionModal/index.tsx | 15 +- .../AddVersionModal/useSplitUpload.ts | 393 ++++++++++++++++++ react-ui/src/services/dataset/index.js | 41 ++ react-ui/src/utils/downloadfile.ts | 1 + 5 files changed, 447 insertions(+), 7 deletions(-) create mode 100644 react-ui/src/pages/Dataset/components/AddVersionModal/useSplitUpload.ts diff --git a/react-ui/package.json b/react-ui/package.json index ae98cad7..fb1bec83 100644 --- a/react-ui/package.json +++ b/react-ui/package.json @@ -93,7 +93,8 @@ "react-dom": "^18.2.0", "react-draggable": "^4.4.6", "react-helmet-async": "^1.3.0", - "react-highlight": "^0.15.0" + "react-highlight": "^0.15.0", + "spark-md5": "~3.0.2" }, "devDependencies": { "@ant-design/pro-cli": "^3.1.0", @@ -119,6 +120,7 @@ "@types/react-dom": "^18.0.11", "@types/react-helmet": "^6.1.5", "@types/react-highlight": "^0.12.5", + "@types/spark-md5": "~3.0.5", "@umijs/lint": "^4.0.66", "@umijs/max": "^4.0.66", "cross-env": "^7.0.3", diff --git a/react-ui/src/pages/Dataset/components/AddVersionModal/index.tsx b/react-ui/src/pages/Dataset/components/AddVersionModal/index.tsx index cbce7040..b0a29d20 100644 --- a/react-ui/src/pages/Dataset/components/AddVersionModal/index.tsx +++ b/react-ui/src/pages/Dataset/components/AddVersionModal/index.tsx @@ -3,7 +3,7 @@ import KFIcon from '@/components/KFIcon'; import KFModal from '@/components/KFModal'; import { DataSource, ResourceType, resourceConfig } from '@/pages/Dataset/config'; import { to } from '@/utils/promise'; -import { getFileListFromEvent, removeUploadedFile, validateUploadFiles } from '@/utils/ui'; +import { getFileListFromEvent, validateUploadFiles } from '@/utils/ui'; import { Button, Form, @@ -17,6 +17,7 @@ import { import { omit } from 'lodash'; import { useEffect, useState } from 'react'; import styles from '../AddDatasetModal/index.less'; +import useSplitUpload from './useSplitUpload'; interface AddVersionModalProps extends Omit { resourceType: ResourceType; @@ -41,6 +42,7 @@ function AddVersionModal({ const [uuid] = useState(Date.now()); const config = resourceConfig[resourceType]; const [form] = Form.useForm(); + const [sliptUploadRequest, cancelUpload] = useSplitUpload(); useEffect(() => { const getNextVersion = async () => { @@ -67,7 +69,8 @@ function AddVersionModal({ defaultFileList: [], beforeUpload: config.beforeUpload, accept: config.uploadAccept, - onRemove: removeUploadedFile, + customRequest: sliptUploadRequest, + onRemove: cancelUpload, }; // 上传请求 @@ -85,11 +88,11 @@ function AddVersionModal({ const fileList: UploadFile[] = formData['fileList'] ?? []; if (validateUploadFiles(fileList)) { const version_vos = fileList.map((item) => { - const data = item.response?.data?.[0] ?? {}; + const data = item.response?.data ?? {}; return { - file_name: data.fileName, - file_size: data.fileSize, - url: data.url, + file_name: data.filename, + file_size: data.totalSize, + url: data.location, }; }); const params = { diff --git a/react-ui/src/pages/Dataset/components/AddVersionModal/useSplitUpload.ts b/react-ui/src/pages/Dataset/components/AddVersionModal/useSplitUpload.ts new file mode 100644 index 00000000..f6e1a68a --- /dev/null +++ b/react-ui/src/pages/Dataset/components/AddVersionModal/useSplitUpload.ts @@ -0,0 +1,393 @@ +import { getUploadChunkReq, mergeChunkReq, uploadChunkReq } from '@/services/dataset'; +import { generateUUID } from '@/utils'; +import { to } from '@/utils/promise'; +import { type UploadFile, type UploadProps } from 'antd'; +import PQueue from 'p-queue'; +import { useRef } from 'react'; +import SparkMD5 from 'spark-md5'; + +type OnProgress = ({ percent }: { percent: number }) => void; +type FileUploadChunkQueueValue = { + stop: () => void; + cancelFlag: boolean; +}; + +const useSplitUpload = () => { + // 使用 useRef 来持久化存储队列对象，避免重新渲染时丢失 + const fileUploadChunkQueue = useRef>({}); + // 分片大小 + const DEFAULT_SIZE = 10 * 1024 * 1024; + // 重试次数 + const RETRY_COUNT = 2; + // 并发数 + const CONCURRENCY_COUNT = 5; + + const cancelUpload = (file: UploadFile) => { + // 停止特定文件的队列 + const fileUid = file.uid; + if (fileUploadChunkQueue.current[fileUid]) { + fileUploadChunkQueue.current[fileUid].cancelFlag = true; + fileUploadChunkQueue.current[fileUid].stop(); + console.log(`已停止文件 ${file.name} 的上传`); + + // 从队列中移除 + delete fileUploadChunkQueue.current[fileUid]; + return true; + } else { + console.warn(`未找到文件 ${file.name} 的上传队列`); + return true; + } + }; + + /** + * 取消所有文件的上传 + */ + const cancelAllUploads = () => { + // 停止所有上传队列 + Object.keys(fileUploadChunkQueue.current).forEach((fileUid) => { + fileUploadChunkQueue.current[fileUid].cancelFlag = true; + fileUploadChunkQueue.current[fileUid].stop(); + console.log(`已停止文件 ${fileUid} 的上传`); + }); + + // 清空所有队列 + fileUploadChunkQueue.current = {}; + return true; + }; + + /** + * 获取指定文件的取消状态 + */ + const getCancelFlag = (fileUid: string): boolean => { + return fileUploadChunkQueue.current[fileUid]?.cancelFlag || false; + }; + + /** + * 重置指定文件的取消状态 + */ + const resetCancelFlag = (fileUid: string) => { + if (fileUploadChunkQueue.current[fileUid]) { + fileUploadChunkQueue.current[fileUid].cancelFlag = false; + } + }; + + /** + * 获取一个上传任务，没有则初始化一个 + */ + const getTaskInfo = async (file: any, identifier: string) => { + const res = await getUploadChunkReq({ + chunkNumber: 1, + chunkSize: DEFAULT_SIZE, + currentChunkSize: Math.min(DEFAULT_SIZE, file.size), + totalSize: file.size, + identifier: identifier, + filename: file.name, + relativePath: file.name, + totalChunks: Math.ceil(file.size / DEFAULT_SIZE), + }); + return res; + }; + + /** + * 上传逻辑处理，如果文件已经上传完成（完成分块合并操作），则不会进入到此方法中 + */ + const handleUpload = async ( + file: any, + taskRecord: any, + chunkNum: number, + identifier: string, + onProgress: OnProgress, + ) => { + let uploadedSize = 0; // 已上传的大小 + const totalSize = file.size || 0; // 文件总大小 + const failedChunks = new Set(); // 存储失败任务 + const partMsgList: any = []; // 分片文件集 + const { uploaded_chunks } = taskRecord; + const uploadedChunk = uploaded_chunks || []; + + // 创建队列实例，设置并发数为5 + const queue = new PQueue({ + concurrency: CONCURRENCY_COUNT, + }); + + const uploadChunk = async (partNumber: number) => { + // 检查该文件的取消标志 + if (getCancelFlag(file.uid)) { + throw new Error('上传已取消'); + } + + const start = Number(DEFAULT_SIZE) * (partNumber - 1); + const end = Math.min(start + Number(DEFAULT_SIZE), totalSize); + const blob = file.slice(start, end); + const currentChunkSize = end - start; + + const uploadData = new FormData(); + uploadData.append('chunkNumber', `${partNumber}`); + uploadData.append('chunkSize', String(DEFAULT_SIZE)); + uploadData.append('currentChunkSize', String(currentChunkSize)); + uploadData.append('filename', file.name); + uploadData.append('relativePath', file.name); + uploadData.append('identifier', identifier); + uploadData.append('totalChunks', String(chunkNum)); + uploadData.append('totalSize', totalSize); + uploadData.append('upfile', new File([blob], `${partNumber}`)); + + const [res, error] = await to(uploadChunkReq(uploadData)); + if (error || !res) { + failedChunks.add(partNumber); + console.log(`分片${partNumber}上传失败`, error); + throw new Error(`分片${partNumber}上传失败`); + } + + partMsgList.push({ + chunkIndex: partNumber, + chunkSize: blob.size, + etag: res.data, + }); + + failedChunks.delete(partNumber); + return blob.size; + }; + + /** + * 更新上传进度 + * @param increment 为已上传的进度增加的字节量 + */ + const updateProcess = (increment: number) => { + uploadedSize += increment; + const percent = Math.min(100, (uploadedSize / totalSize) * 100); + onProgress({ percent: Number(percent.toFixed(2)) }); + }; + + // 处理已上传的分片 + uploadedChunk.forEach((partNumber: number) => { + const chunkSize = + partNumber === chunkNum + ? totalSize % Number(DEFAULT_SIZE) || Number(DEFAULT_SIZE) + : Number(DEFAULT_SIZE); + updateProcess(chunkSize); + }); + + // 获取待上传的分片 + const pendingChunks = Array.from({ length: chunkNum }, (_, i) => i + 1).filter( + (partNumber) => !uploadedChunk.includes(partNumber), + ); + + if (pendingChunks.length === 0) return []; + + // 设置取消功能 - 更新队列中的 stop 方法 + fileUploadChunkQueue.current[file.uid] = { + ...fileUploadChunkQueue.current[file.uid], + stop: () => { + // 只设置当前文件的取消标志 + fileUploadChunkQueue.current[file.uid].cancelFlag = true; + queue.clear(); // 清空未开始的任务 + queue.pause(); // 暂停队列 + console.log(`文件 ${file.uid} 的上传已被取消`); + }, + }; + + try { + // 第一轮上传 + const firstRoundTasks = pendingChunks.map((partNumber) => async () => { + // 每次任务开始前检查该文件的取消状态 + if (getCancelFlag(file.uid)) { + throw new Error('上传已取消'); + } + + try { + const chunkSize = await uploadChunk(partNumber); + updateProcess(chunkSize); + } catch (error) { + // 如果是取消错误，直接抛出 + if ((error as Error).message === '上传已取消') { + throw error; + } + console.warn(`分片${partNumber}上传失败:`, (error as Error).message); + } + }); + + await queue.addAll(firstRoundTasks); + + // 检查该文件是否被取消 + if (getCancelFlag(file.uid)) { + throw new Error('上传已取消'); + } + + // 检查失败率 + const failureRate = failedChunks.size / pendingChunks.length; + if (failureRate > 0.2) { + throw new Error(`上传失败率过高 (${Math.round(failureRate * 100)}%)，请检查网络连接`); + } + + // 重试失败的分片（最多重试2次） + if (failedChunks.size > 0) { + console.log(`开始重试 ${failedChunks.size} 个失败分片`); + + for (let retry = 1; retry <= RETRY_COUNT && failedChunks.size > 0; retry++) { + // 每次重试前检查取消状态 + if (getCancelFlag(file.uid)) { + throw new Error('上传已取消'); + } + + const retryTasks = Array.from(failedChunks).map((partNumber) => async () => { + // 每个任务开始前检查取消状态 + if (getCancelFlag(file.uid)) { + throw new Error('上传已取消'); + } + + try { + const chunkSize = await uploadChunk(partNumber); + updateProcess(chunkSize); + console.log(`分片${partNumber}第${retry}次重试成功`); + } catch (error) { + // 如果是取消错误，直接抛出 + if ((error as Error).message === '上传已取消') { + throw error; + } + console.warn(`分片${partNumber}第${retry}次重试失败`); + } + }); + + await queue.addAll(retryTasks); + } + } + + // 最终检查取消状态 + if (getCancelFlag(file.uid)) { + throw new Error('上传已取消'); + } + + return Array.from(failedChunks); + } finally { + // 上传完成或取消后，清理队列引用 + delete fileUploadChunkQueue.current[file.uid]; + } + }; + + const handleHttpRequest: UploadProps['customRequest'] = async ({ + onProgress, + onError, + onSuccess, + file: uploadFile, + }) => { + const file = uploadFile as UploadFile; + const md5 = await computeMD5(file); + + // 开始新上传前初始化该文件的取消状态 + if (!fileUploadChunkQueue.current[file.uid]) { + fileUploadChunkQueue.current[file.uid] = { + stop: () => {}, // 先设置一个空函数，后面会覆盖 + cancelFlag: false, + }; + } else { + resetCancelFlag(file.uid); + } + + const [taskRes, taskError] = await to(getTaskInfo(file, md5)); + if (taskError || !taskRes) { + onError?.(new Error('获取分片上传任务失败')); + return; + } + + const task = taskRes.data; + const { skip_upload: skipUpload } = task; + const chunkNum = Math.ceil((file.size ?? 0) / DEFAULT_SIZE); + + // 文件已经上传过的，获取记录 + if (skipUpload) { + onSuccess?.({ + ...taskRes, + data: { + location: taskRes.data.location, + filename: file.name, + totalSize: file.size, + }, + }); + return; + } + + // 处理取消情况 + if (getCancelFlag(file.uid)) { + onError?.(new Error('用户已取消')); + return; + } + + const [failedChuncks, uploadError] = await to( + handleUpload(file, task, chunkNum, md5, onProgress as OnProgress), + ); + + // 处理取消情况 + if (getCancelFlag(file.uid)) { + onError?.(new Error('用户已取消')); + return; + } + + if (uploadError || !failedChuncks) { + onError?.(new Error('上传文件异常')); + return; + } + + if (failedChuncks.length !== 0) { + onError?.(new Error('部分分片上传失败')); + return; + } + + const [mergeRes] = await to( + mergeChunkReq({ + fileType: file.type, + name: file.name, + relativePath: file.name, + size: file.size, + uniqueIdentifier: md5, + refProjectId: '123456789', // refProjectId为预留字段，可关联附件所属目标，例如所属档案，所属工程等 + }), + ); + + if (mergeRes) { + onSuccess?.(mergeRes); + } else { + onError?.(new Error('文件合并失败')); + } + }; + + return [handleHttpRequest, cancelUpload, cancelAllUploads] as const; +}; + +// 计算文件的md5，考虑性能问题，只算第一个分片的md5，如果失败获取一个uuid +function computeMD5(file: any): Promise { + let fileReader = new FileReader(); + let time = new Date().getTime(); + let blobSlice = File.prototype.slice; + const chunkSize = 10 * 1024 * 1024; + let chunks = Math.ceil(file.size / chunkSize); + let spark = new SparkMD5.ArrayBuffer(); + + return new Promise((resolve) => { + fileReader.onload = (e) => { + if (e.target?.result) { + spark.append(e.target.result as ArrayBuffer); + const md5 = spark.end(); + resolve(md5); + console.log( + `MD5计算完毕：${file.name} \nMD5：${md5} \n分片：${chunks} 大小:${file.size} 用时：${ + new Date().getTime() - time + } ms`, + ); + } else { + resolve(generateUUID()); + } + }; + + fileReader.onerror = function () { + resolve(generateUUID()); + }; + + //由于计算整个文件的Md5太慢，因此采用只计算第1块文件的md5的方式 + let start = 0; + let end = chunkSize >= file.size ? file.size : start + chunkSize; + fileReader.readAsArrayBuffer(blobSlice.call(file, start, end)); + }); +} + +export default useSplitUpload; diff --git a/react-ui/src/services/dataset/index.js b/react-ui/src/services/dataset/index.js index 9af0ed49..38d6042c 100644 --- a/react-ui/src/services/dataset/index.js +++ b/react-ui/src/services/dataset/index.js @@ -269,4 +269,45 @@ export function getModelFileUrlReq(params) { }); } +// ----------------------------大文件上传--------------------------------- +// 创建上传任务 +export function getUploadChunkReq(params) { + return request(`/api/mmp/uploader/chunk`, { + method: 'GET', + + params, + skipLoading: true, + }); +} + +// 上传分片 +export function uploadChunkReq(data) { + return request(`/api/mmp/uploader/chunk`, { + method: 'POST', + data, + skipLoading: true, + skipErrorHandler: true, + timeout: 600 * 1000, // 10分钟 + }); +} + +// 合并分片 +export function mergeChunkReq(data) { + return request(`/api/mmp/uploader/mergeFile`, { + method: 'POST', + + data, + skipLoading: true, + }); +} + +// 获取上传文件列表 +export function getUploadFileListReq(params) { + return request(`/api/mmp/uploader/selectFileList`, { + method: 'GET', + + params, + skipLoading: true, + }); +} diff --git a/react-ui/src/utils/downloadfile.ts b/react-ui/src/utils/downloadfile.ts index 024c7db3..7ca74401 100644 --- a/react-ui/src/utils/downloadfile.ts +++ b/react-ui/src/utils/downloadfile.ts @@ -43,6 +43,7 @@ export function downLoadZip(url: string, params?: any) { params, responseType: 'blob', getResponse: true, + timeout: 3600 * 1000, // 1小时 }).then((res) => { resolveBlob(res, MimeType.ZIP); });