diff --git a/react-ui/config/config.ts b/react-ui/config/config.ts index 4ac775d9..01e35e73 100644 --- a/react-ui/config/config.ts +++ b/react-ui/config/config.ts @@ -179,20 +179,20 @@ export default defineConfig({ { name: 'flint', // 数据服务微应用 // entry: 'http://localhost:8083', // 开发环境地址 - entry: '/flint', // 生产环境地址 - activeRule: '/flint', // 激活规则 + entry: '/tools/flint', // 生产环境地址 + activeRule: '/tools/flint', // 激活规则 }, { name: 'trainfer', // 服务管理 // entry: 'http://localhost:8081', // 开发环境地址 - entry: '/trainfer', // 生产环境地址 - activeRule: '/trainfer', // 激活规则 + entry: '/tools/trainfer', // 生产环境地址 + activeRule: '/tools/trainfer', // 激活规则 }, { name: 'portal', // 服务管理 // entry: 'http://localhost:8082', // 开发环境地址 - entry: '/portal', // 生产环境地址 - activeRule: '/portal', // 激活规则 + entry: '/tools/portal', // 生产环境地址 + activeRule: '/tools/portal', // 激活规则 }, ] : [], diff --git a/react-ui/config/routes.ts b/react-ui/config/routes.ts index d601956c..73e61f4f 100644 --- a/react-ui/config/routes.ts +++ b/react-ui/config/routes.ts @@ -528,6 +528,12 @@ export default [ key: 'intelligent-manage-execute', component: './Application/IntelligentExec', }, + { + name: 'API 文档', + path: 'api/:versionId', + key: 'api', + component: './Application/Info/components/SwaggerDocs', + }, ], }, ], @@ -765,30 +771,50 @@ export default [ ...(process.env.UMI_APP_HUO_SHI ? [ { - path: '/flint/*', + path: '/tools/flint/*', name: '数据管理', microApp: 'flint', layout: true, }, { - path: '/trainfer/*', + path: '/tools/trainfer/*', name: '火石服务', microApp: 'trainfer', layout: true, }, { - path: '/portal/*', + path: '/tools/portal/*', name: '火石服务', microApp: 'portal', layout: true, }, + { + path: '/tools/flint/data', + redirect: '/tools/flint/data/mark/list', + }, + { + path: '/tools/portal/enterprise', + redirect: '/tools/portal/enterprise/storage/index', + }, + { + path: '/tools/flint/bml', + redirect: '/tools/flint/bml/bml/train/sft', + }, + { + path: '/tools/flint/bml/bml', + redirect: '/tools/flint/bml/bml/train/sft', + }, + { + path: '/tools/trainfer', + redirect: '/tools/trainfer/service/ai', + }, // 数据标注页面 - 使用自定义组件包装微应用 - // { - // path: '/flint-data/mark/list', - // name: '数据标注', - // component: './DatasetPreparation/mark/index', - // layout: true, - // }, + { + path: '/tools/flint/data/mark/list', + name: '数据标注', + component: './DatasetPreparation/mark/index', + layout: true, + }, // // 数据服务子路径 - 直接路由到微应用（排除 mark/list） // { // path: '/flint-data/*', diff --git a/react-ui/package.json b/react-ui/package.json index 4d97d864..2f3e506d 100644 --- a/react-ui/package.json +++ b/react-ui/package.json @@ -9,8 +9,8 @@ "build-with-huoshi": "cross-env UMI_APP_HUO_SHI=true max build", "deploy": "sh ./scripts/deploy.sh", "dev": "npm run start:dev", - "dev-no-sso": "cross-env NO_SSO=true npm run start:dev", - "dev-with-huoshi": "cross-env NO_SSO=true UMI_APP_HUO_SHI=true npm run start:dev", + "dev-no-sso": "cross-env NO_SSO=true npm run start:mock", + "dev-with-huoshi": "cross-env NO_SSO=true UMI_APP_HUO_SHI=true npm run start:mock", "docker-hub:build": "docker build -f Dockerfile.hub -t ant-design-pro ./", "docker-prod:build": "docker-compose -f ./docker/docker-compose.yml build", "docker-prod:dev": "docker-compose -f ./docker/docker-compose.yml up", @@ -101,7 +101,8 @@ "react-helmet-async": "^1.3.0", "react-highlight": "^0.15.0", "react-markdown": "~10.1.0", - "spark-md5": "~3.0.2" + "spark-md5": "~3.0.2", + "swagger-ui-react": "~5.32.0" }, "devDependencies": { "@ant-design/pro-cli": "^3.1.0", @@ -128,6 +129,7 @@ "@types/react-helmet": "^6.1.5", "@types/react-highlight": "^0.12.5", "@types/spark-md5": "~3.0.5", + "@types/swagger-ui-react": "~5.18.0", "@umijs/lint": "^4.0.66", "@umijs/max": "^4.0.66", "cross-env": "^7.0.3", @@ -191,4 +193,4 @@ "public" ] } -} \ No newline at end of file +} diff --git a/react-ui/src/components/RightContent/AvatarDropdown.tsx b/react-ui/src/components/RightContent/AvatarDropdown.tsx index 798cb370..473d4b39 100644 --- a/react-ui/src/components/RightContent/AvatarDropdown.tsx +++ b/react-ui/src/components/RightContent/AvatarDropdown.tsx @@ -84,7 +84,9 @@ const AvatarDropdown: React.FC = ({ menu, isHome = false // 退出数据标注平台 const [res] = await to(getLabelStudioUrl()); if (res && res.data) { - oauthLogout(`${res.data}/oauth/logout`); + const base = res.data; + const url = base.endsWith('/') ? base : `${base}/`; + oauthLogout(`${url}oauth/logout`); } // 至少 2 秒后跳转，希望子系统能完成注销 await Promise.all([logout(), sleep(2000)]); diff --git a/react-ui/src/enums/index.ts b/react-ui/src/enums/index.ts index dc51e411..44755d07 100644 --- a/react-ui/src/enums/index.ts +++ b/react-ui/src/enums/index.ts @@ -209,14 +209,17 @@ export const bigDataTypeOptions = [ export enum BigDataMarkStatus { Unlabel = 0, Labeled = 1, + Labeling = 2, + LabeledComplete = 3, + NoNeedLabel = 4, } export const bigDataMarkStatusOptions = [ { label: '未标注', value: BigDataMarkStatus.Unlabel }, { label: '已标注', value: BigDataMarkStatus.Labeled }, - { label: '标注中', value: 2 }, - { label: '标注完成', value: 3 }, - { label: '不需要标注', value: 4 }, + { label: '标注中', value: BigDataMarkStatus.Labeling }, + { label: '标注完成', value: BigDataMarkStatus.LabeledComplete }, + { label: '不需要标注', value: BigDataMarkStatus.NoNeedLabel }, ]; // 大模型数据集启用预标 @@ -246,54 +249,85 @@ export const bigMarkToolOptions = [ value: 17, tooltip: '支持对同一个问题生成多个（5个）答案，用户可对答案进行打分和排序', }, + { + label: '正负偏好标注', + value: 19, + tooltip: undefined, + }, ]; -// 大模型数据集数据样式 +// 大模型数据集数据样式/SFT export const bigDataPatternOptions_sft = [ { label: 'input+target', value: 'Q_A', - tooltip: '对话类数据，问题（input）和答案（target）一一对应。', + tooltip: `对话类数据，问题（input）和答案（target）一一对应。\n例如：{ "input": "太阳系有哪些行星？", "target": "太阳系有八大行星，分别是水星、金星、地球、火星、木星、土星、天王星和海王星。" }`, }, { label: 'input', value: 'Q', - tooltip: '对话类数据，仅包含问题（input），答案（target）需要通过平台进行标注。', + tooltip: `对话类数据，仅包含问题（input），答案（target）需要通过平台进行标注。\n例如：{ "input": "太阳系有哪些行星？" }`, }, ]; -// 大模型数据集数据样式 +// 大模型数据集数据样式/奖励模型 export const bigDataPatternOptions_rewardmodel = [ { label: 'input+多target（含排序）', value: 'Q_5A', - tooltip: - '对话类数据，单个问题（input）对应多个答案（target），并且每个答案有对应的得分和排序。', + tooltip: `对话类数据，单个问题（input）对应多个答案（target），并且每个答案有对应的得分和排序。 +例如： +{ + "input": "社会科学研究方法的书属于什么类别？", + "targets": [ + { "content": "社会科学研究方法的图书属于C1", "score": 1, "level": "A" }, + { "content": "社会科学研究方法的图书属于C2", "score": 2, "level": "A" } + ] +}`, }, { label: 'input', value: 'Q', - tooltip: - '对话类数据，仅包含问题（input），答案（target）和对应的得分、排序，需要通过平台进行标注。', + tooltip: `对话类数据，仅包含问题（input），答案（target）和对应的得分、排序，需要通过平台进行标注。\n例如：{ "input": "社会科学研究方法的书属于什么类别？" }`, }, ]; -// 大模型数据集数据样式 +// 大模型数据集数据样式/强化学习 export const bigDataPatternOptions_learning = [ { label: 'input', value: 'Q_0', - tooltip: '对话类数据，仅包含问题（input），并且不需要进行标注。', + tooltip: `对话类数据，仅包含问题（input），并且不需要进行标注。\n例如：{ "input": "太阳系有哪些行星？", "target": "0" }`, }, ]; -// 大模型数据集数据样式 +// 大模型数据集数据样式/预训练 export const bigDataPatternOptions_pretraining = [ - { label: '纯文本', value: 'TEXT', tooltip: '纯文本类数据，可直接用于发起预训练任务。' }, + { + label: '纯文本', + value: 'TEXT', + tooltip: + '纯文本类数据，可直接用于发起预训练任务。\n例如：伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。', + }, { label: 'PDF', value: 'PDF', - tooltip: 'PDF类数据，不可直接用于预训练，需要经过“数据格式转换”并转存后，才可以进行预训练。', + tooltip: + 'PDF类数据，不可直接用于预训练，需要经过“数据格式转换”并转存后，才可以进行预训练。\n例如：伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。', + }, +]; + +// 大模型数据集数据样式/SFT +export const bigDataPatternOptions_dpo = [ + { + label: 'input+target+targetlose', + value: 'Q_A_L', + tooltip: `对话类数据，问题（input）、答案（target）、弃用答案（targetlose）三者共同组成一条数据，其中弃用答案表示效果较差的回复。\n例如：{ "input": "一篇文章有960个字。他们分别需要多少分钟才能录完？", "target": "您需要告诉我前置条件，他们每人输入的速度，我才能计算", "targetlose": "我不知道。" }`, + }, + { + label: 'input', + value: 'Q', + tooltip: `对话类数据，仅包含问题（input），答案（target）和弃用答案（targetlose）需要通过平台进行标注。\n例如：{ "input": "一篇文章有960个字。他们分别需要多少分钟才能录完？" }`, }, ]; diff --git a/react-ui/src/global.less b/react-ui/src/global.less index d5625c9a..33edf258 100644 --- a/react-ui/src/global.less +++ b/react-ui/src/global.less @@ -18,9 +18,11 @@ body, .ant-layout { min-height: 100vh; } + .ant-pro-sider.ant-layout-sider.ant-pro-sider-fixed { left: unset; } + canvas { display: block; } @@ -30,10 +32,12 @@ body { -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } + .ant-pro-layout .ant-pro-layout-content { padding: 0; background-color: transparent; } + .ant-pro-layout .ant-pro-layout-bg-list { background: @background-color; } @@ -45,43 +49,55 @@ body { .ant-pro-layout .ant-pro-sider-menu { padding-top: 35px; } + .ant-pro-global-header-logo-mix { padding-left: 12px; } + .ant-pro-layout .ant-pro-sider .ant-layout-sider-children { border-right: unset; } + .ant-pro-base-menu-inline { border-radius: 0px 20px 20px 0px; } + .ant-drawer .ant-drawer-body { padding: 0; } + .ant-drawer .ant-drawer-body .ant-row { padding: 0 24px; } + .ant-drawer .ant-drawer-body .ant-form-item { margin-bottom: 20px; } + .ant-menu .ant-menu-submenu-title .anticon { font-size: 16px; } + .ant-table-wrapper .ant-table-pagination.ant-pagination { margin: 0; padding: 20px 16px; background-color: #fff; } + .ant-pro-global-header-logo img { height: 32px; } + .ant-pro-layout .ant-layout-sider.ant-pro-sider { height: 100vh; padding-top: 60px; } + .ant-pro-layout .ant-pro-layout-container { height: 100vh; overflow-y: hidden; } + .ant-pagination .ant-pagination-item.ant-pagination-item-active { background: @primary-color; border-width: 0; @@ -90,11 +106,13 @@ body { color: #fff; } } + .ant-pagination .ant-pagination-item-active:hover { color: #fff; background: rgba(22, 100, 255, 0.8); border-color: rgba(22, 100, 255, 0.8); } + .ant-pagination .ant-pagination-item { border: 1px solid #e6e6e6; } @@ -104,17 +122,20 @@ body { height: 5px; background: transparent; } + ::-webkit-scrollbar-thumb { width: 5px; height: 5px; background: rgba(0, 0, 0, 0.2); border-radius: 99px; } + ::-webkit-scrollbar-track { width: 5px; height: 5px; background: transparent; } + ul, ol { list-style: none; @@ -124,12 +145,15 @@ ol { .ant-table { width: 100%; overflow-x: auto; - &-thead > tr, - &-tbody > tr { - > th, - > td { + + &-thead>tr, + &-tbody>tr { + + >th, + >td { white-space: pre; - > span { + + >span { display: block; } } @@ -176,9 +200,10 @@ input:-webkit-autofill { padding-top: 100px; padding-bottom: 30px; } + // qiankun 微前端容器样式 .qiankun-micro-app-container, -.qiankun-micro-app-container > div { +.qiankun-micro-app-container>div { width: 100%; height: 100%; overflow: hidden; @@ -187,6 +212,7 @@ input:-webkit-autofill { .form-list-add-btn:not(:disabled) { border-color: .addAlpha(@primary-color, 0.5) []; box-shadow: none !important; + &:hover { border-style: solid; } @@ -219,3 +245,10 @@ input:-webkit-autofill { width: 100%; } } + +.pre-line-tooltip { + .ant-tooltip-inner { + white-space: pre-line; + width: 500px; + } +} \ No newline at end of file diff --git a/react-ui/src/huoshi.less b/react-ui/src/huoshi.less index 3b9d04ff..fdf741e8 100644 --- a/react-ui/src/huoshi.less +++ b/react-ui/src/huoshi.less @@ -371,4 +371,15 @@ .qiankun-micro-app-container{ padding: 70px 24px 24px; -} \ No newline at end of file +} + +.el-dropdown-link { + outline: none; + color: var(--el-color-primary); + display: flex; + align-items: center; + cursor: pointer; + .el-icon { + margin-left: 4px; + } +} diff --git a/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.less b/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.less index 6e64f168..2ebf291c 100644 --- a/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.less +++ b/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.less @@ -1,6 +1,23 @@ -.swagger-api { - height: 100%; - overflow-y: scroll; +.swagger-docs { + &--component { + height: 100%; + background-color: white; + border-radius: 0px 10px 10px 10px; + overflow-y: auto; + } + + &--page { + height: 100%; + padding: @page-padding-top @content-padding @page-padding-bottom; + .backgroundPageImage(url(@/assets/img/app/app-bg-2.png)); + } + + &--page &__content { + height: 100%; + background-color: white; + border-radius: 10px; + overflow-y: auto; + } :global { .swagger-ui .scheme-container { diff --git a/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.tsx b/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.tsx index 676a8c0f..289665de 100644 --- a/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.tsx +++ b/react-ui/src/pages/Application/Info/components/SwaggerDocs/index.tsx @@ -1,34 +1,52 @@ import { ApplicationSource } from '@/pages/Application/types'; -import { useTypedSourceContext } from '@/pages/Application/useSourceContext'; import { getTaskDocsReq } from '@/services/modelDeployment'; import { to } from '@/utils/promise'; +import { useParams, useSearchParams } from '@umijs/max'; +import classNames from 'classnames'; import { useEffect, useState } from 'react'; import SwaggerUI from 'swagger-ui-react'; import 'swagger-ui-react/swagger-ui.css'; import styles from './index.less'; export type DrawerTitleProps = { - serviceId: number; + versionId: number; + source: ApplicationSource; + isComponent?: boolean; }; -function SwaggerDocs({ serviceId }: DrawerTitleProps) { - const { source = ApplicationSource.Platform } = useTypedSourceContext(); +function SwaggerDocs({ versionId, source, isComponent = false }: DrawerTitleProps) { const [spec, setSpec] = useState(); + const params = useParams(); + const paramsVersionId = params.versionId; + const [searchParams] = useSearchParams(); + const serviceSource = searchParams.get('source'); + + const uniId = versionId ?? paramsVersionId; + const uniSource = source ?? serviceSource; useEffect(() => { const getTaskDocs = async () => { - const [res] = await to(getTaskDocsReq(source, serviceId)); - if (res && res.data && res.data.docs) { - setSpec(res.data.docs); + const [res] = await to(getTaskDocsReq(uniSource, uniId)); + if (res && res.data) { + setSpec(res.data); } }; getTaskDocs(); - }, [source, serviceId]); + }, [uniSource, uniId]); // url="https://petstore.swagger.io/v2/swagger.json" - return
{spec && }
; + return ( +
+
{spec && }
+
+ ); } export default SwaggerDocs; diff --git a/react-ui/src/pages/Application/Info/components/VersionList/index.tsx b/react-ui/src/pages/Application/Info/components/VersionList/index.tsx index e8b42e54..2a2f15dd 100644 --- a/react-ui/src/pages/Application/Info/components/VersionList/index.tsx +++ b/react-ui/src/pages/Application/Info/components/VersionList/index.tsx @@ -113,7 +113,7 @@ function VersionList({ minWidth: 100, }, operation: { - fixedWidth: 280, + fixedWidth: 340, }, }), [], @@ -276,6 +276,11 @@ function VersionList({ } }; + // api 文档 + const toAPIDocs = (record: ServiceVersionData) => { + onNavigate(`api/${record.id}?source=${serviceSource}`, getParams()); + }; + // 分页切换 const handleTableChange: TableProps['onChange'] = ( pagination, @@ -459,6 +464,17 @@ function VersionList({ > 测试 + + ), }, diff --git a/react-ui/src/pages/Application/Info/index.tsx b/react-ui/src/pages/Application/Info/index.tsx index 93b25714..530f0496 100644 --- a/react-ui/src/pages/Application/Info/index.tsx +++ b/react-ui/src/pages/Application/Info/index.tsx @@ -150,6 +150,7 @@ function AppInfo() { const handleDetailChange = (value: string | undefined) => { updateService({ detail: value ?? '', + service_name: serviceInfo?.service_name, }); }; @@ -157,6 +158,7 @@ function AppInfo() { const handleManualChange = (value: string | undefined) => { updateService({ manual: value ?? '', + service_name: serviceInfo?.service_name, }); }; @@ -172,6 +174,7 @@ function AppInfo() { is_public, public_version_id: publishVersion, source: serviceInfo!.source, + service_name: serviceInfo?.service_name, }); }, }); @@ -184,6 +187,7 @@ function AppInfo() { updateService({ is_public, public_version_id: serviceInfo?.public_version_id, + service_name: serviceInfo?.service_name, }); }, }); @@ -293,11 +297,19 @@ function AppInfo() { > ), }, - serviceInfo!.source === ApplicationSource.Platform + serviceInfo!.source === ApplicationSource.Platform && + serviceInfo!.is_public === true && + serviceInfo!.public_version_id ? { key: AppInfoTab.Docs, label: `API 文档`, - children: , + children: ( + + ), } : undefined, ].filter((v) => v !== undefined); diff --git a/react-ui/src/pages/Authorize/index.tsx b/react-ui/src/pages/Authorize/index.tsx index d9b71f8f..b7018e44 100644 --- a/react-ui/src/pages/Authorize/index.tsx +++ b/react-ui/src/pages/Authorize/index.tsx @@ -42,7 +42,7 @@ function Authorize() { message.success('登录成功！'); if (uapLogoutUri) { - SessionStorage.setItem(uapLogoutUri, SessionStorage.uapLogoutUrl); + SessionStorage.setItem(SessionStorage.uapLogoutUrl, uapLogoutUri); } if (redirectUrl) { location.replace(redirectUrl); diff --git a/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/index.tsx b/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/index.tsx index ec6c8390..74459362 100644 --- a/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/index.tsx +++ b/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/index.tsx @@ -5,6 +5,7 @@ import { BigDataPremarkStatus, BigDataSource, BigDataType, + bigDataPatternOptions_dpo, bigDataPatternOptions_learning, bigDataPatternOptions_pretraining, bigDataPatternOptions_rewardmodel, @@ -19,8 +20,8 @@ import { getBigDatasetMarkListReq, getBigDatasetNextVersionReq, getFileTemplateUrl, + getHuoshiUserInfoReq, getPromptTemplateUrl, - getUserReq, } from '@/services/bigModel'; import { to } from '@/utils/promise'; import { getFileListFromEvent, limitUploadFileType, validateUploadFiles } from '@/utils/ui'; @@ -45,7 +46,7 @@ import useSplitUpload from './useSplitUpload'; import { fileUploadRules, uploadSmallFileRequest } from './utils'; interface AddBigDatasetVersionModalProps extends Omit { - tenantId: string; // 火石租户ID, + uapId?: string; resourceType: ResourceType; resourceId: string; resoureName: string; @@ -53,7 +54,7 @@ interface AddBigDatasetVersionModalProps extends Omit { } function AddBigDatasetVersionModal({ - tenantId, + uapId, resourceType, resourceId, resoureName, @@ -64,7 +65,7 @@ function AddBigDatasetVersionModal({ const [form] = Form.useForm(); const [markSevices, setMarkServices] = useState([]); const [patterns, setPatterns] = useState(bigDataPatternOptions_sft); - const [currentUser, setCurrentUser] = useState(undefined); + const [huoshiUser, setHuoshiUser] = useState(undefined); const [markToolOptions, setMarkToolOptions] = useState(bigMarkToolOptions.slice(0, 1)); const dataType = (Form.useWatch('dataType', form) as BigDataType) ?? BigDataType.SFT; const markStatus = Form.useWatch('markStatus', form) ?? BigDataMarkStatus.Unlabel; @@ -73,12 +74,48 @@ function AddBigDatasetVersionModal({ const rule = fileUploadRules[dataType]; const [sliptUploadRequest, cancelUpload] = useSplitUpload(); - const jsonlUrl = getFileTemplateUrl(dataType, markStatus, '.jsonl', pattern === 'PDF'); - const jsonUrl = getFileTemplateUrl(dataType, markStatus, '.jsonl', pattern === 'PDF'); - const xlsxUrl = getFileTemplateUrl(dataType, markStatus, '.xlsx', pattern === 'PDF'); - const xlsUrl = getFileTemplateUrl(dataType, markStatus, '.xls', pattern === 'PDF'); - const txtUrl = getFileTemplateUrl(dataType, markStatus, '.txt', pattern === 'PDF'); - const zipUrl = getFileTemplateUrl(dataType, markStatus, '.zip', pattern === 'PDF'); + const jsonlUrl = getFileTemplateUrl( + 'LANGUAGE_MODEL', + dataType, + markStatus, + '.jsonl', + pattern === 'PDF', + ); + const jsonUrl = getFileTemplateUrl( + 'LANGUAGE_MODEL', + dataType, + markStatus, + '.jsonl', + pattern === 'PDF', + ); + const xlsxUrl = getFileTemplateUrl( + 'LANGUAGE_MODEL', + dataType, + markStatus, + '.xlsx', + pattern === 'PDF', + ); + const xlsUrl = getFileTemplateUrl( + 'LANGUAGE_MODEL', + dataType, + markStatus, + '.xls', + pattern === 'PDF', + ); + const txtUrl = getFileTemplateUrl( + 'LANGUAGE_MODEL', + dataType, + markStatus, + '.txt', + pattern === 'PDF', + ); + const zipUrl = getFileTemplateUrl( + 'LANGUAGE_MODEL', + dataType, + markStatus, + '.zip', + pattern === 'PDF', + ); const promptUrl = getPromptTemplateUrl(dataType, markStatus); const urls = [ @@ -109,14 +146,16 @@ function AddBigDatasetVersionModal({ ]; useEffect(() => { - const getCurrentUser = async () => { - const [res] = await to(getUserReq(tenantId)); + const getHuoshiUser = async () => { + const [res] = await to(getHuoshiUserInfoReq(uapId)); if (res && res.data) { - setCurrentUser(res.data); + setHuoshiUser(res.data); } }; - getCurrentUser(); - }, [tenantId]); + if (uapId) { + getHuoshiUser(); + } + }, [uapId]); useEffect(() => { const getNextVersion = async () => { @@ -180,7 +219,7 @@ function AddBigDatasetVersionModal({ source: BigDataSource.Create, oriDataPath: uploadedFiles[0], modelCategory: 'LANGUAGE_MODEL', - users: [currentUser], + users: [huoshiUser], }; createDatasetVersion(params); } @@ -196,7 +235,7 @@ function AddBigDatasetVersionModal({ form.setFieldValue('pattern', bigDataPatternOptions_sft[0].value); } else if (value === BigDataType.RewardModel) { setPatterns(bigDataPatternOptions_rewardmodel); - setMarkToolOptions(bigMarkToolOptions.slice(1)); + setMarkToolOptions(bigMarkToolOptions.slice(1, 2)); form.setFieldValue('pattern', bigDataPatternOptions_rewardmodel[0].value); } else if (value === BigDataType.Pretraining) { setPatterns(bigDataPatternOptions_pretraining); @@ -204,6 +243,10 @@ function AddBigDatasetVersionModal({ } else if (value === BigDataType.Learning) { setPatterns(bigDataPatternOptions_learning); form.setFieldValue('pattern', bigDataPatternOptions_learning[0].value); + } else if (value === BigDataType.DPO) { + setPatterns(bigDataPatternOptions_dpo); + form.setFieldValue('pattern', bigDataPatternOptions_dpo[0].value); + setMarkToolOptions(bigMarkToolOptions.slice(2, 3)); } }; @@ -342,7 +385,8 @@ function AddBigDatasetVersionModal({ const dataType = getFieldValue('dataType'); return dataType === BigDataType.SFT || dataType === BigDataType.RewardModel || - dataType === BigDataType.Test ? ( + dataType === BigDataType.Test || + dataType === BigDataType.DPO ? ( ( {v.label}{' '} - + @@ -388,7 +432,8 @@ function AddBigDatasetVersionModal({ return (dataType === BigDataType.SFT || dataType === BigDataType.RewardModel || - dataType === BigDataType.Test) && + dataType === BigDataType.Test || + dataType === BigDataType.DPO) && markStatus === BigDataMarkStatus.Unlabel ? ( <> ( {v.label}{' '} - - - + {v.tooltip && ( + + + + )} ))} - - - 不启用 - 启用 - - + {dataType !== BigDataType.DPO && ( + <> + + + 不启用 + 启用 + + + + {({ getFieldValue }) => { + const premarkStatus = getFieldValue('premarkStatus'); + return premarkStatus === BigDataPremarkStatus.Enabled ? ( + + - - ) : null; - }} - - - {/* - - */} - - -
- {`1.Prompt中必须包含一个{{}} + +
+ {`1.Prompt中必须包含一个{{}} 用来标识用户输入的问题在指令中的位置，否则会导致预标失败。`} - 下载Prompt编写指南。 -
-
- {`2、参考案例：输出内容应该合法合规，不能出现违法暴力等相关词汇，积极正能量，回答要尽量简洁，不超过100字。我的输入内容是 + 下载Prompt编写指南。 +
+
+ {`2、参考案例：输出内容应该合法合规，不能出现违法暴力等相关词汇，积极正能量，回答要尽量简洁，不超过100字。我的输入内容是 {{}}。`} -
-
- ), - overlayStyle: { maxWidth: 500 }, - }} - > - - + + + ), + overlayStyle: { maxWidth: 500 }, + }} + > + + + + )} ) : null; }} diff --git a/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/utils.ts b/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/utils.ts index 8b49a9ce..3f62a8a3 100644 --- a/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/utils.ts +++ b/react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/utils.ts @@ -57,6 +57,14 @@ export const fileUploadRules: Record = { fleSizeUnit: 'GB', template: [5], }, + [BigDataType.DPO]: { + tips: '支持 .jsonl、.json、.xlsx、.xls、.txt 格式文件或含上述文件类型的 .zip 文件', + accept: '.jsonl,.json,.xlsx,.xls,.txt,.zip', + fileType: '.jsonl、.json、.xlsx、.xls、.txt 或 .zip', + fleSize: 500, + fleSizeUnit: 'MB', + template: [0, 1, 2, 3, 4], + }, }; // 上传小文件 diff --git a/react-ui/src/pages/Dataset/components/AddBigModelVersionModal/index.tsx b/react-ui/src/pages/Dataset/components/AddBigModelVersionModal/index.tsx index 5417371d..61e95743 100644 --- a/react-ui/src/pages/Dataset/components/AddBigModelVersionModal/index.tsx +++ b/react-ui/src/pages/Dataset/components/AddBigModelVersionModal/index.tsx @@ -42,7 +42,10 @@ export interface Condition { status: string; } -type FileUploadType = 0 | 1; +enum FileUploadType { + Picker = 0, + Upload = 1, +} function AddBigModelVersionModal({ resourceType, @@ -99,18 +102,24 @@ function AddBigModelVersionModal({ const fileList: UploadFile[] = formData['fileList'] ?? []; const uploadType = formData['uploadType'] as FileUploadType; const modelPath = formData['model_path']; - if (validateUploadFiles(fileList)) { - const uploadedFiles = fileList.map((item) => { - const data = item.response?.data; - return data; - }); + if ( + (uploadType === FileUploadType.Upload && validateUploadFiles(fileList)) || + uploadType === FileUploadType.Picker + ) { + const uploadedFiles = + uploadType === FileUploadType.Upload + ? fileList.map((item) => { + const data = item.response?.data; + return data; + }) + : [modelPath]; const params = { ...omit(formData, ['fileList', 'uploadType', 'model_path']), modelManagerId: resourceId, name: resoureName, - modelPath: uploadType === 1 ? uploadedFiles[0] : modelPath, - information: uploadType === 1 ? uploadedFiles[0] : modelPath, + modelPath: uploadedFiles[0], + information: uploadedFiles[0], source: DataSource.Create, trainType: 1, }; @@ -210,8 +219,8 @@ function AddBigModelVersionModal({ style={{ marginBottom: 10 }} > - 选择文件 - 上传文件 + 选择文件 + 上传文件 diff --git a/react-ui/src/pages/Dataset/components/AddDatasetModal/index.less b/react-ui/src/pages/Dataset/components/AddDatasetModal/index.less index ca11b792..9e1b2316 100644 --- a/react-ui/src/pages/Dataset/components/AddDatasetModal/index.less +++ b/react-ui/src/pages/Dataset/components/AddDatasetModal/index.less @@ -29,3 +29,12 @@ text-decoration: underline; } } + +.form-item-tooltip { + :global { + .ant-tooltip-inner { + white-space: pre-wrap; + width: 500px; + } + } +} \ No newline at end of file diff --git a/react-ui/src/pages/Dataset/components/ResourceInfo/index.tsx b/react-ui/src/pages/Dataset/components/ResourceInfo/index.tsx index bb3a2e0c..0a1d4c08 100644 --- a/react-ui/src/pages/Dataset/components/ResourceInfo/index.tsx +++ b/react-ui/src/pages/Dataset/components/ResourceInfo/index.tsx @@ -248,7 +248,7 @@ const ResourceInfo = ({ resourceType }: ResourceInfoProps) => { if (isBigModel) { if (resourceType === ResourceType.Dataset) { const { close } = openAntdModal(AddBigDatasetVersionModal, { - tenantId: tenantId ?? resourceInfo.tenantId ?? '', + uapId: currentUser?.uapId, resourceType: resourceType, resourceId: hotStoneId, resoureName: name, @@ -300,7 +300,7 @@ const ResourceInfo = ({ resourceType }: ResourceInfoProps) => { is_public, name, resourceInfo.tenantId, - currentUser?.userName, + currentUser, getVersionList, ], ); diff --git a/react-ui/src/pages/Dataset/components/ResourceIntro/index.tsx b/react-ui/src/pages/Dataset/components/ResourceIntro/index.tsx index c55c4d6e..ec91b158 100644 --- a/react-ui/src/pages/Dataset/components/ResourceIntro/index.tsx +++ b/react-ui/src/pages/Dataset/components/ResourceIntro/index.tsx @@ -2,7 +2,9 @@ import BasicTableInfo, { BasicInfoData } from '@/components/BasicTableInfo'; import KFMdEditor from '@/components/KFMdEditor'; import SubAreaTitle from '@/components/SubAreaTitle'; import { + BigDataMarkStatus, bigDataMarkStatusOptions, + bigDataPatternOptions_dpo, bigDataPatternOptions_learning, bigDataPatternOptions_pretraining, bigDataPatternOptions_rewardmodel, @@ -19,7 +21,7 @@ import { ResourceType, resourceConfig, } from '@/pages/Dataset/config'; -import { getBigDatasetMarkListReq } from '@/services/bigModel'; +import { getBigDatasetMarkListReq, getTaskListReq } from '@/services/bigModel'; import { EnumOptions, formatCodeConfig, @@ -29,8 +31,10 @@ import { formatTrainTask, } from '@/utils/format'; import { to } from '@/utils/promise'; +import { useModel, useNavigate } from '@umijs/max'; +import { Button, Flex, message } from 'antd'; import classNames from 'classnames'; -import { useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import styles from './index.less'; type ResourceIntroProps = { @@ -50,6 +54,7 @@ const getDatasetDatas = ( isBigModel: boolean, data: DatasetData, markSevices: EnumOptions[], + gotoMarkTask: (info: BigDatasetVersionInfo) => void, ): BasicInfoData[] => { const arr: BasicInfoData[] = [ // { @@ -90,6 +95,7 @@ const getDatasetDatas = ( ...bigDataPatternOptions_rewardmodel, ...bigDataPatternOptions_learning, ...bigDataPatternOptions_pretraining, + ...bigDataPatternOptions_dpo, ]), }, { @@ -114,8 +120,25 @@ const getDatasetDatas = ( }, { label: '标注状态', - value: bigData.markStatus, - format: formatEnum(bigDataMarkStatusOptions), + value: ( + + {formatEnum(bigDataMarkStatusOptions)(bigData.markStatus)} + {(bigData.markStatus === BigDataMarkStatus.Unlabel || + bigData.markStatus === BigDataMarkStatus.LabeledComplete || + bigData.markStatus === BigDataMarkStatus.Labeling) && + bigData.markTaskId && + bigData.markToolId && ( + + )} + + ), }, { label: '提示', @@ -253,11 +276,46 @@ function ResourceIntro({ }: ResourceIntroProps) { const [markSevices, setMarkServices] = useState([]); const config = resourceConfig[resourceType]; + const navigate = useNavigate(); + const { initialState } = useModel('@@initialState'); + const { currentUser } = initialState || {}; + + const goToMarkTask = useCallback( + async (info: BigDatasetVersionInfo) => { + const jumoToMark = () => { + navigate( + `/tools/flint/data/mark/detail?id=${info.markTaskId}&markToolId=${info.markToolId}`, + ); + }; + + // 当前任务我的标注量为0时 + if (info.markedCount === 0) { + // 若无法拉取到新的未标数据，也不可进入标注详情页 + const [res] = await to( + getTaskListReq({ + taskId: info.markTaskId, + userId: currentUser?.userName, + next: true, + markToolId: info.markToolId, + }), + ); + if (res && res.data && Array.isArray(res.data) && res.data.length > 0) { + jumoToMark(); + } else { + message.error('当前标注任务已全部被领取，无法获取标注数据'); + } + } else { + jumoToMark(); + } + }, + [navigate, currentUser?.userName], + ); + const basicDatas: BasicInfoData[] = useMemo(() => { return resourceType === ResourceType.Dataset - ? getDatasetDatas(isBigModel, info as DatasetData, markSevices) + ? getDatasetDatas(isBigModel, info as DatasetData, markSevices, goToMarkTask) : getModelDatas(isBigModel, info as ModelData); - }, [resourceType, isBigModel, info, markSevices]); + }, [resourceType, isBigModel, info, markSevices, goToMarkTask]); useEffect(() => { const getBigDatasetMarkList = async () => { diff --git a/react-ui/src/pages/Dataset/components/ResourceVersion/index.tsx b/react-ui/src/pages/Dataset/components/ResourceVersion/index.tsx index 09f972f8..e44b2aee 100644 --- a/react-ui/src/pages/Dataset/components/ResourceVersion/index.tsx +++ b/react-ui/src/pages/Dataset/components/ResourceVersion/index.tsx @@ -245,16 +245,17 @@ function ResourceVersion({ name, resourceType, info, isBig }: ResourceVersionPro dataIndex: 'option', width: 200, key: 'option', - render: (_: any, record: ResourceFileData) => ( - - - {progress > 0 && progress < 100 && ( - - )} - - ), + render: (_: any, record: ResourceFileData) => + !record.isDir && ( + + + {progress > 0 && progress < 100 && ( + + )} + + ), }, ]; diff --git a/react-ui/src/pages/DatasetPreparation/mark/index.tsx b/react-ui/src/pages/DatasetPreparation/mark/index.tsx index dba3ee7b..72a61ccf 100644 --- a/react-ui/src/pages/DatasetPreparation/mark/index.tsx +++ b/react-ui/src/pages/DatasetPreparation/mark/index.tsx @@ -52,7 +52,7 @@ const MarkPage = () => { {/* 大模型标注内容（微应用） */} {activeTab === 'datasetManagement' && (
- +
)} diff --git a/react-ui/src/pages/Workspace/components/Notebooks/index.tsx b/react-ui/src/pages/Workspace/components/Notebooks/index.tsx index ed588123..fb48df50 100644 --- a/react-ui/src/pages/Workspace/components/Notebooks/index.tsx +++ b/react-ui/src/pages/Workspace/components/Notebooks/index.tsx @@ -42,7 +42,7 @@ function NotebookComponent() { }, [getEditorList]); const gotoDevEnv = () => { - navigate(`/pipeline/developmentEnvironment`); + navigate(`/developmentEnvironment`); }; const gotoDevEnvItem = (item: EditorData) => { diff --git a/react-ui/src/services/bigModel/index.ts b/react-ui/src/services/bigModel/index.ts index 984f5992..1730318e 100644 --- a/react-ui/src/services/bigModel/index.ts +++ b/react-ui/src/services/bigModel/index.ts @@ -183,15 +183,16 @@ export function mergeFile(data: any) { // 数据集上传的文件模板 export const getFileTemplateUrl = ( + modelCategory: string, dataType: BigDataType, isMarked: number, extend: string, isPDF: boolean = false, ) => { if (isPDF) { - return `${DatasetAPIPrefix}/file/download/demo?dataType=${dataType}&isMarked=${isMarked}&pattern=PDF&extend=${extend}`; + return `${DatasetAPIPrefix}/file/download/demo?modelCategory=${modelCategory}&dataType=${dataType}&isMarked=${isMarked}&pattern=PDF&extend=${extend}&language=zh_CN`; } - return `${DatasetAPIPrefix}/file/download/demo?dataType=${dataType}&isMarked=${isMarked}&extend=${extend}`; + return `${DatasetAPIPrefix}/file/download/demo?modelCategory=${modelCategory}&dataType=${dataType}&isMarked=${isMarked}&extend=${extend}&language=zh_CN`; }; // 数据集prompt模板 @@ -214,7 +215,7 @@ export function unpraiseBigDatasetReq(id: any) { } // 获取用户信息 -export function getUserReq(userId: any) { +export function getHuoshiUserInfoReq(userId: any) { return request(`${DatasetAPIPrefix}/flintDatasets/getUser`, { method: 'GET', params: { @@ -223,6 +224,14 @@ export function getUserReq(userId: any) { }); } +// 获取标注任务列表 +export function getTaskListReq(params: any) { + return request(`${DatasetAPIPrefix}/task-item/list`, { + method: 'GET', + params, + }); +} + // ----------------------------------模型--------------------------------------- // 获取大模型列表 export function getBigModelListReq(params: any) { diff --git a/react-ui/src/services/typings.d.ts b/react-ui/src/services/typings.d.ts index c1deaf05..8f3599f3 100644 --- a/react-ui/src/services/typings.d.ts +++ b/react-ui/src/services/typings.d.ts @@ -204,6 +204,7 @@ declare namespace API { address?: string; phone?: string; roleNames?: string[]; + uapId?: string; }; type ErrorResponse = { diff --git a/ruoyi-auth/src/main/java/com/ruoyi/auth/controller/TokenController.java b/ruoyi-auth/src/main/java/com/ruoyi/auth/controller/TokenController.java index 24f0f365..2413c706 100644 --- a/ruoyi-auth/src/main/java/com/ruoyi/auth/controller/TokenController.java +++ b/ruoyi-auth/src/main/java/com/ruoyi/auth/controller/TokenController.java @@ -68,25 +68,14 @@ public class TokenController { // Casdoor 配置 @Value("${casdoor.endpoint:}") private String casdoorEndpoint; - @Value("${casdoor.client-id:}") private String casdoorClientId; - @Value("${casdoor.client-secret:}") private String casdoorClientSecret; - - @Value("${casdoor.organization-name:built-in}") - private String casdoorOrganizationName; - - @Value("${casdoor.application-name:app-built-in}") - private String casdoorApplicationName; - @Value("${casdoor.redirect-uri:}") private String casdoorRedirectUri; - @Value("${casdoor.frontend-redirect-uri:}") private String frontendRedirectUri; - @Value("${casdoor.scope:openid profile email}") private String casdoorScope; @@ -243,13 +232,14 @@ public class TokenController { // 5. 生成若依token Map result = tokenService.createToken(userInfo); result.put("checkTokenUri", casdoorEndpoint + "/api/validate-token"); - result.put("logoutUri", casdoorEndpoint + "/logout"); + result.put("logoutUri", casdoorEndpoint + "/api/logout?redirect_uri=http://172.20.32.121:31213"); result.put("oauth2AccessToken", accessToken); result.put("idToken", idToken); result.put("provider", "casdoor"); result.put("token_type", tokenJson.getString("token_type")); result.put("expires_in", tokenJson.getInteger("expires_in")); result.put("scope", tokenJson.getString("scope")); + result.put("redirectUri", casdoorRedirectUri); if (uapUse && userInfo.getSysUser() != null) { result.put("uapLogoutUri", uapUrl + logoutUrl); @@ -340,8 +330,7 @@ public class TokenController { result.put("state", state); result.put("clientId", casdoorClientId); result.put("redirectUri", casdoorRedirectUri); - result.put("logoutUri", oauth2ClientProperties.getLogoutUri()); - + result.put("logoutUri", casdoorEndpoint + "/api/logout?redirect_uri=http://172.20.32.121:31213"); return R.ok(result); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java index d1c36fa8..d3f07dff 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java @@ -2,6 +2,7 @@ package com.ruoyi.platform.scheduling; import cn.hutool.core.lang.UUID; import cn.hutool.core.map.MapUtil; +import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; @@ -203,10 +204,8 @@ public class AppScheduleTask { String zipFileName = "batch_download_" + taskId + ".zip"; String zipFilePath = savePath + "/" + zipFileName; - HashMap upload = MapUtil.newHashMap(); - upload.put("input", (String) taskInfo.getByPath("input_file.direct_url")); - upload.put("output", saveFilePath); - csCollectService.allFileUpload(upload, savePath + "/" + zipFileName); + List inputFilePaths = readInputFilePath(taskInfo); + csCollectService.allFileUpload(inputFilePaths, saveFilePath, zipFilePath); HashMap batch_download = MapUtil.newHashMap(); batch_download.put("direct_url", zipFilePath); batch_download.put("filename", zipFileName); @@ -228,4 +227,28 @@ public class AppScheduleTask { // trainingTaskDao.update(trainingTask); // } //} + + /** + * 获取输入路径 + * + * @param taskInfo + * @return + */ + private List readInputFilePath(JSONObject taskInfo) { + List empty = new ArrayList<>();; + taskInfo.forEach((key, value) -> { + if (value instanceof JSONObject) { + JSONObject valueObj = (JSONObject) value; + if (valueObj.containsKey("direct_url")) { + + String directUrl = valueObj.getStr("direct_url"); + String filename = valueObj.getStr("filename"); + + String suffix = StrUtil.removeSuffix(directUrl, filename); + empty.add(suffix); + } + } + }); + return empty; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java index aef41d8b..7a6aa599 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java @@ -5,7 +5,7 @@ import cn.hutool.json.JSONObject; import com.ruoyi.platform.domain.TrainingTask; import org.springframework.web.multipart.MultipartFile; -import java.util.HashMap; +import java.util.List; public interface CSCollectService { @@ -23,5 +23,5 @@ public interface CSCollectService { JSONObject singleFileDownload(Integer objectId, String savePath,String fileName) throws Exception; - void allFileUpload(HashMap upload, String zipFilePath) throws Exception; + void allFileUpload(List inputs, String output, String zipFilePath) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java index 7556d4c4..b7e339ed 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java @@ -101,8 +101,7 @@ public class CSCollectServiceImpl implements CSCollectService { .contentType("application/json") .execute() .body(); - JSONObject resData = returnResDataWithOK(res); - return (Integer) getByPath(resData, "object.objectID"); + return packageId; } /** @@ -123,7 +122,6 @@ public class CSCollectServiceImpl implements CSCollectService { CSAuthInfo csAuthInfo = getZSAuthInfoFromRedis(); List> hpcBindingFiles = buildHpcBindingFiles(taskParam, taskInfo); JSONObject params = convertToJobRequest(trainingTask.getTaskName(),clusterId, jobName, hpcBindingFiles, csAuthInfo.getUserId(), ntasks, nodes); - String url = csUrl + "/jsm/v2/jobs/submit"; String res = HttpUtils.sendBodyPostWithToken(url, params, csAuthInfo.getToken()); JSONObject resData = returnResDataWithOK(res); @@ -191,12 +189,13 @@ public class CSCollectServiceImpl implements CSCollectService { if (fileValue instanceof JSONObject) { JSONObject fileObj = (JSONObject) fileValue; // 提取 file_id 和 filename - Integer objectId = fileObj.getInt("file_id"); + Integer packageId = fileObj.getInt("file_id"); + String filename = fileObj.getStr("filename"); // 构建 binding 元素 Map binding = new HashMap<>(); - binding.put("paramName", fieldName); - binding.put("objectID", objectId); + binding.put("filename", filename); + binding.put("packageID", packageId); bindingFiles.add(binding); } } @@ -246,11 +245,12 @@ public class CSCollectServiceImpl implements CSCollectService { if (hpcBindingFiles != null) { for (Map fileInfo : hpcBindingFiles) { JSONObject binding = new JSONObject(); - binding.set("paramName", fileInfo.get("paramName")); + binding.set("paramName", "inputFile"); JSONObject resource = new JSONObject(); - resource.set("type", "object"); - resource.set("objectID", fileInfo.get("objectID")); + resource.set("type", "path"); + resource.set("path", "/"+fileInfo.get("filename")); + resource.set("packageID", fileInfo.get("packageID")); binding.set("resource", resource); bindingArray.set(binding); @@ -419,16 +419,17 @@ public class CSCollectServiceImpl implements CSCollectService { /** * 所有文件打包上传 * - * @param upload + * @param inputs + * @param output * @param zipFilePath * @throws Exception */ @Override - public void allFileUpload(HashMap upload, String zipFilePath) throws Exception { + public void allFileUpload(List inputs, String output, String zipFilePath) throws Exception { // 异步提交任务 CompletableFuture future = CompletableFuture.supplyAsync(() -> { try { - minioUtil.createZipFromDirectory(bucketName, upload, zipFilePath); + minioUtil.createZipFromDirectory(bucketName, inputs,output, zipFilePath); return "执行成功"; } catch (Exception e) { throw new RuntimeException("异步提交任务失败", e); @@ -436,7 +437,6 @@ public class CSCollectServiceImpl implements CSCollectService { }); // 处理异步结果 future.thenAcceptAsync(result -> { - log.info("所有文件打包上传成功，result: {}", result); }).exceptionally(ex -> { log.error("异步处理任务失败", ex); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSServiceVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSServiceVersionServiceImpl.java index ef1056c1..a06189af 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSServiceVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSServiceVersionServiceImpl.java @@ -187,11 +187,11 @@ public class CSServiceVersionServiceImpl implements ServiceVersionService { ServiceVersion serviceVersion = csServiceDao.getServiceVersionById(serviceVersionId); com.ruoyi.platform.domain.service.Service service = serviceDao.getServiceById(serviceVersion.getServiceId()); ServiceTemp serviceTemp = serviceTempService.getServiceTemp(service.getServiceTempId()); - Integer objectID = csCollectService.uploadFile(serviceTemp.getName(), file); + Integer packageId = csCollectService.uploadFile(serviceTemp.getName(), file); String savePath = "/mini-model-platform-data/cs-data/upload/"+serviceVersionId+ "/" + UUID.fastUUID()+"/"+file.getOriginalFilename(); minioService.uploadFile(bucketName, savePath, file); HashMap result = MapUtil.newHashMap(); - result.put("file_id", objectID); + result.put("file_id", packageId); result.put("direct_url", savePath); result.put("filename", file.getOriginalFilename()); return result; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java index a3e398df..454cad3a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java @@ -480,17 +480,12 @@ public class ServiceServiceImpl implements ServiceService { @Override public Map getServiceVersionDocs(Long id) { ServiceVersion serviceVersion = serviceDao.getServiceVersionById(id); - HashMap paramMap = new HashMap<>(); - paramMap.put("deployment_name", serviceVersion.getDeploymentName()); - String req = HttpUtils.sendPost(argoUrl + modelService + "/getDocs", JSON.toJSONString(paramMap)); + String req = HttpUtils.sendGet(serviceVersion.getUrl() + "/docs/openapi.json", null); if (StringUtils.isNotEmpty(req)) { Map reqMap = JacksonUtil.parseJSONStr2Map(req); - if (reqMap.get("code").equals(200)) { - Map data = (Map) reqMap.get("data"); - return data; - } else { - throw new RuntimeException("获取日志失败：" + reqMap.get("msg")); - } + List> servers = (ArrayList) reqMap.get("servers"); + servers.get(0).put("url", serviceVersion.getUrl()); + return reqMap; } else { throw new RuntimeException("获取日志失败"); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java index 757a118f..99dae844 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java @@ -1,8 +1,9 @@ package com.ruoyi.platform.utils; +import com.ruoyi.common.core.utils.StringUtils; import io.minio.*; -import io.minio.errors.MinioException; +import io.minio.errors.*; import io.minio.http.Method; import io.minio.messages.DeleteObject; import io.minio.messages.Item; @@ -16,6 +17,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.*; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -516,43 +518,69 @@ public class MinioUtil { /** * 创建目录的压缩包 */ - public void createZipFromDirectory(String bucket, Map dirMap, String zipPath) + public void createZipFromDirectory(String bucket, List inputs,String output, String zipPath) throws Exception { - // 创建内存中的ZIP流 - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Path tempFile = Files.createTempFile("zip-", ".tmp"); + try { + // 写入临时文件 + try (FileOutputStream fos = new FileOutputStream(tempFile.toFile()); + ZipOutputStream zos = new ZipOutputStream(fos)) { - try (ZipOutputStream zos = new ZipOutputStream(baos)) { + for (String input : inputs) { + if (StringUtils.isNotBlank(input)) { + writeBuffer("input", input, bucket, zos); + } + } + writeBuffer("output", output, bucket, zos); + } - // 遍历每个目录映射 - for (Map.Entry entry : dirMap.entrySet()) { - String folderName = entry.getKey(); // 压缩包内的文件夹名 - String sourcePath = entry.getValue(); // MinIO中的路径 + // 上传临时文件 + try (InputStream is = Files.newInputStream(tempFile)) { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(zipPath) + .stream(is, Files.size(tempFile), -1) + .contentType("application/zip") + .build() + ); + } + } finally { + // 清理临时文件 + Files.deleteIfExists(tempFile); + } + } - // 1. 自动提取目录路径（去除文件名） - String sourceDir = extractDirectoryPath(sourcePath); + /** + * + * @param folderName 压缩包内的文件夹名 + * @param sourcePath MinIO中的路径 + */ + private void writeBuffer(String folderName,String sourcePath,String bucket,ZipOutputStream zos) { + try { - // 2. 规范化文件夹名 + // 1. 规范化文件夹名 if (!folderName.endsWith("/")) { folderName = folderName + "/"; } - // 3. 获取目录下的所有文件 + // 2. 获取目录下的所有文件 Iterable> items = minioClient.listObjects( ListObjectsArgs.builder() .bucket(bucket) - .prefix(sourceDir) + .prefix(sourcePath) .recursive(true) .build() ); - // 4. 添加文件到ZIP + // 3. 添加文件到ZIP for (Result result : items) { Item item = result.get(); if (item.isDir()) continue; String objectName = item.objectName(); - String relativePath = objectName.substring(sourceDir.length()-1); + String relativePath = objectName.substring(sourcePath.length() - 1); // 使用自定义的文件夹名 String zipEntryName = folderName + relativePath; @@ -576,48 +604,8 @@ public class MinioUtil { zos.closeEntry(); } - } - - zos.finish(); - } - - // 上传压缩包到MinIO - byte[] zipBytes = baos.toByteArray(); - - try (ByteArrayInputStream bais = new ByteArrayInputStream(zipBytes)) { - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucket) - .object(zipPath) - .stream(bais, zipBytes.length, -1) - .contentType("application/zip") - .build() - ); - } - } - - private String extractDirectoryPath(String path) { - path = path.trim(); - - // 如果路径包含点号且不以斜杠结尾，可能是文件 - if (path.contains(".") && !path.endsWith("/")) { - int lastSlash = path.lastIndexOf("/"); - if (lastSlash > 0) { - // 返回目录部分 - return path.substring(0, lastSlash + 1); + }catch (Exception e){ + e.printStackTrace(); } } - - // 如果不是文件路径，直接返回 - if (!path.endsWith("/")) { - return path + "/"; - } - return path; - } - - public static void main(String[] args) { - String objectName = "mini-model-platform-data/cs-data/98/28bbb298-6e21-4361-b00a-ba8dc1643f65/file/log.lammps"; - String relativePath = objectName.substring("/mini-model-platform-data/cs-data/98/28bbb298-6e21-4361-b00a-ba8dc1643f65/file/".length()-1); - System.out.println(relativePath); //输出og.lammps - } } \ No newline at end of file