ci4s
/
ci4sManagement-cloud
0
0
Fork 7
Code Issues Pull Requests 1 Packages Projects Releases 1 Wiki Activity

Merge branch 'dev-opt' of https://gitlink.org.cn/ci4s/ci4sManagement-cloud into dev-opt

This commit is contained in:
chenzhihang 2026-03-11 10:38:07 +08:00
parent 21afd42beb 6353ecdc2c
commit 2077c0ebd2
30 changed files with 594 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
react-ui/config/config.ts
View File

@ -179,20 +179,20 @@ export default defineConfig({
{
name: 'flint', // 数据服务微应用
// entry: 'http://localhost:8083', // 开发环境地址
entry: '/flint', // 生产环境地址
activeRule: '/flint', // 激活规则
entry: '/tools/flint', // 生产环境地址
activeRule: '/tools/flint', // 激活规则
},
{
name: 'trainfer', // 服务管理
// entry: 'http://localhost:8081', // 开发环境地址
entry: '/trainfer', // 生产环境地址
activeRule: '/trainfer', // 激活规则
entry: '/tools/trainfer', // 生产环境地址
activeRule: '/tools/trainfer', // 激活规则
},
{
name: 'portal', // 服务管理
// entry: 'http://localhost:8082', // 开发环境地址
entry: '/portal', // 生产环境地址
activeRule: '/portal', // 激活规则
entry: '/tools/portal', // 生产环境地址
activeRule: '/tools/portal', // 激活规则
},
]
: [],

44
react-ui/config/routes.ts
View File

@ -528,6 +528,12 @@ export default [
key: 'intelligent-manage-execute',
component: './Application/IntelligentExec',
},
{
name: 'API 文档',
path: 'api/:versionId',
key: 'api',
component: './Application/Info/components/SwaggerDocs',
},
],
},
],
@ -765,30 +771,50 @@ export default [
...(process.env.UMI_APP_HUO_SHI
? [
{
path: '/flint/*',
path: '/tools/flint/*',
name: '数据管理',
microApp: 'flint',
layout: true,
},
{
path: '/trainfer/*',
path: '/tools/trainfer/*',
name: '火石服务',
microApp: 'trainfer',
layout: true,
},
{
path: '/portal/*',
path: '/tools/portal/*',
name: '火石服务',
microApp: 'portal',
layout: true,
},
{
path: '/tools/flint/data',
redirect: '/tools/flint/data/mark/list',
},
{
path: '/tools/portal/enterprise',
redirect: '/tools/portal/enterprise/storage/index',
},
{
path: '/tools/flint/bml',
redirect: '/tools/flint/bml/bml/train/sft',
},
{
path: '/tools/flint/bml/bml',
redirect: '/tools/flint/bml/bml/train/sft',
},
{
path: '/tools/trainfer',
redirect: '/tools/trainfer/service/ai',
},
// 数据标注页面 - 使用自定义组件包装微应用
// {
// path: '/flint-data/mark/list',
// name: '数据标注',
// component: './DatasetPreparation/mark/index',
// layout: true,
// },
{
path: '/tools/flint/data/mark/list',
name: '数据标注',
component: './DatasetPreparation/mark/index',
layout: true,
},
// // 数据服务子路径 - 直接路由到微应用（排除 mark/list）
// {
// path: '/flint-data/*',

10
react-ui/package.json
View File

@ -9,8 +9,8 @@
"build-with-huoshi": "cross-env UMI_APP_HUO_SHI=true max build",
"deploy": "sh ./scripts/deploy.sh",
"dev": "npm run start:dev",
"dev-no-sso": "cross-env NO_SSO=true npm run start:dev",
"dev-with-huoshi": "cross-env NO_SSO=true UMI_APP_HUO_SHI=true npm run start:dev",
"dev-no-sso": "cross-env NO_SSO=true npm run start:mock",
"dev-with-huoshi": "cross-env NO_SSO=true UMI_APP_HUO_SHI=true npm run start:mock",
"docker-hub:build": "docker build -f Dockerfile.hub -t ant-design-pro ./",
"docker-prod:build": "docker-compose -f ./docker/docker-compose.yml build",
"docker-prod:dev": "docker-compose -f ./docker/docker-compose.yml up",
@ -101,7 +101,8 @@
"react-helmet-async": "^1.3.0",
"react-highlight": "^0.15.0",
"react-markdown": "~10.1.0",
"spark-md5": "~3.0.2"
"spark-md5": "~3.0.2",
"swagger-ui-react": "~5.32.0"
},
"devDependencies": {
"@ant-design/pro-cli": "^3.1.0",
@ -128,6 +129,7 @@
"@types/react-helmet": "^6.1.5",
"@types/react-highlight": "^0.12.5",
"@types/spark-md5": "~3.0.5",
"@types/swagger-ui-react": "~5.18.0",
"@umijs/lint": "^4.0.66",
"@umijs/max": "^4.0.66",
"cross-env": "^7.0.3",
@ -191,4 +193,4 @@
"public"
]
}
}
}

4
react-ui/src/components/RightContent/AvatarDropdown.tsx
View File

@ -84,7 +84,9 @@ const AvatarDropdown: React.FC<GlobalHeaderRightProps> = ({ menu, isHome = false
// 退出数据标注平台
const [res] = await to(getLabelStudioUrl());
if (res && res.data) {
oauthLogout(`${res.data}/oauth/logout`);
const base = res.data;
const url = base.endsWith('/') ? base : `${base}/`;
oauthLogout(`${url}oauth/logout`);
}
// 至少 2 秒后跳转，希望子系统能完成注销
await Promise.all([logout(), sleep(2000)]);

66
react-ui/src/enums/index.ts
View File

@ -209,14 +209,17 @@ export const bigDataTypeOptions = [
export enum BigDataMarkStatus {
Unlabel = 0,
Labeled = 1,
Labeling = 2,
LabeledComplete = 3,
NoNeedLabel = 4,
}
export const bigDataMarkStatusOptions = [
{ label: '未标注', value: BigDataMarkStatus.Unlabel },
{ label: '已标注', value: BigDataMarkStatus.Labeled },
{ label: '标注中', value: 2 },
{ label: '标注完成', value: 3 },
{ label: '不需要标注', value: 4 },
{ label: '标注中', value: BigDataMarkStatus.Labeling },
{ label: '标注完成', value: BigDataMarkStatus.LabeledComplete },
{ label: '不需要标注', value: BigDataMarkStatus.NoNeedLabel },
];
// 大模型数据集启用预标
@ -246,54 +249,85 @@ export const bigMarkToolOptions = [
value: 17,
tooltip: '支持对同一个问题生成多个（5个）答案，用户可对答案进行打分和排序',
},
{
label: '正负偏好标注',
value: 19,
tooltip: undefined,
},
];
// 大模型数据集数据样式
// 大模型数据集数据样式/SFT
export const bigDataPatternOptions_sft = [
{
label: 'input+target',
value: 'Q_A',
tooltip: '对话类数据，问题（input）和答案（target）一一对应。',
tooltip: `对话类数据，问题（input）和答案（target）一一对应。\n例如：{ "input": "太阳系有哪些行星？", "target": "太阳系有八大行星，分别是水星、金星、地球、火星、木星、土星、天王星和海王星。" }`,
},
{
label: 'input',
value: 'Q',
tooltip: '对话类数据，仅包含问题（input），答案（target）需要通过平台进行标注。',
tooltip: `对话类数据，仅包含问题（input），答案（target）需要通过平台进行标注。\n例如：{ "input": "太阳系有哪些行星？" }`,
},
];
// 大模型数据集数据样式
// 大模型数据集数据样式/奖励模型
export const bigDataPatternOptions_rewardmodel = [
{
label: 'input+多target（含排序）',
value: 'Q_5A',
tooltip:
'对话类数据，单个问题（input）对应多个答案（target），并且每个答案有对应的得分和排序。',
tooltip: `对话类数据，单个问题（input）对应多个答案（target），并且每个答案有对应的得分和排序。
例如：
{
"input": "社会科学研究方法的书属于什么类别？",
"targets": [
{ "content": "社会科学研究方法的图书属于C1", "score": 1, "level": "A" },
{ "content": "社会科学研究方法的图书属于C2", "score": 2, "level": "A" }
]
}`,
},
{
label: 'input',
value: 'Q',
tooltip:
'对话类数据，仅包含问题（input），答案（target）和对应的得分、排序，需要通过平台进行标注。',
tooltip: `对话类数据，仅包含问题（input），答案（target）和对应的得分、排序，需要通过平台进行标注。\n例如：{ "input": "社会科学研究方法的书属于什么类别？" }`,
},
];
// 大模型数据集数据样式
// 大模型数据集数据样式/强化学习
export const bigDataPatternOptions_learning = [
{
label: 'input',
value: 'Q_0',
tooltip: '对话类数据，仅包含问题（input），并且不需要进行标注。',
tooltip: `对话类数据，仅包含问题（input），并且不需要进行标注。\n例如：{ "input": "太阳系有哪些行星？", "target": "0" }`,
},
];
// 大模型数据集数据样式
// 大模型数据集数据样式/预训练
export const bigDataPatternOptions_pretraining = [
{ label: '纯文本', value: 'TEXT', tooltip: '纯文本类数据，可直接用于发起预训练任务。' },
{
label: '纯文本',
value: 'TEXT',
tooltip:
'纯文本类数据，可直接用于发起预训练任务。\n例如：伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。',
},
{
label: 'PDF',
value: 'PDF',
tooltip: 'PDF类数据，不可直接用于预训练，需要经过“数据格式转换”并转存后，才可以进行预训练。',
tooltip:
'PDF类数据，不可直接用于预训练，需要经过“数据格式转换”并转存后，才可以进行预训练。\n例如：伸和扩展人的智能的理论、方法、技术及应用系统的一门新的技术科学。',
},
];
// 大模型数据集数据样式/SFT
export const bigDataPatternOptions_dpo = [
{
label: 'input+target+targetlose',
value: 'Q_A_L',
tooltip: `对话类数据，问题（input）、答案（target）、弃用答案（targetlose）三者共同组成一条数据，其中弃用答案表示效果较差的回复。\n例如：{ "input": "一篇文章有960个字。他们分别需要多少分钟才能录完？", "target": "您需要告诉我前置条件，他们每人输入的速度，我才能计算", "targetlose": "我不知道。" }`,
},
{
label: 'input',
value: 'Q',
tooltip: `对话类数据，仅包含问题（input），答案（target）和弃用答案（targetlose）需要通过平台进行标注。\n例如：{ "input": "一篇文章有960个字。他们分别需要多少分钟才能录完？" }`,
},
];

45
react-ui/src/global.less
View File

@ -18,9 +18,11 @@ body,
.ant-layout {
min-height: 100vh;
}
.ant-pro-sider.ant-layout-sider.ant-pro-sider-fixed {
left: unset;
}
canvas {
display: block;
}
@ -30,10 +32,12 @@ body {
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
.ant-pro-layout .ant-pro-layout-content {
padding: 0;
background-color: transparent;
}
.ant-pro-layout .ant-pro-layout-bg-list {
background: @background-color;
}
@ -45,43 +49,55 @@ body {
.ant-pro-layout .ant-pro-sider-menu {
padding-top: 35px;
}
.ant-pro-global-header-logo-mix {
padding-left: 12px;
}
.ant-pro-layout .ant-pro-sider .ant-layout-sider-children {
border-right: unset;
}
.ant-pro-base-menu-inline {
border-radius: 0px 20px 20px 0px;
}
.ant-drawer .ant-drawer-body {
padding: 0;
}
.ant-drawer .ant-drawer-body .ant-row {
padding: 0 24px;
}
.ant-drawer .ant-drawer-body .ant-form-item {
margin-bottom: 20px;
}
.ant-menu .ant-menu-submenu-title .anticon {
font-size: 16px;
}
.ant-table-wrapper .ant-table-pagination.ant-pagination {
margin: 0;
padding: 20px 16px;
background-color: #fff;
}
.ant-pro-global-header-logo img {
height: 32px;
}
.ant-pro-layout .ant-layout-sider.ant-pro-sider {
height: 100vh;
padding-top: 60px;
}
.ant-pro-layout .ant-pro-layout-container {
height: 100vh;
overflow-y: hidden;
}
.ant-pagination .ant-pagination-item.ant-pagination-item-active {
background: @primary-color;
border-width: 0;
@ -90,11 +106,13 @@ body {
color: #fff;
}
}
.ant-pagination .ant-pagination-item-active:hover {
color: #fff;
background: rgba(22, 100, 255, 0.8);
border-color: rgba(22, 100, 255, 0.8);
}
.ant-pagination .ant-pagination-item {
border: 1px solid #e6e6e6;
}
@ -104,17 +122,20 @@ body {
height: 5px;
background: transparent;
}
::-webkit-scrollbar-thumb {
width: 5px;
height: 5px;
background: rgba(0, 0, 0, 0.2);
border-radius: 99px;
}
::-webkit-scrollbar-track {
width: 5px;
height: 5px;
background: transparent;
}
ul,
ol {
list-style: none;
@ -124,12 +145,15 @@ ol {
.ant-table {
width: 100%;
overflow-x: auto;
&-thead > tr,
&-tbody > tr {
> th,
> td {
&-thead>tr,
&-tbody>tr {
>th,
>td {
white-space: pre;
> span {
>span {
display: block;
}
}
@ -176,9 +200,10 @@ input:-webkit-autofill {
padding-top: 100px;
padding-bottom: 30px;
}
// qiankun 微前端容器样式
.qiankun-micro-app-container,
.qiankun-micro-app-container > div {
.qiankun-micro-app-container>div {
width: 100%;
height: 100%;
overflow: hidden;
@ -187,6 +212,7 @@ input:-webkit-autofill {
.form-list-add-btn:not(:disabled) {
border-color: .addAlpha(@primary-color, 0.5) [];
box-shadow: none !important;
&:hover {
border-style: solid;
}
@ -219,3 +245,10 @@ input:-webkit-autofill {
width: 100%;
}
}
.pre-line-tooltip {
.ant-tooltip-inner {
white-space: pre-line;
width: 500px;
}
}

13
react-ui/src/huoshi.less
View File

@ -371,4 +371,15 @@
.qiankun-micro-app-container{
padding: 70px 24px 24px;
}
}
.el-dropdown-link {
outline: none;
color: var(--el-color-primary);
display: flex;
align-items: center;
cursor: pointer;
.el-icon {
margin-left: 4px;
}
}

23
react-ui/src/pages/Application/Info/components/SwaggerDocs/index.less
View File

@ -1,6 +1,23 @@
.swagger-api {
height: 100%;
overflow-y: scroll;
.swagger-docs {
&--component {
height: 100%;
background-color: white;
border-radius: 0px 10px 10px 10px;
overflow-y: auto;
}
&--page {
height: 100%;
padding: @page-padding-top @content-padding @page-padding-bottom;
.backgroundPageImage(url(@/assets/img/app/app-bg-2.png));
}
&--page &__content {
height: 100%;
background-color: white;
border-radius: 10px;
overflow-y: auto;
}
:global {
.swagger-ui .scheme-container {

36
react-ui/src/pages/Application/Info/components/SwaggerDocs/index.tsx
View File

@ -1,34 +1,52 @@
import { ApplicationSource } from '@/pages/Application/types';
import { useTypedSourceContext } from '@/pages/Application/useSourceContext';
import { getTaskDocsReq } from '@/services/modelDeployment';
import { to } from '@/utils/promise';
import { useParams, useSearchParams } from '@umijs/max';
import classNames from 'classnames';
import { useEffect, useState } from 'react';
import SwaggerUI from 'swagger-ui-react';
import 'swagger-ui-react/swagger-ui.css';
import styles from './index.less';
export type DrawerTitleProps = {
serviceId: number;
versionId: number;
source: ApplicationSource;
isComponent?: boolean;
};
function SwaggerDocs({ serviceId }: DrawerTitleProps) {
const { source = ApplicationSource.Platform } = useTypedSourceContext();
function SwaggerDocs({ versionId, source, isComponent = false }: DrawerTitleProps) {
const [spec, setSpec] = useState();
const params = useParams();
const paramsVersionId = params.versionId;
const [searchParams] = useSearchParams();
const serviceSource = searchParams.get('source');
const uniId = versionId ?? paramsVersionId;
const uniSource = source ?? serviceSource;
useEffect(() => {
const getTaskDocs = async () => {
const [res] = await to(getTaskDocsReq(source, serviceId));
if (res && res.data && res.data.docs) {
setSpec(res.data.docs);
const [res] = await to(getTaskDocsReq(uniSource, uniId));
if (res && res.data) {
setSpec(res.data);
}
};
getTaskDocs();
}, [source, serviceId]);
}, [uniSource, uniId]);
// url="https://petstore.swagger.io/v2/swagger.json"
return <div className={styles['swagger-api']}>{spec && <SwaggerUI spec={spec} />}</div>;
return (
<div
className={classNames(
styles['swagger-docs'],
!isComponent ? styles['swagger-docs--page'] : styles['swagger-docs--component'],
)}
>
<div className={styles['swagger-docs__content']}>{spec && <SwaggerUI spec={spec} />}</div>
</div>
);
}
export default SwaggerDocs;

18
react-ui/src/pages/Application/Info/components/VersionList/index.tsx
View File

@ -113,7 +113,7 @@ function VersionList({
minWidth: 100,
},
operation: {
fixedWidth: 280,
fixedWidth: 340,
},
}),
[],
@ -276,6 +276,11 @@ function VersionList({
}
};
// api 文档
const toAPIDocs = (record: ServiceVersionData) => {
onNavigate(`api/${record.id}?source=${serviceSource}`, getParams());
};
// 分页切换
const handleTableChange: TableProps<ServiceVersionData>['onChange'] = (
pagination,
@ -459,6 +464,17 @@ function VersionList({
>
测试
</Button>
<Divider type="vertical" className="table-operation-divider" />
<Button
type="link"
size="small"
key="api"
// icon={<KFIcon type="icon-bianji" />}
onClick={() => toAPIDocs(record)}
disabled={record.run_state !== ServiceRunStatus.Running}
>
API 文档
</Button>
</div>
),
},

16
react-ui/src/pages/Application/Info/index.tsx
View File

@ -150,6 +150,7 @@ function AppInfo() {
const handleDetailChange = (value: string | undefined) => {
updateService({
detail: value ?? '',
service_name: serviceInfo?.service_name,
});
};
@ -157,6 +158,7 @@ function AppInfo() {
const handleManualChange = (value: string | undefined) => {
updateService({
manual: value ?? '',
service_name: serviceInfo?.service_name,
});
};
@ -172,6 +174,7 @@ function AppInfo() {
is_public,
public_version_id: publishVersion,
source: serviceInfo!.source,
service_name: serviceInfo?.service_name,
});
},
});
@ -184,6 +187,7 @@ function AppInfo() {
updateService({
is_public,
public_version_id: serviceInfo?.public_version_id,
service_name: serviceInfo?.service_name,
});
},
});
@ -293,11 +297,19 @@ function AppInfo() {
></Discussion>
),
},
serviceInfo!.source === ApplicationSource.Platform
serviceInfo!.source === ApplicationSource.Platform &&
serviceInfo!.is_public === true &&
serviceInfo!.public_version_id
? {
key: AppInfoTab.Docs,
label: `API 文档`,
children: <SwaggerDocs serviceId={serviceId}></SwaggerDocs>,
children: (
<SwaggerDocs
versionId={serviceInfo!.public_version_id}
source={serviceInfo!.source}
isComponent={true}
></SwaggerDocs>
),
}
: undefined,
].filter((v) => v !== undefined);

2
react-ui/src/pages/Authorize/index.tsx
View File

@ -42,7 +42,7 @@ function Authorize() {
message.success('登录成功！');
if (uapLogoutUri) {
SessionStorage.setItem(uapLogoutUri, SessionStorage.uapLogoutUrl);
SessionStorage.setItem(SessionStorage.uapLogoutUrl, uapLogoutUri);
}
if (redirectUrl) {
location.replace(redirectUrl);

241
react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/index.tsx
View File

@ -5,6 +5,7 @@ import {
BigDataPremarkStatus,
BigDataSource,
BigDataType,
bigDataPatternOptions_dpo,
bigDataPatternOptions_learning,
bigDataPatternOptions_pretraining,
bigDataPatternOptions_rewardmodel,
@ -19,8 +20,8 @@ import {
getBigDatasetMarkListReq,
getBigDatasetNextVersionReq,
getFileTemplateUrl,
getHuoshiUserInfoReq,
getPromptTemplateUrl,
getUserReq,
} from '@/services/bigModel';
import { to } from '@/utils/promise';
import { getFileListFromEvent, limitUploadFileType, validateUploadFiles } from '@/utils/ui';
@ -45,7 +46,7 @@ import useSplitUpload from './useSplitUpload';
import { fileUploadRules, uploadSmallFileRequest } from './utils';
interface AddBigDatasetVersionModalProps extends Omit<ModalProps, 'onOk'> {
tenantId: string; // 火石租户ID,
uapId?: string;
resourceType: ResourceType;
resourceId: string;
resoureName: string;
@ -53,7 +54,7 @@ interface AddBigDatasetVersionModalProps extends Omit<ModalProps, 'onOk'> {
}
function AddBigDatasetVersionModal({
tenantId,
uapId,
resourceType,
resourceId,
resoureName,
@ -64,7 +65,7 @@ function AddBigDatasetVersionModal({
const [form] = Form.useForm();
const [markSevices, setMarkServices] = useState([]);
const [patterns, setPatterns] = useState(bigDataPatternOptions_sft);
const [currentUser, setCurrentUser] = useState(undefined);
const [huoshiUser, setHuoshiUser] = useState(undefined);
const [markToolOptions, setMarkToolOptions] = useState(bigMarkToolOptions.slice(0, 1));
const dataType = (Form.useWatch('dataType', form) as BigDataType) ?? BigDataType.SFT;
const markStatus = Form.useWatch('markStatus', form) ?? BigDataMarkStatus.Unlabel;
@ -73,12 +74,48 @@ function AddBigDatasetVersionModal({
const rule = fileUploadRules[dataType];
const [sliptUploadRequest, cancelUpload] = useSplitUpload();
const jsonlUrl = getFileTemplateUrl(dataType, markStatus, '.jsonl', pattern === 'PDF');
const jsonUrl = getFileTemplateUrl(dataType, markStatus, '.jsonl', pattern === 'PDF');
const xlsxUrl = getFileTemplateUrl(dataType, markStatus, '.xlsx', pattern === 'PDF');
const xlsUrl = getFileTemplateUrl(dataType, markStatus, '.xls', pattern === 'PDF');
const txtUrl = getFileTemplateUrl(dataType, markStatus, '.txt', pattern === 'PDF');
const zipUrl = getFileTemplateUrl(dataType, markStatus, '.zip', pattern === 'PDF');
const jsonlUrl = getFileTemplateUrl(
'LANGUAGE_MODEL',
dataType,
markStatus,
'.jsonl',
pattern === 'PDF',
);
const jsonUrl = getFileTemplateUrl(
'LANGUAGE_MODEL',
dataType,
markStatus,
'.jsonl',
pattern === 'PDF',
);
const xlsxUrl = getFileTemplateUrl(
'LANGUAGE_MODEL',
dataType,
markStatus,
'.xlsx',
pattern === 'PDF',
);
const xlsUrl = getFileTemplateUrl(
'LANGUAGE_MODEL',
dataType,
markStatus,
'.xls',
pattern === 'PDF',
);
const txtUrl = getFileTemplateUrl(
'LANGUAGE_MODEL',
dataType,
markStatus,
'.txt',
pattern === 'PDF',
);
const zipUrl = getFileTemplateUrl(
'LANGUAGE_MODEL',
dataType,
markStatus,
'.zip',
pattern === 'PDF',
);
const promptUrl = getPromptTemplateUrl(dataType, markStatus);
const urls = [
@ -109,14 +146,16 @@ function AddBigDatasetVersionModal({
];
useEffect(() => {
const getCurrentUser = async () => {
const [res] = await to(getUserReq(tenantId));
const getHuoshiUser = async () => {
const [res] = await to(getHuoshiUserInfoReq(uapId));
if (res && res.data) {
setCurrentUser(res.data);
setHuoshiUser(res.data);
}
};
getCurrentUser();
}, [tenantId]);
if (uapId) {
getHuoshiUser();
}
}, [uapId]);
useEffect(() => {
const getNextVersion = async () => {
@ -180,7 +219,7 @@ function AddBigDatasetVersionModal({
source: BigDataSource.Create,
oriDataPath: uploadedFiles[0],
modelCategory: 'LANGUAGE_MODEL',
users: [currentUser],
users: [huoshiUser],
};
createDatasetVersion(params);
}
@ -196,7 +235,7 @@ function AddBigDatasetVersionModal({
form.setFieldValue('pattern', bigDataPatternOptions_sft[0].value);
} else if (value === BigDataType.RewardModel) {
setPatterns(bigDataPatternOptions_rewardmodel);
setMarkToolOptions(bigMarkToolOptions.slice(1));
setMarkToolOptions(bigMarkToolOptions.slice(1, 2));
form.setFieldValue('pattern', bigDataPatternOptions_rewardmodel[0].value);
} else if (value === BigDataType.Pretraining) {
setPatterns(bigDataPatternOptions_pretraining);
@ -204,6 +243,10 @@ function AddBigDatasetVersionModal({
} else if (value === BigDataType.Learning) {
setPatterns(bigDataPatternOptions_learning);
form.setFieldValue('pattern', bigDataPatternOptions_learning[0].value);
} else if (value === BigDataType.DPO) {
setPatterns(bigDataPatternOptions_dpo);
form.setFieldValue('pattern', bigDataPatternOptions_dpo[0].value);
setMarkToolOptions(bigMarkToolOptions.slice(2, 3));
}
};
@ -342,7 +385,8 @@ function AddBigDatasetVersionModal({
const dataType = getFieldValue('dataType');
return dataType === BigDataType.SFT ||
dataType === BigDataType.RewardModel ||
dataType === BigDataType.Test ? (
dataType === BigDataType.Test ||
dataType === BigDataType.DPO ? (
<Form.Item
label="是否标注"
name="markStatus"
@ -370,7 +414,7 @@ function AddBigDatasetVersionModal({
{patterns.map((v) => (
<Radio value={v.value} key={v.value}>
{v.label}{' '}
<Tooltip title={v.tooltip}>
<Tooltip title={v.tooltip} rootClassName={styles['form-item-tooltip']}>
<QuestionCircleOutlined />
</Tooltip>
</Radio>
@ -388,7 +432,8 @@ function AddBigDatasetVersionModal({
return (dataType === BigDataType.SFT ||
dataType === BigDataType.RewardModel ||
dataType === BigDataType.Test) &&
dataType === BigDataType.Test ||
dataType === BigDataType.DPO) &&
markStatus === BigDataMarkStatus.Unlabel ? (
<>
<Form.Item
@ -400,95 +445,87 @@ function AddBigDatasetVersionModal({
{markToolOptions.map((v) => (
<Radio value={v.value} key={v.value}>
{v.label}{' '}
<Tooltip title={v.tooltip}>
<QuestionCircleOutlined />
</Tooltip>
{v.tooltip && (
<Tooltip title={v.tooltip}>
<QuestionCircleOutlined />
</Tooltip>
)}
</Radio>
))}
</Radio.Group>
</Form.Item>
<Form.Item
label="是否启用预标"
name="premarkStatus"
rules={[{ required: true, message: '请选择是否启用预标' }]}
tooltip="若启用预标，系统会自动调用大模型服务，基于上传的原始问题生成对应的答案，辅助用户标注（需要部署星火大模型服务）"
>
<Radio.Group>
<Radio value={BigDataPremarkStatus.Disabled}>不启用</Radio>
<Radio value={BigDataPremarkStatus.Enabled}>启用</Radio>
</Radio.Group>
</Form.Item>
{dataType !== BigDataType.DPO && (
<>
<Form.Item
label="是否启用预标"
name="premarkStatus"
rules={[{ required: true, message: '请选择是否启用预标' }]}
tooltip="若启用预标，系统会自动调用大模型服务，基于上传的原始问题生成对应的答案，辅助用户标注（需要部署星火大模型服务）"
>
<Radio.Group>
<Radio value={BigDataPremarkStatus.Disabled}>不启用</Radio>
<Radio value={BigDataPremarkStatus.Enabled}>启用</Radio>
</Radio.Group>
</Form.Item>
<Form.Item noStyle dependencies={['premarkStatus']}>
{({ getFieldValue }) => {
const premarkStatus = getFieldValue('premarkStatus');
return premarkStatus === BigDataPremarkStatus.Enabled ? (
<Form.Item
label="预标服务"
name="serviceId"
rules={[{ required: true, message: '请选择预标服务' }]}
>
<Select
allowClear
placeholder="请选择预标服务"
options={markSevices}
fieldNames={{ label: 'name', value: 'id' }}
optionFilterProp="name"
showSearch
/>
</Form.Item>
) : null;
}}
</Form.Item>
<Form.Item noStyle dependencies={['premarkStatus']}>
{({ getFieldValue }) => {
const premarkStatus = getFieldValue('premarkStatus');
return premarkStatus === BigDataPremarkStatus.Enabled ? (
<Form.Item
label="预标服务"
name="serviceId"
rules={[{ required: true, message: '请选择预标服务' }]}
>
<Select
allowClear
placeholder="请选择预标服务"
options={markSevices}
fieldNames={{ label: 'name', value: 'id' }}
optionFilterProp="name"
showSearch
/>
</Form.Item>
) : null;
}}
</Form.Item>
{/* <Form.Item
label="标注人员"
name="user"
rules={[
{
required: false,
message: '请选择',
},
]}
>
<Button type="primary">点击添加</Button>
</Form.Item> */}
<Form.Item
label="提示"
name="prompt"
rules={[
{
required: false,
message: '请输入',
},
]}
tooltip={{
title: (
<div className={styles['form-item-tooltip']}>
<div>
{`1.Prompt中必须包含一个{{}}
<Form.Item
label="提示"
name="prompt"
rules={[
{
required: false,
message: '请输入',
},
]}
tooltip={{
title: (
<div className={styles['form-item-tooltip']}>
<div>
{`1.Prompt中必须包含一个{{}}
用来标识用户输入的问题在指令中的位置，否则会导致预标失败。`}
<a href={promptUrl}>下载Prompt编写指南</a>。
</div>
<div>
{`2、参考案例：输出内容应该合法合规，不能出现违法暴力等相关词汇，积极正能量，回答要尽量简洁，不超过100字。我的输入内容是
<a href={promptUrl}>下载Prompt编写指南</a>。
</div>
<div>
{`2、参考案例：输出内容应该合法合规，不能出现违法暴力等相关词汇，积极正能量，回答要尽量简洁，不超过100字。我的输入内容是
{{}}。`}
</div>
</div>
),
overlayStyle: { maxWidth: 500 },
}}
>
<Input.TextArea
placeholder="请输入"
autoSize={{ minRows: 4, maxRows: 6 }}
maxLength={500}
showCount
allowClear
disabled={premarkStatus === BigDataPremarkStatus.Disabled}
/>
</Form.Item>
</div>
</div>
),
overlayStyle: { maxWidth: 500 },
}}
>
<Input.TextArea
placeholder="请输入"
autoSize={{ minRows: 4, maxRows: 6 }}
maxLength={500}
showCount
allowClear
disabled={premarkStatus === BigDataPremarkStatus.Disabled}
/>
</Form.Item>
</>
)}
</>
) : null;
}}

8
react-ui/src/pages/Dataset/components/AddBigDatasetVersionModal/utils.ts
View File

@ -57,6 +57,14 @@ export const fileUploadRules: Record<BigDataType, FileUploadRulesValue> = {
fleSizeUnit: 'GB',
template: [5],
},
[BigDataType.DPO]: {
tips: '支持 .jsonl、.json、.xlsx、.xls、.txt 格式文件或含上述文件类型的 .zip 文件',
accept: '.jsonl,.json,.xlsx,.xls,.txt,.zip',
fileType: '.jsonl、.json、.xlsx、.xls、.txt 或 .zip',
fleSize: 500,
fleSizeUnit: 'MB',
template: [0, 1, 2, 3, 4],
},
};
// 上传小文件

29
react-ui/src/pages/Dataset/components/AddBigModelVersionModal/index.tsx
View File

@ -42,7 +42,10 @@ export interface Condition {
status: string;
}
type FileUploadType = 0 | 1;
enum FileUploadType {
Picker = 0,
Upload = 1,
}
function AddBigModelVersionModal({
resourceType,
@ -99,18 +102,24 @@ function AddBigModelVersionModal({
const fileList: UploadFile[] = formData['fileList'] ?? [];
const uploadType = formData['uploadType'] as FileUploadType;
const modelPath = formData['model_path'];
if (validateUploadFiles(fileList)) {
const uploadedFiles = fileList.map((item) => {
const data = item.response?.data;
return data;
});
if (
(uploadType === FileUploadType.Upload && validateUploadFiles(fileList)) ||
uploadType === FileUploadType.Picker
) {
const uploadedFiles =
uploadType === FileUploadType.Upload
? fileList.map((item) => {
const data = item.response?.data;
return data;
})
: [modelPath];
const params = {
...omit(formData, ['fileList', 'uploadType', 'model_path']),
modelManagerId: resourceId,
name: resoureName,
modelPath: uploadType === 1 ? uploadedFiles[0] : modelPath,
information: uploadType === 1 ? uploadedFiles[0] : modelPath,
modelPath: uploadedFiles[0],
information: uploadedFiles[0],
source: DataSource.Create,
trainType: 1,
};
@ -210,8 +219,8 @@ function AddBigModelVersionModal({
style={{ marginBottom: 10 }}
>
<Radio.Group>
<Radio value={0}>选择文件</Radio>
<Radio value={1}>上传文件</Radio>
<Radio value={FileUploadType.Picker}>选择文件</Radio>
<Radio value={FileUploadType.Upload}>上传文件</Radio>
</Radio.Group>
</Form.Item>

9
react-ui/src/pages/Dataset/components/AddDatasetModal/index.less
View File

@ -29,3 +29,12 @@
text-decoration: underline;
}
}
.form-item-tooltip {
:global {
.ant-tooltip-inner {
white-space: pre-wrap;
width: 500px;
}
}
}

4
react-ui/src/pages/Dataset/components/ResourceInfo/index.tsx
View File

@ -248,7 +248,7 @@ const ResourceInfo = ({ resourceType }: ResourceInfoProps) => {
if (isBigModel) {
if (resourceType === ResourceType.Dataset) {
const { close } = openAntdModal(AddBigDatasetVersionModal, {
tenantId: tenantId ?? resourceInfo.tenantId ?? '',
uapId: currentUser?.uapId,
resourceType: resourceType,
resourceId: hotStoneId,
resoureName: name,
@ -300,7 +300,7 @@ const ResourceInfo = ({ resourceType }: ResourceInfoProps) => {
is_public,
name,
resourceInfo.tenantId,
currentUser?.userName,
currentUser,
getVersionList,
],
);

70
react-ui/src/pages/Dataset/components/ResourceIntro/index.tsx
View File

@ -2,7 +2,9 @@ import BasicTableInfo, { BasicInfoData } from '@/components/BasicTableInfo';
import KFMdEditor from '@/components/KFMdEditor';
import SubAreaTitle from '@/components/SubAreaTitle';
import {
BigDataMarkStatus,
bigDataMarkStatusOptions,
bigDataPatternOptions_dpo,
bigDataPatternOptions_learning,
bigDataPatternOptions_pretraining,
bigDataPatternOptions_rewardmodel,
@ -19,7 +21,7 @@ import {
ResourceType,
resourceConfig,
} from '@/pages/Dataset/config';
import { getBigDatasetMarkListReq } from '@/services/bigModel';
import { getBigDatasetMarkListReq, getTaskListReq } from '@/services/bigModel';
import {
EnumOptions,
formatCodeConfig,
@ -29,8 +31,10 @@ import {
formatTrainTask,
} from '@/utils/format';
import { to } from '@/utils/promise';
import { useModel, useNavigate } from '@umijs/max';
import { Button, Flex, message } from 'antd';
import classNames from 'classnames';
import { useEffect, useMemo, useState } from 'react';
import { useCallback, useEffect, useMemo, useState } from 'react';
import styles from './index.less';
type ResourceIntroProps = {
@ -50,6 +54,7 @@ const getDatasetDatas = (
isBigModel: boolean,
data: DatasetData,
markSevices: EnumOptions[],
gotoMarkTask: (info: BigDatasetVersionInfo) => void,
): BasicInfoData[] => {
const arr: BasicInfoData[] = [
// {
@ -90,6 +95,7 @@ const getDatasetDatas = (
...bigDataPatternOptions_rewardmodel,
...bigDataPatternOptions_learning,
...bigDataPatternOptions_pretraining,
...bigDataPatternOptions_dpo,
]),
},
{
@ -114,8 +120,25 @@ const getDatasetDatas = (
},
{
label: '标注状态',
value: bigData.markStatus,
format: formatEnum(bigDataMarkStatusOptions),
value: (
<Flex align="center">
<span>{formatEnum(bigDataMarkStatusOptions)(bigData.markStatus)}</span>
{(bigData.markStatus === BigDataMarkStatus.Unlabel ||
bigData.markStatus === BigDataMarkStatus.LabeledComplete ||
bigData.markStatus === BigDataMarkStatus.Labeling) &&
bigData.markTaskId &&
bigData.markToolId && (
<Button
type="primary"
size="small"
style={{ marginLeft: 8 }}
onClick={() => gotoMarkTask(bigData)}
>
去标注
</Button>
)}
</Flex>
),
},
{
label: '提示',
@ -253,11 +276,46 @@ function ResourceIntro({
}: ResourceIntroProps) {
const [markSevices, setMarkServices] = useState([]);
const config = resourceConfig[resourceType];
const navigate = useNavigate();
const { initialState } = useModel('@@initialState');
const { currentUser } = initialState || {};
const goToMarkTask = useCallback(
async (info: BigDatasetVersionInfo) => {
const jumoToMark = () => {
navigate(
`/tools/flint/data/mark/detail?id=${info.markTaskId}&markToolId=${info.markToolId}`,
);
};
// 当前任务我的标注量为0时
if (info.markedCount === 0) {
// 若无法拉取到新的未标数据，也不可进入标注详情页
const [res] = await to(
getTaskListReq({
taskId: info.markTaskId,
userId: currentUser?.userName,
next: true,
markToolId: info.markToolId,
}),
);
if (res && res.data && Array.isArray(res.data) && res.data.length > 0) {
jumoToMark();
} else {
message.error('当前标注任务已全部被领取，无法获取标注数据');
}
} else {
jumoToMark();
}
},
[navigate, currentUser?.userName],
);
const basicDatas: BasicInfoData[] = useMemo(() => {
return resourceType === ResourceType.Dataset
? getDatasetDatas(isBigModel, info as DatasetData, markSevices)
? getDatasetDatas(isBigModel, info as DatasetData, markSevices, goToMarkTask)
: getModelDatas(isBigModel, info as ModelData);
}, [resourceType, isBigModel, info, markSevices]);
}, [resourceType, isBigModel, info, markSevices, goToMarkTask]);
useEffect(() => {
const getBigDatasetMarkList = async () => {

21
react-ui/src/pages/Dataset/components/ResourceVersion/index.tsx
View File

@ -245,16 +245,17 @@ function ResourceVersion({ name, resourceType, info, isBig }: ResourceVersionPro
dataIndex: 'option',
width: 200,
key: 'option',
render: (_: any, record: ResourceFileData) => (
<Flex className="table-operation-column" align="center" gap={10}>
<Button type="link" size="small" key="download" onClick={() => downloadAlone(record)}>
下载
</Button>
{progress > 0 && progress < 100 && (
<Progress style={{ flex: 1 }} key="progress" percent={progress} status="active" />
)}
</Flex>
),
render: (_: any, record: ResourceFileData) =>
!record.isDir && (
<Flex className="table-operation-column" align="center" gap={10}>
<Button type="link" size="small" key="download" onClick={() => downloadAlone(record)}>
下载
</Button>
{progress > 0 && progress < 100 && (
<Progress style={{ flex: 1 }} key="progress" percent={progress} status="active" />
)}
</Flex>
),
},
];

2
react-ui/src/pages/DatasetPreparation/mark/index.tsx
View File

@ -52,7 +52,7 @@ const MarkPage = () => {
{/* 大模型标注内容（微应用） */}
{activeTab === 'datasetManagement' && (
<div className={styles['resource-page__content']} id="dataset-mark-content">
<MicroApp key={microAppKey} name="flint-data" />
<MicroApp key={microAppKey} name="flint" />
</div>
)}
</div>

2
react-ui/src/pages/Workspace/components/Notebooks/index.tsx
View File

@ -42,7 +42,7 @@ function NotebookComponent() {
}, [getEditorList]);
const gotoDevEnv = () => {
navigate(`/pipeline/developmentEnvironment`);
navigate(`/developmentEnvironment`);
};
const gotoDevEnvItem = (item: EditorData) => {

15
react-ui/src/services/bigModel/index.ts
View File

@ -183,15 +183,16 @@ export function mergeFile(data: any) {
// 数据集上传的文件模板
export const getFileTemplateUrl = (
modelCategory: string,
dataType: BigDataType,
isMarked: number,
extend: string,
isPDF: boolean = false,
) => {
if (isPDF) {
return `${DatasetAPIPrefix}/file/download/demo?dataType=${dataType}&isMarked=${isMarked}&pattern=PDF&extend=${extend}`;
return `${DatasetAPIPrefix}/file/download/demo?modelCategory=${modelCategory}&dataType=${dataType}&isMarked=${isMarked}&pattern=PDF&extend=${extend}&language=zh_CN`;
}
return `${DatasetAPIPrefix}/file/download/demo?dataType=${dataType}&isMarked=${isMarked}&extend=${extend}`;
return `${DatasetAPIPrefix}/file/download/demo?modelCategory=${modelCategory}&dataType=${dataType}&isMarked=${isMarked}&extend=${extend}&language=zh_CN`;
};
// 数据集prompt模板
@ -214,7 +215,7 @@ export function unpraiseBigDatasetReq(id: any) {
}
// 获取用户信息
export function getUserReq(userId: any) {
export function getHuoshiUserInfoReq(userId: any) {
return request(`${DatasetAPIPrefix}/flintDatasets/getUser`, {
method: 'GET',
params: {
@ -223,6 +224,14 @@ export function getUserReq(userId: any) {
});
}
// 获取标注任务列表
export function getTaskListReq(params: any) {
return request(`${DatasetAPIPrefix}/task-item/list`, {
method: 'GET',
params,
});
}
// ----------------------------------模型---------------------------------------
// 获取大模型列表
export function getBigModelListReq(params: any) {

1
react-ui/src/services/typings.d.ts vendored
View File

@ -204,6 +204,7 @@ declare namespace API {
address?: string;
phone?: string;
roleNames?: string[];
uapId?: string;
};
type ErrorResponse = {

17
ruoyi-auth/src/main/java/com/ruoyi/auth/controller/TokenController.java
View File

@ -68,25 +68,14 @@ public class TokenController {
// Casdoor 配置
@Value("${casdoor.endpoint:}")
private String casdoorEndpoint;
@Value("${casdoor.client-id:}")
private String casdoorClientId;
@Value("${casdoor.client-secret:}")
private String casdoorClientSecret;
@Value("${casdoor.organization-name:built-in}")
private String casdoorOrganizationName;
@Value("${casdoor.application-name:app-built-in}")
private String casdoorApplicationName;
@Value("${casdoor.redirect-uri:}")
private String casdoorRedirectUri;
@Value("${casdoor.frontend-redirect-uri:}")
private String frontendRedirectUri;
@Value("${casdoor.scope:openid profile email}")
private String casdoorScope;
@ -243,13 +232,14 @@ public class TokenController {
// 5. 生成若依token
Map<String, Object> result = tokenService.createToken(userInfo);
result.put("checkTokenUri", casdoorEndpoint + "/api/validate-token");
result.put("logoutUri", casdoorEndpoint + "/logout");
result.put("logoutUri", casdoorEndpoint + "/api/logout?redirect_uri=http://172.20.32.121:31213");
result.put("oauth2AccessToken", accessToken);
result.put("idToken", idToken);
result.put("provider", "casdoor");
result.put("token_type", tokenJson.getString("token_type"));
result.put("expires_in", tokenJson.getInteger("expires_in"));
result.put("scope", tokenJson.getString("scope"));
result.put("redirectUri", casdoorRedirectUri);
if (uapUse && userInfo.getSysUser() != null) {
result.put("uapLogoutUri", uapUrl + logoutUrl);
@ -340,8 +330,7 @@ public class TokenController {
result.put("state", state);
result.put("clientId", casdoorClientId);
result.put("redirectUri", casdoorRedirectUri);
result.put("logoutUri", oauth2ClientProperties.getLogoutUri());
result.put("logoutUri", casdoorEndpoint + "/api/logout?redirect_uri=http://172.20.32.121:31213");
return R.ok(result);
}

31
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AppScheduleTask.java
View File

@ -2,6 +2,7 @@ package com.ruoyi.platform.scheduling;
import cn.hutool.core.lang.UUID;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
@ -203,10 +204,8 @@ public class AppScheduleTask {
String zipFileName = "batch_download_" + taskId + ".zip";
String zipFilePath = savePath + "/" + zipFileName;
HashMap<String, String> upload = MapUtil.newHashMap();
upload.put("input", (String) taskInfo.getByPath("input_file.direct_url"));
upload.put("output", saveFilePath);
csCollectService.allFileUpload(upload, savePath + "/" + zipFileName);
List<String> inputFilePaths = readInputFilePath(taskInfo);
csCollectService.allFileUpload(inputFilePaths, saveFilePath, zipFilePath);
HashMap<String, String> batch_download = MapUtil.newHashMap();
batch_download.put("direct_url", zipFilePath);
batch_download.put("filename", zipFileName);
@ -228,4 +227,28 @@ public class AppScheduleTask {
// trainingTaskDao.update(trainingTask);
// }
//}
/**
* 获取输入路径
*
* @param taskInfo
* @return
*/
private List<String> readInputFilePath(JSONObject taskInfo) {
List<String> empty = new ArrayList<>();;
taskInfo.forEach((key, value) -> {
if (value instanceof JSONObject) {
JSONObject valueObj = (JSONObject) value;
if (valueObj.containsKey("direct_url")) {
String directUrl = valueObj.getStr("direct_url");
String filename = valueObj.getStr("filename");
String suffix = StrUtil.removeSuffix(directUrl, filename);
empty.add(suffix);
}
}
});
return empty;
}
}

4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CSCollectService.java
View File

@ -5,7 +5,7 @@ import cn.hutool.json.JSONObject;
import com.ruoyi.platform.domain.TrainingTask;
import org.springframework.web.multipart.MultipartFile;
import java.util.HashMap;
import java.util.List;
public interface CSCollectService {
@ -23,5 +23,5 @@ public interface CSCollectService {
JSONObject singleFileDownload(Integer objectId, String savePath,String fileName) throws Exception;
void allFileUpload(HashMap<String, String> upload, String zipFilePath) throws Exception;
void allFileUpload(List<String> inputs, String output, String zipFilePath) throws Exception;
}

26
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSCollectServiceImpl.java
View File

@ -101,8 +101,7 @@ public class CSCollectServiceImpl implements CSCollectService {
.contentType("application/json")
.execute()
.body();
JSONObject resData = returnResDataWithOK(res);
return (Integer) getByPath(resData, "object.objectID");
return packageId;
}
/**
@ -123,7 +122,6 @@ public class CSCollectServiceImpl implements CSCollectService {
CSAuthInfo csAuthInfo = getZSAuthInfoFromRedis();
List<Map<String, Object>> hpcBindingFiles = buildHpcBindingFiles(taskParam, taskInfo);
JSONObject params = convertToJobRequest(trainingTask.getTaskName(),clusterId, jobName, hpcBindingFiles, csAuthInfo.getUserId(), ntasks, nodes);
String url = csUrl + "/jsm/v2/jobs/submit";
String res = HttpUtils.sendBodyPostWithToken(url, params, csAuthInfo.getToken());
JSONObject resData = returnResDataWithOK(res);
@ -191,12 +189,13 @@ public class CSCollectServiceImpl implements CSCollectService {
if (fileValue instanceof JSONObject) {
JSONObject fileObj = (JSONObject) fileValue;
// 提取 file_id 和 filename
Integer objectId = fileObj.getInt("file_id");
Integer packageId = fileObj.getInt("file_id");
String filename = fileObj.getStr("filename");
// 构建 binding 元素
Map<String, Object> binding = new HashMap<>();
binding.put("paramName", fieldName);
binding.put("objectID", objectId);
binding.put("filename", filename);
binding.put("packageID", packageId);
bindingFiles.add(binding);
}
}
@ -246,11 +245,12 @@ public class CSCollectServiceImpl implements CSCollectService {
if (hpcBindingFiles != null) {
for (Map<String, Object> fileInfo : hpcBindingFiles) {
JSONObject binding = new JSONObject();
binding.set("paramName", fileInfo.get("paramName"));
binding.set("paramName", "inputFile");
JSONObject resource = new JSONObject();
resource.set("type", "object");
resource.set("objectID", fileInfo.get("objectID"));
resource.set("type", "path");
resource.set("path", "/"+fileInfo.get("filename"));
resource.set("packageID", fileInfo.get("packageID"));
binding.set("resource", resource);
bindingArray.set(binding);
@ -419,16 +419,17 @@ public class CSCollectServiceImpl implements CSCollectService {
/**
* 所有文件打包上传
*
* @param upload
* @param inputs
* @param output
* @param zipFilePath
* @throws Exception
*/
@Override
public void allFileUpload(HashMap<String, String> upload, String zipFilePath) throws Exception {
public void allFileUpload(List<String> inputs, String output, String zipFilePath) throws Exception {
// 异步提交任务
CompletableFuture<String> future = CompletableFuture.supplyAsync(() -> {
try {
minioUtil.createZipFromDirectory(bucketName, upload, zipFilePath);
minioUtil.createZipFromDirectory(bucketName, inputs,output, zipFilePath);
return "执行成功";
} catch (Exception e) {
throw new RuntimeException("异步提交任务失败", e);
@ -436,7 +437,6 @@ public class CSCollectServiceImpl implements CSCollectService {
});
// 处理异步结果
future.thenAcceptAsync(result -> {
log.info("所有文件打包上传成功，result: {}", result);
}).exceptionally(ex -> {
log.error("异步处理任务失败", ex);

4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CSServiceVersionServiceImpl.java
View File

@ -187,11 +187,11 @@ public class CSServiceVersionServiceImpl implements ServiceVersionService {
ServiceVersion serviceVersion = csServiceDao.getServiceVersionById(serviceVersionId);
com.ruoyi.platform.domain.service.Service service = serviceDao.getServiceById(serviceVersion.getServiceId());
ServiceTemp serviceTemp = serviceTempService.getServiceTemp(service.getServiceTempId());
Integer objectID = csCollectService.uploadFile(serviceTemp.getName(), file);
Integer packageId = csCollectService.uploadFile(serviceTemp.getName(), file);
String savePath = "/mini-model-platform-data/cs-data/upload/"+serviceVersionId+ "/" + UUID.fastUUID()+"/"+file.getOriginalFilename();
minioService.uploadFile(bucketName, savePath, file);
HashMap<String, Object> result = MapUtil.newHashMap();
result.put("file_id", objectID);
result.put("file_id", packageId);
result.put("direct_url", savePath);
result.put("filename", file.getOriginalFilename());
return result;

13
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java
View File

@ -480,17 +480,12 @@ public class ServiceServiceImpl implements ServiceService {
@Override
public Map<String, Object> getServiceVersionDocs(Long id) {
ServiceVersion serviceVersion = serviceDao.getServiceVersionById(id);
HashMap<String, Object> paramMap = new HashMap<>();
paramMap.put("deployment_name", serviceVersion.getDeploymentName());
String req = HttpUtils.sendPost(argoUrl + modelService + "/getDocs", JSON.toJSONString(paramMap));
String req = HttpUtils.sendGet(serviceVersion.getUrl() + "/docs/openapi.json", null);
if (StringUtils.isNotEmpty(req)) {
Map<String, Object> reqMap = JacksonUtil.parseJSONStr2Map(req);
if (reqMap.get("code").equals(200)) {
Map<String, Object> data = (Map<String, Object>) reqMap.get("data");
return data;
} else {
throw new RuntimeException("获取日志失败：" + reqMap.get("msg"));
}
List<Map<String, Object>> servers = (ArrayList) reqMap.get("servers");
servers.get(0).put("url", serviceVersion.getUrl());
return reqMap;
} else {
throw new RuntimeException("获取日志失败");
}

104
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java
View File

@ -1,8 +1,9 @@
package com.ruoyi.platform.utils;
import com.ruoyi.common.core.utils.StringUtils;
import io.minio.*;
import io.minio.errors.MinioException;
import io.minio.errors.*;
import io.minio.http.Method;
import io.minio.messages.DeleteObject;
import io.minio.messages.Item;
@ -16,6 +17,7 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
@ -516,43 +518,69 @@ public class MinioUtil {
/**
* 创建目录的压缩包
*/
public void createZipFromDirectory(String bucket, Map<String,String> dirMap, String zipPath)
public void createZipFromDirectory(String bucket, List<String> inputs,String output, String zipPath)
throws Exception {
// 创建内存中的ZIP流
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Path tempFile = Files.createTempFile("zip-", ".tmp");
try {
// 写入临时文件
try (FileOutputStream fos = new FileOutputStream(tempFile.toFile());
ZipOutputStream zos = new ZipOutputStream(fos)) {
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
for (String input : inputs) {
if (StringUtils.isNotBlank(input)) {
writeBuffer("input", input, bucket, zos);
}
}
writeBuffer("output", output, bucket, zos);
}
// 遍历每个目录映射
for (Map.Entry<String, String> entry : dirMap.entrySet()) {
String folderName = entry.getKey(); // 压缩包内的文件夹名
String sourcePath = entry.getValue(); // MinIO中的路径
// 上传临时文件
try (InputStream is = Files.newInputStream(tempFile)) {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(bucket)
.object(zipPath)
.stream(is, Files.size(tempFile), -1)
.contentType("application/zip")
.build()
);
}
} finally {
// 清理临时文件
Files.deleteIfExists(tempFile);
}
}
// 1. 自动提取目录路径（去除文件名）
String sourceDir = extractDirectoryPath(sourcePath);
/**
*
* @param folderName 压缩包内的文件夹名
* @param sourcePath MinIO中的路径
*/
private void writeBuffer(String folderName,String sourcePath,String bucket,ZipOutputStream zos) {
try {
// 2. 规范化文件夹名
// 1. 规范化文件夹名
if (!folderName.endsWith("/")) {
folderName = folderName + "/";
}
// 3. 获取目录下的所有文件
// 2. 获取目录下的所有文件
Iterable<Result<Item>> items = minioClient.listObjects(
ListObjectsArgs.builder()
.bucket(bucket)
.prefix(sourceDir)
.prefix(sourcePath)
.recursive(true)
.build()
);
// 4. 添加文件到ZIP
// 3. 添加文件到ZIP
for (Result<Item> result : items) {
Item item = result.get();
if (item.isDir()) continue;
String objectName = item.objectName();
String relativePath = objectName.substring(sourceDir.length()-1);
String relativePath = objectName.substring(sourcePath.length() - 1);
// 使用自定义的文件夹名
String zipEntryName = folderName + relativePath;
@ -576,48 +604,8 @@ public class MinioUtil {
zos.closeEntry();
}
}
zos.finish();
}
// 上传压缩包到MinIO
byte[] zipBytes = baos.toByteArray();
try (ByteArrayInputStream bais = new ByteArrayInputStream(zipBytes)) {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(bucket)
.object(zipPath)
.stream(bais, zipBytes.length, -1)
.contentType("application/zip")
.build()
);
}
}
private String extractDirectoryPath(String path) {
path = path.trim();
// 如果路径包含点号且不以斜杠结尾，可能是文件
if (path.contains(".") && !path.endsWith("/")) {
int lastSlash = path.lastIndexOf("/");
if (lastSlash > 0) {
// 返回目录部分
return path.substring(0, lastSlash + 1);
}catch (Exception e){
e.printStackTrace();
}
}
// 如果不是文件路径，直接返回
if (!path.endsWith("/")) {
return path + "/";
}
return path;
}
public static void main(String[] args) {
String objectName = "mini-model-platform-data/cs-data/98/28bbb298-6e21-4361-b00a-ba8dc1643f65/file/log.lammps";
String relativePath = objectName.substring("/mini-model-platform-data/cs-data/98/28bbb298-6e21-4361-b00a-ba8dc1643f65/file/".length()-1);
System.out.println(relativePath); //输出og.lammps
}
}