forked from ci4s/pipeline-convert
新增杂项
This commit is contained in:
parent
7fbcd0ebd3
commit
5f11e1a6b8
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"category_id": 2,
|
||||
"component_name": "通用模型训练",
|
||||
"component_label": "通用模型训练组件",
|
||||
"description": "通用模型训练组件介绍",
|
||||
"image": "镜像",
|
||||
"working_directory": "工作目录",
|
||||
"command": "启动命令",
|
||||
"mount_path": "挂载目录",
|
||||
"in_parameters": {
|
||||
"--dataset": {
|
||||
"type": "ref",
|
||||
"item_type": "dataset",
|
||||
"label": "选择数据集",
|
||||
"require": 1,
|
||||
"choice": [],
|
||||
"default": "",
|
||||
"placeholder": "",
|
||||
"describe": "选择数据集",
|
||||
"editable": 1,
|
||||
"condition": ""
|
||||
},
|
||||
"--model_name": {
|
||||
"type": "ref",
|
||||
"item_type": "model",
|
||||
"label": "选择模型",
|
||||
"require": 0,
|
||||
"choice": [],
|
||||
"range": "$min,$max",
|
||||
"default": "",
|
||||
"placeholder": "",
|
||||
"describe": "这里是这个参数的描述和备注",
|
||||
"editable": 1,
|
||||
"condition": "",
|
||||
"form_info": {"name": "mnist", "path": "/mnt/e/xxxx"}
|
||||
}
|
||||
},
|
||||
"out_parameters": {
|
||||
"--model_output": {
|
||||
"type": "str",
|
||||
"path": "/model"
|
||||
}
|
||||
},
|
||||
"env_variables": {}
|
||||
}
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
{
|
||||
"category_id": 3,
|
||||
"component_name": "model_export",
|
||||
"component_label": "模型导出",
|
||||
"description": "模型导出到平台模型库",
|
||||
"image": "镜像", //自动关联镜像仓库,可选框, 可以为空，用户可以在流水线中再填写
|
||||
"working_directory": "工作目录", //工作目录，不填写将直接使用镜像默认的工作目录
|
||||
"command": "启动命令", //镜像的入口命令，直接写成单行字符串，例如python xx.py，无需添加
|
||||
"mount_path": "挂载目录", //支持pvc挂载, 需要确保pvc和pv绑定成功
|
||||
"control_strategy": {
|
||||
//组件的控制策略，模版注册时候不可见
|
||||
"max_run_time": {
|
||||
"type": "str",
|
||||
"item_type": "",
|
||||
// type为list时,list元素的类型
|
||||
"label": "超时中断",
|
||||
// 中文名
|
||||
"require": 1,
|
||||
// 是否必须
|
||||
"choice": [],
|
||||
// type为list时,可选的值
|
||||
"default": "0",
|
||||
// 默认值
|
||||
"placeholder": "",
|
||||
// 输入提示内容
|
||||
"describe": "组件运行时长，支持s(秒)，m(分钟)，h(小时)，d(天)，示例：3h，代表3个小时超时，0表示不限制时长",
|
||||
"editable": 1,
|
||||
// 是否可修改
|
||||
},
|
||||
"retry_time": {
|
||||
"type": "int",
|
||||
"item_type": "",
|
||||
// type为list时,list元素的类型
|
||||
"label": "重试次数",
|
||||
// 中文名
|
||||
"require": 1,
|
||||
// 是否必须
|
||||
"choice": [],
|
||||
// type为list时,可选的值
|
||||
"default": "0",
|
||||
// 默认值
|
||||
"placeholder": "",
|
||||
// 输入提示内容
|
||||
"describe": "组件运行失败自动重试次数",
|
||||
"editable": 1,
|
||||
// 是否可修改
|
||||
}
|
||||
},
|
||||
|
||||
"resources_standard": { //资源规格，模版注册的时候不可见
|
||||
"label":"资源规格", // 中文名
|
||||
"choice":[
|
||||
{"name":"CPU/GPU", "type":"list", "value":["GPU 0*v100,CPU:4,内存32GB","GPU 1*v100,CPU:4,内存16GB","GPU 1*A100,CPU:4,内存32GB"]},
|
||||
{"name":"NPU", "type":"list", "value":["NPU 1*Ascend 910 CPU:24,显存:32GB,内存256GB", "NPU 2*Ascend 910 CPU:48,显存:32GB,内存512GB", "NPU 4*Ascend 910 CPU:96,显存:32GB,内存1024GB"]}], // type为enum/multiple时，可选值
|
||||
"placeholder":"", // 目前支持CPU/GPU/NPU
|
||||
"describe":"资源规格，目前支持CPU/GPU/NPU",
|
||||
"editable":1, // 是否可修改
|
||||
"condition":"", // 显示的条件 {"name":"", "value":"xxxxxx"}
|
||||
},
|
||||
|
||||
"in_parameters":{
|
||||
"--model_name":{ // "参数名"
|
||||
"type":"str",
|
||||
"item_type": "", //支持 str,ref, int, list
|
||||
"label":"模型名称", // 中文名
|
||||
"require":1, // 是否必须
|
||||
"choice":[], // type为list时,可选的值
|
||||
"default":"", // 默认值
|
||||
"placeholder":"模型名称", // 输入提示内容
|
||||
"describe":"模型名称",
|
||||
"editable":1, // 是否可修改
|
||||
"condition":"", // 显示的条件
|
||||
},
|
||||
"--model_type":{ // "参数名"
|
||||
"type":"str",
|
||||
"item_type": "", //支持 str,ref, int, list
|
||||
"label":"模型类型", // 中文名
|
||||
"require":1, // 是否必须
|
||||
"choice":[], // type为list时,可选的值
|
||||
"default":"", // 默认值
|
||||
"placeholder":"模型类型", // 输入提示内容
|
||||
"describe":"模型类型",
|
||||
"editable":1, // 是否可修改
|
||||
"condition":"", // 显示的条件
|
||||
},
|
||||
"--model_path": { // 参数名
|
||||
"type":"str", //支持 str,ref, int, list
|
||||
"item_type": "", // 在type为list时每个子参数的类型
|
||||
"label":"模型路径", // 中文名
|
||||
"require":0, // 是否必须
|
||||
"choice":[], // type为list时，可选值
|
||||
"default":"master", // 默认值
|
||||
"placeholder":"", // 输入提示内容
|
||||
"describe":"模型路径",
|
||||
"editable":1, // 是否可修改
|
||||
"condition":"", // 显示的条件 表单信息：{"name":"mnist","path":"/mnt/e/xxxx"}
|
||||
},
|
||||
"--model_access": { // 参数名
|
||||
"type":"str", //支持 str,ref, int, list
|
||||
"item_type": "", // 在type为list时每个子参数的类型
|
||||
"label":"模型权限", // 中文名
|
||||
"require":0, // 是否必须
|
||||
"choice":[], // type为list时，可选值
|
||||
"default":"1", // 默认值
|
||||
"placeholder":"", // 输入提示内容
|
||||
"describe":"模型访问权限，private 私有，public 公开",
|
||||
"editable":1, // 是否可修改
|
||||
"condition":"", // 显示的条件 表单信息：{"name":"mnist","path":"/mnt/e/xxxx"}
|
||||
},
|
||||
},
|
||||
|
||||
"env_virables": {
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# ====== project dir config ======
|
||||
SCRIPT_PATH=$(realpath "$0")
|
||||
SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
|
||||
MAIN_DIR=$(dirname "$SCRIPT_DIR")
|
||||
#add main_dir to python dir
|
||||
|
||||
|
||||
# ====== model import dir ======
|
||||
export PYTHONPATH=$MAIN_DIR:$PYTHONPATH
|
||||
|
||||
|
||||
# ====== servers for openi ======
|
||||
#servers=( $VC_TASK0_HOSTS $VC_TASK1_HOSTS $VC_TASK2_HOSTS $VC_TASK3_HOSTS $VC_TASK4_HOSTS $VC_TASK5_HOSTS $VC_TASK6_HOSTS $VC_TASK7_HOSTS )
|
||||
for i in {0..11}
|
||||
do
|
||||
server="VC_TASK${i}_HOSTS"
|
||||
servers[$i]=${!server}
|
||||
done
|
||||
echo "servers=${servers[@]}"
|
||||
|
||||
|
||||
# ====== run command on each worker ======
|
||||
#comand="/opt/conda/bin/pip install transformers"
|
||||
commands=( "nvidia-smi | grep M" )
|
||||
#commands=( "/opt/conda/bin/pip install transformers" )
|
||||
for command in "${commands[@]}"
|
||||
do
|
||||
for server in "${servers[@]}"
|
||||
do
|
||||
echo "Executing command on $server"
|
||||
ssh $server "cd $SCRIPT_DIR; $command"
|
||||
done
|
||||
done
|
||||
|
||||
|
||||
# ====== Environment ======
|
||||
# - NCCL & IB
|
||||
export NCCL_DEBUG=ERROR #INFO ERROR
|
||||
export NCCL_IB_DISABLE=0
|
||||
#export NCCL_SOCKET_IFNAME=eth0
|
||||
export NCCL_IB_GID_INDEX=3
|
||||
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
||||
export CUDA_HOME="/usr/local/cuda"
|
||||
export PATH=/usr/local/cuda/bin:$PATH
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
|
||||
#export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7
|
||||
|
||||
|
||||
# ====== hostfile ======
|
||||
# HOSTFILE=/userhome/codeGeeX/hostfile
|
||||
MASTER_IP=$VC_TASK0_HOSTS
|
||||
rm $SCRIPT_DIR/hostfile
|
||||
for server in "${servers[@]}"
|
||||
do
|
||||
echo "$server slots=8" >> $SCRIPT_DIR/hostfile
|
||||
done
|
||||
cat $SCRIPT_DIR/hostfile
|
||||
echo "MASTER_IP=$MASTER_IP"
|
||||
|
||||
|
||||
# ====== dir config ======
|
||||
#DATASET_1="/userhome/dataset/7b_GPU_pretrain_0615/zh_bin/text_document"
|
||||
#DATASET_2="/userhome/dataset/7b_GPU_pretrain_0615/en_bin/text_document"
|
||||
#DATASET_3="/userhome/dataset/7b_GPU_pretrain_0615/code_bin/text_document"
|
||||
#DATA_PATH="10 ${DATASET_1} 4 ${DATASET_2} 1 ${DATASET_3}"
|
||||
DATA_PATH=/userhome/dataset/7b_GPU_260GB_stage9_0814_0814_fixVocab_bin/text_document
|
||||
|
||||
# ====== Parameters ======
|
||||
# - 13B
|
||||
#NLAYERS=39
|
||||
#HIDDEN=5120
|
||||
#NATTN_HEAD=40
|
||||
#SEQLEN=2048
|
||||
#TRIAL_TAG="13B-64a100-800g-0316-v1"
|
||||
|
||||
|
||||
## - 350M
|
||||
#NLAYERS=24 #39
|
||||
#HIDDEN=1024
|
||||
#NATTN_HEAD=16
|
||||
#SEQLEN=1024
|
||||
#TRIAL_TAG="350M-64a100-1T-0330-v1-test-time"
|
||||
|
||||
# 7B
|
||||
NLAYERS=32 #39
|
||||
HIDDEN=4096
|
||||
NATTN_HEAD=32
|
||||
SEQLEN=2048
|
||||
TRIAL_TAG="7B-0814"
|
||||
|
||||
|
||||
# ====== trial dir ======
|
||||
TRIAL_NAME="pretrain-wanli"
|
||||
# - logging & output
|
||||
NOW=$(date +"%Y%m%d_%H%M%S")
|
||||
OUTPUT_DIR=/userhome/model/wanli/$TRIAL_NAME-$TRIAL_TAG
|
||||
TB_DIR=$OUTPUT_DIR/tb$NOW
|
||||
TB_DIR=$OUTPUT_DIR/tb20230814_221138
|
||||
mkdir -p $OUTPUT_DIR
|
||||
mkdir -p $TB_DIR
|
||||
|
||||
|
||||
# ====== run ======
|
||||
echo "Launching deepspeed"
|
||||
deepspeed \
|
||||
--hostfile hostfile \
|
||||
--master_addr $MASTER_IP \
|
||||
$MAIN_DIR/pretrain_gpt.py \
|
||||
--tensor-model-parallel-size 4 \
|
||||
--pipeline-model-parallel-size 1 \
|
||||
--num-layers $NLAYERS \
|
||||
--hidden-size $HIDDEN \
|
||||
--make-vocab-size-divisible-by 128 \
|
||||
--num-attention-heads $NATTN_HEAD \
|
||||
--seq-length $SEQLEN \
|
||||
--max-position-embeddings $SEQLEN \
|
||||
--micro-batch-size 8 \
|
||||
--global-batch-size 1920 \
|
||||
--lr 4e-6 \
|
||||
--min-lr 1e-6 \
|
||||
--lr-decay-style cosine \
|
||||
--train-samples 42340948 \
|
||||
--log-interval 1 \
|
||||
--data-path $DATA_PATH \
|
||||
--vocab-file $MAIN_DIR/megatron/tokenizer/llama_zh_hf \
|
||||
--merge-file $MAIN_DIR/megatron/tokenizer/spm_13w/data_dict.128k.txt \
|
||||
--save-interval 800 \
|
||||
--save $OUTPUT_DIR \
|
||||
--load /userhome/model/wanli/pretrain-wanli-7B-0807 \
|
||||
--override-opt_param-scheduler \
|
||||
--split 99,1,0 \
|
||||
--clip-grad 1.0 \
|
||||
--weight-decay 0.1 \
|
||||
--adam-beta1 0.9 \
|
||||
--adam-beta2 0.99 \
|
||||
--bf16 \
|
||||
--finetune \
|
||||
--recompute-activations \
|
||||
--use-flash-attn \
|
||||
--dataloader-type 'single' \
|
||||
--attention-softmax-in-fp32 \
|
||||
--reset-position-ids \
|
||||
--reset-attention-mask \
|
||||
--eod-mask-loss \
|
||||
--no-gradient-accumulation-fusion \
|
||||
--no-async-tensor-model-parallel-allreduce \
|
||||
--tokenizer-type LlamazhTokenizer \
|
||||
--no-query-key-layer-scaling \
|
||||
--timing-log-level 2 \
|
||||
--data-cache-path /userhome/_megatron_cache/ \
|
||||
--tensorboard-dir $TB_DIR |& tee ${OUTPUT_DIR}/$NOW.log
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/google/go-containerregistry/pkg/authn"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// 替换为你的 Harbor 仓库地址和镜像名称
|
||||
harborRepository := "fzrj.harbor.com"
|
||||
imageName := "fzrj.harbor.com/machine-learning/git:202312071000"
|
||||
|
||||
// 构建镜像的完整名称
|
||||
imageRef, err := name.ParseReference(imageName)
|
||||
if err != nil {
|
||||
log.Fatalf("Error parsing image reference: %v", err)
|
||||
}
|
||||
|
||||
// 使用 Harbor 仓库的地址构建远程镜像引用
|
||||
remoteImageRef, err := name.NewTag(fmt.Sprintf("%s/%s", harborRepository, imageRef.Identifier()), name.WeakValidation)
|
||||
if err != nil {
|
||||
log.Fatalf("Error constructing remote image reference: %v", err)
|
||||
}
|
||||
|
||||
// 获取 Docker 配置文件的路径（通常在用户主目录下）
|
||||
//dockerConfigPath := os.ExpandEnv("$HOME/.docker/config.json")
|
||||
|
||||
// 从 Docker 配置文件中获取认证信息
|
||||
authenticator, err := authn.DefaultKeychain.Resolve(remoteImageRef.Context().Registry)
|
||||
if err != nil {
|
||||
log.Fatalf("Error resolving Docker credentials: %v", err)
|
||||
}
|
||||
|
||||
// 使用允许不安全传输的方式
|
||||
options := []remote.Option{
|
||||
remote.WithAuth(authenticator),
|
||||
remote.WithTransport(http.DefaultTransport),
|
||||
}
|
||||
// 获取镜像的信息
|
||||
image, err := remote.Image(remoteImageRef, options...)
|
||||
if err != nil {
|
||||
log.Fatalf("Error fetching image: %v", err)
|
||||
}
|
||||
|
||||
// 获取镜像的配置信息
|
||||
config, err := image.ConfigFile()
|
||||
if err != nil {
|
||||
log.Fatalf("Error fetching image config: %v", err)
|
||||
}
|
||||
|
||||
// 输出镜像信息
|
||||
fmt.Printf("Image Name: %s\n", imageRef.Name())
|
||||
fmt.Printf("Image Digest: %s\n", imageRef.Identifier())
|
||||
fmt.Printf("Image Architecture: %s\n", config.Architecture)
|
||||
fmt.Printf("Image OS: %s\n", config.OS)
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
myminio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"log"
|
||||
)
|
||||
|
||||
harborRepository := "fzrj.harbor.com"
|
||||
imageName := "fzrj.harbor.com/machine-learning/git:202312071000"
|
||||
|
||||
func main() {
|
||||
endpoint := "minio.argo:9000" // 例如: "localhost:9000"
|
||||
accessKeyID := "admin"
|
||||
secretAccessKey := "qazxc123456."
|
||||
useSSL := false // 如果MinIO部署在HTTPS上则设置为true
|
||||
|
||||
// 初始化MinIO客户端
|
||||
minioClient, err := myminio.New(endpoint, &myminio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
})
|
||||
|
||||
// 要上传的文件
|
||||
filePath := "git-clone.yaml"
|
||||
bucketName := "platform-data"
|
||||
objectName := "somuns/dataset/caffe/git-clone.yaml"
|
||||
|
||||
// 使用FPutObject上传文件
|
||||
info, err := minioClient.FPutObject(context.Background(), bucketName, objectName, filePath, myminio.PutObjectOptions{ContentType: "application/octet-stream"})
|
||||
if err != nil {
|
||||
fmt.Print("upload err:", err.Error())
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
log.Printf("Successfully uploaded %s of size %d\n", objectName, info)
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
package parse
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
FROM bitnami/python
|
||||
WORKDIR /app
|
||||
|
||||
COPY model_export.py /app
|
||||
COPY sources.list /etc/apt/
|
||||
|
||||
# 更新 sources.list 后安装 ssh
|
||||
RUN apt-get update && apt-get install -y ssh
|
||||
|
||||
ENTRYPOINT ["python", "git_clone.py"]
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
tag=$(date +'%Y%m%d%H%M')
|
||||
docker build \
|
||||
--build-arg "HTTP_PROXY=http://172.20.32.253:3128" \
|
||||
--build-arg "HTTPS_PROXY=http://172.20.32.253:3128" \
|
||||
-t fzrj.harbor.com/machine-learning/git:${tag} .
|
||||
|
||||
docker push fzrj.harbor.com/machine-learning/git:${tag}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM2VENDQWRHZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQ0FYRFRJek1URXdNekF4TURJMU1sb1lEekl4TWpNeE1ERXdNREV3TWpVeVdqQVZNUk13RVFZRApWUVFERXdwcmRXSmxjbTVsZEdWek1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBCjJmTjg3MGg5N2RlTlh4ank5T0p5R1ZIcnVmd1BtM05Ma0F5VkwzUUhCUjROVXFQc2N5a1FpTFNUSE1TZWp6L2sKWUsrOWFQVjM5ekJiNjZlSmFaM3V6QitKcktYZXVDcUlZMHNNWlNSN3VxUWVWRVlYSDExL1F0WXdZRG5qQ1dNRgpvQ2RERkk2L0MzTm9zTEU3b3hxb2RvSGhkZFVRUno3L3U0UERxNngwVlN5U29uQTIwbEc4eXFCNmpudGduczkwCjdkNkFTclJRME5ZeHpNTXczTXI0dVJNdTBYY0FkQVFaVmc1MDcyU2hSQk1RVW56bnQzRkNxcGhLWjM1Ny9tWDUKZG55Uml4aEd5MGNOWThacjhmWXJnMGYwQ0plbTJaSGd6MEp1STZ1MUNEYlRMaVQ0dUFLRUdsYXpCNDY2eU01TwpjQWpjVytxVGJzaTBQOGZpTGx6d1pRSURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQXFRd0R3WURWUjBUCkFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVQVDU1Zk5UVmtTUVJzOGdDcTZWZi8yZWNaY2t3RFFZSktvWkkKaHZjTkFRRUxCUUFEZ2dFQkFGVWVRVEVId3dJY2J2cEllU2ZZSnJ0WDhaZUttcmVLZnROQ2VRT0dpOWhwUW1VTwpnU2JSVjdGZkRHaktJUWtTVEU4UkZ4L3haRHltdTVIRmdrVVpoc0ptVjFFSU9XcXFiMzRDeWJPOGR2Sm9QeXpqClBYV1BxSWEvRVNFNWVqRUZqUXoyZHJIZDVacVowMldFY01iclVDYXhmeDJPUkNKa0pIYXNkQWNyNG5LSWlsankKemp2TGJ5NDFtT0hhMWREdlEvUi9lT2lSV2dISEszYkQ2REV3bjk4TEhCUUJtSW1XdDVHcFVJTnFxN2w2ZVNKOQpxSnkwZDNoM0FQMmlyeDFBNEVDSVEwWVFrNS9IMU0weEMrSENkNUt6UkUvdHpvenlCU3FNSWFUMENmN0t1Y1l0CjNDU0hKQWtCSkpoRkRFcElrVkw0a25iNnl6MFF6RjBYbCt4ZGhlMD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
|
||||
server: https://apiserver.cluster.local:6443
|
||||
name: kubernetes
|
||||
contexts:
|
||||
- context:
|
||||
cluster: kubernetes
|
||||
user: kubernetes-admin
|
||||
name: kubernetes-admin@kubernetes
|
||||
current-context: kubernetes-admin@kubernetes
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users:
|
||||
- name: kubernetes-admin
|
||||
user:
|
||||
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGVENDQWYyZ0F3SUJBZ0lJU05BVDZEcHlHSEV3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWdGdzB5TXpFeE1ETXdNVEF5TlRKYUdBOHlNVEl6TVRBeE1EQXhNREkxTTFvdwpOREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhHVEFYQmdOVkJBTVRFR3QxWW1WeWJtVjBaWE10CllXUnRhVzR3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRHFBMHAvTmNaMEpXU0UKRUFCTVNISktjbHViNjF3Tzd0VUYrd3BhNFRVQXNyU3IrZnp4Z01BUzFVenplem16ajQ1UElxSmRyZEZIY3Y0TApwRnFkK2o0dHNaZzlJRXMvU2ozL3ZEdWhBb3gydnByUC9iK3N6dWs5OHkyd3lOeHNIN3V2RHh2ZjFFMVRxSzdXCmxocks5RVRyWkFKNEhnY1dLbkVzMW44Z09iZ2F5aDdiVkJWVmhCbFY4VDVpSTBPNWZlbHdMaVQvMXhNVUk4TEwKd3ZUalQzQngxd2duUlBBdjNvc2s3VHJobjlCQysyZUZCWWwzeHI4NnRmc2JDUktHWmpNZm1lc0FLNmFzTkpFLwpyUmo0RXhNd2NYeGxpNmRRdk5QaHNQbnBzQ3JpZ21BdkdyblFPdldTaHRlbXRjTEdMUldvaTZ4Q0ViZU93QUlKCjg4UlROT1hwQWdNQkFBR2pTREJHTUE0R0ExVWREd0VCL3dRRUF3SUZvREFUQmdOVkhTVUVEREFLQmdnckJnRUYKQlFjREFqQWZCZ05WSFNNRUdEQVdnQlE5UG5sODFOV1JKQkd6eUFLcnBWLy9aNXhseVRBTkJna3Foa2lHOXcwQgpBUXNGQUFPQ0FRRUExWCthNXowUWx5NEFsQWsrZDJSaFFrVFB1T3BVRVExajlNVGNLK1FFdEphbmZzbVhVNmZCCmZmdGIxeHdrQzE5QWsvS2N6QkEyU3BKMDExUWhBZFJTRk1QVVhVUmU4a1F3WWpGK0J3QUF1ZUordEV0VVdmSXQKdnFoWjI0ekxaMU84NHJMRkRkZThYYkFvTFJWM29BTThucDEranI0cGZZVEU4aHNrZXZ5TzAxWnFOMzhIODdIagphNjg3TytwRjg4YXhsMEx3ZkpFK2Vja1FaT25DekF1WSticmZkV3FuQllYUlhyVGxiUzQxMHE2V0pPSldjZUNTClllenpudVM0MWxJdHBGQ0UrYzFIajRrajUyZncrUWd6dWhBb1JnRXY3S1ptbWNwbVlIZ0dvMVNKWWpQUTNiVW4KZmdNdHBrcldKL2RGN25xaTA3K3RPR0pnc05EYWgweDJhdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
|
||||
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcGdJQkFBS0NBUUVBNmdOS2Z6WEdkQ1ZraEJBQVRFaHlTbkpibSt0Y0R1N1ZCZnNLV3VFMUFMSzBxL244CjhZREFFdFZNODNzNXM0K09UeUtpWGEzUlIzTCtDNlJhbmZvK0xiR1lQU0JMUDBvOS83dzdvUUtNZHI2YXovMi8Kck03cFBmTXRzTWpjYkIrN3J3OGIzOVJOVTZpdTFwWWF5dlJFNjJRQ2VCNEhGaXB4TE5aL0lEbTRHc29lMjFRVgpWWVFaVmZFK1lpTkR1WDNwY0M0ay85Y1RGQ1BDeThMMDQwOXdjZGNJSjBUd0w5NkxKTzA2NFovUVF2dG5oUVdKCmQ4YS9Pclg3R3drU2htWXpINW5yQUN1bXJEU1JQNjBZK0JNVE1IRjhaWXVuVUx6VDRiRDU2YkFxNG9KZ0x4cTUKMERyMWtvYlhwclhDeGkwVnFJdXNRaEczanNBQ0NmUEVVelRsNlFJREFRQUJBb0lCQVFEZFFpRlU1L1pzNk5JQwpIOUw0VFdOcXluZWRrOHBtQUNHQVZ4MHlVazg0Q3IvUy96bk82aW94WkdZdXJLc2hBTEhRVnZIbjJ5bk9BWWI4CllQaWc5L09FWnErSDFCbnhqSFBVY2hoN0l4ckdSMisvQ3JBSloxV1hFbTk0QUdLUkJPd0RHQ2tGMkFrSSthdmUKRHV6a1NsTkRzVHE0Y2NQSGpIZVhxbk5tZDh4LzNoNnVWeHE2bVgzSFJ1SHpPeDEwUGYzV0tuRnFaRW1vcDA5QgpXbEhDQW1TTkd5V1ZsVSs0cDVacGFBbkk2SHZIblJnbDErKzA1NTZ3VFk2ZHd5WWNzcW1oMkx2aDlPWDQ2dVc3CmNTNHlpeGt2c0VkUzRWUVBZRTNTVXdTajdlWVhXWndHR0ZzL3ViRWFacXhZOEdYZlVBY013eFZ3eExZTEk2NTEKSVVLL3hOTEZBb0dCQVBmeC82QjhRLy9ITWlpMEZPeDliTmtldnJIbGd0OEp5MXZOeUE3YlVzWTBEWmNKdXA4SwpSbnVZdkpMejg1SEFycnptbkJoZXhsOE9vc0tSQ0pzOERmYmVJMUZMdlJ0WnJhMUZzcFlsTldvOUpOeEgzalZBCmZpbFFwM0ZzN2VwY3BkcVc5ZFBnblkzS2VMUzU2VUVzQWhLZW81aTRLcEo0amZFb1JWdHNnY3lmQW9HQkFQR2QKYk53eEdscW9EbnIxdCtqbnNyNk95UEw0SVhZdkRYamRwRG5vcitmMElEc1B6YzlRa2tCeldzVXpDSCt6bGxFNgpqV1VRUzVyNVpRV3ZUZ3lrM1NHNTR3ekNyWHRTaGZoKzNVQUVQQVJ1bkVEWm9xaW9FaWlFVVVNVnN4NW5zZ2Q3ClE2ckxrSmhCSHlSa3F0VVFzNDlyVUxnRVpiR0haNFBidEZGd1pEaDNBb0dCQU5GQzNZeUZWSjY3Y0FvS2N0NWwKdlVoQ09vL0tOUG9wT2dqeFZMZTFROWNrZDVQU0VqL0lYRlJVUkZPQ3NUK0NiL2ZlYWovblBXVmM1T0Z4cHVIZApmYXV0TUFHYytxM3FzanBiYUs1d0dQVGhSZ3JrWFJaMHFBWVduRHVPdy8zWTArSDVKdUQ3NGYyWnRPWlRRaHBQCkdqQzgwelpCdVN5RnNzYW1SbU15NFJuaEFvR0JBTDB0MS9pb2Njd2QzZ2lrc0hVcVJhZ3l3R0NOckJXUFRlc2YKRCtGUHNidmFTQW8ybTUwQnB5SFRzSFRMT3BKU21SWE0xTDBXL3p1K25qelBRRHRpTXNxZmM0bWdQSmxjeWcwQgp1RmJSZWZTMmJwM1FTaTNvT3grQ1N1Wk43VXVYUWF2ZDllT2JOQUlpcXpHaHFLamZSTmk4M0J6WlJJNjJTSFRUCjUrSUVNRlNOQW9HQkFNdWo2L1VKRnFuYlAzaTM1TTlzc1dySmFSRlVjUkUyZ0w0dnk0ZFgrWHp6ZlU0NTVXdGsKbGpSZnlPZS9iZzFQTk9ScTJ0cVpNb0I1UXZFYUxKRHRpdGdOcWpRT2gwOS9neFErcGFIVlFTYk95MkJhN2Vacgo4S1B6TlFkSmNKU1JDeVRiRmYrOFk0MmhmMHVRbTBhbEpIM05YTmxiQ3dQamoxSGx2MjBWay9RaQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
import argparse
|
||||
import os
|
||||
from kubernetes import client, config
|
||||
from minio import Minio
|
||||
# from minio.error import ResponseError
|
||||
from minio.error import S3Error
|
||||
|
||||
|
||||
def get_minio_credentials(secret_name, access_key_key, secret_key_key):
|
||||
# 从 Secret 中获取 MinIO 的 AccessKey 和 SecretKey
|
||||
kubeconfig_path = 'kubeconfig'
|
||||
config.load_kube_config(kubeconfig_path) # 加载默认的 kubeconfig 文件
|
||||
v1 = client.CoreV1Api()
|
||||
|
||||
namespace = "argo"
|
||||
|
||||
secret = v1.read_namespaced_secret(secret_name, namespace)
|
||||
|
||||
access_key = secret.data[access_key_key].decode("utf-8")
|
||||
secret_key = secret.data[secret_key_key].decode("utf-8")
|
||||
|
||||
return access_key, secret_key
|
||||
|
||||
|
||||
def upload_model_to_minio(minio_endpoint, bucket_name, model_name, model_path, model_type, model_access):
|
||||
# 初始化 MinIO 客户端
|
||||
#access_key, secret_key = get_minio_credentials("my-minio-cred", "accesskey", "secretkey")
|
||||
|
||||
access_key = "admin"
|
||||
secret_key = "qazxc123456."
|
||||
minio_client = Minio(minio_endpoint,
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
secure=False)
|
||||
|
||||
# 递归上传目录下的所有文件
|
||||
for root, dirs, files in os.walk(model_path):
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
relative_path = os.path.relpath(file_path, model_path)
|
||||
object_name = os.path.join(model_type, model_name, relative_path).replace(os.sep, "/")
|
||||
|
||||
# 设置上传选项
|
||||
content_type = "application/octet-stream"
|
||||
user_metadata = {'x-amz-acl': model_access}
|
||||
|
||||
# 读取文件内容
|
||||
with open(file_path, 'rb') as file_data:
|
||||
try:
|
||||
# 使用 MinIO 客户端上传文件
|
||||
minio_client.put_object(bucket_name, object_name, file_data, os.path.getsize(file_path),
|
||||
content_type=content_type, metadata=user_metadata)
|
||||
except S3Error as err:
|
||||
print(f"Error uploading {file_path} to MinIO: {err}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Upload model files to MinIO')
|
||||
|
||||
parser.add_argument('--model-name', required=True, help='Model name')
|
||||
parser.add_argument('--model-path', required=True, help='Local path to model directory')
|
||||
parser.add_argument('--model-type', required=True, help='Model type')
|
||||
parser.add_argument('--model-access', required=True, help='Model access')
|
||||
|
||||
minio_endpoint = "minio.argo.svc.cluster.local:9000"
|
||||
bucket_name = "platform-data"
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
upload_model_to_minio(minio_endpoint, bucket_name, args.model_name, args.model_path, args.model_type,
|
||||
args.model_access)
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"category_id": 4,
|
||||
"component_name": "distributed-model-train",
|
||||
"component_label": "分布式模型训练组件",
|
||||
"description": "分布式模型训练组件，支持deepspeed, metatron, horovod",
|
||||
"image": "镜像",
|
||||
"working_directory": "工作目录",
|
||||
"command": "启动命令",
|
||||
"mount_path": "挂载目录",
|
||||
"in_parameters": {
|
||||
"--dataset": {
|
||||
"type": "ref",
|
||||
"item_type": "dataset",
|
||||
"label": "选择数据集",
|
||||
"require": 1,
|
||||
"choice": [],
|
||||
"default": "",
|
||||
"placeholder": "",
|
||||
"describe": "选择数据集",
|
||||
"editable": 1,
|
||||
"condition": ""
|
||||
},
|
||||
"--model_name": {
|
||||
"type": "ref",
|
||||
"item_type": "model",
|
||||
"label": "选择模型",
|
||||
"require": 0,
|
||||
"choice": [],
|
||||
"range": "$min,$max",
|
||||
"default": "",
|
||||
"placeholder": "",
|
||||
"describe": "这里是这个参数的描述和备注",
|
||||
"editable": 1,
|
||||
"condition": "",
|
||||
"form_info": {"name": "mnist", "path": "/mnt/e/xxxx"}
|
||||
}
|
||||
},
|
||||
"out_parameters": {
|
||||
"--model_output": {
|
||||
"type": "str",
|
||||
"path": "/model"
|
||||
}
|
||||
},
|
||||
"env_variables": {}
|
||||
}
|
||||
Loading…
Reference in New Issue