forked from ci4s/pipeline-convert
141 lines
3.8 KiB
Go
141 lines
3.8 KiB
Go
package parse
|
|
|
|
import (
|
|
"github.com/gin-gonic/gin"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"path"
|
|
"pipeline-convert/configs"
|
|
"pipeline-convert/internal/model"
|
|
"pipeline-convert/internal/utils/stringutils"
|
|
)
|
|
|
|
type parse struct {
|
|
}
|
|
|
|
func NewParse() Parse {
|
|
return &parse{}
|
|
}
|
|
|
|
const (
|
|
DistributedTrainType = "distributed-model-train"
|
|
ModelDeployType = "model-deploy"
|
|
FrameLogMountPrefix = "argo-workflow/workflow-run-time-data"
|
|
FrameLogMountPath = "/tensorboard-logs"
|
|
GenernalTrain = "model-train"
|
|
DistributedTrain = "distributed-model-train"
|
|
ModelEvaluate = "model-evaluate"
|
|
ModelExport = "model-export"
|
|
DatasetExport = "dataset-export"
|
|
GeneralDataProcess = "general-data-process"
|
|
GitClone = "git-clone"
|
|
AutoMlCategoryID = 8
|
|
GitCloneCategoryID = 1
|
|
AutoML = "auto-ml"
|
|
HyperparameterOptimization = "auto-hpo"
|
|
ExperimentRemoteRepo = "EXPERIMENT_REMOTE_REPO"
|
|
ExperimentRunID = "EXPERIMENT_RUN_ID"
|
|
ExperimentRunName = "EXPERIMENT_RUN_NAME"
|
|
ExperimentRunIDPlacholder = "${EXPERIMENT_RUN_ID}"
|
|
ExperimentRunNamePlacholder = "${EXPERIMENT_RUN_NAME}"
|
|
DatasetKey = "--dataset"
|
|
ModelKey = "--model"
|
|
ModelSourceKey = "--model_id"
|
|
DatasetSourceKey = "--dataset_id"
|
|
ModelVersionKey = "--version"
|
|
CodePathKey = "--code_path"
|
|
BranchNameKey = "--branch"
|
|
ModelNameKey = "--model_name"
|
|
ArgoTaskType = "argo-task"
|
|
VcJobTaskType = "vcjob-task"
|
|
CodeConfigKey = "--code_config"
|
|
JuicefsStorageType = "juicefs"
|
|
)
|
|
|
|
var (
|
|
FrameLogMountType = []string{
|
|
GenernalTrain,
|
|
DistributedTrain,
|
|
}
|
|
|
|
MetricEnvType = []string{
|
|
GenernalTrain,
|
|
DistributedTrain,
|
|
ModelEvaluate,
|
|
}
|
|
)
|
|
|
|
func (p *parse) Parse(c *gin.Context, cs model.Component, d *model.Dag) (*WorkflowTemplate, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
//func GetParse(c *gin.Context, name string) Parse {
|
|
// switch name {
|
|
// case DistributedTrainType:
|
|
// return NewDistributedTrainParse()
|
|
// case ModelDeployType:
|
|
// return NewModelDeployParse(NewGeneralParse())
|
|
// case GenernalTrain:
|
|
// return NewGeneralParse()
|
|
// default:
|
|
// return nil
|
|
// }
|
|
// return nil
|
|
//}
|
|
|
|
//func GetComponentsParser(c *gin.Context, name string) Parse {
|
|
// taskType := configs.RConfig.GetConfig().ArgoTaskConfig.TaskType
|
|
// if taskType == "" || taskType == ArgoTaskType {
|
|
// return GetArgoTaskParser(c, name)
|
|
// }
|
|
// return GetVcJobTaskParser(c, name)
|
|
//}
|
|
//
|
|
//func GetArgoTaskParser(c *gin.Context, name string) Parse {
|
|
// switch name {
|
|
// case DistributedTrainType:
|
|
// return NewDistributedTrainParse()
|
|
// case ModelDeployType:
|
|
// return NewModelDeployParse(NewGeneralParse())
|
|
// default:
|
|
// return NewArgoTaskBuilder()
|
|
// }
|
|
// return nil
|
|
//}
|
|
|
|
//func GetVcJobTaskParser(c *gin.Context, name string) Parse {
|
|
// return NewVcJobTaskBuilder()
|
|
//}
|
|
|
|
// GenerateFrameLogMountSubPath 生成输入的挂载路径
|
|
func GenerateFrameLogMountSubPath(prefix string, keyName string) string {
|
|
return path.Join(prefix, WorkflowName, stringutils.RemovePrefix(keyName, []string{"-", "--"}), "log")
|
|
}
|
|
|
|
func AddCommonEnv(env []corev1.EnvVar, storageConfig *configs.StorageConfig) []corev1.EnvVar {
|
|
env = append(env, corev1.EnvVar{
|
|
Name: "POD_NAME",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "metadata.name",
|
|
},
|
|
},
|
|
},
|
|
corev1.EnvVar{
|
|
Name: "POD_NAMESPACE",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "metadata.namespace",
|
|
},
|
|
},
|
|
},
|
|
corev1.EnvVar{
|
|
Name: "STORAGE_TYPE",
|
|
Value: storageConfig.ResourceType,
|
|
},
|
|
corev1.EnvVar{
|
|
Name: "STORAGE_BUCKET",
|
|
Value: storageConfig.BucktName,
|
|
})
|
|
return env
|
|
}
|