fit:modelarts

This commit is contained in:
qiwang 2023-02-24 17:47:13 +08:00
parent 552acf1d2c
commit a23d2d078e
15 changed files with 529 additions and 1800 deletions

View File

@ -5,4 +5,5 @@ info(
desc: "AI core微服务"
author: "wanqgi"
email: "1364512070@qq.com"
)
)

View File

@ -0,0 +1,31 @@
package logic
import (
"context"
"PCM/adaptor/AIComputing/modelarts/rpc/internal/svc"
"PCM/adaptor/AIComputing/modelarts/rpc/modelarts"
"github.com/zeromicro/go-zero/core/logx"
)
type FindDatasetListLogic struct {
ctx context.Context
svcCtx *svc.ServiceContext
logx.Logger
}
func NewFindDatasetListLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FindDatasetListLogic {
return &FindDatasetListLogic{
ctx: ctx,
svcCtx: svcCtx,
Logger: logx.WithContext(ctx),
}
}
// find modelarts Token
func (l *FindDatasetListLogic) FindDatasetList(in *modelarts.FindDatasetReq) (*modelarts.FindDatasetResp, error) {
// todo: add your logic here and delete this line
return &modelarts.FindDatasetResp{}, nil
}

View File

@ -1,7 +1,13 @@
package logic
import (
"bytes"
"context"
"encoding/json"
"io/ioutil"
"log"
"net/http"
"time"
"PCM/adaptor/AIComputing/modelarts/rpc/internal/svc"
"PCM/adaptor/AIComputing/modelarts/rpc/modelarts"
@ -23,9 +29,39 @@ func NewGetTokenLogic(ctx context.Context, svcCtx *svc.ServiceContext) *GetToken
}
}
// ListJob list all jobs
// getToken
func (l *GetTokenLogic) GetToken(in *modelarts.TokenReq) (*modelarts.TokenResp, error) {
// todo: add your logic here and delete this line
var resp modelarts.TokenResp
TokenUrl := "v3/auth/tokens"
jsonStr, _ := json.Marshal(in)
reqUrl, err := http.NewRequest("POST", "https://iam.cn-north-4.myhuaweicloud.com/"+TokenUrl, bytes.NewBuffer(jsonStr))
if err != nil {
log.Fatal(err)
}
defer reqUrl.Body.Close()
c := http.Client{Timeout: time.Duration(3) * time.Second}
respUrl, err := c.Do(reqUrl)
print(respUrl)
defer respUrl.Body.Close()
if err != nil {
log.Fatal(err)
}
result, _ := ioutil.ReadAll(respUrl.Body)
json.Unmarshal([]byte(string(result)), &resp)
//X-Auth-Token := respUrl.Header().Get("X-Subject-Token")
//return resp, nil
return &modelarts.TokenResp{}, nil
}

View File

@ -22,8 +22,14 @@ func NewModelArtsServer(svcCtx *svc.ServiceContext) *ModelArtsServer {
}
}
// ListJob list all jobs
// get modelarts Token
func (s *ModelArtsServer) GetToken(ctx context.Context, in *modelarts.TokenReq) (*modelarts.TokenResp, error) {
l := logic.NewGetTokenLogic(ctx, s.svcCtx)
return l.GetToken(in)
}
// find modelarts Token
func (s *ModelArtsServer) FindDatasetList(ctx context.Context, in *modelarts.FindDatasetReq) (*modelarts.FindDatasetResp, error) {
l := logic.NewFindDatasetListLogic(ctx, s.svcCtx)
return l.FindDatasetList(in)
}

View File

@ -16,7 +16,7 @@ import (
"google.golang.org/grpc/reflection"
)
var configFile = flag.String("f", "etc/modelarts.yaml", "the config file")
var configFile = flag.String("f", "adaptor/AIComputing/modelarts/rpc/etc/modelarts.yaml", "the config file")
func main() {
flag.Parse()

View File

@ -514,6 +514,242 @@ func (*TokenResp) Descriptor() ([]byte, []int) {
return file_modelarts_proto_rawDescGZIP(), []int{9}
}
type Datasets struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
DatasetId string `protobuf:"bytes,1,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"`
DatasetName string `protobuf:"bytes,2,opt,name=dataset_name,json=datasetName,proto3" json:"dataset_name,omitempty"`
DatasetType string `protobuf:"bytes,3,opt,name=dataset_type,json=datasetType,proto3" json:"dataset_type,omitempty"`
Status string `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"`
DataSources *DataSources `protobuf:"bytes,5,opt,name=dataSources,proto3" json:"dataSources,omitempty"`
}
func (x *Datasets) Reset() {
*x = Datasets{}
if protoimpl.UnsafeEnabled {
mi := &file_modelarts_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Datasets) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Datasets) ProtoMessage() {}
func (x *Datasets) ProtoReflect() protoreflect.Message {
mi := &file_modelarts_proto_msgTypes[10]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Datasets.ProtoReflect.Descriptor instead.
func (*Datasets) Descriptor() ([]byte, []int) {
return file_modelarts_proto_rawDescGZIP(), []int{10}
}
func (x *Datasets) GetDatasetId() string {
if x != nil {
return x.DatasetId
}
return ""
}
func (x *Datasets) GetDatasetName() string {
if x != nil {
return x.DatasetName
}
return ""
}
func (x *Datasets) GetDatasetType() string {
if x != nil {
return x.DatasetType
}
return ""
}
func (x *Datasets) GetStatus() string {
if x != nil {
return x.Status
}
return ""
}
func (x *Datasets) GetDataSources() *DataSources {
if x != nil {
return x.DataSources
}
return nil
}
type FindDatasetReq struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` // @gotags: copier:"projectId"
}
func (x *FindDatasetReq) Reset() {
*x = FindDatasetReq{}
if protoimpl.UnsafeEnabled {
mi := &file_modelarts_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FindDatasetReq) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FindDatasetReq) ProtoMessage() {}
func (x *FindDatasetReq) ProtoReflect() protoreflect.Message {
mi := &file_modelarts_proto_msgTypes[11]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FindDatasetReq.ProtoReflect.Descriptor instead.
func (*FindDatasetReq) Descriptor() ([]byte, []int) {
return file_modelarts_proto_rawDescGZIP(), []int{11}
}
func (x *FindDatasetReq) GetProjectId() string {
if x != nil {
return x.ProjectId
}
return ""
}
type FindDatasetResp struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
TotalNumber int32 `protobuf:"varint,1,opt,name=total_number,json=totalNumber,proto3" json:"total_number,omitempty"` // @gotags: copier:"totalNumber"
Datasets *Datasets `protobuf:"bytes,2,opt,name=datasets,proto3" json:"datasets,omitempty"`
}
func (x *FindDatasetResp) Reset() {
*x = FindDatasetResp{}
if protoimpl.UnsafeEnabled {
mi := &file_modelarts_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FindDatasetResp) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FindDatasetResp) ProtoMessage() {}
func (x *FindDatasetResp) ProtoReflect() protoreflect.Message {
mi := &file_modelarts_proto_msgTypes[12]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FindDatasetResp.ProtoReflect.Descriptor instead.
func (*FindDatasetResp) Descriptor() ([]byte, []int) {
return file_modelarts_proto_rawDescGZIP(), []int{12}
}
func (x *FindDatasetResp) GetTotalNumber() int32 {
if x != nil {
return x.TotalNumber
}
return 0
}
func (x *FindDatasetResp) GetDatasets() *Datasets {
if x != nil {
return x.Datasets
}
return nil
}
type DataSources struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
DataType int32 `protobuf:"varint,1,opt,name=data_type,json=dataType,proto3" json:"data_type,omitempty"` // @gotags: copier:"totalNumber"
DataPath string `protobuf:"bytes,2,opt,name=data_path,json=dataPath,proto3" json:"data_path,omitempty"`
}
func (x *DataSources) Reset() {
*x = DataSources{}
if protoimpl.UnsafeEnabled {
mi := &file_modelarts_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DataSources) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DataSources) ProtoMessage() {}
func (x *DataSources) ProtoReflect() protoreflect.Message {
mi := &file_modelarts_proto_msgTypes[13]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DataSources.ProtoReflect.Descriptor instead.
func (*DataSources) Descriptor() ([]byte, []int) {
return file_modelarts_proto_rawDescGZIP(), []int{13}
}
func (x *DataSources) GetDataType() int32 {
if x != nil {
return x.DataType
}
return 0
}
func (x *DataSources) GetDataPath() string {
if x != nil {
return x.DataPath
}
return ""
}
var File_modelarts_proto protoreflect.FileDescriptor
var file_modelarts_proto_rawDesc = []byte{
@ -554,11 +790,42 @@ var file_modelarts_proto_rawDesc = []byte{
0x52, 0x65, 0x71, 0x12, 0x23, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x0f, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73, 0x2e, 0x61, 0x75,
0x74, 0x68, 0x52, 0x04, 0x61, 0x75, 0x74, 0x68, 0x22, 0x0b, 0x0a, 0x09, 0x54, 0x6f, 0x6b, 0x65,
0x6e, 0x52, 0x65, 0x73, 0x70, 0x32, 0x42, 0x0a, 0x09, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x41, 0x72,
0x74, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x13,
0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73, 0x2e, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73, 0x2e,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x42, 0x0c, 0x5a, 0x0a, 0x2f, 0x6d, 0x6f,
0x6e, 0x52, 0x65, 0x73, 0x70, 0x22, 0xc1, 0x01, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65,
0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x69, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x49,
0x64, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74,
0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f,
0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x61, 0x74, 0x61,
0x73, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75,
0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12,
0x38, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x05,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73,
0x2e, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x0b, 0x64, 0x61,
0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x2f, 0x0a, 0x0e, 0x66, 0x69, 0x6e,
0x64, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x65, 0x71, 0x12, 0x1d, 0x0a, 0x0a, 0x70,
0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x0f, 0x66, 0x69,
0x6e, 0x64, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, 0x21, 0x0a,
0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20,
0x01, 0x28, 0x05, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72,
0x12, 0x2f, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73, 0x2e, 0x64,
0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x73, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74,
0x73, 0x22, 0x47, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73,
0x12, 0x1b, 0x0a, 0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x05, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a,
0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x74, 0x68, 0x32, 0x8c, 0x01, 0x0a, 0x09, 0x4d,
0x6f, 0x64, 0x65, 0x6c, 0x41, 0x72, 0x74, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x13, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73,
0x2e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x6f, 0x64, 0x65,
0x6c, 0x61, 0x72, 0x74, 0x73, 0x2e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x12,
0x48, 0x0a, 0x0f, 0x66, 0x69, 0x6e, 0x64, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x4c, 0x69,
0x73, 0x74, 0x12, 0x19, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73, 0x2e, 0x66,
0x69, 0x6e, 0x64, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x1a, 0x2e,
0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x44, 0x61,
0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x42, 0x0c, 0x5a, 0x0a, 0x2f, 0x6d, 0x6f,
0x64, 0x65, 0x6c, 0x61, 0x72, 0x74, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
@ -574,35 +841,43 @@ func file_modelarts_proto_rawDescGZIP() []byte {
return file_modelarts_proto_rawDescData
}
var file_modelarts_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
var file_modelarts_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
var file_modelarts_proto_goTypes = []interface{}{
(*Auth)(nil), // 0: modelarts.auth
(*Identity)(nil), // 1: modelarts.identity
(*Scope)(nil), // 2: modelarts.scope
(*Project)(nil), // 3: modelarts.project
(*Methods)(nil), // 4: modelarts.methods
(*Password)(nil), // 5: modelarts.password
(*User)(nil), // 6: modelarts.user
(*Domain)(nil), // 7: modelarts.domain
(*TokenReq)(nil), // 8: modelarts.TokenReq
(*TokenResp)(nil), // 9: modelarts.TokenResp
(*Auth)(nil), // 0: modelarts.auth
(*Identity)(nil), // 1: modelarts.identity
(*Scope)(nil), // 2: modelarts.scope
(*Project)(nil), // 3: modelarts.project
(*Methods)(nil), // 4: modelarts.methods
(*Password)(nil), // 5: modelarts.password
(*User)(nil), // 6: modelarts.user
(*Domain)(nil), // 7: modelarts.domain
(*TokenReq)(nil), // 8: modelarts.TokenReq
(*TokenResp)(nil), // 9: modelarts.TokenResp
(*Datasets)(nil), // 10: modelarts.datasets
(*FindDatasetReq)(nil), // 11: modelarts.findDatasetReq
(*FindDatasetResp)(nil), // 12: modelarts.findDatasetResp
(*DataSources)(nil), // 13: modelarts.dataSources
}
var file_modelarts_proto_depIdxs = []int32{
1, // 0: modelarts.auth.identity:type_name -> modelarts.identity
2, // 1: modelarts.auth.scope:type_name -> modelarts.scope
4, // 2: modelarts.identity.methods:type_name -> modelarts.methods
5, // 3: modelarts.identity.password:type_name -> modelarts.password
3, // 4: modelarts.scope.project:type_name -> modelarts.project
6, // 5: modelarts.password.user:type_name -> modelarts.user
7, // 6: modelarts.user.domain:type_name -> modelarts.domain
0, // 7: modelarts.TokenReq.auth:type_name -> modelarts.auth
8, // 8: modelarts.ModelArts.GetToken:input_type -> modelarts.TokenReq
9, // 9: modelarts.ModelArts.GetToken:output_type -> modelarts.TokenResp
9, // [9:10] is the sub-list for method output_type
8, // [8:9] is the sub-list for method input_type
8, // [8:8] is the sub-list for extension type_name
8, // [8:8] is the sub-list for extension extendee
0, // [0:8] is the sub-list for field type_name
1, // 0: modelarts.auth.identity:type_name -> modelarts.identity
2, // 1: modelarts.auth.scope:type_name -> modelarts.scope
4, // 2: modelarts.identity.methods:type_name -> modelarts.methods
5, // 3: modelarts.identity.password:type_name -> modelarts.password
3, // 4: modelarts.scope.project:type_name -> modelarts.project
6, // 5: modelarts.password.user:type_name -> modelarts.user
7, // 6: modelarts.user.domain:type_name -> modelarts.domain
0, // 7: modelarts.TokenReq.auth:type_name -> modelarts.auth
13, // 8: modelarts.datasets.dataSources:type_name -> modelarts.dataSources
10, // 9: modelarts.findDatasetResp.datasets:type_name -> modelarts.datasets
8, // 10: modelarts.ModelArts.GetToken:input_type -> modelarts.TokenReq
11, // 11: modelarts.ModelArts.findDatasetList:input_type -> modelarts.findDatasetReq
9, // 12: modelarts.ModelArts.GetToken:output_type -> modelarts.TokenResp
12, // 13: modelarts.ModelArts.findDatasetList:output_type -> modelarts.findDatasetResp
12, // [12:14] is the sub-list for method output_type
10, // [10:12] is the sub-list for method input_type
10, // [10:10] is the sub-list for extension type_name
10, // [10:10] is the sub-list for extension extendee
0, // [0:10] is the sub-list for field type_name
}
func init() { file_modelarts_proto_init() }
@ -731,6 +1006,54 @@ func file_modelarts_proto_init() {
return nil
}
}
file_modelarts_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Datasets); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_modelarts_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FindDatasetReq); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_modelarts_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FindDatasetResp); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_modelarts_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DataSources); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
@ -738,7 +1061,7 @@ func file_modelarts_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_modelarts_proto_rawDesc,
NumEnums: 0,
NumMessages: 10,
NumMessages: 14,
NumExtensions: 0,
NumServices: 1,
},

View File

@ -22,8 +22,10 @@ const _ = grpc.SupportPackageIsVersion7
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type ModelArtsClient interface {
//ListJob list all jobs
//get modelarts Token
GetToken(ctx context.Context, in *TokenReq, opts ...grpc.CallOption) (*TokenResp, error)
//find modelarts Token
FindDatasetList(ctx context.Context, in *FindDatasetReq, opts ...grpc.CallOption) (*FindDatasetResp, error)
}
type modelArtsClient struct {
@ -43,12 +45,23 @@ func (c *modelArtsClient) GetToken(ctx context.Context, in *TokenReq, opts ...gr
return out, nil
}
func (c *modelArtsClient) FindDatasetList(ctx context.Context, in *FindDatasetReq, opts ...grpc.CallOption) (*FindDatasetResp, error) {
out := new(FindDatasetResp)
err := c.cc.Invoke(ctx, "/modelarts.ModelArts/findDatasetList", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// ModelArtsServer is the server API for ModelArts service.
// All implementations must embed UnimplementedModelArtsServer
// for forward compatibility
type ModelArtsServer interface {
//ListJob list all jobs
//get modelarts Token
GetToken(context.Context, *TokenReq) (*TokenResp, error)
//find modelarts Token
FindDatasetList(context.Context, *FindDatasetReq) (*FindDatasetResp, error)
mustEmbedUnimplementedModelArtsServer()
}
@ -59,6 +72,9 @@ type UnimplementedModelArtsServer struct {
func (UnimplementedModelArtsServer) GetToken(context.Context, *TokenReq) (*TokenResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetToken not implemented")
}
func (UnimplementedModelArtsServer) FindDatasetList(context.Context, *FindDatasetReq) (*FindDatasetResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method FindDatasetList not implemented")
}
func (UnimplementedModelArtsServer) mustEmbedUnimplementedModelArtsServer() {}
// UnsafeModelArtsServer may be embedded to opt out of forward compatibility for this service.
@ -90,6 +106,24 @@ func _ModelArts_GetToken_Handler(srv interface{}, ctx context.Context, dec func(
return interceptor(ctx, in, info, handler)
}
func _ModelArts_FindDatasetList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FindDatasetReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ModelArtsServer).FindDatasetList(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/modelarts.ModelArts/findDatasetList",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ModelArtsServer).FindDatasetList(ctx, req.(*FindDatasetReq))
}
return interceptor(ctx, in, info, handler)
}
// ModelArts_ServiceDesc is the grpc.ServiceDesc for ModelArts service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -101,6 +135,10 @@ var ModelArts_ServiceDesc = grpc.ServiceDesc{
MethodName: "GetToken",
Handler: _ModelArts_GetToken_Handler,
},
{
MethodName: "findDatasetList",
Handler: _ModelArts_FindDatasetList_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "modelarts.proto",

View File

@ -13,20 +13,26 @@ import (
)
type (
Auth = modelarts.Auth
Domain = modelarts.Domain
Identity = modelarts.Identity
Methods = modelarts.Methods
Password = modelarts.Password
Project = modelarts.Project
Scope = modelarts.Scope
TokenReq = modelarts.TokenReq
TokenResp = modelarts.TokenResp
User = modelarts.User
Auth = modelarts.Auth
DataSources = modelarts.DataSources
Datasets = modelarts.Datasets
Domain = modelarts.Domain
FindDatasetReq = modelarts.FindDatasetReq
FindDatasetResp = modelarts.FindDatasetResp
Identity = modelarts.Identity
Methods = modelarts.Methods
Password = modelarts.Password
Project = modelarts.Project
Scope = modelarts.Scope
TokenReq = modelarts.TokenReq
TokenResp = modelarts.TokenResp
User = modelarts.User
ModelArts interface {
// ListJob list all jobs
// get modelarts Token
GetToken(ctx context.Context, in *TokenReq, opts ...grpc.CallOption) (*TokenResp, error)
// find modelarts Token
FindDatasetList(ctx context.Context, in *FindDatasetReq, opts ...grpc.CallOption) (*FindDatasetResp, error)
}
defaultModelArts struct {
@ -40,8 +46,14 @@ func NewModelArts(cli zrpc.Client) ModelArts {
}
}
// ListJob list all jobs
// get modelarts Token
func (m *defaultModelArts) GetToken(ctx context.Context, in *TokenReq, opts ...grpc.CallOption) (*TokenResp, error) {
client := modelarts.NewModelArtsClient(m.cli.Conn())
return client.GetToken(ctx, in, opts...)
}
// find modelarts Token
func (m *defaultModelArts) FindDatasetList(ctx context.Context, in *FindDatasetReq, opts ...grpc.CallOption) (*FindDatasetResp, error) {
client := modelarts.NewModelArtsClient(m.cli.Conn())
return client.FindDatasetList(ctx, in, opts...)
}

View File

@ -48,11 +48,42 @@ message TokenResp{
}
/******************auth end*************************/
/******************find datasetList start*************************/
message datasets{
string dataset_id =1;
string dataset_name =2;
string dataset_type =3;
string status =4;
dataSources dataSources =5;
}
message findDatasetReq{
string project_id = 1; // @gotags: copier:"projectId"
}
message findDatasetResp{
int32 total_number = 1; // @gotags: copier:"totalNumber"
datasets datasets = 2;
}
message dataSources{
int32 data_type = 1; // @gotags: copier:"totalNumber"
string data_path = 2;
}
/******************find datasetList end*************************/
// Slurm Services for Shuguang Branch
service ModelArts {
//ListJob list all jobs
//get modelarts Token
rpc GetToken(TokenReq) returns (TokenResp);
//find modelarts Token
rpc findDatasetList(findDatasetReq) returns (findDatasetResp);
}

View File

@ -1,5 +0,0 @@
package service
func getToken() *ListJobLogic {
}

View File

@ -1,155 +0,0 @@
package logic
/*
#cgo LDFLAGS: -lslurmdb
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <slurm/slurm.h>
#include <slurm/slurmdb.h>
#include <slurm/slurm_errno.h>
#include <memory.h>
#include <malloc.h>
slurmdb_job_rec_t *get_all_slurmdb_job() {
slurmdb_job_cond_t *job_cond = NULL;
void *conn = slurmdb_connection_get();
List joblist = slurmdb_jobs_get(conn, job_cond);
uint16_t listsize = slurm_list_count(joblist);
//qosinfo.record_count = size;
slurmdb_job_rec_t *jobarray = malloc(listsize * sizeof(slurmdb_job_rec_t));
ListIterator itr = slurm_list_iterator_create(joblist);
slurmdb_job_rec_t *rec = NULL;
int i = 0;
while ((rec = slurm_list_next(itr))) {
jobarray[i] = *rec;
i++;
}
slurmdb_connection_close(&conn);
slurm_list_destroy(joblist);
//int arraysize = sizeof(jobarray);
//printf("%d\n", arraysize);
return jobarray;
}
slurmdb_job_rec_t *job_from_array(slurmdb_job_rec_t *job_rec_array, int i) {
return (slurmdb_job_rec_t *) &(job_rec_array[i]);
}
slurmdb_job_rec_t *job_from_array_by_id(slurmdb_job_rec_t *job_rec_array, int job_id) {
int i;
int arraysize = sizeof(job_rec_array);
for (i=0; i<arraysize; i++)
{
//printf("var: %d : %p : %d \n", var, &(job_rec_array[var]), (job_rec_array[var]).stats.cpu_min);
//printf("%d \n",(job_rec_array[i]).stats.cpu_min);
//printf("var: %d : %p : %d \n", var, &(job_rec_array[var]), (job_rec_array[var]).jobid);
if (job_id==(job_rec_array[i]).jobid)
{
break;
} else {
return NULL;
}
}
return (slurmdb_job_rec_t *) &(job_rec_array[i]);
}
int getLength(slurmdb_job_rec_t *job_rec_array) {
return sizeof(job_rec_array);
}
*/
import "C"
import (
"context"
"PCM/adaptor/slurm/slurmTianhe/rpc/internal/svc"
"PCM/adaptor/slurm/slurmTianhe/rpc/slurmTianhe"
"github.com/zeromicro/go-zero/core/logx"
)
type ListHistoryJobLogic struct {
ctx context.Context
svcCtx *svc.ServiceContext
logx.Logger
}
func NewListHistoryJobLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ListHistoryJobLogic {
return &ListHistoryJobLogic{
ctx: ctx,
svcCtx: svcCtx,
Logger: logx.WithContext(ctx),
}
}
// ListHistoryJob list all jobs from slurmdb
func (l *ListHistoryJobLogic) ListHistoryJob(in *slurmTianhe.ListHistoryJobReq) (*slurmTianhe.ListHistoryJobResp, error) {
var dbJobResp slurmTianhe.ListHistoryJobResp
jobInfos := C.get_all_slurmdb_job()
size := uint32(C.getLength(jobInfos))
dbJobResp.RecordCount = size
for i := uint32(0); i < size; i++ {
cJob := C.job_from_array(jobInfos, C.int(i))
goJob := SlurmdbJobConvertCToGo(cJob)
dbJobResp.HistoryJobs = append(dbJobResp.HistoryJobs, &goJob)
}
dbJobResp.Code = 200
dbJobResp.Msg = "success"
return &dbJobResp, nil
}
func SlurmdbJobConvertCToGo(cStructJob *C.slurmdb_job_rec_t) slurmTianhe.HistoryJob {
var goStruct slurmTianhe.HistoryJob
goStruct.Account = C.GoString(cStructJob.account)
goStruct.Associd = uint32(cStructJob.associd)
goStruct.AllocCpu = uint32(cStructJob.alloc_cpus)
goStruct.AllocNodes = uint32(cStructJob.alloc_nodes)
goStruct.Blockid = C.GoString(cStructJob.blockid)
goStruct.Cluster = C.GoString(cStructJob.cluster)
goStruct.DerivedEs = C.GoString(cStructJob.derived_es)
goStruct.DerivedEc = uint32(cStructJob.derived_ec)
goStruct.End = int64(cStructJob.end)
goStruct.Elapsed = uint32(cStructJob.elapsed)
goStruct.Eligible = int64(cStructJob.eligible)
goStruct.Exitcode = uint32(cStructJob.exitcode)
goStruct.Gid = uint32(cStructJob.gid)
goStruct.Jobid = uint32(cStructJob.jobid)
goStruct.Jobname = C.GoString(cStructJob.jobname)
goStruct.Lft = uint32(cStructJob.lft)
goStruct.Nodes = C.GoString(cStructJob.nodes)
goStruct.Priority = uint32(cStructJob.priority)
goStruct.Partition = C.GoString(cStructJob.partition)
goStruct.Qosid = uint32(cStructJob.qosid)
goStruct.Resvid = uint32(cStructJob.resvid)
goStruct.Requid = uint32(cStructJob.requid)
goStruct.ReqCpus = uint32(cStructJob.req_cpus)
goStruct.ReqMem = uint32(cStructJob.req_mem)
goStruct.Start = int64(cStructJob.start)
goStruct.State = uint32(cStructJob.state)
goStruct.SysCpuUsec = uint32(cStructJob.sys_cpu_usec)
goStruct.SysCpuSec = uint32(cStructJob.sys_cpu_sec)
goStruct.Submit = int64(cStructJob.submit)
goStruct.Suspended = uint32(cStructJob.suspended)
goStruct.ShowFull = uint32(cStructJob.show_full)
//var stats_adrs []*pbslurm.SlurmdbStatsT
//var stats pbslurm.SlurmdbStatsT
//stat.ActCpufreq = float64(c_struct_job.stats.act_cpufreq)
//stats.CpuMin = uint32((c_struct_job.stats).cpu_min)
//stat.CpuAve = float64(c_struct_job.stats.cpu_ave)
//stats_adrs = append(stats_adrs, &stats)
//go_struct.Stats = stats
return goStruct
}

View File

@ -1,31 +0,0 @@
package logic
import (
"context"
"PCM/adaptor/slurm/slurmTianhe/rpc/internal/svc"
"PCM/adaptor/slurm/slurmTianhe/rpc/slurmTianhe"
"github.com/zeromicro/go-zero/core/logx"
)
type ListJobLogic struct {
ctx context.Context
svcCtx *svc.ServiceContext
logx.Logger
}
func NewListJobLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ListJobLogic {
return &ListJobLogic{
ctx: ctx,
svcCtx: svcCtx,
Logger: logx.WithContext(ctx),
}
}
// ListJob list all jobs
func (l *ListJobLogic) ListJob(in *slurmTianhe.ListJobReq) (*slurmTianhe.ListJobResp, error) {
// todo: add your logic here and delete this line
return &slurmTianhe.ListJobResp{}, nil
}

88
go.mod
View File

@ -1,88 +0,0 @@
module PCM
go 1.19
require (
github.com/bitly/go-simplejson v0.5.0
github.com/jinzhu/copier v0.3.5
github.com/pkg/errors v0.9.1
github.com/shopspring/decimal v1.3.1
github.com/sony/sonyflake v1.1.0
github.com/zeromicro/go-zero v1.4.4
google.golang.org/grpc v1.50.1
google.golang.org/protobuf v1.28.1
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/fgprof v0.9.3 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mattn/go-colorable v0.1.9 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/openzipkin/zipkin-go v0.4.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/prometheus/client_golang v1.13.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v3 v3.5.5 // indirect
go.opentelemetry.io/otel v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.10.0 // indirect
go.opentelemetry.io/otel/sdk v1.10.0 // indirect
go.opentelemetry.io/otel/trace v1.10.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/automaxprocs v1.5.1 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20221111202108-142d8a6fa32e // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.22.9 // indirect
k8s.io/apimachinery v0.22.9 // indirect
k8s.io/client-go v0.22.9 // indirect
k8s.io/klog/v2 v2.80.1 // indirect
k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)

1470
go.sum

File diff suppressed because it is too large Load Diff