♻ P端状态映射C端状态

This commit is contained in:
zhouqunjie 2023-11-07 19:56:28 +08:00
parent 576530b96f
commit f74106c47d
9 changed files with 77 additions and 99 deletions

View File

@ -3,7 +3,8 @@ ListenOn: 0.0.0.0:2007
# core rpc
PcmCoreRpcConf:
Endpoints:
- 0.0.0.0:2004
#- localhost:2004
- dev.jointcloud.net:32456
NonBlock: true
RestUrl: http://192.168.249.122:6820
SSH:
@ -38,7 +39,31 @@ Path:
Ping: /slurm/v0.0.38/ping
Association: /slurmdb/v0.0.38/association
Associations: /slurmdb/v0.0.38/associations
SlurmStatus:
PENDING: Pending
RUNNING: Running
CONFIGURING: Running
COMPLETING: Running
COMPLETED: Completed
FAILED: Failed
TIMEOUT: Failed
DEADLINE: pending
BOOT_FAIL: Failed
OUT_OF_MEMORY: Failed
NODE_FAIL: Failed
CANCELLED: Canceled
SPECIAL_EXIT: Failed
SUSPENDED: Pending
STOPPED: Failed
RESIZING: Failed
PREEMPTED: Failed
REVOKED: Failed
SIGNALING: Pending
RESV_DEL_HOLD: Pending
REQUEUE_FED: Pending
REQUEUE_HOLD: Pending
REQUEUED: Pending
STAGE_OUT: Failed
SlurmRestUser: slurmrestd
ParticipantId: 1706858330967773184
ParticipantId: 1706858330967773111
SlurmToken: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTkyNjE1MDIsImlhdCI6MTY5OTI1OTcwMiwic3VuIjoic2x1cm1yZXN0ZCJ9.sty-5_R761ycPFwl--J2Dp6m9yVHpojugLzZpaw-4w0

7
go.mod
View File

@ -8,14 +8,17 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/zeromicro/go-zero v1.6.0
gitlink.org.cn/jcce-pcm/pcm-coordinator v0.1.9
gitlink.org.cn/jcce-pcm/utils v0.0.2
golang.org/x/crypto v0.14.0
google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.31.0
)
require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
@ -40,6 +43,7 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect
github.com/jinzhu/copier v0.3.5 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
@ -53,8 +57,10 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nacos-group/nacos-sdk-go/v2 v2.2.3 // indirect
github.com/openzipkin/zipkin-go v0.4.2 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
@ -87,6 +93,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/gorm v1.25.3 // indirect

13
go.sum
View File

@ -1,3 +1,5 @@
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
@ -6,6 +8,8 @@ github.com/alicebob/miniredis/v2 v2.31.0 h1:ObEFUNlJwoIiyjxdrYF0QIDE7qXcLc7D3WpS
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
@ -71,6 +75,8 @@ github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@ -109,6 +115,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/nacos-group/nacos-sdk-go/v2 v2.2.3 h1:sUQx4f1bXDeeOOEQZjGAitzxYApbYY9fVDbxVCaBW+I=
github.com/nacos-group/nacos-sdk-go/v2 v2.2.3/go.mod h1:UL4U89WYdnyajgKJUMpuT1Rr6iNmbjrxOO40JRgtA00=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
@ -118,6 +126,7 @@ github.com/openzipkin/zipkin-go v0.4.2/go.mod h1:ZeVkFjuuBiSy13y8vpSDCjMi9GoI3hP
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
@ -155,6 +164,8 @@ github.com/zeromicro/go-zero v1.6.0 h1:UwSOR1lGZ2g7L0S07PM8RoneAcubtd5x//EfbuNuc
github.com/zeromicro/go-zero v1.6.0/go.mod h1:E9GCFPb0SwsTKFBcFr9UynGvXiDMmfc6fI5F15vqvAQ=
gitlink.org.cn/jcce-pcm/pcm-coordinator v0.1.9 h1:BMlZgSUnbYKcVIKEX4hBXb8rjnNfElQxBXZsCcF3ABU=
gitlink.org.cn/jcce-pcm/pcm-coordinator v0.1.9/go.mod h1:5KV9tMTNyHhh5nPT4mCA/Zt6VUL21hFrhPovs1hDmRs=
gitlink.org.cn/jcce-pcm/utils v0.0.2 h1:Stif8W9C9TOCS2hw4g+OlOywDrsVYNrkiyKfBrWkT0w=
gitlink.org.cn/jcce-pcm/utils v0.0.2/go.mod h1:u8PTlBpzUyOlbQJgfSiutq91q/JtrJIQiPNDe4S/pGs=
go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k=
go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI=
go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0=
@ -278,6 +289,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

View File

@ -50,9 +50,10 @@ type Path struct {
// SlurmConf slurm 相关URL配置
type SlurmConf struct {
RestUrl string `json:"RestUrl"`
SSH SSH `json:"SSH"`
Path Path `json:"Path"`
SlurmRestUser string `json:"SlurmRestUser"`
SlurmToken string `json:"SlurmToken"`
RestUrl string `json:"RestUrl"`
SSH SSH `json:"SSH"`
Path Path `json:"Path"`
SlurmRestUser string `json:"SlurmRestUser"`
SlurmToken string `json:"SlurmToken"`
SlurmStatus map[string]string `json:"SlurmStatus"`
}

View File

@ -33,11 +33,10 @@ func (l *SubmitJobLogic) SubmitJob(in *slurm.SubmitJobReq) (*slurm.SubmitJobResp
}
slurmHttpRequest := httputils.GetHttpRequest()
result, _ := slurmHttpRequest.SetHeader(httputils.ContentType, httputils.ApplicationJson).
slurmHttpRequest.SetHeader(httputils.ContentType, httputils.ApplicationJson).
SetHeader("X-SLURM-USER-NAME", l.svcCtx.Config.SlurmRestUser).
SetHeader("X-SLURM-USER-TOKEN", l.svcCtx.Config.SlurmToken).
SetBody(in).
SetResult(&submitJobResp).Post(l.svcCtx.Config.RestUrl + l.svcCtx.Config.Path.JobSubmit)
println(result.Body())
return &submitJobResp, nil
}

View File

@ -15,7 +15,7 @@ func ReportHeartbeat(participantRpc participantservice.ParticipantService) {
}
resp, err := participantRpc.ReportHeartbeat(context.Background(), &participantservice.ParticipantHeartbeatReq{
ParticipantId: participantId,
Address: "10.101.15.3:6443",
Address: "192.168.249.122",
})
if err != nil {
logx.Error(err)

View File

@ -3,6 +3,7 @@ package cron
import (
"context"
"github.com/zeromicro/go-zero/core/logx"
"gitlink.org.cn/jcce-pcm/pcm-coordinator/pkg/constants"
"gitlink.org.cn/jcce-pcm/pcm-coordinator/pkg/utils"
"gitlink.org.cn/jcce-pcm/pcm-coordinator/rpc/pcmCore"
"gitlink.org.cn/jcce-pcm/pcm-slurm/internal/logic"
@ -15,6 +16,7 @@ func SyncTask(svc *svc.ServiceContext) {
if err != nil {
return
}
// 查询core端分发下来的任务列表
infoReq := pcmCore.InfoListReq{
ParticipantId: participantId,
@ -35,91 +37,28 @@ func SyncTask(svc *svc.ServiceContext) {
if len(infoList.HpcInfoList) != 0 {
for index := range infoList.HpcInfoList {
if infoList.HpcInfoList[index].Status == "SAVED" {
if infoList.HpcInfoList[index].Status == constants.Saved {
subReq := slurm.SubmitJobReq{
Script: infoList.HpcInfoList[index].CmdScript,
Job: &slurm.JobProperties{
Account: infoList.HpcInfoList[index].Account,
AccountGatherFrequency: "",
Argv: nil,
Array: "",
BatchFeatures: "",
BeginTime: 0,
BurstBuffer: "",
ClusterConstraint: "",
Comment: "",
Constraints: "",
CoreSpecification: 0,
CoresPerSocket: 0,
CpuBinding: "",
CpuBindingHint: "",
CpuFrequency: "",
CpusPerGpu: "",
CpusPerTask: 0,
CurrentWorkingDirectory: "",
Deadline: "",
DelayBoot: 0,
Dependency: "",
Distribution: "",
CurrentWorkingDirectory: infoList.HpcInfoList[index].WorkDir,
Name: infoList.HpcInfoList[index].Name,
Partition: infoList.HpcInfoList[index].Queue,
StandardError: infoList.HpcInfoList[index].StdErrFile,
StandardInput: infoList.HpcInfoList[index].StdOutFile,
StandardOutput: infoList.HpcInfoList[index].StdOutFile,
Environment: env,
Exclusive: "",
GetUserEnvironment: false,
Gres: "",
GresFlags: "",
GpuBinding: "",
GpuFrequency: "",
Gpus: "",
GpusPerNode: "",
GpusPerSocket: "",
GpusPerTask: "",
Hold: false,
KillOnInvalidDependency: false,
Licenses: "",
MailType: "",
MailUser: "",
McsLabel: "",
MemoryBinding: "",
MemoryPerCpu: 0,
MemoryPerGpu: 0,
MemoryPerNode: 0,
MinimumCpusPerNode: 0,
MinimumNodes: false,
Name: "",
Nice: "",
NoKill: false,
Nodes: nil,
OpenMode: "",
Partition: "",
Priority: "",
Qos: "",
Requeue: false,
Reservation: "",
Signal: "",
SocketsPerNode: 0,
SpreadJob: false,
StandardError: "",
StandardInput: "",
StandardOutput: "",
Tasks: 0,
TasksPerCore: 0,
TasksPerNode: 0,
TasksPerSocket: 0,
ThreadSpecification: 0,
ThreadsPerCore: 0,
TimeLimit: 0,
TimeMinimum: 0,
WaitAllNodes: false,
Wckey: "",
},
Jobs: nil,
}
jobResult, err := submitJobLogic.SubmitJob(&subReq)
if jobResult.JobSubmitUserMsg == "0" {
infoList.HpcInfoList[index].Status = "PENDING"
if len(jobResult.Errors) == 0 {
infoList.HpcInfoList[index].Status = "Pending"
infoList.HpcInfoList[index].JobId = jobResult.JobSubmitUserMsg
} else {
infoList.HpcInfoList[index].Status = "FAILED"
infoList.HpcInfoList[index].Status = "Failed"
infoList.HpcInfoList[index].Result = jobResult.JobSubmitUserMsg
}
// 同步信息到core端
@ -134,21 +73,17 @@ func SyncTask(svc *svc.ServiceContext) {
if err != nil {
return
}
} else if infoList.HpcInfoList[index].Status == "PENDING" || infoList.HpcInfoList[index].Status == "RUNNING" {
} else if infoList.HpcInfoList[index].Status == "Pending" || infoList.HpcInfoList[index].Status == "Running" {
// 查询P端实际的任务列表
listReq := slurm.ListJobReq{}
resp, _ := listLogic.ListJob(&listReq)
for _, job := range resp.Jobs {
if job.JobId == infoList.HpcInfoList[index].JobId {
if job.JobState == "RUNNING" || resp.Jobs[0].JobState == "CONFIGURING" {
infoList.HpcInfoList[index].Status = "RUNNING"
} else if job.JobState == "COMPLETED" || resp.Jobs[0].JobState == "statC" || resp.Jobs[0].JobState == "statH" {
infoList.HpcInfoList[index].Status = "COMPLETED"
} else if job.JobState == "FAILED" || resp.Jobs[0].JobState == "TIMEOUT" || resp.Jobs[0].JobState == "DEADLINE" {
infoList.HpcInfoList[index].Status = "FAILED"
} else if job.JobState == "COMPLETING" {
infoList.HpcInfoList[index].Status = "OTHER"
var pcmState = svc.Config.SlurmStatus[job.JobState]
if pcmState == "" {
infoList.HpcInfoList[index].Status = "Other"
} else {
infoList.HpcInfoList[index].Status = pcmState
}
}
}

View File

@ -11,7 +11,6 @@ import (
"gitlink.org.cn/jcce-pcm/pcm-coordinator/rpc/client/participantservice"
"gitlink.org.cn/jcce-pcm/pcm-slurm/internal/logic"
"gitlink.org.cn/jcce-pcm/pcm-slurm/internal/svc"
"gitlink.org.cn/jcce-pcm/utils/tool"
"time"
)
@ -77,7 +76,7 @@ func NodesDynamicInfo(svc *svc.ServiceContext) ([]*model.ScNodeAvailInfo, error)
}
var nodeAvailInfo []*participantservice.NodeAvailInfo
tool.Convert(nodes, &nodeAvailInfo)
utils.Convert(nodes, &nodeAvailInfo)
if err != nil {
return nil, err

View File

@ -15,7 +15,6 @@ import (
"gitlink.org.cn/jcce-pcm/pcm-slurm/internal/server"
"gitlink.org.cn/jcce-pcm/pcm-slurm/internal/svc"
"gitlink.org.cn/jcce-pcm/pcm-slurm/slurm"
"gitlink.org.cn/jcce-pcm/utils/tool"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
"strconv"
@ -67,7 +66,7 @@ func PushParticipantInfo(address string, participantService participantservice.P
req := participantservice.ParticipantPhyReq{
ParticipantId: participantId,
Address: address,
Type: "2",
Type: "HPC",
TenantId: 2,
TenantName: "slurm-cs",
LabelInfo: labels,
@ -78,5 +77,5 @@ func PushParticipantInfo(address string, participantService participantservice.P
}
// 更新本地配置文件ParticipantId
tool.UpdateParticipantId(*configFile, strconv.FormatInt(resp.ParticipantId, 10))
utils.UpdateParticipantId(*configFile, strconv.FormatInt(resp.ParticipantId, 10))
}