Compare commits
58 Commits
master
...
feature/re
| Author | SHA1 | Date |
|---|---|---|
|
|
ec05a7c969 | |
|
|
3cce0f8ba9 | |
|
|
8ebc515bd4 | |
|
|
d4adf4a42c | |
|
|
92c377246e | |
|
|
cd4085240f | |
|
|
1cc61f4a18 | |
|
|
223799ae00 | |
|
|
bdd947eb73 | |
|
|
8e9c83d88b | |
|
|
6948743c8c | |
|
|
d2074e8d2f | |
|
|
97b21dfd8b | |
|
|
990df7bf26 | |
|
|
8b448fa6f1 | |
|
|
eb71615a91 | |
|
|
017f9992a4 | |
|
|
87ce597b8f | |
|
|
cc0244969d | |
|
|
c57db5b241 | |
|
|
8b322ee724 | |
|
|
b263edb79f | |
|
|
8dfcbb023b | |
|
|
9572fad958 | |
|
|
580050a9db | |
|
|
fb0b8b0538 | |
|
|
6e523ba99e | |
|
|
34706622ce | |
|
|
493ac4ae6f | |
|
|
132fe85e3c | |
|
|
deb878467b | |
|
|
3dcde360e6 | |
|
|
35a6259a49 | |
|
|
a6a2b7221f | |
|
|
089a05ba3d | |
|
|
1249a8df5f | |
|
|
fe9d5fd614 | |
|
|
d4811916a4 | |
|
|
baca6c909e | |
|
|
1dec100f72 | |
|
|
f40bc45a42 | |
|
|
02a82b890d | |
|
|
83c2980bc8 | |
|
|
97e931eed9 | |
|
|
b11eb0b4ce | |
|
|
7cb871dc13 | |
|
|
b6723a0d10 | |
|
|
8854abd0b1 | |
|
|
249a21bfd2 | |
|
|
ab787bf46b | |
|
|
bb03b1ccc3 | |
|
|
3b7a8054c1 | |
|
|
e98bc40203 | |
|
|
00a4798150 | |
|
|
4079d420df | |
|
|
7e9de07c34 | |
|
|
6d8c6f3a9a | |
|
|
a0de9ff891 |
|
|
@ -2,5 +2,6 @@
|
|||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
<mapping directory="$PROJECT_DIR$/code" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
34
Dockerfile
34
Dockerfile
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
|
||||
# 第一阶段:构建阶段
|
||||
FROM 172.20.32.187/pipeline-service/golang:1.24.5-alpine3.22 AS builder
|
||||
FROM 172.20.32.159/pipeline-service/golang:1.24.5-alpine3.22 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
|
@ -17,20 +17,36 @@ WORKDIR /app
|
|||
COPY go.mod go.sum ./
|
||||
# 复制源代码
|
||||
COPY . .
|
||||
#RUN go env -w GOPROXY=https://goproxy.cn,direct
|
||||
RUN go env -w GOPROXY=http://172.20.32.233:30005/repository/hnxjy-goproxy/
|
||||
RUN go env -w GOPROXY=https://goproxy.cn,direct
|
||||
#RUN go env -w GOPROXY=http://172.20.32.233:30005/repository/hnxjy-goproxy/
|
||||
RUN go mod tidy && go mod download
|
||||
# 构建应用
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o remote-task-executor-cli
|
||||
|
||||
# 第二阶段:运行阶段
|
||||
FROM 172.20.32.187/pipeline-service/pipeline-convert:running
|
||||
FROM 172.20.32.159/pipeline-service/pipeline-convert:running
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories \
|
||||
&& apk update \
|
||||
&& apk add python3 py3-pip
|
||||
|
||||
#RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories \
|
||||
# && apk update \
|
||||
# && apk add python3 py3-pip
|
||||
|
||||
# 接收构建参数
|
||||
ARG NACOS_HOST=172.20.32.121
|
||||
ARG NACOS_PORT=31203
|
||||
|
||||
|
||||
ARG HTTP_PROXY
|
||||
ARG HTTPS_PROXY
|
||||
ARG NO_PROXY
|
||||
|
||||
# 设置代理(如果传入)
|
||||
ENV http_proxy=${HTTP_PROXY}
|
||||
ENV https_proxy=${HTTPS_PROXY}
|
||||
ENV no_proxy=${NO_PROXY}
|
||||
|
||||
|
||||
# 从构建阶段复制可执行文件
|
||||
COPY --from=builder /app/remote-task-executor-cli /app/
|
||||
|
|
@ -39,6 +55,10 @@ COPY --from=builder /app/merge_aim.py /app/
|
|||
# 设置执行权限
|
||||
RUN chmod a+x /app/remote-task-executor-cli && chmod a+x /app/merge_aim.py
|
||||
|
||||
# 设置 Nacos 环境变量(构建时必需,从 ARG 传递)
|
||||
ENV NACOS_HOST=${NACOS_HOST}
|
||||
ENV NACOS_PORT=${NACOS_PORT}
|
||||
|
||||
# 设置非 root 用户
|
||||
#RUN addgroup -S appgroup && adduser -S appuser -G appgroup
|
||||
# 健康检查
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
# 第一阶段:构建阶段FROM 172.20.32.187/pipeline-service/golang:1.24.5-alpine3.22 AS builder
|
||||
FROM 172.20.32.159/pipeline-service/golang:1.24.5-alpine3.22 AS builder
|
||||
WORKDIR /app
|
||||
|
||||
# 先复制依赖文件,利用缓存
|
||||
COPY go.mod go.sum ./
|
||||
# 复制源代码
|
||||
COPY . .
|
||||
# 设置 Go 代理
|
||||
RUN go env -w GOPROXY=https://goproxy.cn,direct
|
||||
RUN go mod tidy && go mod download
|
||||
# 构建应用(server 命令)
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o remote-task-executor-server -ldflags="-w -s" .
|
||||
|
||||
# 第二阶段:运行阶段
|
||||
FROM 172.20.32.159/pipeline-service/pipeline-convert:running
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装必要的工具(如果需要)
|
||||
#RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories \
|
||||
# && apk update \
|
||||
# && apk add --no-cache ca-certificates tzdata
|
||||
|
||||
# 设置时区
|
||||
ENV TZ=Asia/Shanghai
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
# 从构建阶段复制可执行文件
|
||||
COPY --from=builder /app/remote-task-executor-server /app/
|
||||
|
||||
# 设置执行权限
|
||||
RUN chmod a+x /app/remote-task-executor-server
|
||||
|
||||
# 暴露端口
|
||||
EXPOSE 8080
|
||||
|
||||
# 健康检查
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1
|
||||
|
||||
# 入口点(启动 server)
|
||||
ENTRYPOINT ["/app/remote-task-executor-server", "server", "--port", "8080"]
|
||||
|
||||
|
|
@ -1,10 +1,31 @@
|
|||
#!/bin/bash
|
||||
|
||||
tag=$(date +'%Y%m%d%H%M')
|
||||
image=172.20.32.187/pipeline-service/remote-task-cli:${tag}
|
||||
image=172.20.32.159/pipeline-service/remote-task-cli:${tag}
|
||||
|
||||
# 从环境变量读取 Nacos 配置,如果没有则使用默认值
|
||||
NACOS_HOST=${NACOS_HOST:-172.20.32.121}
|
||||
NACOS_PORT=${NACOS_PORT:-31203}
|
||||
|
||||
|
||||
# 代理
|
||||
HTTP_PROXY=${HTTP_PROXY:-http://192.168.20.150:10324}
|
||||
HTTPS_PROXY=${HTTPS_PROXY:-http://192.168.20.150:10324}
|
||||
|
||||
# 不走代理地址
|
||||
NO_PROXY=${NO_PROXY:-localhost,127.0.0.1,172.20.0.0/16,192.168.0.0/16}
|
||||
|
||||
echo "构建镜像,使用 Nacos 配置: Host=${NACOS_HOST}, Port=${NACOS_PORT}"
|
||||
|
||||
docker build \
|
||||
--build-arg NACOS_HOST=${NACOS_HOST} \
|
||||
--build-arg NACOS_PORT=${NACOS_PORT} \
|
||||
--build-arg HTTP_PROXY=${HTTP_PROXY} \
|
||||
--build-arg HTTPS_PROXY=${HTTPS_PROXY} \
|
||||
--build-arg NO_PROXY=${NO_PROXY} \
|
||||
-t ${image} .
|
||||
|
||||
docker build -t ${image} .
|
||||
docker push ${image}
|
||||
|
||||
docker tag ${image} ccr.ccs.tencentyun.com/somunslotus/remote-task-cli:${tag}
|
||||
docker push ccr.ccs.tencentyun.com/somunslotus/remote-task-cli:${tag}
|
||||
docker push ccr.ccs.tencentyun.com/somunslotus/remote-task-cli:${tag}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
#!/bin/bash
|
||||
|
||||
# 构建和推送 server 镜像的脚本
|
||||
|
||||
set -e
|
||||
|
||||
# 配置变量
|
||||
IMAGE_NAME="remote-task-executor-server"
|
||||
IMAGE_TAG="${1:-latest}"
|
||||
REGISTRY="172.20.32.159/pipeline-service"
|
||||
FULL_IMAGE_NAME="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}"
|
||||
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}构建 Server 镜像${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "镜像名称: ${YELLOW}${FULL_IMAGE_NAME}${NC}"
|
||||
echo ""
|
||||
|
||||
# 检查 Dockerfile 是否存在
|
||||
if [ ! -f "Dockerfile.server" ]; then
|
||||
echo -e "${RED}错误: Dockerfile.server 不存在${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 构建镜像
|
||||
echo -e "${GREEN}[1/3] 构建 Docker 镜像...${NC}"
|
||||
docker build -f Dockerfile.server -t ${FULL_IMAGE_NAME} .
|
||||
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "${RED}构建失败${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}构建成功${NC}"
|
||||
echo ""
|
||||
|
||||
# 标记为 latest(如果指定了其他 tag)
|
||||
if [ "${IMAGE_TAG}" != "latest" ]; then
|
||||
echo -e "${GREEN}[2/3] 标记为 latest...${NC}"
|
||||
docker tag ${FULL_IMAGE_NAME} ${REGISTRY}/${IMAGE_NAME}:latest
|
||||
echo -e "${GREEN}标记成功${NC}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# 推送镜像
|
||||
echo -e "${GREEN}[3/3] 推送镜像到仓库...${NC}"
|
||||
docker push ${FULL_IMAGE_NAME}
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "${RED}推送失败${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${IMAGE_TAG}" != "latest" ]; then
|
||||
docker push ${REGISTRY}/${IMAGE_NAME}:latest
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}推送成功${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}构建完成!${NC}"
|
||||
echo -e "${GREEN}镜像: ${FULL_IMAGE_NAME}${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
|
||||
52
cmd/run.go
52
cmd/run.go
|
|
@ -4,7 +4,6 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"log"
|
||||
"os"
|
||||
"remote-task-excutor-cli/pkg/config"
|
||||
|
|
@ -16,6 +15,8 @@ import (
|
|||
"remote-task-excutor-cli/pkg/service/trainlog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var runCmd = &cobra.Command{
|
||||
|
|
@ -39,8 +40,8 @@ var runCmd = &cobra.Command{
|
|||
defer cancel()
|
||||
// 1. 配置Nacos连接
|
||||
nacosCfg := config.NacosConfig{
|
||||
Host: config.Host,
|
||||
Port: config.Port,
|
||||
Host: config.GetNacosHost(),
|
||||
Port: config.GetNacosPort(),
|
||||
NamespaceID: config.Namespace,
|
||||
Group: config.Group,
|
||||
DataID: config.DataID,
|
||||
|
|
@ -82,12 +83,12 @@ var runCmd = &cobra.Command{
|
|||
func printParams(config *models.RunConfig) {
|
||||
// 执行工作流
|
||||
fmt.Println("Starting remote ML task with configuration:")
|
||||
fmt.Printf("Code Config: %s\n", config.CodeConfig)
|
||||
fmt.Printf("Resource: %s\n", config.Resource)
|
||||
fmt.Printf("Code Config: %+v\n", config.CodeConfig)
|
||||
fmt.Printf("Resource: %+v\n", config.Resource)
|
||||
fmt.Printf("Image: %d\n", config.Image)
|
||||
fmt.Printf("Command: %s\n", config.Command)
|
||||
fmt.Printf("Dataset: %s\n", config.Dataset)
|
||||
fmt.Printf("Model Name: %s\n", config.ModelName)
|
||||
fmt.Printf("Dataset: %+v\n", config.Dataset)
|
||||
fmt.Printf("Model Name: %+v\n", config.ModelName)
|
||||
fmt.Printf("Run Args: %v\n", config.RunArgs)
|
||||
}
|
||||
|
||||
|
|
@ -102,6 +103,7 @@ var (
|
|||
modelName string
|
||||
taskOutput string
|
||||
resourceType string
|
||||
remotelog string
|
||||
)
|
||||
|
||||
func parseParams() (*models.RunConfig, error) {
|
||||
|
|
@ -134,13 +136,16 @@ func parseParams() (*models.RunConfig, error) {
|
|||
}
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(runArgs), &rs); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal runArgsConfig:%s, error:%v", runArgs, err)
|
||||
}
|
||||
var params map[string]string
|
||||
if runArgs != "" {
|
||||
if err := json.Unmarshal([]byte(runArgs), &rs); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal runArgsConfig:%s, error:%v", runArgs, err)
|
||||
}
|
||||
|
||||
params, err := ParseRunArgs(rs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse runArgsConfig:%s, error:%v", runArgs, err)
|
||||
params, err = ParseRunArgs(rs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse runArgsConfig:%s, error:%v", runArgs, err)
|
||||
}
|
||||
}
|
||||
|
||||
return &models.RunConfig{
|
||||
|
|
@ -155,6 +160,21 @@ func parseParams() (*models.RunConfig, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
var exclude_key = []string{
|
||||
"dataset",
|
||||
"model_name",
|
||||
"model_output",
|
||||
}
|
||||
|
||||
func isContainExcludeKey(key string, excludeKeys []string) bool {
|
||||
for _, excludeKey := range excludeKeys {
|
||||
if key == excludeKey {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseRunArgs 解析运行参数字符串为键值对
|
||||
func ParseRunArgs(input []string) (map[string]string, error) {
|
||||
result := make(map[string]string)
|
||||
|
|
@ -170,6 +190,9 @@ func ParseRunArgs(input []string) (map[string]string, error) {
|
|||
}
|
||||
key := strings.TrimPrefix(strings.TrimSpace(kv[0]), "--")
|
||||
value := strings.TrimSpace(kv[1])
|
||||
if isContainExcludeKey(key, exclude_key) {
|
||||
continue
|
||||
}
|
||||
result[key] = value
|
||||
}
|
||||
|
||||
|
|
@ -203,7 +226,8 @@ func init() {
|
|||
"选择模型")
|
||||
runCmd.Flags().StringVarP(&taskOutput, "task_output", "o", "",
|
||||
"任务输出目录")
|
||||
|
||||
runCmd.Flags().StringVarP(&remotelog, "remote_log", "l", "",
|
||||
"任务输出目录")
|
||||
// 设置必需参数
|
||||
runCmd.MarkFlagRequired("resource")
|
||||
runCmd.MarkFlagRequired("image")
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,6 @@
|
|||
train --dataset='{"id":1015,"name":"原子掺杂识别数据集","path":"fanshuai/datasets/1015/fanshuai_dataset_20250723103621/v0723/dataset","owner":"fanshuai","version":"v0723","identifier":"fanshuai_dataset_20250723103621","mount_path":"C:/code/remote-execute-cli
|
||||
/debugdata/dataset"}' \ --code_config='{"id":57,"code_repo_name":"云际训练任务调试代码","code_repo_vis":1,"is_public":true,"git_url":"https://openi.pcl.ac.cn/somunslotus/egnn.git","git_branch":"master","verify_mode":null,"git_user_name":null,"git_password":null,"ssh_key":null,"create_by":"fanshuai","create_time":"2025-08-12T14:26:53.000+08:00","update_by":"fanshuai","update_time":"2025-08-12T14:26:53.000+08:00","state":1,"mount_path":"C:/code/remote-execute-cli
|
||||
/debugdata/code"}' \ --image=37 \ --resource='{"clusterId":"1865927992266461184","baseResourceSpecs":[{"type":"STORAGE","name":"disk","number":1024}, {"type":"CPU","name":"CPU","number":8}, {"type":"MEMORY","name":"RAM","number":50}, {"type":"MEMORY","name":"VRAM","number":40}]}' \ --model_name='{"id":1017,"name":"原子掺杂模型0723","path":"fanshuai/model/1017/fanshuai_model_20250723143044/v1/model","owner":"fanshuai","version":"v1","identifier":"fanshuai_model_20250723143044","mount_path":"C:/code/remote-execute-cli
|
||||
/debugdata/model"}' \ --command="recognize_dophant/egnn/train_pl_vor.py" \ --resource_type="GPU" \ --run_args="[\"--dataset=/{{workflow.name}}/dataset\",\"--model_name=/{{workflow.name}}/model\",\"--model_output=/model\",\"--bs=32\",\"--nw=8\",\"--wd=5e-4\",\"--rf=0.9\",\"--lr=0.01\",\"--epochs=1000\"]" \ --task_output="C:/code/remote-execute-cli/result
|
||||
/debugdata/result"
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
// Package docs 自动生成,请勿手动修改
|
||||
// 使用 swag init 命令生成
|
||||
package docs
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,804 @@
|
|||
basePath: /api/v1
|
||||
definitions:
|
||||
handler.Response:
|
||||
properties:
|
||||
code:
|
||||
description: 状态码
|
||||
type: integer
|
||||
data:
|
||||
description: 数据对象
|
||||
message:
|
||||
description: 错误信息
|
||||
type: string
|
||||
type: object
|
||||
models.BaseResourceSpec:
|
||||
properties:
|
||||
availableUnit:
|
||||
type: string
|
||||
availableValue:
|
||||
type: integer
|
||||
createTime:
|
||||
type: string
|
||||
id:
|
||||
type: integer
|
||||
name:
|
||||
type: string
|
||||
resourceSpecId:
|
||||
type: integer
|
||||
totalUnit:
|
||||
type: string
|
||||
totalValue:
|
||||
type: integer
|
||||
type:
|
||||
type: string
|
||||
updateTime:
|
||||
type: string
|
||||
userId:
|
||||
type: string
|
||||
type: object
|
||||
models.Card:
|
||||
properties:
|
||||
card:
|
||||
type: string
|
||||
originImageID:
|
||||
type: string
|
||||
type: object
|
||||
models.Cluster:
|
||||
properties:
|
||||
clusterID:
|
||||
type: string
|
||||
code:
|
||||
$ref: '#/definitions/models.CodeInfo'
|
||||
resources:
|
||||
items:
|
||||
$ref: '#/definitions/models.ResourcesItem'
|
||||
type: array
|
||||
runtime:
|
||||
$ref: '#/definitions/models.Runtime'
|
||||
type: object
|
||||
models.ClusterImage:
|
||||
properties:
|
||||
cards:
|
||||
items:
|
||||
$ref: '#/definitions/models.Card'
|
||||
type: array
|
||||
clusterID:
|
||||
type: string
|
||||
imageID:
|
||||
type: integer
|
||||
originImageID:
|
||||
type: string
|
||||
originImageName:
|
||||
type: string
|
||||
originImageType:
|
||||
type: string
|
||||
type: object
|
||||
models.CodeConfig:
|
||||
properties:
|
||||
code_repo_name:
|
||||
type: string
|
||||
code_repo_vis:
|
||||
type: integer
|
||||
git_branch:
|
||||
type: string
|
||||
git_password:
|
||||
type: string
|
||||
git_url:
|
||||
type: string
|
||||
git_user_name:
|
||||
type: string
|
||||
id:
|
||||
type: integer
|
||||
is_public:
|
||||
type: boolean
|
||||
mount_path:
|
||||
type: string
|
||||
ssh_key:
|
||||
type: string
|
||||
verify_mode:
|
||||
type: string
|
||||
required:
|
||||
- code_repo_name
|
||||
- git_branch
|
||||
- git_url
|
||||
- id
|
||||
- mount_path
|
||||
type: object
|
||||
models.CodeInfo:
|
||||
properties:
|
||||
bindingID:
|
||||
type: integer
|
||||
type:
|
||||
type: string
|
||||
type: object
|
||||
models.DataResourceConfig:
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
identifier:
|
||||
type: string
|
||||
mount_path:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
owner:
|
||||
type: string
|
||||
path:
|
||||
type: string
|
||||
version:
|
||||
type: string
|
||||
required:
|
||||
- id
|
||||
- identifier
|
||||
- mount_path
|
||||
- name
|
||||
- owner
|
||||
- path
|
||||
- version
|
||||
type: object
|
||||
models.FileBinding:
|
||||
properties:
|
||||
bindingID:
|
||||
type: integer
|
||||
type:
|
||||
type: string
|
||||
type: object
|
||||
models.ImageBinding:
|
||||
properties:
|
||||
imageID:
|
||||
type: integer
|
||||
type:
|
||||
type: string
|
||||
type: object
|
||||
models.ImageInfo:
|
||||
properties:
|
||||
clusterImages:
|
||||
items:
|
||||
$ref: '#/definitions/models.ClusterImage'
|
||||
type: array
|
||||
createTime:
|
||||
type: string
|
||||
imageID:
|
||||
type: integer
|
||||
name:
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceAsyncSubmitRespData:
|
||||
properties:
|
||||
task_id:
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceAsyncSubmitResponse:
|
||||
properties:
|
||||
code:
|
||||
type: integer
|
||||
data:
|
||||
$ref: '#/definitions/models.InferenceAsyncSubmitRespData'
|
||||
msg:
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceAsyncTaskStatusData:
|
||||
properties:
|
||||
error:
|
||||
type: string
|
||||
job_set_id:
|
||||
type: string
|
||||
progress:
|
||||
description: 进度描述
|
||||
type: string
|
||||
result:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.InferenceSubmitTaskResponse'
|
||||
description: 任务完成后的结果
|
||||
status:
|
||||
description: PENDING/RUNNING/SUBMITTING/COMPLETED/FAILED
|
||||
type: string
|
||||
task_id:
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceAsyncTaskStatusRequest:
|
||||
properties:
|
||||
task_id:
|
||||
type: string
|
||||
required:
|
||||
- task_id
|
||||
type: object
|
||||
models.InferenceAsyncTaskStatusResponse:
|
||||
properties:
|
||||
code:
|
||||
type: integer
|
||||
data:
|
||||
$ref: '#/definitions/models.InferenceAsyncTaskStatusData'
|
||||
msg:
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceResultInfo:
|
||||
properties:
|
||||
jobSetID:
|
||||
description: 任务集ID
|
||||
type: string
|
||||
localJobID:
|
||||
description: 本地任务ID
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceSubmitTaskRequest:
|
||||
properties:
|
||||
code_config:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.CodeConfig'
|
||||
description: 代码配置
|
||||
command:
|
||||
description: 启动命令
|
||||
type: string
|
||||
description:
|
||||
description: 描述
|
||||
type: string
|
||||
image:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.ImageInfo'
|
||||
description: 镜像信息
|
||||
model:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.DataResourceConfig'
|
||||
description: 模型信息
|
||||
resource:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.RemoteSourceConfig'
|
||||
description: 资源配置
|
||||
resource_type:
|
||||
description: 资源类型
|
||||
type: string
|
||||
sub_model:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.DataResourceConfig'
|
||||
description: 增量模型信息(可选)
|
||||
version:
|
||||
description: 版本
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceSubmitTaskResponse:
|
||||
properties:
|
||||
code:
|
||||
description: 状态码
|
||||
type: integer
|
||||
data:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.InferenceSubmitTaskResponseData'
|
||||
description: 响应数据
|
||||
msg:
|
||||
description: 消息
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceSubmitTaskResponseData:
|
||||
properties:
|
||||
resultInfo:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.InferenceResultInfo'
|
||||
description: 结果信息
|
||||
taskInfo:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.SubtaskRequest'
|
||||
description: 任务信息
|
||||
type: object
|
||||
models.InferenceTaskStatusData:
|
||||
properties:
|
||||
status:
|
||||
description: 任务状态
|
||||
type: string
|
||||
url:
|
||||
description: 推理URL
|
||||
type: string
|
||||
type: object
|
||||
models.InferenceTaskStatusRequest:
|
||||
properties:
|
||||
jobSetID:
|
||||
description: 任务集ID
|
||||
type: string
|
||||
localJobID:
|
||||
description: 本地任务ID
|
||||
type: string
|
||||
required:
|
||||
- jobSetID
|
||||
- localJobID
|
||||
type: object
|
||||
models.InferenceTaskStatusResponse:
|
||||
properties:
|
||||
code:
|
||||
description: 状态码
|
||||
type: integer
|
||||
data:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.InferenceTaskStatusData'
|
||||
description: 任务状态数据
|
||||
msg:
|
||||
description: 消息
|
||||
type: string
|
||||
type: object
|
||||
models.InputParams:
|
||||
properties:
|
||||
ClusterID:
|
||||
type: string
|
||||
Output:
|
||||
type: string
|
||||
PackageName:
|
||||
type: string
|
||||
type: object
|
||||
models.Job:
|
||||
properties:
|
||||
description:
|
||||
type: string
|
||||
files:
|
||||
$ref: '#/definitions/models.JobFiles'
|
||||
jobResources:
|
||||
$ref: '#/definitions/models.JobResources'
|
||||
jobSetID:
|
||||
description: 停止任务时需要
|
||||
type: string
|
||||
localJobID:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
onlyCreate:
|
||||
type: boolean
|
||||
targetJob:
|
||||
items:
|
||||
$ref: '#/definitions/models.TargetJob'
|
||||
type: array
|
||||
type:
|
||||
type: string
|
||||
type: object
|
||||
models.JobFiles:
|
||||
properties:
|
||||
dataset:
|
||||
$ref: '#/definitions/models.FileBinding'
|
||||
image:
|
||||
$ref: '#/definitions/models.ImageBinding'
|
||||
model:
|
||||
$ref: '#/definitions/models.FileBinding'
|
||||
sub_model:
|
||||
$ref: '#/definitions/models.FileBinding'
|
||||
type: object
|
||||
models.JobResources:
|
||||
properties:
|
||||
clusters:
|
||||
items:
|
||||
$ref: '#/definitions/models.Cluster'
|
||||
type: array
|
||||
scheduleStrategy:
|
||||
type: string
|
||||
type: object
|
||||
models.JobSetInfo:
|
||||
properties:
|
||||
jobs:
|
||||
items:
|
||||
$ref: '#/definitions/models.Job'
|
||||
type: array
|
||||
type: object
|
||||
models.RemoteSourceConfig:
|
||||
properties:
|
||||
availableCount:
|
||||
type: integer
|
||||
baseResourceSpecs:
|
||||
items:
|
||||
$ref: '#/definitions/models.BaseResourceSpec'
|
||||
type: array
|
||||
changeType:
|
||||
type: integer
|
||||
clusterId:
|
||||
type: string
|
||||
costPerUnit:
|
||||
type: integer
|
||||
costType:
|
||||
type: string
|
||||
createTime:
|
||||
type: string
|
||||
id:
|
||||
type: integer
|
||||
name:
|
||||
type: string
|
||||
region:
|
||||
type: string
|
||||
sourceKey:
|
||||
type: string
|
||||
status:
|
||||
type: integer
|
||||
tag:
|
||||
type: string
|
||||
totalCount:
|
||||
type: integer
|
||||
type:
|
||||
type: string
|
||||
updateTime:
|
||||
type: string
|
||||
userId:
|
||||
type: string
|
||||
type: object
|
||||
models.ResourceConfig:
|
||||
properties:
|
||||
availableCount:
|
||||
type: integer
|
||||
clusterID:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
resources:
|
||||
items:
|
||||
$ref: '#/definitions/models.ResourcesItem'
|
||||
type: array
|
||||
totalCount:
|
||||
type: integer
|
||||
type:
|
||||
type: string
|
||||
required:
|
||||
- clusterID
|
||||
type: object
|
||||
models.ResourcesItem:
|
||||
properties:
|
||||
availableValue:
|
||||
type: integer
|
||||
name:
|
||||
type: string
|
||||
number:
|
||||
type: integer
|
||||
type:
|
||||
type: string
|
||||
required:
|
||||
- availableValue
|
||||
- name
|
||||
- type
|
||||
type: object
|
||||
models.RunConfig:
|
||||
properties:
|
||||
code_config:
|
||||
$ref: '#/definitions/models.CodeConfig'
|
||||
command:
|
||||
type: string
|
||||
dataset:
|
||||
$ref: '#/definitions/models.DataResourceConfig'
|
||||
image:
|
||||
type: integer
|
||||
model_name:
|
||||
$ref: '#/definitions/models.DataResourceConfig'
|
||||
resource:
|
||||
$ref: '#/definitions/models.ResourceConfig'
|
||||
run_args:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
task_output:
|
||||
type: string
|
||||
type: object
|
||||
models.Runtime:
|
||||
properties:
|
||||
envs:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
params:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
models.StopInferenceTaskRequest:
|
||||
properties:
|
||||
jobSetID:
|
||||
description: 任务集ID
|
||||
type: string
|
||||
localJobID:
|
||||
description: 本地任务ID
|
||||
type: string
|
||||
required:
|
||||
- jobSetID
|
||||
- localJobID
|
||||
type: object
|
||||
models.StopInferenceTaskResponse:
|
||||
properties:
|
||||
code:
|
||||
description: 状态码
|
||||
type: integer
|
||||
message:
|
||||
description: 消息
|
||||
type: string
|
||||
type: object
|
||||
models.SubmitTaskRequest:
|
||||
properties:
|
||||
code_id:
|
||||
description: 上传代码返回的ID
|
||||
type: integer
|
||||
dataset_id:
|
||||
description: 数据集ID(可选)
|
||||
type: integer
|
||||
model_id:
|
||||
description: 上传模型返回的ID
|
||||
type: integer
|
||||
run_config:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.RunConfig'
|
||||
description: 运行配置
|
||||
type: object
|
||||
models.SubtaskRequest:
|
||||
properties:
|
||||
jobSetInfo:
|
||||
$ref: '#/definitions/models.JobSetInfo'
|
||||
userID:
|
||||
type: integer
|
||||
type: object
|
||||
models.TargetJob:
|
||||
properties:
|
||||
inputParams:
|
||||
$ref: '#/definitions/models.InputParams'
|
||||
targetJobID:
|
||||
type: string
|
||||
type: object
|
||||
models.UploadCodeRequest:
|
||||
properties:
|
||||
run_config:
|
||||
allOf:
|
||||
- $ref: '#/definitions/models.RunConfig'
|
||||
description: 运行配置
|
||||
type: object
|
||||
host: localhost:8080
|
||||
info:
|
||||
contact:
|
||||
email: support@example.com
|
||||
name: API Support
|
||||
url: http://www.example.com/support
|
||||
description: 推理任务执行服务 API 文档
|
||||
license:
|
||||
name: Apache 2.0
|
||||
url: http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
termsOfService: http://swagger.io/terms/
|
||||
title: Remote Task Executor API
|
||||
version: "1.0"
|
||||
paths:
|
||||
/api/v1/getInferenceTaskAsync:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 查询异步推理任务的处理状态、进度和最终结果
|
||||
parameters:
|
||||
- description: 查询参数
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceAsyncTaskStatusRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceAsyncTaskStatusResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
summary: 查询异步任务状态
|
||||
tags:
|
||||
- 任务查询
|
||||
/api/v1/getTaskStatus:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 查询推理任务的执行状态和推理URL
|
||||
parameters:
|
||||
- description: 查询参数
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceTaskStatusRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceTaskStatusResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
summary: 查询任务状态
|
||||
tags:
|
||||
- 任务查询
|
||||
/api/v1/stopInferenceTask:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 停止正在运行的推理任务
|
||||
parameters:
|
||||
- description: 停止任务参数
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/models.StopInferenceTaskRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/models.StopInferenceTaskResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
summary: 停止推理任务
|
||||
tags:
|
||||
- 任务控制
|
||||
/api/v1/submitInferenceTaskAsync:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 异步提交推理任务,立即返回 task_id,任务在后台处理
|
||||
parameters:
|
||||
- description: 推理任务配置
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceSubmitTaskRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceAsyncSubmitResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
summary: 异步提交推理任务
|
||||
tags:
|
||||
- 任务管理
|
||||
/api/v1/submitTask:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 同步提交推理任务,包含代码、模型、增量模型的准备步骤,等待完成后返回结果
|
||||
parameters:
|
||||
- description: 推理任务配置
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceSubmitTaskRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/models.InferenceSubmitTaskResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
summary: 提交推理任务(同步)
|
||||
tags:
|
||||
- 任务管理
|
||||
/api/v1/uploadCode:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 上传代码资源到远程存储
|
||||
parameters:
|
||||
- description: 代码配置
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/models.UploadCodeRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: '#/definitions/handler.Response'
|
||||
- properties:
|
||||
data:
|
||||
properties:
|
||||
codeID:
|
||||
type: integer
|
||||
type: object
|
||||
type: object
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
summary: 上传代码资源
|
||||
tags:
|
||||
- 资源上传
|
||||
/api/v1/uploadModel:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 上传模型资源到远程存储
|
||||
parameters:
|
||||
- description: 模型配置
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/models.DataResourceConfig'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: '#/definitions/handler.Response'
|
||||
- properties:
|
||||
data:
|
||||
properties:
|
||||
modelID:
|
||||
type: integer
|
||||
type: object
|
||||
type: object
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/handler.Response'
|
||||
summary: 上传模型资源
|
||||
tags:
|
||||
- 资源上传
|
||||
/health:
|
||||
get:
|
||||
description: 检查服务健康状态
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: '#/definitions/handler.Response'
|
||||
- properties:
|
||||
data:
|
||||
properties:
|
||||
status:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
summary: 健康检查
|
||||
tags:
|
||||
- 系统
|
||||
schemes:
|
||||
- http
|
||||
- https
|
||||
swagger: "2.0"
|
||||
23
go.mod
23
go.mod
|
|
@ -1,24 +1,30 @@
|
|||
module remote-task-excutor-cli
|
||||
|
||||
go 1.24
|
||||
|
||||
toolchain go1.24.5
|
||||
go 1.24.5
|
||||
|
||||
require (
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-git/go-git/v5 v5.16.2
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/mholt/archiver/v3 v3.5.1
|
||||
github.com/nacos-group/nacos-sdk-go/v2 v2.3.2
|
||||
github.com/otiai10/copy v1.14.1
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/swaggo/files v1.0.1
|
||||
github.com/swaggo/gin-swagger v1.6.1
|
||||
golang.org/x/sync v0.16.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.1.6 // indirect
|
||||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||
github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.6 // indirect
|
||||
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5 // indirect
|
||||
github.com/alibabacloud-go/darabonba-array v0.1.0 // indirect
|
||||
|
|
@ -57,6 +63,10 @@ require (
|
|||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.6 // indirect
|
||||
github.com/go-openapi/spec v0.20.4 // indirect
|
||||
github.com/go-openapi/swag v0.19.15 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.27.0 // indirect
|
||||
|
|
@ -69,12 +79,14 @@ require (
|
|||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.6 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
|
|
@ -93,6 +105,7 @@ require (
|
|||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
|
||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/swaggo/swag v1.8.12 // indirect
|
||||
github.com/tjfoc/gmsm v1.4.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
|
|
@ -109,9 +122,9 @@ require (
|
|||
golang.org/x/net v0.42.0 // indirect
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/text v0.27.0 // indirect
|
||||
golang.org/x/time v0.1.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.34.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
google.golang.org/grpc v1.56.3 // indirect
|
||||
google.golang.org/protobuf v1.36.9 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
|
|
|
|||
47
go.sum
47
go.sum
|
|
@ -1,13 +1,21 @@
|
|||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
|
||||
github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
|
||||
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
|
||||
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.6 h1:eIf+iGJxdU4U9ypaUfbtOWCsZSbTb8AUHvyPrxu6mAA=
|
||||
github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.6/go.mod h1:4EUIoxs/do24zMOGGqYVWgw0s9NtiylnJglOeEB5UJo=
|
||||
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc=
|
||||
|
|
@ -98,6 +106,7 @@ github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI
|
|||
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
|
||||
github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -118,6 +127,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
|
|||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
|
||||
github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
|
||||
github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
|
||||
|
|
@ -132,6 +143,16 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj
|
|||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM=
|
||||
github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
|
||||
github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
|
||||
github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
|
||||
github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
|
||||
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
|
|
@ -140,6 +161,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
|||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4=
|
||||
github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
|
||||
|
|
@ -182,6 +205,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
|
|||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
|
|
@ -208,6 +233,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
|||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
|
||||
|
|
@ -280,12 +309,19 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
|||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
|
||||
github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
|
||||
github.com/swaggo/gin-swagger v1.6.1 h1:Ri06G4gc9N4t4k8hekMigJ9zKTFSlqj/9paAQCQs7cY=
|
||||
github.com/swaggo/gin-swagger v1.6.1/go.mod h1:LQ+hJStHakCWRiK/YNYtJOu4mR2FP+pxLnILT/qNiTw=
|
||||
github.com/swaggo/swag v1.8.12 h1:pctzkNPu0AlQP2royqX3apjKCQonAnf7KGoxeO4y64w=
|
||||
github.com/swaggo/swag v1.8.12/go.mod h1:lNfm6Gg+oAq3zRJQNEMBE66LIJKM44mxFqhEEgy2its=
|
||||
github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w=
|
||||
github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho=
|
||||
github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=
|
||||
|
|
@ -358,6 +394,7 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/
|
|||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
|
|
@ -391,6 +428,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
@ -430,8 +468,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
|||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
|
||||
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
|
||||
golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA=
|
||||
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
|
|
@ -454,8 +492,8 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
|
|||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
|
||||
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
|
|
@ -491,6 +529,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
# Kubernetes 部署说明
|
||||
|
||||
## 文件说明
|
||||
|
||||
- `deployment.yaml`: Deployment 和 Service 配置(包含 hostPath 挂载到 /remote-task/data)
|
||||
- `secret.yaml.example`: Secret 配置示例(需要创建真实的 secret.yaml)
|
||||
|
||||
## 部署步骤
|
||||
|
||||
### 1. 构建镜像
|
||||
|
||||
在项目根目录执行:
|
||||
|
||||
```bash
|
||||
# 使用默认 latest 标签
|
||||
./scripts/build-server-image.sh
|
||||
|
||||
# 或指定标签
|
||||
./scripts/build-server-image.sh v1.0.0
|
||||
```
|
||||
|
||||
### 2. 创建 Secret(如果需要)
|
||||
|
||||
如果需要使用 MySQL 或其他敏感配置,创建 Secret:
|
||||
|
||||
```bash
|
||||
# 方式1: 使用 kubectl 命令创建
|
||||
kubectl create secret generic remote-task-executor-secret \
|
||||
--from-literal=mysql-dsn='user:password@tcp(host:port)/remote-task?charset=utf8mb4&parseTime=true&loc=Local' \
|
||||
--namespace=default
|
||||
|
||||
# 方式2: 复制示例文件并修改后应用
|
||||
cp k8s/secret.yaml.example k8s/secret.yaml
|
||||
# 编辑 k8s/secret.yaml,填入真实值
|
||||
kubectl apply -f k8s/secret.yaml
|
||||
```
|
||||
|
||||
### 3. 部署到 Kubernetes
|
||||
|
||||
```bash
|
||||
# 部署到默认命名空间
|
||||
./scripts/deploy-server.sh
|
||||
|
||||
# 或指定命名空间
|
||||
./scripts/deploy-server.sh production
|
||||
```
|
||||
|
||||
### 4. 验证部署
|
||||
|
||||
```bash
|
||||
# 查看 Deployment 状态
|
||||
kubectl get deployment remote-task-executor-server
|
||||
|
||||
# 查看 Pod 状态
|
||||
kubectl get pods -l app=remote-task-executor-server
|
||||
|
||||
# 查看 Service
|
||||
kubectl get svc remote-task-executor-server
|
||||
|
||||
# 查看日志
|
||||
kubectl logs -f deployment/remote-task-executor-server
|
||||
```
|
||||
|
||||
### 5. 卸载(如果需要)
|
||||
|
||||
```bash
|
||||
./scripts/undeploy-server.sh
|
||||
```
|
||||
|
||||
## 配置说明
|
||||
|
||||
### 环境变量
|
||||
|
||||
可以通过修改 `deployment.yaml` 中的 `env` 部分添加环境变量。
|
||||
|
||||
### 资源限制
|
||||
|
||||
当前配置:
|
||||
- 请求: 512Mi 内存, 500m CPU
|
||||
- 限制: 2Gi 内存, 2000m CPU
|
||||
|
||||
可根据实际需求调整 `deployment.yaml` 中的 `resources` 部分。
|
||||
|
||||
### 副本数
|
||||
|
||||
默认副本数为 2,可在 `deployment.yaml` 中修改 `spec.replicas`。
|
||||
|
||||
### 健康检查
|
||||
|
||||
- Liveness Probe: 检查应用是否存活
|
||||
- Readiness Probe: 检查应用是否就绪
|
||||
- Startup Probe: 检查应用是否启动完成
|
||||
|
||||
所有探针都使用 `/health` 端点。
|
||||
|
||||
## 访问服务
|
||||
|
||||
### 集群内访问
|
||||
|
||||
```bash
|
||||
# 获取 Service 的 ClusterIP
|
||||
kubectl get svc remote-task-executor-server
|
||||
|
||||
# 在集群内通过 Service 名称访问
|
||||
curl http://remote-task-executor-server/health
|
||||
```
|
||||
|
||||
### 外部访问
|
||||
|
||||
如果需要外部访问,可以:
|
||||
|
||||
1. 使用 NodePort 或 LoadBalancer 类型的 Service
|
||||
2. 配置 Ingress(参考 deployment.yaml 中的注释部分)
|
||||
|
||||
## 故障排查
|
||||
|
||||
### 查看 Pod 日志
|
||||
|
||||
```bash
|
||||
kubectl logs -f deployment/remote-task-executor-server
|
||||
```
|
||||
|
||||
### 进入 Pod 调试
|
||||
|
||||
```bash
|
||||
kubectl exec -it deployment/remote-task-executor-server -- /bin/sh
|
||||
```
|
||||
|
||||
### 查看事件
|
||||
|
||||
```bash
|
||||
kubectl get events --sort-by='.lastTimestamp'
|
||||
```
|
||||
|
||||
### 查看 Pod 详细信息
|
||||
|
||||
```bash
|
||||
kubectl describe pod <pod-name>
|
||||
```
|
||||
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: remote-task-executor-server
|
||||
namespace: default
|
||||
labels:
|
||||
app: remote-task-executor-server
|
||||
version: v1
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: remote-task-executor-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: remote-task-executor-server
|
||||
version: v1
|
||||
spec:
|
||||
containers:
|
||||
- name: server
|
||||
image: 172.20.32.187/pipeline-service/remote-task-executor-server:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: TZ
|
||||
value: "Asia/Shanghai"
|
||||
- name: NACOS_HOST
|
||||
value: "172.20.32.121" # Nacos 服务器地址
|
||||
- name: NACOS_PORT
|
||||
value: "31203" # Nacos 服务器端口
|
||||
# 挂载 hostPath 卷
|
||||
volumeMounts:
|
||||
- name: data-volume
|
||||
mountPath: /remote-task/data
|
||||
# 资源限制
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "2000m"
|
||||
# 健康检查
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
# 启动探针
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 0
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 12
|
||||
# 定义 volumes
|
||||
volumes:
|
||||
- name: data-volume
|
||||
hostPath:
|
||||
path: /remote-task/data
|
||||
type: DirectoryOrCreate
|
||||
# 镜像拉取密钥(如果需要)
|
||||
# imagePullSecrets:
|
||||
# - name: registry-secret
|
||||
# 重启策略
|
||||
restartPolicy: Always
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: remote-task-executor-server
|
||||
namespace: default
|
||||
labels:
|
||||
app: remote-task-executor-server
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app: remote-task-executor-server
|
||||
---
|
||||
# 如果需要外部访问,可以使用 Ingress
|
||||
# apiVersion: networking.k8s.io/v1
|
||||
# kind: Ingress
|
||||
# metadata:
|
||||
# name: remote-task-executor-server
|
||||
# namespace: default
|
||||
# annotations:
|
||||
# kubernetes.io/ingress.class: nginx
|
||||
# nginx.ingress.kubernetes.io/rewrite-target: /
|
||||
# spec:
|
||||
# rules:
|
||||
# - host: remote-task-executor.example.com
|
||||
# http:
|
||||
# paths:
|
||||
# - path: /
|
||||
# pathType: Prefix
|
||||
# backend:
|
||||
# service:
|
||||
# name: remote-task-executor-server
|
||||
# port:
|
||||
# number: 80
|
||||
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# 这是一个示例文件,实际使用时需要创建真实的 Secret
|
||||
# 创建 Secret 的命令:
|
||||
# kubectl create secret generic remote-task-executor-secret \
|
||||
# --from-literal=mysql-dsn='user:password@tcp(host:port)/database?charset=utf8mb4&parseTime=true&loc=Local' \
|
||||
# --namespace=default
|
||||
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: remote-task-executor-secret
|
||||
namespace: default
|
||||
type: Opaque
|
||||
stringData:
|
||||
# MySQL DSN(示例,实际使用时替换为真实值)
|
||||
mysql-dsn: "user:password@tcp(mysql-host:3306)/remote-task?charset=utf8mb4&parseTime=true&loc=Local"
|
||||
# 其他敏感信息可以在这里添加
|
||||
|
||||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -19,6 +20,7 @@ type HTTPClient struct {
|
|||
Client *http.Client
|
||||
BaseURL string
|
||||
Headers map[string]string
|
||||
mu sync.RWMutex // 保护 Headers 的并发访问
|
||||
}
|
||||
|
||||
// NewHTTPClient 创建新的HTTP客户端实例
|
||||
|
|
@ -32,11 +34,31 @@ func NewHTTPClient(baseURL string, timeout time.Duration) *HTTPClient {
|
|||
}
|
||||
}
|
||||
|
||||
// SetHeader 设置请求头
|
||||
// SetHeader 设置请求头(线程安全)
|
||||
func (c *HTTPClient) SetHeader(key, value string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.Headers[key] = value
|
||||
}
|
||||
|
||||
// GetHeader 获取请求头(线程安全)
|
||||
func (c *HTTPClient) GetHeader(key string) string {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.Headers[key]
|
||||
}
|
||||
|
||||
// GetHeaders 获取所有请求头的副本(线程安全)
|
||||
func (c *HTTPClient) GetHeaders() map[string]string {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
headers := make(map[string]string, len(c.Headers))
|
||||
for k, v := range c.Headers {
|
||||
headers[k] = v
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
// Get 发送GET请求,支持查询参数
|
||||
func (c *HTTPClient) Get(path string, queryParams ...map[string]string) ([]byte, error) {
|
||||
// 构建完整URL
|
||||
|
|
@ -60,14 +82,16 @@ func (c *HTTPClient) Get(path string, queryParams ...map[string]string) ([]byte,
|
|||
fullURL = u.String()
|
||||
}
|
||||
|
||||
fmt.Println("request url:", fullURL)
|
||||
// 创建请求
|
||||
req, err := http.NewRequest("GET", fullURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 设置请求头
|
||||
for k, v := range c.Headers {
|
||||
// 设置请求头(使用线程安全的方法)
|
||||
headers := c.GetHeaders()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
|
|
@ -106,7 +130,8 @@ func (c *HTTPClient) PostJSON(path string, data interface{}) ([]byte, error) {
|
|||
|
||||
// 设置请求头
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range c.Headers {
|
||||
headers := c.GetHeaders()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
|
|
@ -134,7 +159,7 @@ func (c *HTTPClient) UploadFiles(
|
|||
files map[string][]string, // 字段名 -> 多个文件路径
|
||||
formFields map[string]string, // 额外表单字段
|
||||
) ([]byte, error) {
|
||||
fullURL := c.BaseURL + path
|
||||
fullURL := path
|
||||
|
||||
// 创建multipart writer
|
||||
body := &bytes.Buffer{}
|
||||
|
|
@ -166,13 +191,13 @@ func (c *HTTPClient) UploadFiles(
|
|||
}
|
||||
}
|
||||
|
||||
// 添加额外表单字段
|
||||
for key, value := range formFields {
|
||||
err := writer.WriteField(key, value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("添加表单字段 %s 失败: %w", key, err)
|
||||
}
|
||||
}
|
||||
//// 添加额外表单字段
|
||||
//for key, value := range formFields {
|
||||
// err := writer.WriteField(key, value)
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("添加表单字段 %s 失败: %w", key, err)
|
||||
// }
|
||||
//}
|
||||
|
||||
// 关闭writer以完成multipart消息
|
||||
err := writer.Close()
|
||||
|
|
@ -190,8 +215,9 @@ func (c *HTTPClient) UploadFiles(
|
|||
contentType := writer.FormDataContentType()
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
|
||||
// 添加自定义头
|
||||
for k, v := range c.Headers {
|
||||
// 添加自定义头(使用线程安全的方法)
|
||||
headers := c.GetHeaders()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
|
|
@ -221,17 +247,37 @@ func (c *HTTPClient) UploadFiles(
|
|||
}
|
||||
|
||||
func removePathPrefix(fullPath, prefix string) (string, error) {
|
||||
// 使用系统文件分隔符标准化路径
|
||||
prefix = filepath.Clean(prefix) + string(filepath.Separator)
|
||||
fullPath = filepath.Clean(fullPath)
|
||||
// 确保两个路径都是绝对路径并标准化
|
||||
absFullPath, err := filepath.Abs(fullPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("获取绝对路径失败 [%s]: %w", fullPath, err)
|
||||
}
|
||||
absPrefix, err := filepath.Abs(prefix)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("获取绝对路径失败 [%s]: %w", prefix, err)
|
||||
}
|
||||
|
||||
// 标准化路径(处理 .. 和 . 等)
|
||||
absFullPath = filepath.Clean(absFullPath)
|
||||
absPrefix = filepath.Clean(absPrefix)
|
||||
|
||||
// 转换为 Linux 风格的路径分隔符(统一使用 /)
|
||||
unixFullPath := filepath.ToSlash(absFullPath)
|
||||
unixPrefix := filepath.ToSlash(absPrefix)
|
||||
|
||||
// 确保前缀以 / 结尾
|
||||
if !strings.HasSuffix(unixPrefix, "/") {
|
||||
unixPrefix += "/"
|
||||
}
|
||||
|
||||
// 检查路径是否有指定的前缀
|
||||
if strings.HasPrefix(fullPath, prefix) {
|
||||
// 返回去除前缀的路径
|
||||
return fullPath[len(prefix):], nil
|
||||
if strings.HasPrefix(unixFullPath, unixPrefix) {
|
||||
// 返回去除前缀的路径(已经是 Linux 风格)
|
||||
return unixFullPath[len(unixPrefix):], nil
|
||||
}
|
||||
// 如果不匹配,返回原始路径(或者可以根据需要返回错误)
|
||||
return "", fmt.Errorf("错误的filepath和prefix, filepath:%v, prefix:%v", fullPath, prefix)
|
||||
|
||||
// 如果不匹配,返回详细错误信息
|
||||
return "", fmt.Errorf("路径前缀不匹配 [文件路径: %s, 前缀: %s]", unixFullPath, unixPrefix)
|
||||
}
|
||||
|
||||
func (c *HTTPClient) encodeFilepath(paths []string) []string {
|
||||
|
|
@ -250,22 +296,22 @@ func (c *HTTPClient) encodeFilepath(paths []string) []string {
|
|||
// queryParams:可选查询参数
|
||||
func (c *HTTPClient) DownloadFile(path, localPath string, queryParams ...map[string]string) (string, error) {
|
||||
// 构建完整URL
|
||||
fullURL := c.BaseURL + path
|
||||
fullURL := path
|
||||
|
||||
// 处理查询参数
|
||||
if len(queryParams) > 0 {
|
||||
u, err := url.Parse(fullURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("解析URL失败: %w", err)
|
||||
}
|
||||
|
||||
q := u.Query()
|
||||
for key, value := range queryParams[0] {
|
||||
q.Add(key, value)
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
fullURL = u.String()
|
||||
}
|
||||
//// 处理查询参数
|
||||
//if queryParams != nil {
|
||||
// u, err := url.Parse(fullURL)
|
||||
// if err != nil {
|
||||
// return "", fmt.Errorf("解析URL失败: %w", err)
|
||||
// }
|
||||
//
|
||||
// q := u.Query()
|
||||
// for key, value := range queryParams[0] {
|
||||
// q.Add(key, value)
|
||||
// }
|
||||
// u.RawQuery = q.Encode()
|
||||
// fullURL = u.String()
|
||||
//}
|
||||
|
||||
// 创建请求
|
||||
req, err := http.NewRequest("GET", fullURL, nil)
|
||||
|
|
@ -273,8 +319,9 @@ func (c *HTTPClient) DownloadFile(path, localPath string, queryParams ...map[str
|
|||
return "", fmt.Errorf("创建请求失败: %w", err)
|
||||
}
|
||||
|
||||
// 设置请求头
|
||||
for k, v := range c.Headers {
|
||||
// 设置请求头(使用线程安全的方法)
|
||||
headers := c.GetHeaders()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,22 +4,43 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/nacos-group/nacos-sdk-go/v2/clients"
|
||||
"github.com/nacos-group/nacos-sdk-go/v2/clients/config_client"
|
||||
"github.com/nacos-group/nacos-sdk-go/v2/common/constant"
|
||||
"github.com/nacos-group/nacos-sdk-go/v2/vo"
|
||||
"github.com/pkg/errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
Group = "ARGO"
|
||||
DataID = "remote-task-config"
|
||||
Host = "172.20.32.197"
|
||||
Port = 31203
|
||||
Namespace = "public"
|
||||
)
|
||||
|
||||
// GetNacosHost 从环境变量获取 Nacos Host,如果不存在则使用默认值
|
||||
func GetNacosHost() string {
|
||||
if host := os.Getenv("NACOS_HOST"); host != "" {
|
||||
return host
|
||||
}
|
||||
return "172.20.32.121" // 默认值
|
||||
}
|
||||
|
||||
// GetNacosPort 从环境变量获取 Nacos Port,如果不存在则使用默认值
|
||||
func GetNacosPort() uint64 {
|
||||
if portStr := os.Getenv("NACOS_PORT"); portStr != "" {
|
||||
if port, err := strconv.ParseUint(portStr, 10, 64); err == nil {
|
||||
return port
|
||||
}
|
||||
}
|
||||
return 31203 // 默认值
|
||||
}
|
||||
|
||||
// NacosConfig Nacos 配置结构
|
||||
type NacosConfig struct {
|
||||
Host string `json:"host" yaml:"host"`
|
||||
|
|
@ -42,6 +63,9 @@ type APIConfig struct {
|
|||
TestURL string `json:"testUrl" yaml:"testUrl"`
|
||||
TokenTTL int `json:"tokenTtl" yaml:"tokenTtl"` // token 有效期(秒)
|
||||
Timeout int `json:"timeout" yaml:"timeout"`
|
||||
HostPath string `json:"hostPath" yaml:"hostPath"`
|
||||
SubPath string `json:"subPath" yaml:"subPath"`
|
||||
MySQLDSN string `json:"mysqlDsn" yaml:"mysqlDsn"`
|
||||
}
|
||||
|
||||
// ConfigReader Nacos 配置读取器
|
||||
|
|
@ -57,12 +81,16 @@ type ConfigReader struct {
|
|||
|
||||
var DefaultApiConfig = &APIConfig{
|
||||
AuthURL: "jcc-admin/admin/login",
|
||||
Username: "hnxjy-super",
|
||||
Username: "hnxjy-super1",
|
||||
Password: "h1n2x3j4y5@",
|
||||
BaseURL: "https://jcc.jointcloud.net/",
|
||||
BaseURL: "https://ai4m.jointcloud.net/",
|
||||
TestURL: "",
|
||||
TokenTTL: 0,
|
||||
Timeout: 10,
|
||||
HostPath: "D:\\code", //"/platform-data",
|
||||
SubPath: "remote-task-excutor-cli",
|
||||
// 例子:user:pass@tcp(127.0.0.1:3306)/remote_task?charset=utf8mb4&parseTime=true&loc=Local
|
||||
MySQLDSN: "root:qazxc123456.@tcp(172.20.32.121:31306)/remote-task?charset=utf8mb4&parseTime=true&loc=Local",
|
||||
}
|
||||
|
||||
// NewConfigReader 创建新的 Nacos 配置读取器
|
||||
|
|
@ -111,43 +139,43 @@ func NewConfigReader(ctx context.Context, cfg NacosConfig) (*ConfigReader, error
|
|||
return nil, errors.Wrap(err, "初始化配置加载失败")
|
||||
}
|
||||
|
||||
// 启动配置监听
|
||||
go reader.startListener()
|
||||
//// 启动配置监听
|
||||
//go reader.startListener()
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
// loadConfig 从Nacos加载配置(带上下文)
|
||||
func (r *ConfigReader) loadConfig(ctx context.Context) error {
|
||||
// 使用上下文限制超时
|
||||
//content, err := r.client.GetConfig(vo.ConfigParam{
|
||||
// DataId: r.dataID,
|
||||
// Group: r.group,
|
||||
//})
|
||||
//
|
||||
//if err != nil {
|
||||
// return errors.Wrap(err, "获取Nacos配置失败")
|
||||
//}
|
||||
//
|
||||
//if content == "" {
|
||||
// return errors.New("从Nacos获取的配置为空")
|
||||
//}
|
||||
//
|
||||
//var apiCfg APIConfig
|
||||
//if err := yaml.Unmarshal([]byte(content), &apiCfg); err != nil {
|
||||
// return errors.Wrapf(err, "解析配置失败: %s", content)
|
||||
//}
|
||||
//
|
||||
//// 设置默认值
|
||||
//if apiCfg.TokenTTL == 0 {
|
||||
// apiCfg.TokenTTL = 3600 // 默认1小时
|
||||
//}
|
||||
//
|
||||
//// 更新配置
|
||||
//r.cfgMutex.Lock()
|
||||
//r.apiCfg = &apiCfg
|
||||
//r.cfgMutex.Unlock()
|
||||
r.apiCfg = DefaultApiConfig
|
||||
//使用上下文限制超时
|
||||
content, err := r.client.GetConfig(vo.ConfigParam{
|
||||
DataId: r.dataID,
|
||||
Group: r.group,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "获取Nacos配置失败")
|
||||
}
|
||||
|
||||
if content == "" {
|
||||
return errors.New("从Nacos获取的配置为空")
|
||||
}
|
||||
|
||||
var apiCfg APIConfig
|
||||
if err := yaml.Unmarshal([]byte(content), &apiCfg); err != nil {
|
||||
return errors.Wrapf(err, "解析配置失败: %s", content)
|
||||
}
|
||||
|
||||
// 设置默认值
|
||||
if apiCfg.TokenTTL == 0 {
|
||||
apiCfg.TokenTTL = 3600 // 默认1小时
|
||||
}
|
||||
|
||||
// 更新配置
|
||||
r.cfgMutex.Lock()
|
||||
r.apiCfg = &apiCfg
|
||||
r.cfgMutex.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -168,12 +196,31 @@ func (r *ConfigReader) startListener() {
|
|||
// onConfigChange 配置变更回调
|
||||
func (r *ConfigReader) onConfigChange(namespace, group, dataId, data string) {
|
||||
if dataId == r.dataID && group == r.group {
|
||||
var apiCfg APIConfig
|
||||
if err := json.Unmarshal([]byte(data), &apiCfg); err != nil {
|
||||
fmt.Printf("解析变更的配置失败: %v\n", err)
|
||||
// 如果配置为空,跳过
|
||||
if data == "" {
|
||||
fmt.Printf("配置变更回调收到空配置,跳过更新\n")
|
||||
return
|
||||
}
|
||||
|
||||
var apiCfg APIConfig
|
||||
var err error
|
||||
|
||||
// 尝试使用 YAML 解析(与 loadConfig 保持一致)
|
||||
// YAML 支持注释,更灵活
|
||||
if err = yaml.Unmarshal([]byte(data), &apiCfg); err != nil {
|
||||
// 如果 YAML 解析失败,尝试 JSON 解析(兼容性处理)
|
||||
if jsonErr := json.Unmarshal([]byte(data), &apiCfg); jsonErr != nil {
|
||||
fmt.Printf("解析变更的配置失败 (YAML和JSON都失败): YAML错误=%v, JSON错误=%v, 配置内容前100字符=%s\n",
|
||||
err, jsonErr, truncateString(data, 100))
|
||||
return
|
||||
}
|
||||
// JSON 解析成功
|
||||
fmt.Printf("使用JSON格式解析配置变更\n")
|
||||
} else {
|
||||
// YAML 解析成功
|
||||
fmt.Printf("使用YAML格式解析配置变更\n")
|
||||
}
|
||||
|
||||
// 设置默认值
|
||||
if apiCfg.TokenTTL == 0 {
|
||||
apiCfg.TokenTTL = 3600 // 默认1小时
|
||||
|
|
@ -184,10 +231,18 @@ func (r *ConfigReader) onConfigChange(namespace, group, dataId, data string) {
|
|||
r.apiCfg = &apiCfg
|
||||
r.cfgMutex.Unlock()
|
||||
|
||||
fmt.Printf("API配置已更新: authURL=%s\n", apiCfg.AuthURL)
|
||||
fmt.Println("API配置已更新: ", apiCfg)
|
||||
}
|
||||
}
|
||||
|
||||
// truncateString 截断字符串,用于日志输出
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
|
||||
// GetAPIConfig 获取API配置(简化版)
|
||||
func (r *ConfigReader) GetAPIConfig() (*APIConfig, error) {
|
||||
r.cfgMutex.RLock()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"remote-task-excutor-cli/pkg/models"
|
||||
"remote-task-excutor-cli/pkg/service"
|
||||
|
|
@ -29,12 +30,23 @@ func NewHandler(
|
|||
}
|
||||
|
||||
// UploadModel 处理模型上传请求
|
||||
// @Summary 上传模型资源
|
||||
// @Description 上传模型资源到远程存储
|
||||
// @Tags 资源上传
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.DataResourceConfig true "模型配置"
|
||||
// @Success 200 {object} Response{data=object{modelID=int}}
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/uploadModel [post]
|
||||
func (h *Handler) UploadModel(c *gin.Context) {
|
||||
var config models.DataResourceConfig
|
||||
if err := c.ShouldBindJSON(&config); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[UploadModel] 请求参数: %+v\n", config)
|
||||
|
||||
// 获取认证信息
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
|
|
@ -53,22 +65,37 @@ func (h *Handler) UploadModel(c *gin.Context) {
|
|||
// 准备模型
|
||||
modelID, err := h.preparationService.PrepareModel(c.Request.Context(), authData, config, clusterID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("准备模型失败: "+err.Error()))
|
||||
response := InternalServerErrorResponse("准备模型失败: " + err.Error())
|
||||
fmt.Printf("[UploadModel] 返回错误: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, SuccessResponse(gin.H{
|
||||
response := SuccessResponse(gin.H{
|
||||
"modelID": modelID,
|
||||
}))
|
||||
})
|
||||
fmt.Printf("[UploadModel] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// UploadCode 处理代码上传请求
|
||||
// @Summary 上传代码资源
|
||||
// @Description 上传代码资源到远程存储
|
||||
// @Tags 资源上传
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.UploadCodeRequest true "代码配置"
|
||||
// @Success 200 {object} Response{data=object{codeID=int}}
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/uploadCode [post]
|
||||
func (h *Handler) UploadCode(c *gin.Context) {
|
||||
var request models.UploadCodeRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[UploadCode] 请求参数: %+v\n", request)
|
||||
|
||||
// 获取认证信息
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
|
|
@ -87,22 +114,38 @@ func (h *Handler) UploadCode(c *gin.Context) {
|
|||
// 准备代码
|
||||
codeID, err := h.preparationService.PrepareCode(c.Request.Context(), authData, &request.RunConfig, clusterID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("准备代码失败: "+err.Error()))
|
||||
response := InternalServerErrorResponse("准备代码失败: " + err.Error())
|
||||
fmt.Printf("[UploadCode] 返回错误: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, SuccessResponse(gin.H{
|
||||
response := SuccessResponse(gin.H{
|
||||
"codeID": codeID,
|
||||
}))
|
||||
})
|
||||
fmt.Printf("[UploadCode] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// SubmitTask 处理任务提交请求
|
||||
// @Summary 提交任务(已废弃,请使用 submitTask)
|
||||
// @Description 提交任务(使用已上传的代码和模型ID)
|
||||
// @Tags 任务管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.SubmitTaskRequest true "任务配置"
|
||||
// @Success 200 {object} Response{data=object{jobSetID=string}}
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/submitTask [post]
|
||||
// @Deprecated
|
||||
func (h *Handler) SubmitTask(c *gin.Context) {
|
||||
var request models.SubmitTaskRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[SubmitTask] 请求参数: %+v\n", request)
|
||||
|
||||
// 获取认证信息
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
|
|
@ -128,24 +171,120 @@ func (h *Handler) SubmitTask(c *gin.Context) {
|
|||
// 提交任务
|
||||
jobSetID, err := h.inferenceService.SubmitTask(c.Request.Context(), authData, &request.RunConfig, clusterID, bindResultSet)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("提交任务失败: "+err.Error()))
|
||||
response := InternalServerErrorResponse("提交任务失败: " + err.Error())
|
||||
fmt.Printf("[SubmitTask] 返回错误: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, SuccessResponse(gin.H{
|
||||
response := SuccessResponse(gin.H{
|
||||
"jobSetID": jobSetID,
|
||||
}))
|
||||
})
|
||||
fmt.Printf("[SubmitTask] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// QueryStatus 处理状态查询请求
|
||||
// @Summary 查询任务状态
|
||||
// @Description 查询推理任务的执行状态和推理URL
|
||||
// @Tags 任务查询
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.InferenceTaskStatusRequest true "查询参数"
|
||||
// @Success 200 {object} models.InferenceTaskStatusResponse
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/getTaskStatus [post]
|
||||
func (h *Handler) QueryStatus(c *gin.Context) {
|
||||
var request struct {
|
||||
JobSetID string `json:"jobSetID" binding:"required"`
|
||||
}
|
||||
var request models.InferenceTaskStatusRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[QueryStatus] 请求参数: %+v\n", request)
|
||||
|
||||
// 获取认证信息
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("获取认证信息失败: "+err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// 查询任务状态
|
||||
status, err := h.inferenceService.GetTaskStatus(c.Request.Context(), authData, request.LocalJobID, request.JobSetID)
|
||||
if err != nil {
|
||||
response := InternalServerErrorResponse("查询任务状态失败: " + err.Error())
|
||||
fmt.Printf("[QueryStatus] 返回错误: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("[QueryStatus] 返回结果: %+v\n", status)
|
||||
c.JSON(http.StatusOK, status)
|
||||
}
|
||||
|
||||
// StopInferenceTask 处理停止推理任务请求
|
||||
// @Summary 停止推理任务
|
||||
// @Description 停止正在运行的推理任务
|
||||
// @Tags 任务控制
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.StopInferenceTaskRequest true "停止任务参数"
|
||||
// @Success 200 {object} models.StopInferenceTaskResponse
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/stopInferenceTask [post]
|
||||
func (h *Handler) StopInferenceTask(c *gin.Context) {
|
||||
var request models.StopInferenceTaskRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[StopInferenceTask] 请求参数: %+v\n", request)
|
||||
|
||||
// 获取认证信息
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("获取认证信息失败: "+err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// 停止任务
|
||||
response, err := h.inferenceService.StopTask(c.Request.Context(), authData, request.LocalJobID, request.JobSetID)
|
||||
if err != nil {
|
||||
errResponse := InternalServerErrorResponse("停止任务失败: " + err.Error())
|
||||
fmt.Printf("[StopInferenceTask] 返回错误: %+v\n", errResponse)
|
||||
c.JSON(http.StatusOK, errResponse)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("[StopInferenceTask] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// ReSubmitTask 处理重启推理任务请求
|
||||
// @Summary 重启推理任务
|
||||
// @Description 将入参原样转发给第三方 /jsm/v2/jobs/submit,返回新的 jobSetID 与入参中的 localJobID
|
||||
// @Tags 任务控制
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.SubtaskRequest true "重启任务参数(userID、jobSetInfo.jobs)"
|
||||
// @Success 200 {object} models.ReSubmitTaskResponse
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/reSubmitTask [post]
|
||||
func (h *Handler) ReSubmitTask(c *gin.Context) {
|
||||
var request models.SubtaskRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[ReSubmitTask] 请求参数: %+v\n", request)
|
||||
|
||||
if len(request.JobSetInfo.Jobs) == 0 {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("jobSetInfo.jobs 不能为空"))
|
||||
return
|
||||
}
|
||||
|
||||
// 获取认证信息
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
|
|
@ -154,19 +293,183 @@ func (h *Handler) QueryStatus(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// 查询任务状态
|
||||
status, err := h.inferenceService.GetTaskStatus(c.Request.Context(), authData, request.JobSetID)
|
||||
response, err := h.inferenceService.ReSubmitTask(c.Request.Context(), authData, &request)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("查询任务状态失败: "+err.Error()))
|
||||
errResponse := InternalServerErrorResponse("重启任务失败: " + err.Error())
|
||||
fmt.Printf("[ReSubmitTask] 返回错误: %+v\n", errResponse)
|
||||
c.JSON(http.StatusOK, errResponse)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, SuccessResponse(status))
|
||||
fmt.Printf("[ReSubmitTask] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// SubmitInferenceTask 处理推理任务提交请求
|
||||
// @Summary 提交推理任务(同步)
|
||||
// @Description 同步提交推理任务,包含代码、模型、增量模型的准备步骤,等待完成后返回结果
|
||||
// @Tags 任务管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.InferenceSubmitTaskRequest true "推理任务配置"
|
||||
// @Success 200 {object} models.InferenceSubmitTaskResponse
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/submitTask [post]
|
||||
func (h *Handler) SubmitInferenceTask(c *gin.Context) {
|
||||
var request models.InferenceSubmitTaskRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[SubmitInferenceTask] 请求参数: %+v\n", request)
|
||||
|
||||
// 参数校验:检查 model.Path 是否为空
|
||||
if request.Model.Path == "" {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("model.Path 不能为空"))
|
||||
return
|
||||
}
|
||||
|
||||
// 参数校验:如果提供了 subModel,检查 subModel.Path 是否为空
|
||||
if request.SubModel != nil && request.SubModel.Path == "" {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("subModel.Path 不能为空"))
|
||||
return
|
||||
}
|
||||
|
||||
// 获取认证信息
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("获取认证信息失败: "+err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// 获取集群ID(从 resource 中获取,如果没有则使用默认值)
|
||||
clusterID := request.Resource.ClusterID
|
||||
if clusterID == "" {
|
||||
clusterID, err = h.authService.GetClusterID(c.Request.Context(), "default")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("获取集群ID失败: "+err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 提交推理任务(包含准备步骤)
|
||||
response, err := h.inferenceService.SubmitInferenceTask(c.Request.Context(), authData, &request, clusterID)
|
||||
if err != nil {
|
||||
errResponse := InternalServerErrorResponse("提交推理任务失败: " + err.Error())
|
||||
fmt.Printf("[SubmitInferenceTask] 返回错误: %+v\n", errResponse)
|
||||
c.JSON(http.StatusOK, errResponse)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("[SubmitInferenceTask] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// SubmitInferenceTaskAsync 异步提交推理任务:立刻返回 task_id
|
||||
// @Summary 异步提交推理任务
|
||||
// @Description 异步提交推理任务,立即返回 task_id,任务在后台处理
|
||||
// @Tags 任务管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.InferenceSubmitTaskRequest true "推理任务配置"
|
||||
// @Success 200 {object} models.InferenceAsyncSubmitResponse
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/submitInferenceTaskAsync [post]
|
||||
func (h *Handler) SubmitInferenceTaskAsync(c *gin.Context) {
|
||||
var request models.InferenceSubmitTaskRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[SubmitInferenceTaskAsync] 请求参数: %+v\n", request)
|
||||
|
||||
// 参数校验:检查 model.Path 是否为空
|
||||
if request.Model.Path == "" {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("model.Path 不能为空"))
|
||||
return
|
||||
}
|
||||
|
||||
// 参数校验:如果提供了 subModel,检查 subModel.Path 是否为空
|
||||
if request.SubModel != nil && request.SubModel.Path == "" {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("subModel.Path 不能为空"))
|
||||
return
|
||||
}
|
||||
|
||||
authData, err := h.authService.GetToken(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("获取认证信息失败: "+err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
clusterID := request.Resource.ClusterID
|
||||
if clusterID == "" {
|
||||
clusterID, err = h.authService.GetClusterID(c.Request.Context(), "default")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, InternalServerErrorResponse("获取集群ID失败: "+err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
taskID, err := h.inferenceService.SubmitInferenceTaskAsync(c.Request.Context(), authData, &request, clusterID)
|
||||
if err != nil {
|
||||
errResponse := InternalServerErrorResponse("异步提交推理任务失败: " + err.Error())
|
||||
fmt.Printf("[SubmitInferenceTaskAsync] 返回错误: %+v\n", errResponse)
|
||||
c.JSON(http.StatusOK, errResponse)
|
||||
return
|
||||
}
|
||||
|
||||
response := models.InferenceAsyncSubmitResponse{
|
||||
Code: http.StatusOK,
|
||||
Msg: "",
|
||||
Data: models.InferenceAsyncSubmitRespData{TaskID: taskID},
|
||||
}
|
||||
fmt.Printf("[SubmitInferenceTaskAsync] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// GetInferenceTaskAsync 查询异步推理任务状态/结果
|
||||
// @Summary 查询异步任务状态
|
||||
// @Description 查询异步推理任务的处理状态、进度和最终结果
|
||||
// @Tags 任务查询
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body models.InferenceAsyncTaskStatusRequest true "查询参数"
|
||||
// @Success 200 {object} models.InferenceAsyncTaskStatusResponse
|
||||
// @Failure 400 {object} Response
|
||||
// @Failure 500 {object} Response
|
||||
// @Router /api/v1/getInferenceTaskAsync [post]
|
||||
func (h *Handler) GetInferenceTaskAsync(c *gin.Context) {
|
||||
var request models.InferenceAsyncTaskStatusRequest
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusOK, BadRequestResponse("请求参数错误: "+err.Error()))
|
||||
return
|
||||
}
|
||||
fmt.Printf("[GetInferenceTaskAsync] 请求参数: %+v\n", request)
|
||||
resp, err := h.inferenceService.GetInferenceTaskAsync(c.Request.Context(), request.TaskID)
|
||||
if err != nil {
|
||||
errResponse := InternalServerErrorResponse("查询异步任务失败: " + err.Error())
|
||||
fmt.Printf("[GetInferenceTaskAsync] 返回错误: %+v\n", errResponse)
|
||||
c.JSON(http.StatusOK, errResponse)
|
||||
return
|
||||
}
|
||||
fmt.Printf("[GetInferenceTaskAsync] 返回结果: %+v\n", resp)
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// HealthCheck 处理健康检查请求
|
||||
// @Summary 健康检查
|
||||
// @Description 检查服务健康状态
|
||||
// @Tags 系统
|
||||
// @Produce json
|
||||
// @Success 200 {object} Response{data=object{status=string}}
|
||||
// @Router /health [get]
|
||||
func (h *Handler) HealthCheck(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, SuccessResponse(gin.H{
|
||||
//fmt.Printf("[HealthCheck] 请求参数: 无\n")
|
||||
response := SuccessResponse(gin.H{
|
||||
"status": "UP",
|
||||
}))
|
||||
})
|
||||
//fmt.Printf("[HealthCheck] 返回结果: %+v\n", response)
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,36 +1,38 @@
|
|||
package handler
|
||||
|
||||
// Response 统一的API响应格式
|
||||
import "net/http"
|
||||
|
||||
// Response 统一的 API 响应格式,成功与失败均使用 code、msg、data;接口失败时错误信息通过 msg 返回。
|
||||
type Response struct {
|
||||
Code int `json:"code"` // 状态码
|
||||
Message string `json:"message"` // 错误信息
|
||||
Data interface{} `json:"data"` // 数据对象
|
||||
Code int `json:"code"` // 状态码
|
||||
Msg string `json:"msg"` // 提示信息(成功可为空或 success,失败时为错误信息)
|
||||
Data interface{} `json:"data"` // 数据对象,失败时为 null
|
||||
}
|
||||
|
||||
// SuccessResponse 成功响应
|
||||
func SuccessResponse(data interface{}) Response {
|
||||
return Response{
|
||||
Code: 200,
|
||||
Message: "success",
|
||||
Data: data,
|
||||
Code: http.StatusOK,
|
||||
Msg: "success",
|
||||
Data: data,
|
||||
}
|
||||
}
|
||||
|
||||
// ErrorResponse 错误响应
|
||||
func ErrorResponse(code int, message string) Response {
|
||||
// ErrorResponse 错误响应,错误信息通过 msg 返回
|
||||
func ErrorResponse(code int, msg string) Response {
|
||||
return Response{
|
||||
Code: code,
|
||||
Message: message,
|
||||
Data: nil,
|
||||
Code: code,
|
||||
Msg: msg,
|
||||
Data: nil,
|
||||
}
|
||||
}
|
||||
|
||||
// BadRequestResponse 400错误响应
|
||||
func BadRequestResponse(message string) Response {
|
||||
return ErrorResponse(400, message)
|
||||
// BadRequestResponse 400 错误响应,错误信息通过 msg 返回
|
||||
func BadRequestResponse(msg string) Response {
|
||||
return ErrorResponse(http.StatusBadRequest, msg)
|
||||
}
|
||||
|
||||
// InternalServerErrorResponse 500错误响应
|
||||
func InternalServerErrorResponse(message string) Response {
|
||||
return ErrorResponse(500, message)
|
||||
// InternalServerErrorResponse 500 错误响应,错误信息通过 msg 返回
|
||||
func InternalServerErrorResponse(msg string) Response {
|
||||
return ErrorResponse(http.StatusInternalServerError, msg)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,15 +30,32 @@ type DataResourceConfig struct {
|
|||
Identifier string `json:"identifier" validate:"required"`
|
||||
Owner string `json:"owner" validate:"required"`
|
||||
MountPath string `json:"mount_path" validate:"required"`
|
||||
GitID int `json:"git_id" validate:"required"`
|
||||
}
|
||||
|
||||
type ResourceConfig struct {
|
||||
ClusterID string `json:"clusterID" validate:"required"`
|
||||
Resources []ResourcesItem `json:"resources"`
|
||||
ClusterID string `json:"clusterID" validate:"required"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
TotalCount int `json:"totalCount"`
|
||||
AvailableCount int `json:"availableCount"`
|
||||
Resources []ResourcesItem `json:"resources"`
|
||||
}
|
||||
|
||||
type ResourcesItem struct {
|
||||
Type string `json:"type" validate:"required"`
|
||||
Name string `json:"name" validate:"required"`
|
||||
AvailableValue int `json:"availableValue" validate:"required"`
|
||||
Number int `json:"number"`
|
||||
}
|
||||
|
||||
type TaskResource struct {
|
||||
ClusterID string `json:"clusterID" validate:"required"`
|
||||
Resources []TaskResourcesItem `json:"baseResourceSpecs"`
|
||||
}
|
||||
|
||||
type TaskResourcesItem struct {
|
||||
Type string `json:"type" validate:"required"`
|
||||
Name string `json:"name" validate:"required"`
|
||||
Number int `json:"number" validate:"required"`
|
||||
Number int `json:"number"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ type AuthResponse struct {
|
|||
// AuthData 认证数据主体
|
||||
type AuthData struct {
|
||||
TokenHead string `json:"tokenHead"`
|
||||
ExpiresIn string `json:"expiresIn"`
|
||||
ExpiresIn int `json:"expiresIn"`
|
||||
JsmUserInfo UserInfo `json:"jsmUserInfo"`
|
||||
TokenTimeout string `json:"tokenTimeout"`
|
||||
TokenTimeout int `json:"tokenTimeout"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// SubmitTaskRequest 提交任务的请求结构体
|
||||
type SubmitTaskRequest struct {
|
||||
RunConfig RunConfig `json:"run_config"` // 运行配置
|
||||
|
|
@ -12,3 +14,74 @@ type SubmitTaskRequest struct {
|
|||
type UploadCodeRequest struct {
|
||||
RunConfig RunConfig `json:"run_config"` // 运行配置
|
||||
}
|
||||
|
||||
// InferenceSubmitTaskRequest 推理任务提交请求结构体
|
||||
type InferenceSubmitTaskRequest struct {
|
||||
Version string `json:"version"` // 版本
|
||||
Description string `json:"description"` // 描述
|
||||
Model DataResourceConfig `json:"model"` // 模型信息
|
||||
SubModel *DataResourceConfig `json:"sub_model"` // 增量模型信息(可选)
|
||||
Image ImageInfo `json:"image"` // 镜像信息
|
||||
CodeConfig CodeConfig `json:"code_config"` // 代码配置
|
||||
ResourceType string `json:"resource_type"` // 资源类型
|
||||
Resource RemoteSourceConfig `json:"resource"` // 资源配置
|
||||
Command string `json:"command"` // 启动命令
|
||||
}
|
||||
|
||||
type RemoteSourceConfig struct {
|
||||
Id int `json:"id"`
|
||||
SourceKey string `json:"sourceKey"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
TotalCount int `json:"totalCount"`
|
||||
AvailableCount int `json:"availableCount"`
|
||||
ChangeType int `json:"changeType"`
|
||||
Status int `json:"status"`
|
||||
Region string `json:"region"`
|
||||
ClusterID string `json:"clusterId"`
|
||||
CostPerUnit int `json:"costPerUnit"`
|
||||
CostType string `json:"costType"`
|
||||
Tag string `json:"tag"`
|
||||
UserId string `json:"userId"`
|
||||
CreateTime time.Time `json:"createTime"`
|
||||
UpdateTime time.Time `json:"updateTime"`
|
||||
BaseResourceSpecs []BaseResourceSpec
|
||||
}
|
||||
|
||||
type BaseResourceSpec struct {
|
||||
Id int `json:"id"`
|
||||
ResourceSpecId int `json:"resourceSpecId"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
TotalValue int `json:"totalValue"`
|
||||
TotalUnit string `json:"totalUnit"`
|
||||
AvailableValue int `json:"availableValue"`
|
||||
AvailableUnit string `json:"availableUnit"`
|
||||
UserId string `json:"userId"`
|
||||
CreateTime time.Time `json:"createTime"`
|
||||
UpdateTime time.Time `json:"updateTime"`
|
||||
}
|
||||
|
||||
// ImageInfo 镜像信息
|
||||
type ImageInfo struct {
|
||||
ImageID int `json:"imageID"`
|
||||
Name string `json:"name"`
|
||||
CreateTime string `json:"createTime"`
|
||||
ClusterImages []ClusterImage `json:"clusterImages"`
|
||||
}
|
||||
|
||||
// ClusterImage 集群镜像信息
|
||||
type ClusterImage struct {
|
||||
ImageID int `json:"imageID"`
|
||||
ClusterID string `json:"clusterID"`
|
||||
OriginImageType string `json:"originImageType"`
|
||||
OriginImageID string `json:"originImageID"`
|
||||
OriginImageName string `json:"originImageName"`
|
||||
Cards []Card `json:"cards"`
|
||||
}
|
||||
|
||||
// Card 卡片信息
|
||||
type Card struct {
|
||||
OriginImageID string `json:"originImageID"`
|
||||
Card string `json:"card"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,6 +44,33 @@ type PackageCreateLoadInfo struct {
|
|||
//LoadToPath []string `json:"loadToPath"`
|
||||
}
|
||||
|
||||
type PresignedPackageCreateUpload struct {
|
||||
UserID int `json:"userID"`
|
||||
Info PresignedPackageCreateUploadInfo `json:"info"`
|
||||
}
|
||||
|
||||
type PresignedPackageCreateUploadResp struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
PresignUrl string `json:"presignUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type PresignedPackageCreateUploadInfo struct {
|
||||
Type string `json:"type"`
|
||||
Param PresignedPackageCreateUploadParam `json:"params"`
|
||||
}
|
||||
|
||||
type PresignedPackageCreateUploadParam struct {
|
||||
BucketID int `json:"bucketID" binding:"required"`
|
||||
Name string `json:"name" binding:"required"`
|
||||
PackageID int `json:"packageID"`
|
||||
Zip bool `json:"zip"`
|
||||
CopyTo []int `json:"copyTo"`
|
||||
CopyPath []string `json:"copyToPath"`
|
||||
}
|
||||
|
||||
type PackageCreateResponse struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
|
|
@ -51,8 +78,9 @@ type PackageCreateResponse struct {
|
|||
}
|
||||
|
||||
type PackageData struct {
|
||||
Package Package `json:"package"`
|
||||
Objects []FileObject `json:"objects"`
|
||||
Package Package `json:"package"`
|
||||
Objects []FileObject `json:"objects"`
|
||||
CopyToFullPaths []string `json:"copiedToFullPaths"`
|
||||
}
|
||||
|
||||
type Package struct {
|
||||
|
|
@ -129,10 +157,12 @@ type ModelBindingInfo struct {
|
|||
|
||||
// ModelAppInfoDetail 应用信息中的Info结构
|
||||
type ModelAppInfoDetail struct {
|
||||
Type string `json:"type"`
|
||||
LocalPath string `json:"localPath"`
|
||||
ObjectIDs []int `json:"objectIDs"`
|
||||
BindingInfo ModelBindingInfo `json:"bindingInfo"`
|
||||
Type string `json:"type"`
|
||||
LocalPath string `json:"localPath"`
|
||||
ObjectIDs []int `json:"objectIDs"`
|
||||
CopiedTo []int `json:"copiedTo"`
|
||||
CopiedToFullRoots []string `json:"copiedToFullRoots"`
|
||||
BindingInfo ModelBindingInfo `json:"bindingInfo"`
|
||||
}
|
||||
|
||||
// ModelAppInfo 应用信息
|
||||
|
|
@ -204,9 +234,10 @@ type BindingResult struct {
|
|||
}
|
||||
|
||||
type BindResultSet struct {
|
||||
BindCodeID int `json:"bind_code_id"`
|
||||
BindDatasetID int `json:"bind_dataset_id"`
|
||||
BindModelID int `json:"bind_model_id"`
|
||||
BindCodeID int `json:"bind_code_id"`
|
||||
BindDatasetID int `json:"bind_dataset_id"`
|
||||
BindModelID int `json:"bind_model_id"`
|
||||
BindSubModelID int `json:"bind_sub_model_id"`
|
||||
}
|
||||
|
||||
type UploadFileConfig struct {
|
||||
|
|
|
|||
|
|
@ -30,16 +30,19 @@ type Job struct {
|
|||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Type string `json:"type"`
|
||||
JobSetID string `json:"jobSetID,omitempty"` // 停止任务时需要
|
||||
Files *JobFiles `json:"files,omitempty"`
|
||||
JobResources *JobResources `json:"jobResources,omitempty"`
|
||||
TargetJob []TargetJob `json:"targetJob,omitempty"`
|
||||
OnlyCreate bool `json:"onlyCreate,omitempty"`
|
||||
}
|
||||
|
||||
// JobFiles 表示任务关联的文件
|
||||
type JobFiles struct {
|
||||
Dataset FileBinding `json:"dataset"`
|
||||
Model FileBinding `json:"model,omitempty"`
|
||||
Image ImageBinding `json:"image"`
|
||||
Dataset FileBinding `json:"dataset"`
|
||||
Model FileBinding `json:"model,omitempty"`
|
||||
Image ImageBinding `json:"image"`
|
||||
SubModel FileBinding `json:"sub_model,omitempty"`
|
||||
}
|
||||
|
||||
// FileBinding 表示文件绑定信息
|
||||
|
|
@ -256,22 +259,191 @@ type ResultObject struct {
|
|||
UpdateTime time.Time `json:"updateTime"` // 更新时间
|
||||
}
|
||||
|
||||
// InferenceTaskStatusRequest 推理任务状态查询请求
|
||||
type InferenceTaskStatusRequest struct {
|
||||
LocalJobID string `json:"localJobID" binding:"required"` // 本地任务ID
|
||||
JobSetID string `json:"jobSetID" binding:"required"` // 任务集ID
|
||||
}
|
||||
|
||||
// InferenceTaskStatusResponse 推理任务状态查询响应
|
||||
type InferenceTaskStatusResponse struct {
|
||||
Code int `json:"code"` // 状态码
|
||||
Msg string `json:"msg"` // 消息
|
||||
Data InferenceTaskStatusData `json:"data"` // 任务状态数据
|
||||
}
|
||||
|
||||
// InferenceTaskStatusData 推理任务状态数据
|
||||
type InferenceTaskStatusData struct {
|
||||
Status string `json:"status"` // 任务状态
|
||||
URL string `json:"url"` // 推理URL
|
||||
}
|
||||
|
||||
// InferenceAsyncSubmitResponse 异步提交推理任务响应
|
||||
type InferenceAsyncSubmitResponse struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data InferenceAsyncSubmitRespData `json:"data"`
|
||||
}
|
||||
|
||||
type InferenceAsyncSubmitRespData struct {
|
||||
TaskID string `json:"task_id"`
|
||||
}
|
||||
|
||||
// InferenceAsyncTaskStatusRequest 异步任务查询请求
|
||||
type InferenceAsyncTaskStatusRequest struct {
|
||||
TaskID string `json:"task_id" binding:"required"`
|
||||
}
|
||||
|
||||
// InferenceAsyncTaskStatusResponse 异步任务查询响应
|
||||
type InferenceAsyncTaskStatusResponse struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data InferenceAsyncTaskStatusData `json:"data"`
|
||||
}
|
||||
|
||||
type InferenceAsyncTaskStatusData struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status string `json:"status"` // PENDING/RUNNING/SUBMITTING/COMPLETED/FAILED
|
||||
Progress string `json:"progress,omitempty"` // 进度描述
|
||||
JobSetID string `json:"job_set_id,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Result *InferenceSubmitTaskResponse `json:"result,omitempty"` // 任务完成后的结果
|
||||
}
|
||||
|
||||
// StopInferenceTaskRequest 停止推理任务请求
|
||||
type StopInferenceTaskRequest struct {
|
||||
LocalJobID string `json:"localJobID" binding:"required"` // 本地任务ID
|
||||
JobSetID string `json:"jobSetID" binding:"required"` // 任务集ID
|
||||
}
|
||||
|
||||
// StopInferenceTaskResponse 停止推理任务响应
|
||||
type StopInferenceTaskResponse struct {
|
||||
Code int `json:"code"` // 状态码
|
||||
Message string `json:"msg"` // 消息
|
||||
}
|
||||
|
||||
// ReSubmitTaskResponse 重启推理任务响应
|
||||
type ReSubmitTaskResponse struct {
|
||||
Code int `json:"code"` // 状态码
|
||||
Message string `json:"msg"` // 消息
|
||||
Data ReSubmitTaskData `json:"data"` // 响应数据
|
||||
}
|
||||
|
||||
// ReSubmitTaskData 重启推理任务响应数据
|
||||
type ReSubmitTaskData struct {
|
||||
JobSetID string `json:"jobSetID"` // 任务集ID(来自第三方接口返回)
|
||||
LocalJobID string `json:"localJobID"` // 本地任务ID(来自入参 jobs)
|
||||
}
|
||||
|
||||
// StopInferenceSubmitRequest 停止推理任务提交给第三方的请求
|
||||
type StopInferenceSubmitRequest struct {
|
||||
UserID int `json:"userID"` // 用户ID
|
||||
JobSetInfo JobSetInfo `json:"jobSetInfo"` // 任务集信息
|
||||
}
|
||||
|
||||
// StopInferenceSubmitResponse 停止推理任务第三方接口响应
|
||||
type StopInferenceSubmitResponse struct {
|
||||
Code string `json:"code"` // 状态码
|
||||
Message string `json:"message"` // 消息
|
||||
Data StopInferenceSubmitData `json:"data"` // 数据
|
||||
}
|
||||
|
||||
// StopInferenceSubmitData 停止推理任务响应数据
|
||||
type StopInferenceSubmitData struct {
|
||||
JobSetID string `json:"jobSetID"` // 任务集ID
|
||||
FilesUploadScheme FilesUploadScheme `json:"filesUploadScheme"` // 文件上传方案
|
||||
Message string `json:"message"` // 消息
|
||||
}
|
||||
|
||||
// FilesUploadScheme 文件上传方案
|
||||
type FilesUploadScheme struct {
|
||||
LocalFileUploadSchemes []interface{} `json:"localFileUploadSchemes"` // 本地文件上传方案
|
||||
}
|
||||
|
||||
// InferenceTaskDetailResponse 推理任务详情响应(第三方接口返回的原始格式)
|
||||
type InferenceTaskDetailResponse struct {
|
||||
Code string `json:"code"` // 状态码
|
||||
Message string `json:"message"` // 消息
|
||||
Data InferenceTaskDetailData `json:"data"` // 任务详情数据
|
||||
}
|
||||
|
||||
// InferenceTaskDetailData 推理任务详情数据(第三方接口返回的原始格式)
|
||||
type InferenceTaskDetailData struct {
|
||||
Type string `json:"type"`
|
||||
InstanceName string `json:"instanceName"`
|
||||
InstanceId string `json:"instanceId"`
|
||||
ModelName string `json:"modelName"`
|
||||
ModelType string `json:"modelType"`
|
||||
InferCard string `json:"inferCard"`
|
||||
InferUrl string `json:"inferUrl"`
|
||||
ClusterName string `json:"clusterName"`
|
||||
ClusterType string `json:"clusterType"`
|
||||
Status string `json:"status"`
|
||||
CreatedTime string `json:"createdTime"`
|
||||
Type string `json:"type"` // 类型
|
||||
Instance InferenceInstance `json:"instance"` // 实例信息
|
||||
}
|
||||
|
||||
// InferenceInstance 推理实例信息
|
||||
type InferenceInstance struct {
|
||||
InstanceName string `json:"InstanceName"` // 实例名称
|
||||
InstanceId string `json:"InstanceId"` // 实例ID
|
||||
ModelName string `json:"ModelName"` // 模型名称
|
||||
ModelType string `json:"ModelType"` // 模型类型
|
||||
InferCard string `json:"InferCard"` // 推理卡类型
|
||||
InferUrl string `json:"InferUrl"` // 推理URL
|
||||
ClusterName string `json:"ClusterName"` // 集群名称
|
||||
ClusterType string `json:"ClusterType"` // 集群类型
|
||||
Status string `json:"Status"` // 状态
|
||||
CreatedTime string `json:"CreatedTime"` // 创建时间
|
||||
}
|
||||
|
||||
// InferenceSubmitTaskResponse 推理任务提交响应
|
||||
type InferenceSubmitTaskResponse struct {
|
||||
Code int `json:"code"` // 状态码
|
||||
Msg string `json:"msg"` // 消息
|
||||
Data InferenceSubmitTaskResponseData `json:"data"` // 响应数据
|
||||
}
|
||||
|
||||
// InferenceSubmitTaskResponseData 推理任务提交响应数据
|
||||
type InferenceSubmitTaskResponseData struct {
|
||||
TaskInfo SubtaskRequest `json:"taskInfo"` // 任务信息
|
||||
ResultInfo InferenceResultInfo `json:"resultInfo"` // 结果信息
|
||||
}
|
||||
|
||||
// InferenceTaskInfo 推理任务信息
|
||||
type InferenceTaskInfo struct {
|
||||
UserID int `json:"userID"` // 用户ID
|
||||
JobSetInfo InferenceJobSetInfo `json:"jobSetInfo"` // 任务集信息
|
||||
}
|
||||
|
||||
// InferenceJobSetInfo 推理任务集信息
|
||||
type InferenceJobSetInfo struct {
|
||||
Jobs []InferenceJob `json:"jobs"` // 任务列表
|
||||
}
|
||||
|
||||
// InferenceJob 推理任务
|
||||
type InferenceJob struct {
|
||||
LocalJobID string `json:"localJobID"` // 本地任务ID
|
||||
Name string `json:"name"` // 任务名称
|
||||
Description string `json:"description"` // 任务描述
|
||||
Type string `json:"type"` // 任务类型
|
||||
Files *InferenceJobFiles `json:"files"` // 文件信息
|
||||
JobResources *InferenceJobResources `json:"jobResources"` // 资源信息
|
||||
}
|
||||
|
||||
// InferenceJobFiles 推理任务文件
|
||||
type InferenceJobFiles struct {
|
||||
Model FileBinding `json:"model"` // 模型绑定
|
||||
SubModel *FileBinding `json:"sub_model,omitempty"` // 增量模型绑定(可选)
|
||||
Image ImageBinding `json:"image"` // 镜像绑定
|
||||
}
|
||||
|
||||
// InferenceJobResources 推理任务资源
|
||||
type InferenceJobResources struct {
|
||||
ScheduleStrategy string `json:"scheduleStrategy"` // 调度策略
|
||||
Clusters []InferenceCluster `json:"clusters"` // 集群列表
|
||||
}
|
||||
|
||||
// InferenceCluster 推理集群信息
|
||||
type InferenceCluster struct {
|
||||
ClusterID string `json:"clusterID"` // 集群ID
|
||||
Runtime Runtime `json:"runtime"` // 运行时配置
|
||||
Code CodeInfo `json:"code"` // 代码信息
|
||||
Resources []ResourcesItem `json:"resources"` // 资源列表
|
||||
}
|
||||
|
||||
// InferenceResultInfo 推理结果信息
|
||||
type InferenceResultInfo struct {
|
||||
LocalJobID string `json:"localJobID"` // 本地任务ID
|
||||
JobSetID string `json:"jobSetID"` // 任务集ID
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,25 +2,109 @@ package router
|
|||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "remote-task-excutor-cli/docs" // swagger docs
|
||||
"remote-task-excutor-cli/pkg/config"
|
||||
"remote-task-excutor-cli/pkg/handler"
|
||||
"remote-task-excutor-cli/pkg/service/auth"
|
||||
"remote-task-excutor-cli/pkg/service/inference"
|
||||
"remote-task-excutor-cli/pkg/service/preparation"
|
||||
"remote-task-excutor-cli/pkg/service/task"
|
||||
"remote-task-excutor-cli/pkg/storage"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
swaggerFiles "github.com/swaggo/files"
|
||||
ginSwagger "github.com/swaggo/gin-swagger"
|
||||
)
|
||||
|
||||
// @title Remote Task Executor API
|
||||
// @version 1.0
|
||||
// @description 推理任务执行服务 API 文档
|
||||
// @termsOfService http://swagger.io/terms/
|
||||
|
||||
// @contact.name API Support
|
||||
// @contact.url http://www.example.com/support
|
||||
// @contact.email support@example.com
|
||||
|
||||
// @license.name Apache 2.0
|
||||
// @license.url http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
|
||||
// @host localhost:8080
|
||||
// @BasePath /api/v1
|
||||
|
||||
// @schemes http https
|
||||
|
||||
// @title Remote Task Executor API
|
||||
// @version 1.0
|
||||
// @description 推理任务执行服务 API 文档
|
||||
// @termsOfService http://swagger.io/terms/
|
||||
|
||||
// @contact.name API Support
|
||||
// @contact.url http://www.example.com/support
|
||||
// @contact.email support@example.com
|
||||
|
||||
// @license.name Apache 2.0
|
||||
// @license.url http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
|
||||
// @host localhost:8080
|
||||
// @BasePath /api/v1
|
||||
|
||||
// @schemes http https
|
||||
|
||||
// customRecoveryMiddleware 自定义 recovery 中间件,捕获 panic 并记录详细信息
|
||||
func customRecoveryMiddleware() gin.HandlerFunc {
|
||||
return gin.CustomRecovery(func(c *gin.Context, recovered interface{}) {
|
||||
// 记录 panic 信息
|
||||
errMsg := fmt.Sprintf("Handler panic: %v", recovered)
|
||||
log.Printf("[PANIC] %s\n请求路径: %s %s\n请求IP: %s\n堆栈信息:\n%s",
|
||||
errMsg,
|
||||
c.Request.Method,
|
||||
c.Request.URL.Path,
|
||||
c.ClientIP(),
|
||||
string(debug.Stack()))
|
||||
|
||||
// 返回统一的错误响应
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"code": 500,
|
||||
"msg": "服务器内部错误,请稍后重试",
|
||||
"data": nil,
|
||||
})
|
||||
c.Abort()
|
||||
})
|
||||
}
|
||||
|
||||
// SetupRouter 设置路由
|
||||
func SetupRouter() *gin.Engine {
|
||||
// 创建gin引擎
|
||||
r := gin.Default()
|
||||
// 创建gin引擎(不使用 Default,手动添加中间件以使用自定义 recovery)
|
||||
r := gin.New()
|
||||
|
||||
// 添加日志中间件
|
||||
r.Use(gin.Logger())
|
||||
|
||||
// 添加自定义 recovery 中间件
|
||||
r.Use(customRecoveryMiddleware())
|
||||
|
||||
// 获取API配置
|
||||
apiCfg := getAPIConfig()
|
||||
fmt.Println("apiCfg is : ", apiCfg)
|
||||
// 初始化 MySQL(如果未配置则不启用异步推理提交)
|
||||
var db *sql.DB
|
||||
if apiCfg.MySQLDSN != "" {
|
||||
mysqlDB, err := storage.NewMySQL(apiCfg.MySQLDSN)
|
||||
if err != nil {
|
||||
log.Printf("初始化MySQL失败: %v", err)
|
||||
panic(err)
|
||||
} else {
|
||||
db = mysqlDB
|
||||
}
|
||||
} else {
|
||||
log.Printf("MySQLDSN 为空:异步推理提交将不可用")
|
||||
}
|
||||
|
||||
// 创建所需的服务
|
||||
authService := auth.NewTokenService(apiCfg)
|
||||
|
|
@ -28,7 +112,7 @@ func SetupRouter() *gin.Engine {
|
|||
taskService := task.NewTaskService(apiCfg.BaseURL, 10*time.Second)
|
||||
|
||||
// 创建inference service
|
||||
inferenceService := inference.NewInferenceService(authService, preparationService, taskService, apiCfg.BaseURL)
|
||||
inferenceService := inference.NewInferenceService(authService, preparationService, taskService, apiCfg.BaseURL, db, apiCfg)
|
||||
|
||||
// 创建Handler实例
|
||||
h := handler.NewHandler(authService, preparationService, inferenceService)
|
||||
|
|
@ -39,13 +123,21 @@ func SetupRouter() *gin.Engine {
|
|||
// 添加inference service相关路由
|
||||
apiV1.POST("/uploadModel", h.UploadModel)
|
||||
apiV1.POST("/uploadCode", h.UploadCode)
|
||||
apiV1.POST("/submitTask", h.SubmitTask)
|
||||
apiV1.POST("/queryStatus", h.QueryStatus)
|
||||
apiV1.POST("/submitTask", h.SubmitInferenceTask)
|
||||
apiV1.POST("/getTaskStatus", h.QueryStatus)
|
||||
apiV1.POST("/stopInferenceTask", h.StopInferenceTask)
|
||||
apiV1.POST("/reSubmitTask", h.ReSubmitTask)
|
||||
// 异步推理任务提交/查询
|
||||
apiV1.POST("/submitInferenceTaskAsync", h.SubmitInferenceTaskAsync)
|
||||
apiV1.POST("/getInferenceTaskAsync", h.GetInferenceTaskAsync)
|
||||
}
|
||||
|
||||
// 添加健康检查路由
|
||||
r.GET("/health", h.HealthCheck)
|
||||
|
||||
// 添加 Swagger 文档路由
|
||||
r.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
|
|
@ -54,8 +146,8 @@ func getAPIConfig() *config.APIConfig {
|
|||
ctx := context.Background()
|
||||
// 1. 配置Nacos连接
|
||||
nacosCfg := config.NacosConfig{
|
||||
Host: config.Host,
|
||||
Port: config.Port,
|
||||
Host: config.GetNacosHost(),
|
||||
Port: config.GetNacosPort(),
|
||||
NamespaceID: config.Namespace,
|
||||
Group: config.Group,
|
||||
DataID: config.DataID,
|
||||
|
|
@ -68,13 +160,7 @@ func getAPIConfig() *config.APIConfig {
|
|||
if err != nil {
|
||||
log.Printf("初始化配置读取器失败: %v, 使用默认配置", err)
|
||||
// 使用默认配置作为fallback
|
||||
return &config.APIConfig{
|
||||
BaseURL: "http://localhost:8080",
|
||||
AuthURL: "/auth/token",
|
||||
Username: "admin",
|
||||
Password: "admin",
|
||||
Timeout: 60,
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
|
|
@ -83,13 +169,7 @@ func getAPIConfig() *config.APIConfig {
|
|||
if err != nil {
|
||||
log.Printf("获取API配置失败: %v, 使用默认配置", err)
|
||||
// 使用默认配置作为fallback
|
||||
return &config.APIConfig{
|
||||
BaseURL: "http://localhost:8080",
|
||||
AuthURL: "/auth/token",
|
||||
Username: "admin",
|
||||
Password: "admin",
|
||||
Timeout: 60,
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return apiCfg
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package runners
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/otiai10/copy"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
|
@ -13,6 +12,8 @@ import (
|
|||
"remote-task-excutor-cli/pkg/service"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/otiai10/copy"
|
||||
)
|
||||
|
||||
type taskRunner struct {
|
||||
|
|
@ -38,7 +39,8 @@ func (r *taskRunner) RunTask(ctx context.Context, config *models.RunConfig) erro
|
|||
return err
|
||||
}
|
||||
fmt.Println("获取token成功")
|
||||
clusterID, err := r.authService.GetClusterID(ctx, "openI")
|
||||
//clusterID, err := r.authService.GetClusterID(ctx, "openI")
|
||||
clusterID := config.Resource.ClusterID
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -68,7 +70,6 @@ func (r *taskRunner) RunTask(ctx context.Context, config *models.RunConfig) erro
|
|||
fmt.Println("获取日志失败:", err)
|
||||
}
|
||||
}()
|
||||
// 轮询任务状态
|
||||
resp, err := r.PollTaskStatusWithBackoff(ctx, authData, jobSetID)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -83,13 +84,13 @@ func (r *taskRunner) RunTask(ctx context.Context, config *models.RunConfig) erro
|
|||
return err
|
||||
}
|
||||
|
||||
if err := r.MergeAimRepo(ctx, config); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := r.MergeTensorboard(ctx, config); err != nil {
|
||||
return err
|
||||
}
|
||||
//if err := r.MergeAimRepo(ctx, config); err != nil {
|
||||
// return err
|
||||
//}
|
||||
//
|
||||
//if err := r.MergeTensorboard(ctx, config); err != nil {
|
||||
// return err
|
||||
//}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -192,21 +193,19 @@ func (r *taskRunner) RecordLogFile(ctx context.Context, authData *models.AuthDat
|
|||
|
||||
func (r *taskRunner) PollTaskStatusWithBackoff(ctx context.Context, authData *models.AuthData, jobSetID string) (*models.TaskDetailResponse, error) {
|
||||
const (
|
||||
baseInterval = 5 * time.Second
|
||||
maxInterval = 1 * time.Minute
|
||||
maxAttempts = 5 // 最多尝试10次
|
||||
baseInterval = 10 * time.Second
|
||||
maxInterval = 1 * time.Minute
|
||||
maxFailAttempts = 5 // 连续失败最多尝试5次
|
||||
maxPollCount = 0 // 0表示不限制轮询次数,可以设置一个最大值
|
||||
)
|
||||
|
||||
var (
|
||||
backoff = baseInterval
|
||||
attemptCount = 0
|
||||
backoff = baseInterval
|
||||
failAttemptCount = 0 // 连续失败次数
|
||||
pollCount = 0 // 总轮询次数
|
||||
)
|
||||
|
||||
for {
|
||||
if attemptCount > maxAttempts {
|
||||
return nil, fmt.Errorf("超出最大查询次数(%d)", maxAttempts)
|
||||
}
|
||||
|
||||
// 检查上下文是否被取消
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
|
|
@ -215,11 +214,15 @@ func (r *taskRunner) PollTaskStatusWithBackoff(ctx context.Context, authData *mo
|
|||
// 获取任务状态
|
||||
status, err := r.taskRunService.GetTaskStatus(ctx, authData, jobSetID)
|
||||
if err != nil {
|
||||
attemptCount++
|
||||
failAttemptCount++
|
||||
// 如果连续失败次数超过限制,返回错误
|
||||
if failAttemptCount > maxFailAttempts {
|
||||
return nil, fmt.Errorf("连续查询失败%d次,最后错误: %w", maxFailAttempts, err)
|
||||
}
|
||||
// 指数回退
|
||||
backoff = mymin(backoff*2, maxInterval)
|
||||
sleepTime := backoff + time.Duration(rand.Int63n(int64(backoff/2))) // 随机抖动避免同步
|
||||
fmt.Printf("任务查询失败(%s), 将在 %s 后重试\n", err, sleepTime)
|
||||
fmt.Printf("任务查询失败(%s), 连续失败%d次,将在 %s 后重试\n", err, failAttemptCount, sleepTime)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
|
|
@ -228,18 +231,37 @@ func (r *taskRunner) PollTaskStatusWithBackoff(ctx context.Context, authData *mo
|
|||
}
|
||||
}
|
||||
|
||||
// 重置回退间隔
|
||||
// 查询成功,重置失败计数和回退间隔
|
||||
failAttemptCount = 0
|
||||
backoff = baseInterval
|
||||
pollCount++
|
||||
|
||||
// 检查返回的状态数据是否有效
|
||||
if status == nil || len(status.Data.SubTaskInfos) == 0 {
|
||||
fmt.Println("任务状态数据为空,继续等待...")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(baseInterval):
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 检查任务状态
|
||||
switch {
|
||||
case IsCompleted(status.Data.SubTaskInfos[0].Status):
|
||||
fmt.Println("任务成功完成!, 完成状态:", status)
|
||||
taskStatus := status.Data.SubTaskInfos[0].Status
|
||||
if IsCompleted(taskStatus) {
|
||||
fmt.Printf("任务完成,状态: %s\n", taskStatus)
|
||||
return status, nil
|
||||
default:
|
||||
fmt.Println("任务运行中: ", status)
|
||||
}
|
||||
|
||||
// 检查任务是否失败
|
||||
if taskStatus == "Failed" {
|
||||
return status, fmt.Errorf("任务执行失败,状态: %s", taskStatus)
|
||||
}
|
||||
|
||||
// 任务运行中,继续轮询
|
||||
fmt.Printf("任务运行中,状态: %s,已轮询 %d 次\n", taskStatus, pollCount)
|
||||
|
||||
// 等待下一次轮询
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
|
@ -250,7 +272,7 @@ func (r *taskRunner) PollTaskStatusWithBackoff(ctx context.Context, authData *mo
|
|||
}
|
||||
|
||||
func IsCompleted(status string) bool {
|
||||
if status == "Completed" || status == "Failed" || status == "Succeed" {
|
||||
if status == "Completed" || status == "Failed" || status == "Succeed" || status == "Stopped" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
|
|
|||
|
|
@ -15,22 +15,31 @@ import (
|
|||
"remote-task-excutor-cli/pkg/config"
|
||||
)
|
||||
|
||||
const tokenRefreshBuffer = 60 * time.Second
|
||||
|
||||
type TokenService struct {
|
||||
mu sync.Mutex
|
||||
authData *models.AuthData
|
||||
authURL string
|
||||
username string
|
||||
password string
|
||||
httpClient *client.HTTPClient
|
||||
mu sync.Mutex
|
||||
authData *models.AuthData
|
||||
tokenObtainedAt time.Time
|
||||
configuredTokenTTL int
|
||||
authURL string
|
||||
username string
|
||||
password string
|
||||
httpClient *client.HTTPClient
|
||||
}
|
||||
|
||||
func NewTokenService(cfg *config.APIConfig) service.AuthService {
|
||||
tokenTTL := cfg.TokenTTL
|
||||
if tokenTTL <= 0 {
|
||||
tokenTTL = 3600
|
||||
}
|
||||
return &TokenService{
|
||||
authURL: cfg.AuthURL,
|
||||
username: cfg.Username,
|
||||
password: cfg.Password,
|
||||
httpClient: client.NewHTTPClient(cfg.BaseURL, time.Duration(cfg.Timeout)*time.Second),
|
||||
authData: &models.AuthData{},
|
||||
authURL: cfg.AuthURL,
|
||||
username: cfg.Username,
|
||||
password: cfg.Password,
|
||||
httpClient: client.NewHTTPClient(cfg.BaseURL, time.Duration(cfg.Timeout)*time.Second),
|
||||
authData: &models.AuthData{},
|
||||
configuredTokenTTL: tokenTTL,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -38,14 +47,41 @@ func (ts *TokenService) GetToken(ctx context.Context) (*models.AuthData, error)
|
|||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
|
||||
if ts.authData.Token != "" {
|
||||
return ts.authData, nil
|
||||
if ts.authData.Token != "" && !ts.isTokenExpiredLocked() {
|
||||
return copyAuthData(ts.authData), nil
|
||||
}
|
||||
|
||||
return ts.refreshToken(ctx)
|
||||
return ts.refreshTokenLocked(ctx)
|
||||
}
|
||||
|
||||
func (ts *TokenService) refreshToken(ctx context.Context) (*models.AuthData, error) {
|
||||
func (ts *TokenService) isTokenExpiredLocked() bool {
|
||||
if ts.authData.Token == "" || ts.tokenObtainedAt.IsZero() {
|
||||
return true
|
||||
}
|
||||
lifetime := tokenLifetime(ts.authData, ts.configuredTokenTTL)
|
||||
expiresAt := ts.tokenObtainedAt.Add(lifetime - tokenRefreshBuffer)
|
||||
return !time.Now().Before(expiresAt)
|
||||
}
|
||||
|
||||
func tokenLifetime(data *models.AuthData, fallbackSeconds int) time.Duration {
|
||||
if data.TokenTimeout > 0 {
|
||||
return time.Duration(data.TokenTimeout) * time.Second
|
||||
}
|
||||
if data.ExpiresIn > 0 {
|
||||
return time.Duration(data.ExpiresIn) * time.Second
|
||||
}
|
||||
return time.Duration(fallbackSeconds) * time.Second
|
||||
}
|
||||
|
||||
func copyAuthData(src *models.AuthData) *models.AuthData {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
cp := *src
|
||||
return &cp
|
||||
}
|
||||
|
||||
func (ts *TokenService) refreshTokenLocked(ctx context.Context) (*models.AuthData, error) {
|
||||
// 认证请求
|
||||
body := map[string]string{
|
||||
"username": ts.username,
|
||||
|
|
@ -69,9 +105,9 @@ func (ts *TokenService) refreshToken(ctx context.Context) (*models.AuthData, err
|
|||
return nil, errors.New("无效的token响应")
|
||||
}
|
||||
|
||||
// 更新 authData
|
||||
ts.authData = &authResponse.Data
|
||||
return &authResponse.Data, nil
|
||||
ts.tokenObtainedAt = time.Now()
|
||||
return copyAuthData(ts.authData), nil
|
||||
}
|
||||
|
||||
func (ts *TokenService) GetClusterID(ctx context.Context, label string) (string, error) {
|
||||
|
|
@ -81,7 +117,7 @@ func (ts *TokenService) GetClusterID(ctx context.Context, label string) (string,
|
|||
}
|
||||
|
||||
// 查询集群ID
|
||||
ts.httpClient.Headers["Authorization"] = "Bearer " + token.Token
|
||||
ts.httpClient.SetHeader("Authorization", "Bearer "+token.Token)
|
||||
params := map[string]string{
|
||||
"pageNum": "1",
|
||||
"pageSize": "30",
|
||||
|
|
@ -112,5 +148,6 @@ func (ts *TokenService) GetClusterID(ctx context.Context, label string) (string,
|
|||
func (ts *TokenService) InvalidateToken() {
|
||||
ts.mu.Lock()
|
||||
defer ts.mu.Unlock()
|
||||
ts.authData.Token = ""
|
||||
ts.authData = &models.AuthData{}
|
||||
ts.tokenObtainedAt = time.Time{}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"remote-task-excutor-cli/pkg/models"
|
||||
)
|
||||
|
||||
func TestTokenLifetimePrefersTokenTimeout(t *testing.T) {
|
||||
data := &models.AuthData{TokenTimeout: 7200, ExpiresIn: 3600}
|
||||
got := tokenLifetime(data, 1800)
|
||||
if got != 2*time.Hour {
|
||||
t.Fatalf("expected 2h, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenLifetimeUsesExpiresInWhenTimeoutMissing(t *testing.T) {
|
||||
data := &models.AuthData{ExpiresIn: 900}
|
||||
got := tokenLifetime(data, 1800)
|
||||
if got != 15*time.Minute {
|
||||
t.Fatalf("expected 15m, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenLifetimeUsesFallback(t *testing.T) {
|
||||
data := &models.AuthData{}
|
||||
got := tokenLifetime(data, 1800)
|
||||
if got != 30*time.Minute {
|
||||
t.Fatalf("expected 30m, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTokenExpiredLocked(t *testing.T) {
|
||||
ts := &TokenService{
|
||||
authData: &models.AuthData{Token: "abc", TokenTimeout: 3600},
|
||||
configuredTokenTTL: 3600,
|
||||
tokenObtainedAt: time.Now().Add(-2 * time.Hour),
|
||||
}
|
||||
|
||||
ts.mu.Lock()
|
||||
expired := ts.isTokenExpiredLocked()
|
||||
ts.mu.Unlock()
|
||||
|
||||
if !expired {
|
||||
t.Fatal("expected token to be expired")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTokenExpiredLockedWithinBuffer(t *testing.T) {
|
||||
ts := &TokenService{
|
||||
authData: &models.AuthData{Token: "abc", TokenTimeout: 120},
|
||||
configuredTokenTTL: 120,
|
||||
tokenObtainedAt: time.Now().Add(-70 * time.Second),
|
||||
}
|
||||
|
||||
ts.mu.Lock()
|
||||
expired := ts.isTokenExpiredLocked()
|
||||
ts.mu.Unlock()
|
||||
|
||||
if !expired {
|
||||
t.Fatal("expected token to be treated as expired within refresh buffer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTokenExpiredLockedStillValid(t *testing.T) {
|
||||
ts := &TokenService{
|
||||
authData: &models.AuthData{Token: "abc", TokenTimeout: 3600},
|
||||
configuredTokenTTL: 3600,
|
||||
tokenObtainedAt: time.Now(),
|
||||
}
|
||||
|
||||
ts.mu.Lock()
|
||||
expired := ts.isTokenExpiredLocked()
|
||||
ts.mu.Unlock()
|
||||
|
||||
if expired {
|
||||
t.Fatal("expected token to still be valid")
|
||||
}
|
||||
}
|
||||
|
|
@ -2,24 +2,47 @@ package common
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"remote-task-excutor-cli/pkg/models"
|
||||
"remote-task-excutor-cli/pkg/service/preparation"
|
||||
)
|
||||
|
||||
// TaskType 任务类型常量
|
||||
const (
|
||||
TrainingTaskType = "AI" // 训练任务类型
|
||||
InferenceTaskType = "Inference" // 推理任务类型
|
||||
TrainingTaskType = "AI" // 训练任务类型
|
||||
InferenceTaskType = "PCM_Inference" // 推理任务类型
|
||||
)
|
||||
|
||||
func formatResource(resource *models.ResourceConfig) models.ResourceConfig {
|
||||
accerlerateItem := models.ResourcesItem{
|
||||
Type: resource.Type,
|
||||
Name: resource.Name,
|
||||
Number: resource.AvailableCount,
|
||||
}
|
||||
|
||||
for i, _ := range resource.Resources {
|
||||
resource.Resources[i].Number = resource.Resources[i].AvailableValue
|
||||
resource.Resources[i].AvailableValue = 0
|
||||
}
|
||||
resource.Resources = append(resource.Resources, accerlerateItem)
|
||||
return *resource
|
||||
}
|
||||
|
||||
// BuildSubmitTaskRequest 构建提交任务请求的通用函数
|
||||
func BuildSubmitTaskRequest(ctx context.Context, authData *models.AuthData, config *models.RunConfig,
|
||||
clusterID string, bindResultSet *models.BindResultSet, taskType string) models.SubtaskRequest {
|
||||
|
||||
// 根据任务类型决定是否包含数据返回任务
|
||||
var jobs []models.Job
|
||||
|
||||
//var resource models.ResourceConfig
|
||||
//fmt.Println("before")
|
||||
//if taskType == InferenceTaskType {
|
||||
// resource = formatResource(&config.Resource)
|
||||
//} else {
|
||||
// resource = config.Resource
|
||||
//}
|
||||
// 主任务
|
||||
fmt.Println("build train req resource is %+v", config.Resource)
|
||||
mainJob := models.Job{
|
||||
LocalJobID: models.MainTaskID,
|
||||
Name: "remote-task" + preparation.GenerateUniqueID(),
|
||||
|
|
@ -38,6 +61,10 @@ func BuildSubmitTaskRequest(ctx context.Context, authData *models.AuthData, conf
|
|||
Type: "Image",
|
||||
ImageID: config.Image,
|
||||
},
|
||||
SubModel: models.FileBinding{
|
||||
Type: preparation.TaskBindingType,
|
||||
BindingID: bindResultSet.BindSubModelID,
|
||||
},
|
||||
},
|
||||
JobResources: &models.JobResources{
|
||||
ScheduleStrategy: "dataLocality",
|
||||
|
|
@ -58,6 +85,10 @@ func BuildSubmitTaskRequest(ctx context.Context, authData *models.AuthData, conf
|
|||
},
|
||||
}
|
||||
|
||||
if taskType == InferenceTaskType {
|
||||
mainJob.OnlyCreate = true
|
||||
}
|
||||
|
||||
jobs = append(jobs, mainJob)
|
||||
|
||||
// 只有训练任务才需要数据返回任务
|
||||
|
|
@ -86,3 +117,57 @@ func BuildSubmitTaskRequest(ctx context.Context, authData *models.AuthData, conf
|
|||
},
|
||||
}
|
||||
}
|
||||
|
||||
//// BuildInferenceSubmitTaskRequest 构建推理任务提交请求
|
||||
//func BuildInferenceSubmitTaskRequest(ctx context.Context, authData *models.AuthData,
|
||||
// codeBindingID, modelBindingID, subModelBindingID int, imageID int, clusterID string,
|
||||
// resourceType string, resources []models.ResourcesItem, command string, description string) models.InferenceSubmitTaskResponseData {
|
||||
//
|
||||
// // 构建文件信息
|
||||
// files := models.InferenceJobFiles{
|
||||
// Model: models.FileBinding{
|
||||
// Type: "Binding",
|
||||
// BindingID: modelBindingID,
|
||||
// },
|
||||
// Image: models.ImageBinding{
|
||||
// Type: "Image",
|
||||
// ImageID: imageID,
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// // 如果有增量模型,添加到文件信息中
|
||||
// if subModelBindingID > 0 {
|
||||
// files.SubModel = &models.FileBinding{
|
||||
// Type: "Binding",
|
||||
// BindingID: subModelBindingID,
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // 构建任务
|
||||
// job := models.InferenceJob{
|
||||
// LocalJobID: models.MainTaskID,
|
||||
// Name: "inference-task" + preparation.GenerateUniqueID(),
|
||||
// Description: description,
|
||||
// Type: "PCM_Inference",
|
||||
// Files: &files,
|
||||
// JobResources: &models.InferenceJobResources{
|
||||
// ScheduleStrategy: "dataLocality",
|
||||
// Clusters: []models.InferenceCluster{
|
||||
// {
|
||||
// ClusterID: clusterID,
|
||||
// Runtime: models.Runtime{
|
||||
// Envs: make(map[string]string),
|
||||
// Params: make(map[string]string),
|
||||
// },
|
||||
// Code: models.CodeInfo{
|
||||
// Type: "Binding",
|
||||
// BindingID: codeBindingID,
|
||||
// },
|
||||
// Resources: resources,
|
||||
// },
|
||||
// },
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// return models.InferenceSubmitTaskResponseData{}
|
||||
//}
|
||||
|
|
|
|||
|
|
@ -22,24 +22,24 @@ func NewTaskSubmitter(httpClient *client.HTTPClient, taskType string) *TaskSubmi
|
|||
}
|
||||
}
|
||||
|
||||
// SubmitTask 提交任务的通用方法
|
||||
// SubmitTask 提交任务的通用方法,返回 jobSetID、提交请求和错误
|
||||
func (ts *TaskSubmitter) SubmitTask(ctx context.Context, authData *models.AuthData, config *models.RunConfig,
|
||||
clusterID string, bindResultSet *models.BindResultSet) (string, error) {
|
||||
clusterID string, bindResultSet *models.BindResultSet) (string, *models.SubtaskRequest, error) {
|
||||
|
||||
// 构建任务请求
|
||||
submitTaskReq := BuildSubmitTaskRequest(ctx, authData, config, clusterID, bindResultSet, ts.taskType)
|
||||
|
||||
// 设置认证头
|
||||
ts.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
ts.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
|
||||
// 打印请求参数
|
||||
fmt.Printf("%s任务提交参数:%+v\n", getTaskTypeName(ts.taskType), submitTaskReq)
|
||||
|
||||
// 提交任务
|
||||
resp, err := ts.httpClient.PostJSON("/jsm/jobSet/submit", submitTaskReq)
|
||||
resp, err := ts.httpClient.PostJSON("/jsm/v2/jobs/submit", submitTaskReq)
|
||||
if err != nil {
|
||||
fmt.Printf("Submit %s task failed: %v\n", getTaskTypeName(ts.taskType), err)
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
fmt.Printf("提交%s任务结果:%s\n", getTaskTypeName(ts.taskType), string(resp))
|
||||
|
|
@ -48,16 +48,16 @@ func (ts *TaskSubmitter) SubmitTask(ctx context.Context, authData *models.AuthDa
|
|||
var submitTaskResp models.SubmitTaskResponse
|
||||
if err := json.Unmarshal(resp, &submitTaskResp); err != nil {
|
||||
fmt.Printf("Submit %s task response unmarshal failed: %v\n", getTaskTypeName(ts.taskType), err)
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
if submitTaskResp.Code != models.ResponseOK {
|
||||
fmt.Printf("Submit %s task failed: %s\n", getTaskTypeName(ts.taskType), submitTaskResp.Code)
|
||||
return "", fmt.Errorf("submit %s task failed: %s", getTaskTypeName(ts.taskType), submitTaskResp.Code)
|
||||
return "", nil, fmt.Errorf("submit %s task failed: %s", getTaskTypeName(ts.taskType), submitTaskResp.Code)
|
||||
}
|
||||
|
||||
fmt.Printf("Submit %s task result: %s\n", getTaskTypeName(ts.taskType), string(resp))
|
||||
return submitTaskResp.Data.JobSetID, nil
|
||||
return submitTaskResp.Data.JobSetID, &submitTaskReq, nil
|
||||
}
|
||||
|
||||
// getTaskTypeName 获取任务类型的中文名称
|
||||
|
|
|
|||
|
|
@ -2,11 +2,20 @@ package inference
|
|||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"remote-task-excutor-cli/pkg/client"
|
||||
"remote-task-excutor-cli/pkg/config"
|
||||
"remote-task-excutor-cli/pkg/models"
|
||||
"remote-task-excutor-cli/pkg/service"
|
||||
"remote-task-excutor-cli/pkg/service/common"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -16,6 +25,9 @@ type inferenceService struct {
|
|||
preparationService service.PreparationService
|
||||
taskService service.TaskService
|
||||
taskSubmitter *common.TaskSubmitter
|
||||
httpClient *client.HTTPClient
|
||||
db *sql.DB
|
||||
apiCfg *config.APIConfig
|
||||
}
|
||||
|
||||
// NewInferenceService 创建新的推理服务实例
|
||||
|
|
@ -24,6 +36,8 @@ func NewInferenceService(
|
|||
preparationService service.PreparationService,
|
||||
taskService service.TaskService,
|
||||
baseURL string,
|
||||
db *sql.DB,
|
||||
apiCfg *config.APIConfig,
|
||||
) service.InferenceService {
|
||||
httpClient := client.NewHTTPClient(baseURL, 10*time.Second)
|
||||
return &inferenceService{
|
||||
|
|
@ -31,24 +45,445 @@ func NewInferenceService(
|
|||
preparationService: preparationService,
|
||||
taskService: taskService,
|
||||
taskSubmitter: common.NewTaskSubmitter(httpClient, common.InferenceTaskType),
|
||||
httpClient: httpClient,
|
||||
db: db,
|
||||
apiCfg: apiCfg,
|
||||
}
|
||||
}
|
||||
|
||||
// SubmitTask 提交推理任务
|
||||
func (s *inferenceService) SubmitTask(ctx context.Context, authData *models.AuthData, config *models.RunConfig, clusterID string,
|
||||
bindResultSet *models.BindResultSet) (string, error) {
|
||||
return s.taskSubmitter.SubmitTask(ctx, authData, config, clusterID, bindResultSet)
|
||||
jobSetID, _, err := s.taskSubmitter.SubmitTask(ctx, authData, config, clusterID, bindResultSet)
|
||||
return jobSetID, err
|
||||
}
|
||||
|
||||
// GetTaskStatus 查询任务状态
|
||||
func (s *inferenceService) GetTaskStatus(ctx context.Context, authData *models.AuthData, jobSetID string) (*models.InferenceTaskDetailResponse, error) {
|
||||
// 这里需要根据实际的TaskDetailResponse和InferenceTaskDetailResponse的差异来实现
|
||||
// 暂时返回nil,需要根据实际需求实现
|
||||
return nil, fmt.Errorf("GetTaskStatus not implemented")
|
||||
// 如果接口超时,返回"启动中"状态,因为对端接口还在准备中
|
||||
func (s *inferenceService) GetTaskStatus(ctx context.Context, authData *models.AuthData, localJobID, jobSetID string) (*models.InferenceTaskStatusResponse, error) {
|
||||
// 设置认证头
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
|
||||
// 构建查询参数
|
||||
params := map[string]string{
|
||||
"localJobID": localJobID,
|
||||
"jobSetID": jobSetID,
|
||||
}
|
||||
|
||||
// 调用第三方接口
|
||||
resp, err := s.httpClient.Get("/jsm/v2/jobs/details", params)
|
||||
if err != nil {
|
||||
// 判断是否是超时错误
|
||||
if isTimeoutError(err) {
|
||||
// 超时了,返回"启动中"状态,因为对端接口还在准备中
|
||||
fmt.Printf("查询任务状态超时,返回启动中状态: %v\n", err)
|
||||
return &models.InferenceTaskStatusResponse{
|
||||
Code: http.StatusOK,
|
||||
Msg: "",
|
||||
Data: models.InferenceTaskStatusData{
|
||||
Status: "Init",
|
||||
URL: "",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
// 非超时错误,返回错误
|
||||
return nil, fmt.Errorf("查询任务状态失败: %w", err)
|
||||
}
|
||||
|
||||
// 解析第三方接口响应
|
||||
var detailResp models.InferenceTaskDetailResponse
|
||||
if err := json.Unmarshal(resp, &detailResp); err != nil {
|
||||
return nil, fmt.Errorf("解析任务状态响应失败: %w", err)
|
||||
}
|
||||
|
||||
// 检查响应状态
|
||||
if detailResp.Code != "OK" {
|
||||
return nil, fmt.Errorf("查询任务状态失败: %s - %s", detailResp.Code, detailResp.Message)
|
||||
}
|
||||
|
||||
// 检查数据是否有效
|
||||
if detailResp.Data.Instance.Status == "" {
|
||||
return nil, fmt.Errorf("任务状态数据无效")
|
||||
}
|
||||
|
||||
// 构建返回响应
|
||||
return &models.InferenceTaskStatusResponse{
|
||||
Code: http.StatusOK,
|
||||
Msg: "",
|
||||
Data: models.InferenceTaskStatusData{
|
||||
Status: detailResp.Data.Instance.Status,
|
||||
URL: detailResp.Data.Instance.InferUrl,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// StopTask 停止任务
|
||||
func (s *inferenceService) StopTask(ctx context.Context, authData *models.AuthData, jobSetID string) error {
|
||||
// 这里需要实现停止任务的逻辑
|
||||
return fmt.Errorf("StopTask not implemented")
|
||||
// isTimeoutError 判断错误是否是超时错误
|
||||
func isTimeoutError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
errStr := err.Error()
|
||||
// 检查常见的超时错误关键词
|
||||
return strings.Contains(errStr, "timeout") ||
|
||||
strings.Contains(errStr, "deadline exceeded") ||
|
||||
strings.Contains(errStr, "context deadline exceeded") ||
|
||||
strings.Contains(errStr, "i/o timeout")
|
||||
}
|
||||
|
||||
// StopTask 停止推理任务
|
||||
func (s *inferenceService) StopTask(ctx context.Context, authData *models.AuthData, localJobID, jobSetID string) (*models.StopInferenceTaskResponse, error) {
|
||||
// 设置认证头
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
|
||||
// 构建停止任务请求
|
||||
stopReq := models.StopInferenceSubmitRequest{
|
||||
UserID: authData.JsmUserInfo.Data.UserID,
|
||||
JobSetInfo: models.JobSetInfo{
|
||||
Jobs: []models.Job{
|
||||
{
|
||||
Type: "StopInference",
|
||||
JobSetID: jobSetID,
|
||||
LocalJobID: localJobID,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// 调用第三方接口
|
||||
resp, err := s.httpClient.PostJSON("/jsm/v2/jobs/submit", stopReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("调用停止任务接口失败: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("停止接口返回响应:", string(resp))
|
||||
// 解析响应
|
||||
var stopResp models.StopInferenceSubmitResponse
|
||||
if err := json.Unmarshal(resp, &stopResp); err != nil {
|
||||
return nil, fmt.Errorf("解析停止任务响应失败: %w", err)
|
||||
}
|
||||
|
||||
// 检查响应状态
|
||||
if stopResp.Code != models.ResponseOK {
|
||||
return nil, fmt.Errorf("停止任务失败: %s - %s", stopResp.Code, stopResp.Message)
|
||||
}
|
||||
|
||||
fmt.Printf("停止推理任务成功,JobSetID: %s, 返回JobSetID: %s\n", jobSetID, stopResp.Data.JobSetID)
|
||||
|
||||
// 返回成功响应
|
||||
return &models.StopInferenceTaskResponse{
|
||||
Code: http.StatusOK,
|
||||
Message: "",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ReSubmitTask 重启推理任务:将入参原样转发给第三方 /jsm/v2/jobs/submit,返回 jobSetID 与入参中的 localJobID
|
||||
func (s *inferenceService) ReSubmitTask(ctx context.Context, authData *models.AuthData, request *models.SubtaskRequest) (*models.ReSubmitTaskResponse, error) {
|
||||
if request == nil || len(request.JobSetInfo.Jobs) == 0 {
|
||||
return nil, fmt.Errorf("jobSetInfo.jobs 不能为空")
|
||||
}
|
||||
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
|
||||
resp, err := s.httpClient.PostJSON("/jsm/v2/jobs/submit", request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("调用重启任务接口失败: %w", err)
|
||||
}
|
||||
|
||||
var submitResp models.SubmitTaskResponse
|
||||
if err := json.Unmarshal(resp, &submitResp); err != nil {
|
||||
return nil, fmt.Errorf("解析重启任务响应失败: %w", err)
|
||||
}
|
||||
|
||||
if submitResp.Code != models.ResponseOK {
|
||||
return nil, fmt.Errorf("重启任务失败: %s - %s", submitResp.Code, submitResp.Message)
|
||||
}
|
||||
|
||||
localJobID := request.JobSetInfo.Jobs[0].LocalJobID
|
||||
fmt.Printf("重启推理任务成功,JobSetID: %s, LocalJobID: %s\n", submitResp.Data.JobSetID, localJobID)
|
||||
|
||||
return &models.ReSubmitTaskResponse{
|
||||
Code: http.StatusOK,
|
||||
Message: "",
|
||||
Data: models.ReSubmitTaskData{
|
||||
JobSetID: submitResp.Data.JobSetID,
|
||||
LocalJobID: localJobID,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *inferenceService) getLocalPath(path string) string {
|
||||
if s.apiCfg == nil {
|
||||
// 如果配置未设置,使用默认值
|
||||
return filepath.Join("D:\\code", "remote-task-excutor-cli", path)
|
||||
}
|
||||
return filepath.Join(s.apiCfg.HostPath, s.apiCfg.SubPath, path)
|
||||
}
|
||||
|
||||
var asyncTaskSeq uint64
|
||||
|
||||
func newAsyncTaskID() string {
|
||||
seq := atomic.AddUint64(&asyncTaskSeq, 1)
|
||||
return fmt.Sprintf("inf_%d_%s", time.Now().UnixMilli(), strconv.FormatUint(seq, 10))
|
||||
}
|
||||
|
||||
func (s *inferenceService) dbMustEnabled() error {
|
||||
if s.db == nil {
|
||||
return fmt.Errorf("mysql 未初始化:请配置 mysqlDsn")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *inferenceService) insertAsyncTask(ctx context.Context, taskID string, req *models.InferenceSubmitTaskRequest, clusterID string) error {
|
||||
reqBytes, _ := json.Marshal(req)
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO inference_async_task
|
||||
(task_id, status, progress, request_json, cluster_id, created_at, updated_at)
|
||||
VALUES
|
||||
(?, 'PENDING', '已接收', ?, ?, NOW(), NOW())
|
||||
`, taskID, string(reqBytes), clusterID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *inferenceService) updateAsyncTask(ctx context.Context, taskID string, status string, progress string, jobSetID string, errMsg string, resultJSON *string) error {
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
UPDATE inference_async_task
|
||||
SET status = ?,
|
||||
progress = ?,
|
||||
job_set_id = ?,
|
||||
error = ?,
|
||||
result_json = COALESCE(?, result_json),
|
||||
updated_at = NOW()
|
||||
WHERE task_id = ?
|
||||
`, status, progress, jobSetID, errMsg, resultJSON, taskID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *inferenceService) GetInferenceTaskAsync(ctx context.Context, taskID string) (*models.InferenceAsyncTaskStatusResponse, error) {
|
||||
if err := s.dbMustEnabled(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var (
|
||||
status string
|
||||
progress sql.NullString
|
||||
jobSetID sql.NullString
|
||||
errMsg sql.NullString
|
||||
resultJSON sql.NullString
|
||||
)
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT status, progress, job_set_id, error, result_json
|
||||
FROM inference_async_task
|
||||
WHERE task_id = ?
|
||||
`, taskID).Scan(&status, &progress, &jobSetID, &errMsg, &resultJSON)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return &models.InferenceAsyncTaskStatusResponse{
|
||||
Code: 404,
|
||||
Msg: "task_id not found",
|
||||
Data: models.InferenceAsyncTaskStatusData{TaskID: taskID},
|
||||
}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result *models.InferenceSubmitTaskResponse
|
||||
if resultJSON.Valid && resultJSON.String != "" {
|
||||
var tmp models.InferenceSubmitTaskResponse
|
||||
if e := json.Unmarshal([]byte(resultJSON.String), &tmp); e == nil {
|
||||
result = &tmp
|
||||
}
|
||||
}
|
||||
|
||||
return &models.InferenceAsyncTaskStatusResponse{
|
||||
Code: http.StatusOK,
|
||||
Msg: "",
|
||||
Data: models.InferenceAsyncTaskStatusData{
|
||||
TaskID: taskID,
|
||||
Status: status,
|
||||
Progress: progress.String,
|
||||
JobSetID: jobSetID.String,
|
||||
Error: errMsg.String,
|
||||
Result: result,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *inferenceService) SubmitInferenceTaskAsync(ctx context.Context, authData *models.AuthData,
|
||||
request *models.InferenceSubmitTaskRequest, clusterID string) (string, error) {
|
||||
if err := s.dbMustEnabled(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
taskID := newAsyncTaskID()
|
||||
if err := s.insertAsyncTask(ctx, taskID, request, clusterID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
bgCtx := context.Background()
|
||||
go func() {
|
||||
// 添加 recover 防止 panic 导致程序崩溃
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
errMsg := fmt.Sprintf("异步推理任务[%s] 发生 panic: %v", taskID, r)
|
||||
fmt.Printf("%s\n", errMsg)
|
||||
// 尝试更新数据库状态为 FAILED
|
||||
if uerr := s.updateAsyncTask(bgCtx, taskID, "Failed", "执行异常", "", errMsg, nil); uerr != nil {
|
||||
fmt.Printf("异步推理任务[%s] panic 后更新状态为 FAILED 失败: %v\n", taskID, uerr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if err := s.updateAsyncTask(bgCtx, taskID, "Init", "开始处理", "", "", nil); err != nil {
|
||||
fmt.Printf("异步推理任务[%s] 更新状态为 Init 失败: %v\n", taskID, err)
|
||||
return
|
||||
}
|
||||
freshAuthData, err := s.authService.GetToken(bgCtx)
|
||||
if err != nil {
|
||||
errMsg := fmt.Sprintf("获取认证信息失败: %v", err)
|
||||
if uerr := s.updateAsyncTask(bgCtx, taskID, "Failed", "执行失败", "", errMsg, nil); uerr != nil {
|
||||
fmt.Printf("异步推理任务[%s] 更新状态为 FAILED 失败: %v\n", taskID, uerr)
|
||||
}
|
||||
return
|
||||
}
|
||||
resp, err := s.SubmitInferenceTask(bgCtx, freshAuthData, request, clusterID)
|
||||
if err != nil {
|
||||
if uerr := s.updateAsyncTask(bgCtx, taskID, "Failed", "执行失败", "", err.Error(), nil); uerr != nil {
|
||||
fmt.Printf("异步推理任务[%s] 更新状态为 FAILED 失败: %v, 原始错误: %v\n", taskID, uerr, err)
|
||||
} else {
|
||||
fmt.Printf("异步推理任务[%s] 执行失败: %v\n", taskID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
resultBytes, _ := json.Marshal(resp)
|
||||
resultStr := string(resultBytes)
|
||||
if err := s.updateAsyncTask(bgCtx, taskID, "Completed", "完成", resp.Data.ResultInfo.JobSetID, "", &resultStr); err != nil {
|
||||
fmt.Printf("异步推理任务[%s] 更新状态为 COMPLETED 失败: %v\n", taskID, err)
|
||||
} else {
|
||||
fmt.Printf("异步推理任务[%s] 已完成,JobSetID=%s\n", taskID, resp.Data.ResultInfo.JobSetID)
|
||||
}
|
||||
}()
|
||||
|
||||
return taskID, nil
|
||||
}
|
||||
|
||||
func (s *inferenceService) convertResourceFormat(resourceConfig models.RemoteSourceConfig) models.ResourceConfig {
|
||||
r := models.ResourceConfig{
|
||||
ClusterID: resourceConfig.ClusterID,
|
||||
}
|
||||
|
||||
for _, resource := range resourceConfig.BaseResourceSpecs {
|
||||
r.Resources = append(r.Resources, models.ResourcesItem{
|
||||
Type: resource.Type,
|
||||
Name: resource.Name,
|
||||
Number: resource.AvailableValue,
|
||||
})
|
||||
}
|
||||
|
||||
acceleratorItem := models.ResourcesItem{
|
||||
Type: resourceConfig.Type,
|
||||
Name: resourceConfig.Name,
|
||||
Number: resourceConfig.AvailableCount,
|
||||
}
|
||||
|
||||
r.Resources = append(r.Resources, acceleratorItem)
|
||||
return r
|
||||
}
|
||||
|
||||
// SubmitInferenceTask 提交推理任务(包含准备步骤)
|
||||
func (s *inferenceService) SubmitInferenceTask(ctx context.Context, authData *models.AuthData,
|
||||
request *models.InferenceSubmitTaskRequest, clusterID string) (*models.InferenceSubmitTaskResponse, error) {
|
||||
|
||||
// 参数校验:检查 model.Path 是否为空
|
||||
if request.Model.Path == "" {
|
||||
return nil, fmt.Errorf("model.Path 不能为空")
|
||||
}
|
||||
|
||||
// 参数校验:如果提供了 subModel,检查 subModel.Path 是否为空
|
||||
if request.SubModel != nil && request.SubModel.Path == "" {
|
||||
return nil, fmt.Errorf("subModel.Path 不能为空")
|
||||
}
|
||||
|
||||
if request.CodeConfig.MountPath == "" {
|
||||
tempDir, err := os.MkdirTemp("", "code-*")
|
||||
if err != nil {
|
||||
fmt.Println("mkdir temp dir err:", err)
|
||||
return nil, err
|
||||
}
|
||||
fmt.Println("mkdir temp dir:", tempDir)
|
||||
request.CodeConfig.MountPath = tempDir
|
||||
}
|
||||
// 1. 准备代码
|
||||
fmt.Println("开始准备代码")
|
||||
codeConfig := models.CodeConfig{
|
||||
GitUrl: request.CodeConfig.GitUrl,
|
||||
GitBranch: request.CodeConfig.GitBranch,
|
||||
MountPath: request.CodeConfig.MountPath,
|
||||
}
|
||||
runConfig := &models.RunConfig{
|
||||
CodeConfig: codeConfig,
|
||||
Image: request.Image.ImageID,
|
||||
Command: request.Command,
|
||||
Resource: s.convertResourceFormat(request.Resource),
|
||||
}
|
||||
codeID, err := s.preparationService.PrepareCode(ctx, authData, runConfig, clusterID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("准备代码失败: %w", err)
|
||||
}
|
||||
fmt.Println("代码准备成功,ID:", codeID)
|
||||
|
||||
// 2. 准备模型
|
||||
fmt.Println("开始准备模型")
|
||||
// 如果 MountPath 为空,使用 Path 作为 MountPath
|
||||
modelConfig := request.Model
|
||||
if modelConfig.MountPath == "" {
|
||||
modelConfig.MountPath = s.getLocalPath(modelConfig.Path)
|
||||
}
|
||||
modelID, err := s.preparationService.PrepareModel(ctx, authData, modelConfig, clusterID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("准备模型失败: %w", err)
|
||||
}
|
||||
fmt.Println("模型准备成功,ID:", modelID)
|
||||
|
||||
// 3. 准备增量模型(如果有)
|
||||
var subModelID int
|
||||
if request.SubModel != nil {
|
||||
fmt.Println("开始准备增量模型")
|
||||
// 如果 MountPath 为空,使用 Path 作为 MountPath
|
||||
subModelConfig := *request.SubModel
|
||||
if subModelConfig.MountPath == "" {
|
||||
subModelConfig.MountPath = s.getLocalPath(subModelConfig.Path)
|
||||
}
|
||||
subModelID, err = s.preparationService.PrepareModel(ctx, authData, subModelConfig, clusterID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("准备增量模型失败: %w", err)
|
||||
}
|
||||
fmt.Println("增量模型准备成功,ID:", subModelID)
|
||||
}
|
||||
|
||||
// 4. 使用 taskSubmitter 提交任务
|
||||
// 注意:taskSubmitter 会使用 BuildSubmitTaskRequest,它构建的是训练任务格式
|
||||
// 对于推理任务,我们需要传入 BindDatasetID=0,这样 Files 中的 Dataset 会是空的
|
||||
jobSetID, submitTaskReq, err := s.taskSubmitter.SubmitTask(ctx, authData, runConfig, clusterID, &models.BindResultSet{
|
||||
BindCodeID: codeID,
|
||||
BindModelID: modelID,
|
||||
BindDatasetID: 0, // 推理任务不需要数据集
|
||||
BindSubModelID: subModelID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("提交任务失败: %w", err)
|
||||
}
|
||||
|
||||
// 5. 构建返回响应数据,使用提交请求中的信息
|
||||
taskData := models.InferenceSubmitTaskResponseData{
|
||||
TaskInfo: *submitTaskReq,
|
||||
ResultInfo: models.InferenceResultInfo{
|
||||
LocalJobID: models.MainTaskID,
|
||||
JobSetID: jobSetID,
|
||||
},
|
||||
}
|
||||
|
||||
return &models.InferenceSubmitTaskResponse{
|
||||
Code: http.StatusOK,
|
||||
Msg: "",
|
||||
Data: taskData,
|
||||
}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,10 +30,21 @@ type TaskService interface {
|
|||
type InferenceService interface {
|
||||
SubmitTask(ctx context.Context, authData *models.AuthData, config *models.RunConfig,
|
||||
clusterID string, bindResultSet *models.BindResultSet) (string, error)
|
||||
GetTaskStatus(ctx context.Context, authData *models.AuthData, jobSetID string) (*models.InferenceTaskDetailResponse, error)
|
||||
StopTask(ctx context.Context, authData *models.AuthData, jobSetID string) error
|
||||
SubmitInferenceTask(ctx context.Context, authData *models.AuthData,
|
||||
request *models.InferenceSubmitTaskRequest, clusterID string) (*models.InferenceSubmitTaskResponse, error)
|
||||
SubmitInferenceTaskAsync(ctx context.Context, authData *models.AuthData,
|
||||
request *models.InferenceSubmitTaskRequest, clusterID string) (string, error)
|
||||
GetInferenceTaskAsync(ctx context.Context, taskID string) (*models.InferenceAsyncTaskStatusResponse, error)
|
||||
GetTaskStatus(ctx context.Context, authData *models.AuthData, localJobID, jobSetID string) (*models.InferenceTaskStatusResponse, error)
|
||||
StopTask(ctx context.Context, authData *models.AuthData, localJobID, jobSetID string) (*models.StopInferenceTaskResponse, error)
|
||||
ReSubmitTask(ctx context.Context, authData *models.AuthData, request *models.SubtaskRequest) (*models.ReSubmitTaskResponse, error)
|
||||
}
|
||||
|
||||
type LogService interface {
|
||||
RecordLogToFile(ctx context.Context, authData *models.AuthData, jobSetID string, done chan bool) error
|
||||
}
|
||||
|
||||
type UpDownService interface {
|
||||
Upload(ctx context.Context, authData *models.AuthData, filesConfig *models.UploadFileConfig, info models.PackageCreateLoadInfo, uploadType string) (*models.PackageCreateResponse, error)
|
||||
DownLoad(ctx context.Context, authData *models.AuthData, path string, packageID int) (string, error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,21 +3,22 @@ package preparation
|
|||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"remote-task-excutor-cli/pkg/client"
|
||||
"remote-task-excutor-cli/pkg/models"
|
||||
"remote-task-excutor-cli/pkg/service"
|
||||
"remote-task-excutor-cli/pkg/service/updown"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -33,16 +34,18 @@ const (
|
|||
RootPath = "/"
|
||||
ImageClassifyType = "image_classification"
|
||||
PytorchType = "pytorch"
|
||||
CreateResourceUri = "/apis/jsm//app/submit"
|
||||
CreateResourceUri = "/apis/jsm/v2/app/submit"
|
||||
)
|
||||
|
||||
type preparationService struct {
|
||||
httpClient *client.HTTPClient
|
||||
httpClient *client.HTTPClient
|
||||
updownService service.UpDownService
|
||||
}
|
||||
|
||||
func NewPreparationService(baseURL string, timeout time.Duration) service.PreparationService {
|
||||
return &preparationService{
|
||||
httpClient: client.NewHTTPClient(baseURL, timeout),
|
||||
httpClient: client.NewHTTPClient(baseURL, timeout),
|
||||
updownService: updown.NewUpDownService(baseURL, timeout),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -144,11 +147,13 @@ func (s *preparationService) PrepareCode(ctx context.Context, authData *models.A
|
|||
fmt.Println("开始准备代码")
|
||||
resul, err := s.uploadCode(ctx, authData, config.CodeConfig)
|
||||
if err != nil {
|
||||
fmt.Println("代码上传失败:", err.Error())
|
||||
return 0, err
|
||||
}
|
||||
fmt.Println("代码上传成功,开始代码定版")
|
||||
id, err := s.bindCode(ctx, authData, config, resul, clusterID)
|
||||
if err != nil {
|
||||
fmt.Println("代码定版失败:", err.Error())
|
||||
return 0, err
|
||||
}
|
||||
fmt.Println("代码定版成功")
|
||||
|
|
@ -224,7 +229,7 @@ func (s *preparationService) bindCode(ctx context.Context, authData *models.Auth
|
|||
},
|
||||
},
|
||||
}
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
resp, err := s.httpClient.PostJSON(CreateResourceUri, request)
|
||||
fmt.Println("代码绑定结果:", string(resp))
|
||||
if err != nil {
|
||||
|
|
@ -255,9 +260,11 @@ func (s *preparationService) bindModel(ctx context.Context, authData *models.Aut
|
|||
Name: appInstanceName,
|
||||
Description: "模型绑定",
|
||||
Info: models.ModelAppInfoDetail{
|
||||
Type: BindingType,
|
||||
LocalPath: RootPath,
|
||||
ObjectIDs: getObjectIDS(uploadResult),
|
||||
Type: BindingType,
|
||||
LocalPath: RootPath,
|
||||
ObjectIDs: getObjectIDS(uploadResult),
|
||||
CopiedTo: []int{23},
|
||||
CopiedToFullRoots: uploadResult.Data.CopyToFullPaths,
|
||||
BindingInfo: models.ModelBindingInfo{
|
||||
Type: ModelType,
|
||||
Name: appInstanceName,
|
||||
|
|
@ -273,7 +280,7 @@ func (s *preparationService) bindModel(ctx context.Context, authData *models.Aut
|
|||
},
|
||||
},
|
||||
}
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
resp, err := s.httpClient.PostJSON(CreateResourceUri, request)
|
||||
fmt.Println("模型绑定结果:", string(resp))
|
||||
if err != nil {
|
||||
|
|
@ -296,42 +303,79 @@ func (s *preparationService) uploadFiles(ctx context.Context, authData *models.A
|
|||
info := models.PackageCreateLoadInfo{
|
||||
UserID: authData.JsmUserInfo.Data.UserID,
|
||||
BucketID: bucketID,
|
||||
Name: config.Name + "-" + GenerateUniqueID(),
|
||||
Name: uploadType + "-" + GenerateUniqueID(),
|
||||
}
|
||||
|
||||
return s.doUpload(ctx, authData, filesConfig, info)
|
||||
return s.updownService.Upload(ctx, authData, filesConfig, info, uploadType)
|
||||
}
|
||||
|
||||
func (s *preparationService) doUpload(ctx context.Context, authData *models.AuthData, filesConfig *models.UploadFileConfig, info models.PackageCreateLoadInfo) (*models.PackageCreateResponse, error) {
|
||||
infoJson, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("序列化PackageCreateLoadInfo失败: %w", err)
|
||||
}
|
||||
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
resp, err := s.httpClient.UploadFiles("/apis/jcs/package/createLoad",
|
||||
filesConfig.FilePrefix,
|
||||
map[string][]string{
|
||||
"files": filesConfig.Files, //s.encodeFilepath(files),
|
||||
func (s *preparationService) getPreSignUrl(ctx context.Context, authData *models.AuthData, info models.PackageCreateLoadInfo) (string, error) {
|
||||
preSignedRequest := models.PresignedPackageCreateUpload{
|
||||
UserID: authData.JsmUserInfo.Data.UserID,
|
||||
Info: models.PresignedPackageCreateUploadInfo{
|
||||
Type: "createUpload",
|
||||
Param: models.PresignedPackageCreateUploadParam{
|
||||
BucketID: info.BucketID,
|
||||
Name: info.Name,
|
||||
},
|
||||
},
|
||||
map[string]string{
|
||||
"info": string(infoJson),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("上传文件失败: %w", err)
|
||||
}
|
||||
|
||||
var result models.PackageCreateResponse
|
||||
err = json.Unmarshal(resp, &result)
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
resp, err := s.httpClient.PostJSON("/apis/jcs/storage/presign", preSignedRequest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析上传结果失败: %w", err)
|
||||
return "", err
|
||||
}
|
||||
if result.Code != "OK" {
|
||||
return nil, fmt.Errorf("上传失败:%v-%v", result.Code, result.Message)
|
||||
var preSignedresp models.PresignedPackageCreateUploadResp
|
||||
if err = json.Unmarshal(resp, &resp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return &result, nil
|
||||
|
||||
if preSignedresp.Code != "OK" {
|
||||
return "", fmt.Errorf("获取presign url失败:%s-%s", preSignedresp.Code, preSignedresp.Message)
|
||||
}
|
||||
|
||||
if len(preSignedresp.Data.PresignUrl) == 0 {
|
||||
return "", fmt.Errorf("获取presign url为空")
|
||||
}
|
||||
|
||||
return preSignedresp.Data.PresignUrl, nil
|
||||
}
|
||||
|
||||
//func (s *preparationService) doUpload(ctx context.Context, authData *models.AuthData, filesConfig *models.UploadFileConfig, info models.PackageCreateLoadInfo) (*models.PackageCreateResponse, error) {
|
||||
// preSignedUrl, err := s.getPreSignUrl(ctx, authData, info)
|
||||
// if err != nil {
|
||||
//
|
||||
// }
|
||||
// infoJson, err := json.Marshal(info)
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("序列化PackageCreateLoadInfo失败: %w", err)
|
||||
// }
|
||||
//
|
||||
// s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
// resp, err := s.httpClient.UploadFiles(preSignedUrl,
|
||||
// filesConfig.FilePrefix,
|
||||
// map[string][]string{
|
||||
// "files": filesConfig.Files, //s.encodeFilepath(files),
|
||||
// },
|
||||
// map[string]string{
|
||||
// "info": string(infoJson),
|
||||
// })
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("上传文件失败: %w", err)
|
||||
// }
|
||||
//
|
||||
// var result models.PackageCreateResponse
|
||||
// err = json.Unmarshal(resp, &result)
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("解析上传结果失败: %w", err)
|
||||
// }
|
||||
// if result.Code != "OK" {
|
||||
// return nil, fmt.Errorf("上传失败:%v-%v", result.Code, result.Message)
|
||||
// }
|
||||
// return &result, nil
|
||||
//}
|
||||
|
||||
func (s *preparationService) uploadCode(ctx context.Context, authData *models.AuthData, config models.CodeConfig) (*models.PackageCreateResponse, error) {
|
||||
// 1 克隆代码仓库
|
||||
if err := CloneRepository(config.GitUrl, config.MountPath, config.GitBranch); err != nil {
|
||||
|
|
@ -346,7 +390,7 @@ func (s *preparationService) uploadCode(ctx context.Context, authData *models.Au
|
|||
BucketID: authData.JsmUserInfo.Data.Buckets.Code,
|
||||
Name: "code" + "-" + GenerateUniqueID(),
|
||||
}
|
||||
return s.doUpload(ctx, authData, files, info)
|
||||
return s.updownService.Upload(ctx, authData, files, info, CodeType)
|
||||
}
|
||||
|
||||
func CloneRepository(url, targetDir, branch string) error {
|
||||
|
|
@ -390,14 +434,14 @@ func (s *preparationService) bindDataset(ctx context.Context, authData *models.A
|
|||
Description: "数据集绑定",
|
||||
Category: ImageCategory,
|
||||
PackageID: uploadResult.Data.Package.PackageID,
|
||||
ClusterIDs: []string{clusterID},
|
||||
ClusterIDs: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
resp, err := s.httpClient.PostJSON(CreateResourceUri, request)
|
||||
fmt.Println("数据集绑定结果:", string(resp))
|
||||
if err != nil {
|
||||
|
|
@ -425,7 +469,7 @@ func (s *preparationService) getBindingID(response []byte) (int, error) {
|
|||
}
|
||||
|
||||
func (s *preparationService) getClusterIDByType(authData models.AuthData, clusterType string) (string, error) {
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
params := map[string]string{
|
||||
"pageNum": "1",
|
||||
"pageSize": "100",
|
||||
|
|
@ -514,11 +558,36 @@ func generateAppInstanceName(name string) string {
|
|||
return name + "-" + GenerateUniqueID()
|
||||
}
|
||||
|
||||
// Base62 字符集
|
||||
const base62Chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
// 将字节数组编码为 Base62 字符串
|
||||
func encodeBase62(data []byte) string {
|
||||
// 取前 8 字节作为 uint64
|
||||
num := binary.BigEndian.Uint64(data[:8])
|
||||
result := make([]byte, 0, 11)
|
||||
|
||||
// 转 Base62
|
||||
for num > 0 {
|
||||
remainder := num % 62
|
||||
result = append([]byte{base62Chars[remainder]}, result...)
|
||||
num /= 62
|
||||
}
|
||||
|
||||
// 补充随机字节(取最后 4 个字节)
|
||||
for _, b := range data[8:] {
|
||||
result = append(result, base62Chars[int(b)%62])
|
||||
}
|
||||
|
||||
return string(result)
|
||||
}
|
||||
|
||||
// 生成唯一 ID
|
||||
func GenerateUniqueID() string {
|
||||
// 获取当前纳秒时间戳
|
||||
// 当前纳秒时间戳
|
||||
now := time.Now().UnixNano()
|
||||
|
||||
// 生成加密安全随机数
|
||||
// 生成 4 字节加密安全随机数
|
||||
randBytes := make([]byte, 4)
|
||||
_, _ = rand.Read(randBytes)
|
||||
|
||||
|
|
@ -527,7 +596,12 @@ func GenerateUniqueID() string {
|
|||
binary.BigEndian.PutUint64(data[0:8], uint64(now))
|
||||
copy(data[8:12], randBytes)
|
||||
|
||||
// 使用Base64 URL编码并截取8位
|
||||
encoded := base64.RawURLEncoding.EncodeToString(data)
|
||||
return encoded[:8]
|
||||
id := encodeBase62(data)
|
||||
|
||||
// 截取前 8 位
|
||||
if len(id) > 8 {
|
||||
id = id[:8]
|
||||
}
|
||||
|
||||
return id
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"remote-task-excutor-cli/pkg/models"
|
||||
"remote-task-excutor-cli/pkg/service"
|
||||
"remote-task-excutor-cli/pkg/service/common"
|
||||
"remote-task-excutor-cli/pkg/service/updown"
|
||||
"time"
|
||||
|
||||
"github.com/mholt/archiver/v3"
|
||||
|
|
@ -17,6 +18,7 @@ import (
|
|||
type taskService struct {
|
||||
httpClient *client.HTTPClient
|
||||
taskSubmitter *common.TaskSubmitter
|
||||
updownService service.UpDownService
|
||||
}
|
||||
|
||||
func NewTaskService(baseURL string, timeout time.Duration) service.TaskService {
|
||||
|
|
@ -24,23 +26,25 @@ func NewTaskService(baseURL string, timeout time.Duration) service.TaskService {
|
|||
return &taskService{
|
||||
httpClient: httpClient,
|
||||
taskSubmitter: common.NewTaskSubmitter(httpClient, common.TrainingTaskType),
|
||||
updownService: updown.NewUpDownService(baseURL, timeout),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *taskService) SubmitTask(ctx context.Context, authData *models.AuthData, config *models.RunConfig,
|
||||
clusterID string, bindResultSet *models.BindResultSet) (string, error) {
|
||||
return s.taskSubmitter.SubmitTask(ctx, authData, config, clusterID, bindResultSet)
|
||||
jobSetID, _, err := s.taskSubmitter.SubmitTask(ctx, authData, config, clusterID, bindResultSet)
|
||||
return jobSetID, err
|
||||
}
|
||||
|
||||
func (s *taskService) GetTaskStatus(ctx context.Context, authData *models.AuthData, jobSetID string) (*models.TaskDetailResponse, error) {
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
params := map[string]string{
|
||||
models.JobSetIDKey: jobSetID,
|
||||
models.LocalJobIDKey: models.MainTaskID,
|
||||
}
|
||||
|
||||
for {
|
||||
resp, err := s.httpClient.Get("/jsm/jobMgr/detail", params)
|
||||
resp, err := s.httpClient.Get("/jsm/v2/jobs/details", params)
|
||||
if err != nil {
|
||||
fmt.Println("Get task status failed: ", err)
|
||||
return nil, err
|
||||
|
|
@ -71,7 +75,7 @@ func (s *taskService) GetTaskResult(ctx context.Context, authData *models.AuthDa
|
|||
return err
|
||||
}
|
||||
|
||||
filename, err := s.downloadTaskResult(ctx, authData, path, result.PcmJobData.ResultFiles[0].Objects[0].PackageID)
|
||||
filename, err := s.updownService.DownLoad(ctx, authData, path, result.PcmJobData.ResultFiles[0].Objects[0].PackageID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -90,64 +94,91 @@ func (s *taskService) GetTaskResult(ctx context.Context, authData *models.AuthDa
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *taskService) downloadTaskResult(ctx context.Context, authData *models.AuthData, path string, packageID int) (string, error) {
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
params := map[string]string{
|
||||
"packageID": fmt.Sprintf("%d", packageID),
|
||||
"userID": fmt.Sprintf("%d", authData.JsmUserInfo.Data.UserID),
|
||||
}
|
||||
filename, err := s.httpClient.DownloadFile("/jcs/v1/package/download", path, params)
|
||||
if err != nil {
|
||||
fmt.Println("Download task result failed: ", err)
|
||||
return "", err
|
||||
}
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
func (s *taskService) getTaskResultDetail(ctx context.Context, authData *models.AuthData,
|
||||
jobSetID string) (*models.TaskResultDetailData, error) {
|
||||
s.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
s.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
queryTaskResultReq := models.TaskQueryRequest{
|
||||
JobSetID: jobSetID,
|
||||
LocalJobID: models.MainTaskID,
|
||||
}
|
||||
|
||||
maxAttemps := 3
|
||||
attemps := 0
|
||||
// 设置10分钟超时
|
||||
const maxQueryDuration = 10 * time.Minute
|
||||
startTime := time.Now()
|
||||
queryInterval := 10 * time.Second
|
||||
|
||||
for {
|
||||
resp, err := s.httpClient.PostJSON("/jsm/jobMgr/result", queryTaskResultReq)
|
||||
// 检查是否超过10分钟
|
||||
elapsed := time.Since(startTime)
|
||||
if elapsed >= maxQueryDuration {
|
||||
return nil, fmt.Errorf("查询任务结果超时:已查询 %v,超过最大查询时间 %v", elapsed, maxQueryDuration)
|
||||
}
|
||||
|
||||
// 检查上下文是否被取消
|
||||
if ctx.Err() != nil {
|
||||
return nil, fmt.Errorf("查询任务结果被取消: %w", ctx.Err())
|
||||
}
|
||||
|
||||
resp, err := s.httpClient.PostJSON("/jsm/v2/jobs/results", queryTaskResultReq)
|
||||
fmt.Println("get result details resp:", string(resp))
|
||||
if err != nil {
|
||||
fmt.Println("Get task result details failed: ", err)
|
||||
return nil, err
|
||||
fmt.Printf("Get task result details failed: %v (已查询 %v)\n", err, elapsed)
|
||||
// 网络错误时等待后重试
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("查询任务结果被取消: %w", ctx.Err())
|
||||
case <-time.After(queryInterval):
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
var taskResultDetails models.TaskResultDetailResponse
|
||||
if err := json.Unmarshal(resp, &taskResultDetails); err != nil {
|
||||
fmt.Println("Get task result details response unmarshal failed: ", err)
|
||||
return nil, err
|
||||
fmt.Printf("Get task result details response unmarshal failed: %v (已查询 %v)\n", err, elapsed)
|
||||
// 解析错误时等待后重试
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("查询任务结果被取消: %w", ctx.Err())
|
||||
case <-time.After(queryInterval):
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if taskResultDetails.Code != models.ResponseOK {
|
||||
if taskResultDetails.Message == "query pcm job: record not found" {
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
} else {
|
||||
if attemps > maxAttemps {
|
||||
fmt.Println("Get task result details failed: ", taskResultDetails.Code)
|
||||
return nil, fmt.Errorf("get task result details failed: %s", taskResultDetails.Code)
|
||||
fmt.Printf("任务结果未找到,继续查询... (已查询 %v)\n", elapsed)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("查询任务结果被取消: %w", ctx.Err())
|
||||
case <-time.After(queryInterval):
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("Get task result details failed: code=%s, message=%s (已查询 %v)\n",
|
||||
taskResultDetails.Code, taskResultDetails.Message, elapsed)
|
||||
// 其他错误也等待后重试,直到超时
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("查询任务结果被取消: %w", ctx.Err())
|
||||
case <-time.After(queryInterval):
|
||||
continue
|
||||
}
|
||||
attemps++
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
if taskResultDetails.Data.PcmJobData.Status == "failed" { // && taskResultDetails.Data.PcmJobData.ErrorMsg == "data return job id is empty" {
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
// 检查任务状态
|
||||
if taskResultDetails.Data.PcmJobData.Status == "failed" {
|
||||
fmt.Printf("任务状态为 failed,继续查询... (已查询 %v)\n", elapsed)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("查询任务结果被取消: %w", ctx.Err())
|
||||
case <-time.After(queryInterval):
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 成功获取结果
|
||||
fmt.Printf("成功获取任务结果 (耗时 %v)\n", elapsed)
|
||||
return &taskResultDetails.Data, nil
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"remote-task-excutor-cli/pkg/client"
|
||||
"remote-task-excutor-cli/pkg/models"
|
||||
"remote-task-excutor-cli/pkg/service"
|
||||
|
|
@ -30,7 +31,7 @@ func NewLogService(baseURL string, filePath string) service.LogService {
|
|||
|
||||
func (l *logService) RecordLogToFile(ctx context.Context, authData *models.AuthData, jobSetID string, done chan bool) error {
|
||||
// 每5秒获取一次日志
|
||||
ticker := time.NewTicker(time.Second * 5)
|
||||
ticker := time.NewTicker(time.Second * 10)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
|
|
@ -51,10 +52,71 @@ func (l *logService) doRecordFile(ctx context.Context, authData *models.AuthData
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logs := l.splitLogs(ctx, s)
|
||||
if err := l.writeLogs(logs); err != nil {
|
||||
return err
|
||||
|
||||
// 写入到 /remote-logs 目录,使用原子操作
|
||||
if err := l.writeLogsAtomically(jobSetID, s); err != nil {
|
||||
return fmt.Errorf("写入日志文件失败: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeLogsAtomically 原子写入日志到 /remote-logs 目录
|
||||
// 每次都是全量覆盖写入
|
||||
func (l *logService) writeLogsAtomically(jobSetID string, logContent string) error {
|
||||
// 确保 /remote-logs 目录存在
|
||||
logDir := "/remote-logs"
|
||||
if err := os.MkdirAll(logDir, 0755); err != nil {
|
||||
return fmt.Errorf("创建日志目录失败: %w", err)
|
||||
}
|
||||
|
||||
// 生成日志文件名(使用 jobSetID)
|
||||
logFileName := fmt.Sprintf("%s.log", jobSetID)
|
||||
logFilePath := filepath.Join(logDir, logFileName)
|
||||
|
||||
// 生成临时文件路径
|
||||
tmpFilePath := logFilePath + ".tmp"
|
||||
|
||||
// 创建临时文件并写入内容
|
||||
tmpFile, err := os.OpenFile(tmpFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建临时文件失败: %w", err)
|
||||
}
|
||||
|
||||
// 使用 bufio 写入以提高性能
|
||||
writer := bufio.NewWriter(tmpFile)
|
||||
if _, err := writer.WriteString(logContent); err != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpFilePath) // 清理临时文件
|
||||
return fmt.Errorf("写入临时文件失败: %w", err)
|
||||
}
|
||||
|
||||
// 刷新缓冲区
|
||||
if err := writer.Flush(); err != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpFilePath)
|
||||
return fmt.Errorf("刷新缓冲区失败: %w", err)
|
||||
}
|
||||
|
||||
// 同步到磁盘
|
||||
if err := tmpFile.Sync(); err != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpFilePath)
|
||||
return fmt.Errorf("同步到磁盘失败: %w", err)
|
||||
}
|
||||
|
||||
// 关闭临时文件
|
||||
if err := tmpFile.Close(); err != nil {
|
||||
os.Remove(tmpFilePath)
|
||||
return fmt.Errorf("关闭临时文件失败: %w", err)
|
||||
}
|
||||
|
||||
// 原子重命名(在大多数文件系统上这是原子操作)
|
||||
if err := os.Rename(tmpFilePath, logFilePath); err != nil {
|
||||
os.Remove(tmpFilePath) // 清理临时文件
|
||||
return fmt.Errorf("重命名文件失败: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -99,12 +161,12 @@ func (l *logService) splitLogs(ctx context.Context, s string) []string {
|
|||
}
|
||||
|
||||
func (l *logService) getTaskLogs(ctx context.Context, authData *models.AuthData, jobSetID string) (string, error) {
|
||||
l.httpClient.Headers["Authorization"] = "Bearer " + authData.Token
|
||||
l.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
params := map[string]string{
|
||||
models.JobSetIDKey: jobSetID,
|
||||
models.LocalJobIDKey: models.MainTaskID,
|
||||
}
|
||||
resp, err := l.httpClient.Get("/jsm/jobMgr/trainlog", params)
|
||||
resp, err := l.httpClient.Get("/jsm/v2/jobs/logs", params)
|
||||
if err != nil {
|
||||
fmt.Println("Get task logs failed: ", err)
|
||||
return "", err
|
||||
|
|
@ -122,6 +184,6 @@ func (l *logService) getTaskLogs(ctx context.Context, authData *models.AuthData,
|
|||
return "", fmt.Errorf("get task logs failed: %s", taskLogResponse.Code)
|
||||
}
|
||||
|
||||
fmt.Println("Get task logs result: ", resp)
|
||||
//fmt.Println("Get task logs result: ", resp)
|
||||
return taskLogResponse.Data, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
package updown
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"remote-task-excutor-cli/pkg/client"
|
||||
"remote-task-excutor-cli/pkg/models"
|
||||
"remote-task-excutor-cli/pkg/service"
|
||||
"time"
|
||||
)
|
||||
|
||||
type updownService struct {
|
||||
httpClient *client.HTTPClient
|
||||
}
|
||||
|
||||
func NewUpDownService(baseURL string, timeout time.Duration) service.UpDownService {
|
||||
return &updownService{
|
||||
httpClient: client.NewHTTPClient(baseURL, 2*24*time.Hour),
|
||||
}
|
||||
}
|
||||
|
||||
func (u *updownService) Upload(ctx context.Context, authData *models.AuthData, filesConfig *models.UploadFileConfig, info models.PackageCreateLoadInfo, uploadType string) (*models.PackageCreateResponse, error) {
|
||||
preSignedUrl, err := u.getUpoadPreSignUrl(ctx, authData, info, uploadType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infoJson, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("序列化PackageCreateLoadInfo失败: %w", err)
|
||||
}
|
||||
|
||||
u.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
resp, err := u.httpClient.UploadFiles(preSignedUrl,
|
||||
filesConfig.FilePrefix,
|
||||
map[string][]string{
|
||||
"files": filesConfig.Files, //s.encodeFilepath(files),
|
||||
},
|
||||
map[string]string{
|
||||
"info": string(infoJson),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("上传文件失败: %w", err)
|
||||
}
|
||||
|
||||
var result models.PackageCreateResponse
|
||||
err = json.Unmarshal(resp, &result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析上传结果失败: %w, 返回结果:%s", err, string(resp))
|
||||
}
|
||||
if result.Code != "OK" {
|
||||
return nil, fmt.Errorf("上传失败:%v-%v", result.Code, result.Message)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (u *updownService) DownLoad(ctx context.Context, authData *models.AuthData, path string,
|
||||
packageID int) (string, error) {
|
||||
downPresignedUrl, err := u.getDownLoadSignUrl(ctx, authData, packageID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("获取下载预签名URL失败: %w", err)
|
||||
}
|
||||
u.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
//params := map[string]string{
|
||||
// "packageID": fmt.Sprintf("%d", packageID),
|
||||
// "userID": fmt.Sprintf("%d", authData.JsmUserInfo.Data.UserID),
|
||||
//}
|
||||
filename, err := u.httpClient.DownloadFile(downPresignedUrl, path, nil)
|
||||
if err != nil {
|
||||
fmt.Println("Download task result failed: ", err)
|
||||
return "", err
|
||||
}
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
func (u *updownService) getUpoadPreSignUrl(ctx context.Context, authData *models.AuthData,
|
||||
info models.PackageCreateLoadInfo, uploadType string) (string, error) {
|
||||
preSignedRequest := &models.PresignedPackageCreateUpload{
|
||||
UserID: authData.JsmUserInfo.Data.UserID,
|
||||
Info: models.PresignedPackageCreateUploadInfo{
|
||||
Type: "createUpload",
|
||||
Param: models.PresignedPackageCreateUploadParam{
|
||||
BucketID: info.BucketID,
|
||||
Name: info.Name,
|
||||
CopyTo: []int{23},
|
||||
CopyPath: []string{uploadType + "/" + info.Name},
|
||||
},
|
||||
},
|
||||
}
|
||||
if uploadType == "code" {
|
||||
preSignedRequest.Info.Param.CopyTo = nil
|
||||
preSignedRequest.Info.Param.CopyPath = nil
|
||||
}
|
||||
|
||||
return u.getPreSignURL(ctx, authData, preSignedRequest)
|
||||
}
|
||||
|
||||
func (u *updownService) getDownLoadSignUrl(ctx context.Context, authData *models.AuthData, packageID int) (string, error) {
|
||||
preSignedRequest := &models.PresignedPackageCreateUpload{
|
||||
UserID: authData.JsmUserInfo.Data.UserID,
|
||||
Info: models.PresignedPackageCreateUploadInfo{
|
||||
Type: "batchDownload",
|
||||
Param: models.PresignedPackageCreateUploadParam{
|
||||
PackageID: packageID,
|
||||
Zip: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
return u.getPreSignURL(ctx, authData, preSignedRequest)
|
||||
}
|
||||
|
||||
func (u *updownService) getPreSignURL(ctx context.Context, authData *models.AuthData,
|
||||
req *models.PresignedPackageCreateUpload) (string, error) {
|
||||
u.httpClient.SetHeader("Authorization", "Bearer "+authData.Token)
|
||||
resp, err := u.httpClient.PostJSON("jsm/v2/storage/presign", req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Println("getPreSignURL resp is ", string(resp))
|
||||
var preSignedresp models.PresignedPackageCreateUploadResp
|
||||
if err = json.Unmarshal(resp, &preSignedresp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if preSignedresp.Code != "OK" {
|
||||
return "", fmt.Errorf("获取presign url失败:%s-%s", preSignedresp.Code, preSignedresp.Message)
|
||||
}
|
||||
|
||||
if len(preSignedresp.Data.PresignUrl) == 0 {
|
||||
return "", fmt.Errorf("获取presign url为空")
|
||||
}
|
||||
|
||||
return preSignedresp.Data.PresignUrl, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func NewMySQL(dsn string) (*sql.DB, error) {
|
||||
if dsn == "" {
|
||||
return nil, fmt.Errorf("mysql dsn is empty")
|
||||
}
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 基本连接池配置(可后续放到配置里)
|
||||
db.SetMaxOpenConns(50)
|
||||
db.SetMaxIdleConns(10)
|
||||
db.SetConnMaxLifetime(30 * time.Minute)
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,88 @@
|
|||
#!/bin/bash
|
||||
|
||||
# 部署 server 到 Kubernetes 的脚本
|
||||
|
||||
set -e
|
||||
|
||||
# 配置变量
|
||||
NAMESPACE="${1:-default}"
|
||||
DEPLOYMENT_NAME="remote-task-executor-server"
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}部署 Server 到 Kubernetes${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "命名空间: ${YELLOW}${NAMESPACE}${NC}"
|
||||
echo -e "部署名称: ${YELLOW}${DEPLOYMENT_NAME}${NC}"
|
||||
echo ""
|
||||
|
||||
# 检查 kubectl 是否可用
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}错误: kubectl 未安装或不在 PATH 中${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查 k8s 目录是否存在
|
||||
if [ ! -d "k8s" ]; then
|
||||
echo -e "${RED}错误: k8s 目录不存在${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查命名空间是否存在,不存在则创建
|
||||
echo -e "${GREEN}[1/4] 检查命名空间...${NC}"
|
||||
if ! kubectl get namespace ${NAMESPACE} &> /dev/null; then
|
||||
echo -e "${YELLOW}命名空间不存在,正在创建...${NC}"
|
||||
kubectl create namespace ${NAMESPACE}
|
||||
echo -e "${GREEN}命名空间创建成功${NC}"
|
||||
else
|
||||
echo -e "${GREEN}命名空间已存在${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 应用 Secret(如果存在,注意:实际使用时需要先创建真实的 Secret)
|
||||
if [ -f "k8s/secret.yaml" ]; then
|
||||
echo -e "${GREEN}[2/3] 应用 Secret...${NC}"
|
||||
kubectl apply -f k8s/secret.yaml -n ${NAMESPACE}
|
||||
echo -e "${GREEN}Secret 应用成功${NC}"
|
||||
echo ""
|
||||
else
|
||||
echo -e "${YELLOW}[2/3] 跳过 Secret(文件不存在,如需使用请参考 secret.yaml.example)${NC}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# 应用 Deployment 和 Service
|
||||
echo -e "${GREEN}[3/3] 应用 Deployment 和 Service...${NC}"
|
||||
kubectl apply -f k8s/deployment.yaml -n ${NAMESPACE}
|
||||
echo -e "${GREEN}Deployment 和 Service 应用成功${NC}"
|
||||
echo ""
|
||||
|
||||
# 等待部署完成
|
||||
echo -e "${GREEN}等待部署就绪...${NC}"
|
||||
kubectl wait --for=condition=available --timeout=300s deployment/${DEPLOYMENT_NAME} -n ${NAMESPACE} || true
|
||||
|
||||
# 显示部署状态
|
||||
echo ""
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}部署完成!${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}部署状态:${NC}"
|
||||
kubectl get deployment ${DEPLOYMENT_NAME} -n ${NAMESPACE}
|
||||
echo ""
|
||||
echo -e "${YELLOW}Pod 状态:${NC}"
|
||||
kubectl get pods -l app=${DEPLOYMENT_NAME} -n ${NAMESPACE}
|
||||
echo ""
|
||||
echo -e "${YELLOW}Service 状态:${NC}"
|
||||
kubectl get svc ${DEPLOYMENT_NAME} -n ${NAMESPACE}
|
||||
echo ""
|
||||
echo -e "${GREEN}查看日志命令:${NC}"
|
||||
echo -e "kubectl logs -f deployment/${DEPLOYMENT_NAME} -n ${NAMESPACE}"
|
||||
echo ""
|
||||
echo -e "${GREEN}进入 Pod 命令:${NC}"
|
||||
echo -e "kubectl exec -it deployment/${DEPLOYMENT_NAME} -n ${NAMESPACE} -- /bin/sh"
|
||||
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
-- 创建数据库(如果不存在)
|
||||
CREATE DATABASE IF NOT EXISTS `remote-task` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
|
||||
|
||||
-- 使用数据库
|
||||
USE `remote-task`;
|
||||
|
||||
-- 创建推理异步任务表
|
||||
CREATE TABLE IF NOT EXISTS `inference_async_task` (
|
||||
`id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '主键ID',
|
||||
`task_id` VARCHAR(64) NOT NULL COMMENT '任务ID(唯一标识)',
|
||||
`status` VARCHAR(32) NOT NULL DEFAULT 'PENDING' COMMENT '任务状态:PENDING-等待处理, RUNNING-处理中, COMPLETED-已完成, FAILED-失败',
|
||||
`progress` VARCHAR(255) DEFAULT NULL COMMENT '进度描述',
|
||||
`cluster_id` VARCHAR(64) DEFAULT NULL COMMENT '集群ID',
|
||||
`job_set_id` VARCHAR(64) DEFAULT NULL COMMENT '任务集ID(提交成功后返回)',
|
||||
`request_json` LONGTEXT NOT NULL COMMENT '请求参数JSON',
|
||||
`result_json` LONGTEXT DEFAULT NULL COMMENT '结果JSON(任务完成后填充)',
|
||||
`error` TEXT DEFAULT NULL COMMENT '错误信息',
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uk_task_id` (`task_id`),
|
||||
KEY `idx_status_updated` (`status`, `updated_at`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='推理异步任务表';
|
||||
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
#!/bin/bash
|
||||
|
||||
# 卸载 server 从 Kubernetes 的脚本
|
||||
|
||||
set -e
|
||||
|
||||
# 配置变量
|
||||
NAMESPACE="${1:-default}"
|
||||
DEPLOYMENT_NAME="remote-task-executor-server"
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${YELLOW}========================================${NC}"
|
||||
echo -e "${YELLOW}卸载 Server 从 Kubernetes${NC}"
|
||||
echo -e "${YELLOW}========================================${NC}"
|
||||
echo -e "命名空间: ${YELLOW}${NAMESPACE}${NC}"
|
||||
echo -e "部署名称: ${YELLOW}${DEPLOYMENT_NAME}${NC}"
|
||||
echo ""
|
||||
|
||||
# 确认
|
||||
read -p "确定要删除部署吗?(y/N): " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo -e "${GREEN}已取消${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 删除 Deployment 和 Service
|
||||
echo -e "${YELLOW}删除 Deployment 和 Service...${NC}"
|
||||
kubectl delete -f k8s/deployment.yaml -n ${NAMESPACE} || true
|
||||
|
||||
# 删除 Secret(可选,通常不建议删除)
|
||||
if [ -f "k8s/secret.yaml" ]; then
|
||||
read -p "是否删除 Secret?(y/N): " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
kubectl delete -f k8s/secret.yaml -n ${NAMESPACE} || true
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}卸载完成!${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
|
||||
Loading…
Reference in New Issue