wangtao
/
training_project
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

华为测试

This commit is contained in:
wangwei10061 2018-11-09 21:30:13 +08:00
parent 8ae7393109
commit 57611e7da3
11 changed files with 240 additions and 643 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -18,7 +18,7 @@
<slf4j.version>1.7.21</slf4j.version>
<fastjson.version>1.2.20</fastjson.version>
<codec.version>1.10</codec.version>
<jackson.version>2.8.6</jackson.version>
<jackson.version>2.9.5</jackson.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

609
src/main/java/com/educoder/bridge/game/controller/GameController.java
View File

@ -41,454 +41,227 @@ import io.swagger.annotations.ApiParam;
@RestController
@RequestMapping("/game")
public class GameController extends BaseController {
@Autowired
private AppConfig appConfig;
@Autowired
private GameService gameService;
@Autowired
private K8sService k8sService;
@Autowired
private AppConfig appConfig;
@Autowired
private GameService gameService;
@Autowired
private K8sService k8sService;
@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
/**
* 开启实训: 克隆版本库
*/
@RequestMapping(path = "/openGameInstance")
@ApiOperation(value = "开启实训", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject openGameInstance(
@ApiParam(name = "tpmGitURL", required = true, value = "Tpm的gitUrl，需要base64编码") @RequestParam String tpmGitURL,
@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiRepoName", required = true, value = "tpiRepoName") @RequestParam String tpiRepoName)
throws Exception {
logger.info("开启实训：tpmGitURL: {}, tpiID: {}, tpiRepoName: {}", tpmGitURL, tpiID, tpiRepoName);
/**
* 开启实训: 克隆版本库
*/
@RequestMapping(path = "/openGameInstance")
@ApiOperation(value = "开启实训", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject openGameInstance(
@ApiParam(name = "tpmGitURL", required = true, value = "Tpm的gitUrl，需要base64编码") @RequestParam String tpmGitURL,
@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiRepoName", required = true, value = "tpiRepoName") @RequestParam String tpiRepoName)
throws Exception {
logger.info("开启实训：tpmGitURL: {}, tpiID: {}, tpiRepoName: {}", tpmGitURL, tpiID, tpiRepoName);
JSONObject response = new JSONObject();
JSONObject response = new JSONObject();
// 设定工作路径为${workspace}/myshixun_${tpiID}
String tpiWorkSpace = appConfig.getWorkspace() + File.separator + "myshixun_" + tpiID;
// 设定工作路径为${workspace}/myshixun_${tpiID}
String tpiWorkSpace = appConfig.getWorkspace() + File.separator + "myshixun_" + tpiID;
// 对当前TPI，从TPM clone 版本库
tpmGitURL = Base64Util.decode(tpmGitURL);
gameService.gitClone(tpiWorkSpace, tpmGitURL, "remote_origin", tpiRepoName);
// 对当前TPI，从TPM clone 版本库
tpmGitURL = Base64Util.decode(tpmGitURL);
// http://10.9.73.116:8889/innov/uznmbg54-2.git
tpmGitURL = "http://testbdgit.educoder.net/" + GameHelper.getIdentifier(tpmGitURL) + "/" + GameHelper.getRepoName(tpmGitURL) + ".git";
// gameService.gitClone(tpiWorkSpace, tpmGitURL, "remote_origin", tpiRepoName);
response.put("code", 0);
response.put("msg", "开启成功");
return response;
}
response.put("code", 0);
response.put("msg", "开启成功");
return response;
}
/**
* 评测： 对每一次评测请求，先判断是否能立即执行还是需要先排队，如果能立即执行，开启一个线程执行以下步骤
* 如果需要排队，将相关参数封装成一个线程，存入redis队列，等待执行，并返回标志位给前台，通知前台进行轮询
*/
@RequestMapping(path = "/gameEvaluate")
@ApiOperation(value = "评测", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject gameEvaluate(
@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiGitURL", required = true, value = "学员对应当前实训的版本库地址，需要base64编码") @RequestParam String tpiGitURL,
@ApiParam(name = "buildID", required = true, value = "本次评测ID") @RequestParam String buildID,
@ApiParam(name = "isPublished", required = false, value = "实训是否已经发布，1代表调试模式，实训未发布") @RequestParam Integer isPublished,
@ApiParam(name = "instanceChallenge", required = true, value = "当前处在第几关") @RequestParam String instanceChallenge,
@ApiParam(name = "testCases", required = true, value = "测试用例") @RequestParam String testCases,
@ApiParam(name = "tpmScript", required = true, value = "tpm评测脚本，需要base64编码") @RequestParam String tpmScript,
@ApiParam(name = "timeLimit", required = false, value = "时间限制") Integer timeLimit,
@ApiParam(name = "resubmit", required = true, value = "是否是重复评测") @RequestParam String resubmit,
@ApiParam(name = "times", required = true, value = "是否是时间轮询请求") @RequestParam Integer times,
@ApiParam(name = "needPortMapping", required = false, value = "容器中需要被映射的端口") Integer needPortMapping,
@ApiParam(name = "podType", required = true, value = "pod类型(0.evaluate，1.webssh，2.evassh)") @RequestParam Integer podType,
@ApiParam(name = "file", required = false, value = "需要传文件的实训，给出文件存放路径(一个目录)及文件类型") String file,
@ApiParam(name = "containers", required = true, value = "需要使用的容器,base64编码") @RequestParam String containers,
@ApiParam(name = "content_modified", required = true, value = "文件是否修改的标志") @RequestParam Integer content_modified)
throws Exception {
logger.info(
"评测：tpiID: {}, tpiGitURL: {}, buildID: {}, isPublished: {}, instanceChallenge: {}, "
+ "testCases: {}, tpmScript: {}, timeLimit: {}, resubmit: {}, "
+ "times: {}, needPortMapping: {}, podType: {}, file: {}, containers: {}, content_modified: {}",
tpiID, tpiGitURL, buildID, isPublished, instanceChallenge, testCases, tpmScript, timeLimit, resubmit, times,
needPortMapping, podType, file, containers, content_modified);
/**
* 评测： 对每一次评测请求，先判断是否能立即执行还是需要先排队，如果能立即执行，开启一个线程执行以下步骤
* 如果需要排队，将相关参数封装成一个线程，存入redis队列，等待执行，并返回标志位给前台，通知前台进行轮询
*/
@RequestMapping(path = "/gameEvaluate")
@ApiOperation(value = "评测", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject gameEvaluate(
@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiGitURL", required = true, value = "学员对应当前实训的版本库地址，需要base64编码") @RequestParam String tpiGitURL,
@ApiParam(name = "buildID", required = true, value = "本次评测ID") @RequestParam String buildID,
@ApiParam(name = "isPublished", required = false, value = "实训是否已经发布，1代表调试模式，实训未发布") @RequestParam Integer isPublished,
@ApiParam(name = "instanceChallenge", required = true, value = "当前处在第几关") @RequestParam String instanceChallenge,
@ApiParam(name = "testCases", required = true, value = "测试用例") @RequestParam String testCases,
@ApiParam(name = "tpmScript", required = true, value = "tpm评测脚本，需要base64编码") @RequestParam String tpmScript,
@ApiParam(name = "timeLimit", required = false, value = "时间限制") Integer timeLimit,
@ApiParam(name = "resubmit", required = true, value = "是否是重复评测") @RequestParam String resubmit,
@ApiParam(name = "times", required = true, value = "是否是时间轮询请求") @RequestParam Integer times,
@ApiParam(name = "needPortMapping", required = false, value = "容器中需要被映射的端口") Integer needPortMapping,
@ApiParam(name = "podType", required = true, value = "pod类型(0.evaluate，1.webssh，2.evassh)") @RequestParam Integer podType,
@ApiParam(name = "file", required = false, value = "需要传文件的实训，给出文件存放路径(一个目录)及文件类型") String file,
@ApiParam(name = "containers", required = true, value = "需要使用的容器,base64编码") @RequestParam String containers,
@ApiParam(name = "content_modified", required = true, value = "文件是否修改的标志") @RequestParam Integer content_modified)
throws Exception {
logger.info(
"评测：tpiID: {}, tpiGitURL: {}, buildID: {}, isPublished: {}, instanceChallenge: {}, "
+ "testCases: {}, tpmScript: {}, timeLimit: {}, resubmit: {}, "
+ "times: {}, needPortMapping: {}, podType: {}, file: {}, containers: {}, content_modified: {}",
tpiID, tpiGitURL, buildID, isPublished, instanceChallenge, testCases, tpmScript, timeLimit, resubmit, times,
needPortMapping, podType, file, containers, content_modified);
// 记录开始时间
String evaluateStartTime = LocalDateTime.now().toString();
JSONObject cost = new JSONObject();
cost.put("evaluateStartTime", evaluateStartTime);
// 记录开始时间
String evaluateStartTime = LocalDateTime.now().toString();
JSONObject cost = new JSONObject();
cost.put("evaluateStartTime", evaluateStartTime);
JedisUtil.set("timeCost:" + tpiID + ":" + buildID, cost.toJSONString());
JSONObject response = new JSONObject();
JSONObject response = new JSONObject();
// 参数处理
tpiGitURL = Base64Util.decode(tpiGitURL);
needPortMapping = needPortMapping == null ? 0 : needPortMapping;
timeLimit = timeLimit == null ? Integer.parseInt(appConfig.getDefaultTimeLimit()) : timeLimit - 10;
testCases = Base64Util.decode(testCases);
containers = Base64Util.decode(containers);
// 参数处理
tpiGitURL = Base64Util.decode(tpiGitURL);
tpiGitURL = "http://testbdgit.educoder.net/" + GameHelper.getIdentifier(tpiGitURL) + "/" + GameHelper.getRepoName(tpiGitURL) + ".git";
needPortMapping = needPortMapping == null ? 0 : needPortMapping;
timeLimit = timeLimit == null ? Integer.parseInt(appConfig.getDefaultTimeLimit()) : timeLimit - 10;
testCases = Base64Util.decode(testCases);
containers = Base64Util.decode(containers);
// 每次评测均生成新TPI评测脚本
tpmScript = Base64Util.decode(tpmScript);
String tpiWorkspace = appConfig.getWorkspace() + File.separator + "myshixun_" + tpiID;
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
gameService.generateTpiEvaluateShellScript(tpmScript, tpiWorkspace, tpiRepoName);
// 每次评测均生成新TPI评测脚本
tpmScript = Base64Util.decode(tpmScript);
String tpiWorkspace = appConfig.getWorkspace() + File.separator + "myshixun_" + tpiID;
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
JSONObject buildParams = new JSONObject(true);
buildParams.put("tpiID", tpiID);
buildParams.put("tpiGitURL", tpiGitURL);
buildParams.put("buildID", buildID);
JSONObject buildParams = new JSONObject(true);
buildParams.put("tpiID", tpiID);
buildParams.put("tpiGitURL", tpiGitURL);
buildParams.put("tpmScript", tpmScript);
buildParams.put("buildID", buildID);
buildParams.put("isPublished", isPublished);
buildParams.put("instanceChallenge", instanceChallenge);
buildParams.put("testCases", testCases);
buildParams.put("timeLimit", timeLimit);
buildParams.put("resubmit", resubmit);
buildParams.put("needPortMapping", needPortMapping);
buildParams.put("podType", podType);
buildParams.put("containers", containers);
buildParams.put("contentModified", content_modified);
buildParams.put("instanceChallenge", instanceChallenge);
buildParams.put("testCases", testCases);
buildParams.put("timeLimit", timeLimit);
buildParams.put("resubmit", resubmit);
buildParams.put("needPortMapping", needPortMapping);
buildParams.put("podType", podType);
buildParams.put("containers", containers);
buildParams.put("contentModified", content_modified);
// 若实训生成文件 todo:这个处理方式好2b，扩展性极差
if (!StringUtils.isEmpty(file)) {
file = Base64Util.decode(file);
buildParams.put("file", file);
// 清空目标文件夹以防止影响此次评测结果
GameHelper.clearFiles(tpiWorkspace + File.separator + tpiRepoName + File.separator
+ JSONObject.parseObject(file).getString("path"));
}
// 若需要端口映射服务，则分配端口，将podName-port键值对存储于redis
String port = "-1";
// podName
String podName = podType != 2 ? "evaluate-" + tpiID : "evassh-" + tpiID;
// 构建任务字符串，便于redis存储
String task = buildParams.toJSONString();
// 若需要端口映射服务，则分配端口，将podName-port键值对存储于redis
String port = "-1";
if (needPortMapping != -1) {
port = JedisUtil.hget("port", podName);
if (port == null) {
port = PortUtil.getPort() + "";
JedisUtil.hset("port", podName, port);
}
}
// 直接执行任务
BuildThread buildThread = gameService.getBuildThread(buildParams);
threadPoolTaskExecutor.execute(buildThread);
// 构建任务字符串，便于redis存储
String task = buildParams.toJSONString();
// 最大running pod数量
int maxRunningPodNum = Integer.parseInt(appConfig.getMaxRunningPodNum());
// 此次请求只为轮询时间，只返回轮询的时间结果
if (times != 1) {
double buildRank;
response.put("ableToCreate", 1);
response.put("costTime", System.currentTimeMillis() - TimeHelper.convertTimeToMillis(evaluateStartTime));
response.put("waitNum", 0);
response.put("code", 0);
response.put("port", port);
response.put("msg", "评测完成");
try {
buildRank = JedisUtil.zrank("task", task);
} catch (Exception e) {
response.put("ableToCreate", 0);
response.put("waitNum", maxRunningPodNum + JedisUtil.zlen("task"));
response.put("code", 0);
response.put("msg", "等待评测");
return response;
}
logger.debug("直接执行任务！task: {}, tpi: {}", task, tpiID);
return response;
}
if (buildRank != Double.MIN_VALUE) {
logger.debug("任务：{} : 还在等待队列中！", task);
/**
* tpm版本库已更新，同步
*/
@RequestMapping(path = "/resetTpmRepository", method = RequestMethod.POST)
@ApiOperation(value = "tpm版本库已更新，同步操作", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject reset(@ApiParam(name = "tpiID", required = true, value = "tpi") @RequestParam String tpiID,
@ApiParam(name = "tpiGitURL", required = true, value = "学员对应当前实训的版本库地址，base64编码") @RequestParam String tpiGitURL,
@ApiParam(name = "tpmGitURL", required = true, value = "学员对应当前实训的tpm版本库地址，base64编码") @RequestParam String tpmGitURL,
@ApiParam(name = "identifier", required = true, value = "push权限") @RequestParam String identifier)
throws Exception {
logger.info("tpm版本库已更新，同步tpi版本库，tpiID: {}, tpiGitURL: {}, tpmGitURL: {}, identifier: {}", tpiID, tpiGitURL,
tpmGitURL, identifier);
response.put("ableToCreate", 0);
response.put("waitNum", buildRank == Double.MAX_VALUE ? maxRunningPodNum + JedisUtil.zlen("task")
: buildRank + maxRunningPodNum);
response.put("code", 0);
response.put("msg", "等待评测");
return response;
} else {
// 轮询时间请求，发现目标任务不在队列，就直接认为其正在运行
logger.debug("任务：{} : 正在执行中！", task);
JSONObject response = new JSONObject();
response.put("ableToCreate", 1);
response.put("waitNum", 0);
response.put("code", 0);
response.put("port", Integer.parseInt(port));
response.put("msg", "正在评测");
return response;
}
}
tpiGitURL = Base64Util.decode(tpiGitURL);
tpmGitURL = Base64Util.decode(tpmGitURL);
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
String path = appConfig.getWorkspace() + File.separator + "myshixun_" + tpiID;
// 获取Running状态的pod数
int runningPodNum = k8sService.getRunningPodNum();
// 判断是否可以立即执行(若pod已经存在，或是，pod不存在但是此时可以创建pod)
boolean executeImmediately = k8sService.isPodRunning("tpiID", tpiID)
|| (runningPodNum < maxRunningPodNum && k8sService.ableToEvaluate());
try {
// // 从tpm远程库拉取代码到myshixun版本库然后强推到tpi远程库
// gameService.gitPullFromTpm(path, tpmGitURL, tpiRepoName);
gameService.gitPushToTpi(path, tpiGitURL, identifier);
if (executeImmediately) {
// 直接执行任务
BuildThread buildThread = gameService.getBuildThread(buildParams);
threadPoolTaskExecutor.execute(buildThread);
logger.debug("tpm库更新内容已同步，tpiID:{}", tpiID);
response.put("code", 0);
response.put("msg", "版本库更新成功");
} catch (GameException e) {
logger.error("tpm库更新内容同步失败, tpiID:{}", tpiID);
response.put("code", -1);
response.put("msg", "版本库同步失败！");
}
response.put("ableToCreate", 1);
response.put("costTime", System.currentTimeMillis() - TimeHelper.convertTimeToMillis(evaluateStartTime));
response.put("waitNum", 0);
response.put("code", 0);
response.put("port", port);
response.put("msg", "评测完成");
return response;
}
logger.debug("直接执行任务！task: {}, tpi: {}", task, tpiID);
} else {
// 否则，将构建任务推入redis
JedisUtil.zpush("task", task);
/**
* tpi版本库head是否存在，若缺失则修复
*
* @param tpiID
* @param tpiGitURL
* @return
* @throws Exception
*/
@RequestMapping(path = "/check")
@ApiOperation(value = "进入实训时做校验", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject check(@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiGitURL", required = true, value = "学员对应当前实训的版本库地址，base64编码") @RequestParam String tpiGitURL)
throws Exception {
logger.info("校验head是否存在，tpiID: {}, tpiGitURL: {}", tpiID, tpiGitURL);
// 评测任务等待执行
response.put("ableToCreate", 0);
// 在等待队列中的位置
response.put("waitNum", JedisUtil.zlen("task") + maxRunningPodNum);
response.put("code", 0);
response.put("msg", "等待评测");
JSONObject response = new JSONObject();
response.put("code", 0);
logger.debug("任务入队等待!task: {}, tpi: {}", task, tpiID);
}
tpiGitURL = Base64Util.decode(tpiGitURL);
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
String identifier = GameHelper.getIdentifier(tpiGitURL);
return response;
}
// 第一个步骤，检查远程TPI版本库是否为空
String command = "ssh -p1122 -o StrictHostKeyChecking=no git@" + appConfig.getGitIP() + " 'ls repositories/"
+ identifier + "/" + tpiRepoName + ".git 2>/dev/null | wc -l'";
JSONObject result = ShellUtil.executeAndGetExitStatus(command);
// 255是连接失败，如果连接成功wc -l总是执行的，对于前面文件夹为空或者不存在的情况，$?总是0，输出结果总是0
if ((result.getInteger("exitStatus") == 0 && result.getString("out").equals("0"))) {
logger.error("check方法，远程TPI版本库为空,tpiID: {}", tpiID);
response.put("code", -2);
response.put("msg", "finished");
return response;
}
/**
* 评测： 对每一次评测请求，调用持续化pods进行处理，不需要排队
*/
@RequestMapping(path = "/persistence/gameEvaluate")
@ApiOperation(value = "持续评测", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject persistenceGameEvaluate(
@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiGitURL", required = true, value = "学员对应当前实训的版本库地址，需要base64编码") @RequestParam String tpiGitURL,
@ApiParam(name = "buildID", required = true, value = "本次评测ID") @RequestParam String buildID,
@ApiParam(name = "instanceChallenge", required = true, value = "当前处在第几关") @RequestParam String instanceChallenge,
@ApiParam(name = "testCases", required = true, value = "测试用例") @RequestParam String testCases,
@ApiParam(name = "tpmScript", required = true, value = "tpm评测脚本，需要base64编码") @RequestParam String tpmScript,
@ApiParam(name = "timeLimit", required = false, value = "时间限制") Integer timeLimit,
@ApiParam(name = "resubmit", required = true, value = "是否是重复评测") @RequestParam String resubmit,
@ApiParam(name = "times", required = true, value = "是否是时间轮询请求") @RequestParam Integer times,
@ApiParam(name = "needPortMapping", required = false, value = "容器中需要被映射的端口") Integer needPortMapping,
@ApiParam(name = "podType", required = true, value = "pod类型(0.evaluate，1.webssh，2.evassh)") @RequestParam Integer podType,
@ApiParam(name = "file", required = false, value = "需要传文件的实训，给出文件存放路径(一个目录)及文件类型") String file,
@ApiParam(name = "containers", required = true, value = "需要使用的容器,base64编码") @RequestParam String containers,
@ApiParam(name = "content_modified", required = true, value = "文件是否修改的标志") @RequestParam Integer content_modified,
@ApiParam(name = "persistenceName", required = true, value = "持续pods名称") @RequestParam String persistenceName)
throws Exception {
logger.info(
"评测：tpiID: {}, tpiGitURL: {}, buildID: {}, instanceChallenge: {}, "
+ "testCases: {}, tpmScript: {}, timeLimit: {}, resubmit: {}, "
+ "times: {}, needPortMapping: {}, podType: {}, file: {}, containers: {}, content_modified: {}",
tpiID, tpiGitURL, buildID, instanceChallenge, testCases, tpmScript, timeLimit, resubmit, times,
needPortMapping, podType, file, containers, content_modified);
// 第二个步骤，检测远程版本库HEAD是否存在 （.git/refs/heads/master没内容就是HEAD丢失了，git branch
// 就会报错）
command = "ssh -p1122 -o StrictHostKeyChecking=no git@" + appConfig.getGitIP() + " 'cd repositories/"
+ identifier + "/" + tpiRepoName + ".git; git branch'";
result = ShellUtil.executeAndGetExitStatus(command);
if (result.getInteger("exitStatus") != 0) {
logger.warn("check方法，远程tpi版本库head丢失，将进行修复，tpiID: {}", tpiID);
// 处理仓库head丢失 ssh -p1122 git@10.9.191.219 'cd
// /home/git/repositories/p79061248/klp26sqc.git/refs/tmp/;
// cd `ls -t | sed -n "2p"`; cat head >../../heads/master; git
// update-ref HEAD `cat head`'
command = "ssh -p1122 -o StrictHostKeyChecking=no git@" + appConfig.getGitIP() + " 'cd repositories/"
+ identifier + "/" + tpiRepoName
+ ".git/refs/tmp; cd `ls -t | sed -n \"2p\"`; cat head >../../heads/master; git update-ref HEAD `cat head`'";
result = ShellUtil.executeAndGetExitStatus(command);
if (result.getInteger("exitStatus") != 0) {
logger.error("check方法，远程tpi版本库head丢失，修复失败，tpiID: {}", tpiID);
response.put("code", -1);
} else {
logger.warn("check方法，远程tpi版本库head丢失，修复成功，tpiID: {}", tpiID);
}
}
// 记录开始时间
String evaluateStartTime = LocalDateTime.now().toString();
JSONObject cost = new JSONObject();
cost.put("evaluateStartTime", evaluateStartTime);
JedisUtil.set("timeCost:" + tpiID + ":" + buildID, cost.toJSONString());
JSONObject response = new JSONObject();
// 参数处理
tpiGitURL = Base64Util.decode(tpiGitURL);
needPortMapping = needPortMapping == null ? 0 : needPortMapping;
timeLimit = timeLimit == null ? Integer.parseInt(appConfig.getDefaultTimeLimit()) : timeLimit - 10;
testCases = Base64Util.decode(testCases);
containers = Base64Util.decode(containers);
// 每次评测均生成新TPI评测脚本
tpmScript = Base64Util.decode(tpmScript);
String tpiWorkspace = appConfig.getWorkspace() + File.separator + "myshixun_" + tpiID;
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
gameService.generateTpiEvaluateShellScript(tpmScript, tpiWorkspace, tpiRepoName);
JSONObject buildParams = new JSONObject(true);
buildParams.put("tpiID", tpiID);
buildParams.put("tpiGitURL", tpiGitURL);
buildParams.put("buildID", buildID);
buildParams.put("instanceChallenge", instanceChallenge);
buildParams.put("testCases", testCases);
buildParams.put("timeLimit", timeLimit);
buildParams.put("resubmit", resubmit);
buildParams.put("needPortMapping", needPortMapping);
buildParams.put("podType", podType);
buildParams.put("containers", containers);
buildParams.put("contentModified", content_modified);
buildParams.put("persistenceName", persistenceName);
// 若实训生成文件 todo:这个处理方式好2b，扩展性极差
if (!StringUtils.isEmpty(file)) {
file = Base64Util.decode(file);
buildParams.put("file", file);
// 清空目标文件夹以防止影响此次评测结果
GameHelper.clearFiles(tpiWorkspace + File.separator + tpiRepoName + File.separator
+ JSONObject.parseObject(file).getString("path"));
}
// podName
String podName = podType != 2 ? "evaluate-" + tpiID : "evassh-" + tpiID;
// 若需要端口映射服务，则分配端口，将podName-port键值对存储于redis
String port = "-1";
if (needPortMapping != -1) {
port = JedisUtil.hget("port", podName);
if (port == null) {
port = PortUtil.getPort() + "";
JedisUtil.hset("port", podName, port);
}
}
// 构建任务字符串，便于redis存储
String task = buildParams.toJSONString();
// 最大running pod数量
int maxRunningPodNum = Integer.parseInt(appConfig.getMaxRunningPodNum());
// 此次请求只为轮询时间，只返回轮询的时间结果
if (times != 1) {
double buildRank;
try {
buildRank = JedisUtil.zrank("task", task);
} catch (Exception e) {
response.put("ableToCreate", 0);
response.put("waitNum", maxRunningPodNum + JedisUtil.zlen("task"));
response.put("code", 0);
response.put("msg", "等待评测");
return response;
}
if (buildRank != Double.MIN_VALUE) {
logger.debug("任务：{} : 还在等待队列中！", task);
response.put("ableToCreate", 0);
response.put("waitNum", buildRank == Double.MAX_VALUE ? maxRunningPodNum + JedisUtil.zlen("task")
: buildRank + maxRunningPodNum);
response.put("code", 0);
response.put("msg", "等待评测");
return response;
} else {
// 轮询时间请求，发现目标任务不在队列，就直接认为其正在运行
logger.debug("任务：{} : 正在执行中！", task);
response.put("ableToCreate", 1);
response.put("waitNum", 0);
response.put("code", 0);
response.put("port", Integer.parseInt(port));
response.put("msg", "正在评测");
return response;
}
}
// 持续化pods直接处理任务
BuildThreadPersistence buildThreadPersistence = (BuildThreadPersistence) BeanFactory
.getObejct("BuildThreadPersistence");
buildThreadPersistence.setBuildParams(buildParams, Thread.currentThread());
threadPoolTaskExecutor.execute(buildThreadPersistence);
try {
// 主线程等待1秒
Thread.sleep(1000);
//如果是自己醒来，设置同步返回失败
buildThreadPersistence.setSyncSuccess(false);
} catch (InterruptedException e) {
logger.info("当前容器同步返回数据成功,tpiID:{}, 总消耗时间{}", tpiID, // 评测总时间(除回传结果给educoder之外的总时间)
// 当前时间 -
// 存储在redis中的开始时间
(System.currentTimeMillis() - TimeHelper.convertTimeToMillis(evaluateStartTime)));
}
response.put("syncResult", buildThreadPersistence.getSyncResult());
response.put("ableToCreate", 1);
response.put("costTime", System.currentTimeMillis() - TimeHelper.convertTimeToMillis(evaluateStartTime));
response.put("waitNum", 0);
response.put("code", 0);
response.put("port", port);
response.put("msg", "评测完成");
logger.debug("直接执行任务！task: {}, tpi: {}", response, tpiID);
return response;
}
/**
* tpm版本库已更新，同步
*/
@RequestMapping(path = "/resetTpmRepository", method = RequestMethod.POST)
@ApiOperation(value = "tpm版本库已更新，同步操作", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject reset(@ApiParam(name = "tpiID", required = true, value = "tpi") @RequestParam String tpiID,
@ApiParam(name = "tpiGitURL", required = true, value = "学员对应当前实训的版本库地址，base64编码") @RequestParam String tpiGitURL,
@ApiParam(name = "tpmGitURL", required = true, value = "学员对应当前实训的tpm版本库地址，base64编码") @RequestParam String tpmGitURL,
@ApiParam(name = "identifier", required = true, value = "push权限") @RequestParam String identifier)
throws Exception {
logger.info("tpm版本库已更新，同步tpi版本库，tpiID: {}, tpiGitURL: {}, tpmGitURL: {}, identifier: {}", tpiID, tpiGitURL,
tpmGitURL, identifier);
JSONObject response = new JSONObject();
tpiGitURL = Base64Util.decode(tpiGitURL);
tpmGitURL = Base64Util.decode(tpmGitURL);
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
String path = appConfig.getWorkspace() + File.separator + "myshixun_" + tpiID;
try {
// 从tpm远程库拉取代码到myshixun版本库然后强推到tpi远程库
gameService.gitPullFromTpm(path, tpmGitURL, tpiRepoName);
gameService.gitPushToTpi(path, tpiGitURL, identifier);
logger.debug("tpm库更新内容已同步，tpiID:{}", tpiID);
response.put("code", 0);
response.put("msg", "版本库更新成功");
} catch (GameException e) {
logger.error("tpm库更新内容同步失败, tpiID:{}", tpiID);
response.put("code", -1);
response.put("msg", "版本库同步失败！");
}
return response;
}
/**
* tpi版本库head是否存在，若缺失则修复
*
* @param tpiID
* @param tpiGitURL
* @return
* @throws Exception
*/
@RequestMapping(path = "/check")
@ApiOperation(value = "进入实训时做校验", httpMethod = "POST", produces = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
public JSONObject check(@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiGitURL", required = true, value = "学员对应当前实训的版本库地址，base64编码") @RequestParam String tpiGitURL)
throws Exception {
logger.info("校验head是否存在，tpiID: {}, tpiGitURL: {}", tpiID, tpiGitURL);
JSONObject response = new JSONObject();
response.put("code", 0);
tpiGitURL = Base64Util.decode(tpiGitURL);
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
String identifier = GameHelper.getIdentifier(tpiGitURL);
// 第一个步骤，检查远程TPI版本库是否为空
String command = "ssh -p1122 -o StrictHostKeyChecking=no git@" + appConfig.getGitIP() + " 'ls repositories/"
+ identifier + "/" + tpiRepoName + ".git 2>/dev/null | wc -l'";
JSONObject result = ShellUtil.executeAndGetExitStatus(command);
// 255是连接失败，如果连接成功wc -l总是执行的，对于前面文件夹为空或者不存在的情况，$?总是0，输出结果总是0
if ((result.getInteger("exitStatus") == 0 && result.getString("out").equals("0"))) {
logger.error("check方法，远程TPI版本库为空,tpiID: {}", tpiID);
response.put("code", -2);
response.put("msg", "finished");
return response;
}
// 第二个步骤，检测远程版本库HEAD是否存在 （.git/refs/heads/master没内容就是HEAD丢失了，git branch
// 就会报错）
command = "ssh -p1122 -o StrictHostKeyChecking=no git@" + appConfig.getGitIP() + " 'cd repositories/"
+ identifier + "/" + tpiRepoName + ".git; git branch'";
result = ShellUtil.executeAndGetExitStatus(command);
if (result.getInteger("exitStatus") != 0) {
logger.warn("check方法，远程tpi版本库head丢失，将进行修复，tpiID: {}", tpiID);
// 处理仓库head丢失 ssh -p1122 git@10.9.191.219 'cd
// /home/git/repositories/p79061248/klp26sqc.git/refs/tmp/;
// cd `ls -t | sed -n "2p"`; cat head >../../heads/master; git
// update-ref HEAD `cat head`'
command = "ssh -p1122 -o StrictHostKeyChecking=no git@" + appConfig.getGitIP() + " 'cd repositories/"
+ identifier + "/" + tpiRepoName
+ ".git/refs/tmp; cd `ls -t | sed -n \"2p\"`; cat head >../../heads/master; git update-ref HEAD `cat head`'";
result = ShellUtil.executeAndGetExitStatus(command);
if (result.getInteger("exitStatus") != 0) {
logger.error("check方法，远程tpi版本库head丢失，修复失败，tpiID: {}", tpiID);
response.put("code", -1);
} else {
logger.warn("check方法，远程tpi版本库head丢失，修复成功，tpiID: {}", tpiID);
}
}
response.put("msg", "finished");
return response;
}
response.put("msg", "finished");
return response;
}
}

209
src/main/java/com/educoder/bridge/game/service/GameService.java
View File

@ -54,132 +54,18 @@ public class GameService {
* 开启实训时第0步构建--克隆版本库
*
* @param tpiWorkSpace 根据TpiID拼接出来的文件路径 /data/workspace/myshixun_${TpiID}
* @param tpmGitURL Tpm版本库地址
* @param remoteName 版本库远端名称
* @param tpiRepoName Tpi版本库名称，需要这个参数的原因是，TPM的仓库名字和TPI的名字并不总是相同的
* @param tpiGitURL Tpi版本库地址
*/
public void gitClone(String tpiWorkSpace, String tpmGitURL, String remoteName, String tpiRepoName)
public void gitClone(String tpiWorkSpace, String tpiGitURL)
throws GameException {
logger.debug("tpiWorkSpace: {}, tpmGitURL: {}, remoteName: {}, tpiRepoName: {}", tpiWorkSpace, tpmGitURL,
remoteName, tpiRepoName);
ShellUtil.execute("sshpass -p 'ArmNative!12' ssh -o StrictHostKeyChecking=no root@49.4.88.47 \"mkdir " + tpiWorkSpace + "\"");
// 如果没有工作目录，先创建
File file = FileUtils.getFile(tpiWorkSpace);
if (!file.exists()) {
try {
FileUtils.forceMkdir(file);
} catch (IOException e) {
logger.error("克隆失败，创建版本库所在工作目录失败：{}", e);
throw new GameException("克隆失败，创建版本库所在工作目录失败：" + e);
}
}
String tpmRepoName = GameHelper.getRepoName(tpmGitURL);
// 如果TPI版本库已经存在，或者说，TPM版本库存在，并且改名为TPI版本库成功，直接返回
if (FileUtils.getFile(tpiWorkSpace, tpiRepoName).exists()
|| (FileUtils.getFile(tpiWorkSpace, tpmRepoName).exists()
&& GameHelper.ensurelocalRepoName(tpiWorkSpace, tpiRepoName, tpmRepoName))) {
logger.info("克隆成功！repoName: {}", tpmRepoName);
return;
}
// 先删除原来的
ShellUtil.execute("sshpass -p 'ArmNative!12' ssh -o StrictHostKeyChecking=no root@49.4.88.47 \"cd " + tpiWorkSpace + " && rm -rf ./*\"");
// 执行克隆操作
JSONObject result = ShellUtil.executeAndGetExitStatus(
"cd " + tpiWorkSpace + " && git config --global user.email educoder@163.com "
+ "&& git config --global user.name educoder && git clone -o " + remoteName + " " + tpmGitURL,
3);
if (result.getInteger("exitStatus") != 0) {
logger.error("克隆失败！e: {}", result);
throw new GameException("克隆失败，版本库地址：" + tpmGitURL);
}
// 克隆成功后，确保本地版本库名字是与tpi版本库一致的
logger.info("克隆成功！repoName: {}", tpmRepoName);
GameHelper.ensurelocalRepoName(tpiWorkSpace, tpiRepoName, tpmRepoName);
}
/**
* 为评测服务的git pull方法 remoteName: origin
*
* @param path
* @param tpiGitURL
* @throws GameException
*/
public void gitPull(String path, String tpiGitURL, Integer contentModified) throws GameException {
logger.debug("path: {}, tpiGitURL: {}", path, tpiGitURL);
String tpiRepoName = GameHelper.getRepoName(tpiGitURL);
String tpiRepoPath = path + File.separator + tpiRepoName;
// pull 操作之前，如果tpi本地版本库不存在，则克隆，主机名为origin
int isRepoExist = ShellUtil.executeAndGetExitStatus("ls " + tpiRepoPath).getInteger("exitStatus");
if (isRepoExist != 0) {
logger.warn("版本库不存在，先克隆！ tpiGitURL: {}", tpiGitURL);
gitClone(path, tpiGitURL, "origin", tpiRepoName);
}
// pull之前，先判断版本库的origin主机是否存在，若不存在，则先创建
JSONObject remoteExist = ShellUtil
.executeAndGetExitStatus("cd " + tpiRepoPath + " && git remote | grep ^origin$");
if (remoteExist.getInteger("exitStatus") != 0) {
if (remoteExist.getString("out").contains("fatal")) {
logger.warn("版本库不完整，先进行init操作");
ShellUtil.execute("cd " + tpiRepoPath + " && git init");
}
logger.warn("origin主机名不存在，先添加主机名及其git地址，remoteName: {}, tpiGitURL: {}", "origin", tpiGitURL);
JSONObject addRemote = ShellUtil
.executeAndGetExitStatus("cd " + tpiRepoPath + " && git remote add origin " + tpiGitURL, 3);
if (addRemote.getInteger("exitStatus") != 0) {
logger.error("origin主机名不存在,添加主机名失败");
throw new GameException("origin主机名不存在，无法完成pull操作");
} else {
logger.debug("origin主机名不存在,添加主机名成功");
}
}
// 执行pull操作
long currentPullStart = System.currentTimeMillis();
logger.debug("##----> gitpull 开始，包括git confit --global; set remote，当前时间为：" + currentPullStart);
String pullCommand = "cd " + tpiRepoPath + " && git config --global user.email educoder@163.com"
+ " && git config --global user.name educoder " + "&& git config --global core.fileMode false" + // 忽略文件权限
"&& git remote set-url origin " + tpiGitURL + " && git reset --hard HEAD && git pull origin master"; // 本地直接清除所有改动
JSONObject result = ShellUtil.executeAndGetExitStatus(pullCommand, 3);
long currentPullEnd = System.currentTimeMillis();
logger.debug("##----> gitpull 结束，总耗时间为：" + (currentPullEnd - currentPullStart));
if (result.getInteger("exitStatus") != 0) {
logger.info("pullCommand: {}", pullCommand);
logger.info("output: {}", result.getString("out"));
backupPullError(path, tpiRepoPath);
// 重新clone
gitClone(path, tpiGitURL, "origin", tpiRepoName);
logger.debug("gitPull冲突，重新clone结束");
} else {
// 判段是否成功拉取到更新
int rePullTimes = 10;
while (contentModified == 1 && rePullTimes > 0 && "Already up-to-date.".equals(result.getString("out"))) {
try {
Thread.sleep(200);
} catch (Exception e) {
}
result = ShellUtil.executeAndGetExitStatus(pullCommand, 1);
rePullTimes--;
}
if (rePullTimes == 0) {
logger.error("版本库有更新，2s之内拉取版本库更新失败！");
} else {
logger.debug("pull成功");
}
}
ShellUtil.execute("sshpass -p 'ArmNative!12' ssh -o StrictHostKeyChecking=no root@49.4.88.47 \"cd " + tpiWorkSpace + " && git config --global user.email educoder@163.com "
+ "&& git config --global user.name educoder && git clone " + tpiGitURL + "\"");
}
private void backupPullError(String path, String tpiRepoPath) {
@ -199,66 +85,6 @@ public class GameService {
}
}
/**
* tpm版本库更新，拉取更新
*
* @param path
* @param tpmGitURL
* @param tpiRepoName
* @throws GameException
*/
public void gitPullFromTpm(String path, String tpmGitURL, String tpiRepoName) throws GameException {
logger.debug("path: {}, tpmGitURL: {}, tpiRepoName: {}", path, tpmGitURL, tpiRepoName);
String tpiRepoPath = path + File.separator + tpiRepoName;
// pull 操作之前，如果本地版本库不存在，则克隆，主机名为remote_origin
int isRepoExist = ShellUtil.executeAndGetExitStatus("ls " + tpiRepoPath).getInteger("exitStatus");
if (isRepoExist != 0) {
logger.warn("版本库不存在，先克隆！ tpmGitURL: {}", tpmGitURL);
gitClone(path, tpmGitURL, "remote_origin", tpiRepoName);
}
JSONObject remoteExist = ShellUtil
.executeAndGetExitStatus("cd " + tpiRepoPath + " && git remote | grep ^remote_origin$");
if (remoteExist.getInteger("exitStatus") != 0) {
if (remoteExist.getString("out").contains("fatal")) {
logger.warn("版本库不完整，先进行init操作");
ShellUtil.execute("cd " + tpiRepoPath + " && git init");
}
logger.warn("remote_origin主机名不存在，先添加主机名及其git地址，remoteName: {}, tpmGitURL: {}", "remote_origin", tpmGitURL);
JSONObject addRemote = ShellUtil
.executeAndGetExitStatus("cd " + tpiRepoPath + " && git remote add remote_origin " + tpmGitURL, 3);
if (addRemote.getInteger("exitStatus") != 0) {
logger.error("remote_origin主机名不存在,添加主机名失败");
throw new GameException("remote_origin主机名不存在，无法完成pull操作");
} else {
logger.debug("remote_origin主机名不存在,添加主机名成功");
}
}
// 执行pull操作
String pullCommand = "cd " + tpiRepoPath + " && git config --global user.email educoder@163.com"
+ " && git config --global user.name educoder " + "&& git config --global core.fileMode false" + // 忽略文件权限
"&& git remote set-url remote_origin " + tpmGitURL
+ " && git reset --hard HEAD && git pull remote_origin master";
JSONObject result = ShellUtil.executeAndGetExitStatus(pullCommand, 3);
if (result.getInteger("exitStatus") != 0) {
logger.info("pullCommand: {}", pullCommand);
logger.info("output: {}", result.getString("out"));
backupPullError(path, tpiRepoPath);
// 重新clone
gitClone(path, tpmGitURL, "remote_origin", tpiRepoName);
logger.debug("gitPull冲突，重新clone结束");
} else {
logger.debug("pull成功");
}
}
/**
* 本地push到tpi
@ -597,22 +423,21 @@ public class GameService {
* @param tpiRepoName TPI版本库名字
* @throws GameException
*/
public void generateTpiEvaluateShellScript(String tpmScript, String tpiWorkspace, String tpiRepoName)
throws GameException {
public void generateTpiEvaluateShellScript(String tpmScript, String tpiWorkspace, String tpiRepoName) {
logger.debug("tpiWorkspace: {}, tpiRepoName: {}", tpiWorkspace, tpiRepoName);
// 设置TPI脚本工作空间
String tpiScript = tpmScript.replace("WORKSPACE", tpiWorkspace + File.separator + tpiRepoName);
try {
// 写脚本到工作空间目录下
FileUtils.deleteQuietly(new File(tpiWorkspace + File.separator + "evaluate.sh"));
FileUtils.writeStringToFile(new File(tpiWorkspace + File.separator + "evaluate.sh"), tpiScript, "UTF-8");
} catch (Exception e) {
logger.error("生成TPI评测脚本失败!{}", e);
throw new GameException(-1, "生成TPI评测脚本失败");
}
// 写脚本到工作空间目录下
ShellUtil.execute("sshpass -p 'ArmNative!12' ssh -o StrictHostKeyChecking=no root@49.4.88.47 \"rm " + tpiWorkspace + "/evaluate.sh_temp\"");
ShellUtil.execute("sshpass -p 'ArmNative!12' ssh -o StrictHostKeyChecking=no root@49.4.88.47 \"rm " + tpiWorkspace + "/evaluate.sh\"");
ShellUtil.execute("sshpass -p 'ArmNative!12' ssh -o StrictHostKeyChecking=no root@49.4.88.47 \"(\n" +
"cat << 'EOF'\n" +
"\n" + Base64Util.encodeNoSafe(tpiScript) + "\n" +
"EOF\n" +
")" + " > " + tpiWorkspace + "/evaluate.sh_temp\"");
ShellUtil.execute("sshpass -p 'ArmNative!12' ssh -o StrictHostKeyChecking=no root@49.4.88.47 \"(cat " + tpiWorkspace + "/evaluate.sh_temp | base64 -d) > " + tpiWorkspace + "/evaluate.sh \"");
}

11
src/main/java/com/educoder/bridge/game/service/K8sService.java
View File

@ -46,8 +46,9 @@ public class K8sService {
.withMasterUrl(appConfig.getK8sServer())
.withNamespace("default")
.withTrustCerts(true)
.withUsername(appConfig.getK8sUser())
.withPassword(appConfig.getK8sPasswd())
.withClientCertFile("/opt/pki/apiserver-kubelet-client.crt")
.withClientKeyFile("/opt/pki/apiserver-kubelet-client.key")
.withOauthToken("23d3cf84c5bbba94")
.build();
try {
client = new DefaultKubernetesClient(config);
@ -142,17 +143,17 @@ public class K8sService {
double cpuUsage = 0.0;
double memoryUsage = 0.0;
/**
* kubectl top nodes 命令输出如下：
* kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true top nodes 命令输出如下：
* NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
*10-9-81-28 296m 15% 2494Mi 43%
*10-9-186-111 144m 7% 1199Mi 32%
* kubectl top nodes | awk '{print $2,$4}' | sed '1d'过滤输出，得到：
* kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true top nodes | awk '{print $2,$4}' | sed '1d'过滤输出，得到：
*138m 1199Mi
*269m 2477Mi
* 行之间以换行分隔，行内的列之间以空格分隔
*/
//执行命令获取输出
String outPut = ShellUtil.execute("kubectl top nodes | awk '{print $2,$4}' | sed '1d'");
String outPut = ShellUtil.execute("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true top nodes | awk '{print $2,$4}' | sed '1d'");
String[] nodesInfo = outPut.split(System.getProperty("line.separator"));
for (String memAndCpu : nodesInfo) {
String[] splitInfo = memAndCpu.split(" ");

8
src/main/java/com/educoder/bridge/game/service/KubernetesService.java
View File

@ -49,7 +49,7 @@ public class KubernetesService {
ins = ins.length() == 0 ? ins : ins.deleteCharAt(ins.length() - 1);
// 拼接出k8s执行命令
String command = "kubectl exec " + podName + " bash "
String command = "kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true exec " + podName + " bash "
+ "/data/workspace/myshixun_" + podName.split("-")[1] + "/evaluate.sh " + instanceChallenge + " " + ins;
logger.debug("evaluate command: {}", command);
@ -115,7 +115,7 @@ public class KubernetesService {
ins = ins.length() == 0 ? ins : ins.deleteCharAt(ins.length() - 1);
int setTimeLimit = timeLimit/2;
// 拼接出k8s执行命令
String command = "timeout " + setTimeLimit + " kubectl exec " + podName + " bash "
String command = "timeout " + setTimeLimit + " kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true exec " + podName + " bash "
+ "/data/workspace/myshixun_" + tpiID + "/evaluate.sh " + instanceChallenge + " " + ins;
logger.debug("evaluate command: {}", command);
@ -176,7 +176,7 @@ public class KubernetesService {
* @return
*/
public boolean svcExist(String podName) {
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl get service " + podName);
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get service " + podName);
if (out.getInteger("exitStatus") == 0) {
return true;
} else {
@ -193,7 +193,7 @@ public class KubernetesService {
podManagerService.cancelDel(podName);
logger.debug("直接删除pod：{}", podName);
String command = "kubectl delete pod " + podName + " --now";
String command = "kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true delete pod " + podName + " --now";
ShellUtil.executeAndGetExitStatus(command);
}
}

6
src/main/java/com/educoder/bridge/game/service/PodManagerService.java
View File

@ -68,15 +68,15 @@ public class PodManagerService {
private synchronized void delPod(String podName) throws Exception {
// 根据podName来删除pod
JSONObject result = ShellUtil.executeAndGetExitStatus("kubectl delete pod " + podName + " --now");
JSONObject result = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true delete pod " + podName + " --now");
logger.info("定时任务: 删除pod-{}结果: {}", podName, result.getString("out"));
// 若存在端口映射service，则一并删除
JSONObject svc = ShellUtil.executeAndGetExitStatus("kubectl get service " + podName);
JSONObject svc = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get service " + podName);
if (result.getInteger("exitStatus") == 0 && svc.getInteger("exitStatus") == 0) {
logger.debug("定时任务: 删除pod对应的service: {}", podName);
// 根据podName删除service
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl delete service " + podName + " --now");
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true delete service " + podName + " --now");
// 移除tpi-port键值对
JedisUtil.hdel("port", podName);

4
src/main/java/com/educoder/bridge/game/thread/BuildThread.java
View File

@ -78,7 +78,7 @@ public class BuildThread extends Thread {
Future<Object> pull = executor.submit(Executors.callable(() -> {
try {
Instant pullStartInstant = Instant.now();
gameService.gitPull(tpiWorkspace, buildParams.getString("tpiGitURL"), buildParams.getInteger("contentModified"));
gameService.gitClone(tpiWorkspace, buildParams.getString("tpiGitURL"));
double pullCost = (double) Duration.between(pullStartInstant, Instant.now()).toMillis() / 1000;
GameHelper.setTimeCostInJedis("timeCost:" + tpiID + ":" + buildID, "pull", String.format("%.3f", pullCost));
} catch (GameException e) {
@ -110,6 +110,8 @@ public class BuildThread extends Thread {
executor.shutdown();
}
gameService.generateTpiEvaluateShellScript(buildParams.getString("tpmScript"), tpiWorkspace, GameHelper.getRepoName(buildParams.getString("tpiGitURL")));
// 通过标签获取pod名
String podName = k8sService.getPod(podInLabels, podNotInLabels);
// 按需创建端口映射svc

6
src/main/java/com/educoder/bridge/game/thread/BuildThreadPersistence.java
View File

@ -91,12 +91,12 @@ public class BuildThreadPersistence extends Thread {
Future<Object> pull = executor.submit(Executors.callable(() -> {
try {
Instant pullStartInstant = Instant.now();
gameService.gitPull(tpiWorkspace, buildParams.getString("tpiGitURL"),
buildParams.getInteger("contentModified"));
// gameService.gitPull(tpiWorkspace, buildParams.getString("tpiGitURL"),
// buildParams.getInteger("contentModified"));
double pullCost = (double) Duration.between(pullStartInstant, Instant.now()).toMillis() / 1000;
logger.debug("pull耗时{}, tpiID{}", pullCost, tpiID);
GameHelper.setTimeCostInJedis("timeCost:" + tpiID + ":" + buildID, "pull", String.format("%.3f", pullCost));
} catch (GameException e) {
} catch (Exception e) {
e.printStackTrace();
}
}));

4
src/main/java/com/educoder/bridge/game/thread/DelZombiePodTask.java
View File

@ -23,8 +23,8 @@ public class DelZombiePodTask {
// @Scheduled(cron = "0 0 4 * * ?")
public void work() {
logger.info("定时任务: 开始删除所有pod");
ShellUtil.execute("kubectl delete pod --all");
ShellUtil.execute("kubectl delete svc `kubectl get svc |grep -wv kubernetes|grep -v NAME | awk -F' ' '{print $1}'`");
ShellUtil.execute("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true delete pod --all");
ShellUtil.execute("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true delete svc `kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get svc |grep -wv kubernetes|grep -v NAME | awk -F' ' '{print $1}'`");
logger.info("定时任务: 删除所有pod完毕");
}
}

16
src/main/java/com/educoder/bridge/webssh/service/WebsshService.java
View File

@ -84,7 +84,7 @@ public class WebsshService {
}
public boolean serviceExist(String podName) {
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl get service " + podName);
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get service " + podName);
if (out.getInteger("exitStatus") == 0) {
return true;
} else {
@ -93,7 +93,7 @@ public class WebsshService {
}
public boolean podExist(String podName) {
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl get pod " + podName);
JSONObject out = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get pod " + podName);
if (out.getInteger("exitStatus") == 0) {
return true;
} else {
@ -103,7 +103,7 @@ public class WebsshService {
public void delete(String tpiID) {
// 获取pod名
String out = ShellUtil.execute("kubectl get pods -l tpiID=" + tpiID + ",type!=evaluate | grep -v NAME");
String out = ShellUtil.execute("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get pods -l tpiID=" + tpiID + ",type!=evaluate | grep -v NAME");
String websshPodName = out.substring(0, out.indexOf(" "));
//当删除队列中已经有删除当前port对应的任务的时候就直接返回
@ -115,7 +115,7 @@ public class WebsshService {
public void deleteNow(String tpiID) throws Exception {
logger.debug("主动不经过定时任务即时删除pod, tpiID: {}", tpiID);
// 获取pod名
String out = ShellUtil.execute("kubectl get pods -l tpiID=" + tpiID + ",type!=evaluate | grep -v NAME");
String out = ShellUtil.execute("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get pods -l tpiID=" + tpiID + ",type!=evaluate | grep -v NAME");
String websshPodName = out.substring(0, out.indexOf(" "));
if (StringUtils.isEmpty(websshPodName)) {
@ -123,16 +123,16 @@ public class WebsshService {
}
// 根据podName来删除pod
JSONObject result = ShellUtil.executeAndGetExitStatus("kubectl delete pod " + websshPodName + " --now");
JSONObject result = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true delete pod " + websshPodName + " --now");
logger.debug("删除pod结果: {}", result.getString("out"));
// 若存在端口映射service，则一并删除
JSONObject svc = ShellUtil.executeAndGetExitStatus("kubectl get service " + websshPodName);
JSONObject svc = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get service " + websshPodName);
if (result.getInteger("exitStatus") == 0 && svc.getInteger("exitStatus") == 0) {
logger.debug("删除pod对应的service: {}", websshPodName);
// 根据podName删除service
JSONObject delSvcOut = ShellUtil.executeAndGetExitStatus("kubectl delete service " + websshPodName + " --now");
JSONObject delSvcOut = ShellUtil.executeAndGetExitStatus("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true delete service " + websshPodName + " --now");
// 移除tpi-port键值对
JedisUtil.hdel("port", websshPodName);
@ -144,7 +144,7 @@ public class WebsshService {
public void active(String tpiID) {
// 获取pod名
String out = ShellUtil.execute("kubectl get pods -l tpiID=" + tpiID + ",type!=evaluate | grep -v NAME");
String out = ShellUtil.execute("kubectl --server='https://49.4.88.47:6443' --client-certificate='/opt/pki/apiserver-kubelet-client.crt' --client-key='/opt/pki/apiserver-kubelet-client.key' --insecure-skip-tls-verify=true get pods -l tpiID=" + tpiID + ",type!=evaluate | grep -v NAME");
String websshPodName = out.substring(0, out.indexOf(" "));

8
src/main/resources/config_test.properties
View File

@ -8,9 +8,7 @@ webssh.userName=root
webssh.passwd=123123
#the k8s server IP that can be accessed from internet
k8s.serverUrl=10.9.81.28
#to execute k8s cmds in slave node ,we have to provide the token
k8s.token=726b127bcf34784b
k8s.serverUrl=https://49.4.88.47:6443
#limit of running pod number
maxRunningPodNum=20000
@ -24,9 +22,7 @@ redisPort=6379
redisAuth=
#测试版kubernetes用户名和密码
k8sServer=127.0.0.1:8080
k8sUser=testk8s
k8sPasswd=testk8s
k8sServer=https://49.4.88.47:6443
#测试版资源总量
cpuAllocatable=4000