Compare commits

...

2 Commits
master ... log

10 changed files with 172 additions and 70 deletions

View File

@ -7,6 +7,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.MediaType;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.web.bind.annotation.RequestMapping;
@ -17,7 +18,6 @@ import org.springframework.web.bind.annotation.RestController;
import com.alibaba.fastjson.JSONObject;
import com.educoder.bridge.common.settings.AppConfig;
import com.educoder.bridge.common.utils.Base64Util;
import com.educoder.bridge.common.utils.BeanFactory;
import com.educoder.bridge.common.utils.GameHelper;
import com.educoder.bridge.common.utils.JedisUtil;
import com.educoder.bridge.common.utils.PortUtil;
@ -25,7 +25,6 @@ import com.educoder.bridge.common.utils.TimeHelper;
import com.educoder.bridge.game.service.GameService;
import com.educoder.bridge.game.service.K8sService;
import com.educoder.bridge.game.thread.BuildThread;
import com.educoder.bridge.game.thread.BuildThreadPersistence;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@ -49,7 +48,8 @@ public class GameController {
private K8sService k8sService;
@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
@Qualifier("buildTaskExecutor")
private ThreadPoolTaskExecutor buildTaskExecutor;
/**
* 开启实训: 克隆版本库
@ -61,8 +61,6 @@ public class GameController {
@ApiParam(name = "tpiID", required = true, value = "实训实例的ID") @RequestParam String tpiID,
@ApiParam(name = "tpiRepoName", required = true, value = "tpiRepoName") @RequestParam String tpiRepoName)
throws Exception {
logger.info("开启实训tpmGitURL: {}, tpiID: {}, tpiRepoName: {}", tpmGitURL, tpiID, tpiRepoName);
JSONObject response = new JSONObject();
// 设定工作路径为${workspace}/myshixun_${tpiID}
@ -102,12 +100,6 @@ public class GameController {
@ApiParam(name = "content_modified", required = true, value = "文件是否修改的标志") @RequestParam Integer content_modified,
@ApiParam(name = "sec_key", required = false, value = "每一次评测的唯一标识") String sec_key)
throws Exception {
logger.info(
"评测tpiID: {}, tpiGitURL: {}, buildID: {}, isPublished: {}, instanceChallenge: {}, "
+ "testCases: {}, tpmScript: {}, timeLimit: {}, resubmit: {}, "
+ "times: {}, needPortMapping: {}, podType: {}, file: {}, containers: {}, content_modified: {}, sec_key: {}",
tpiID, tpiGitURL, buildID, isPublished, instanceChallenge, testCases, tpmScript, timeLimit, resubmit,
times, needPortMapping, podType, file, containers, content_modified, sec_key);
// 记录开始时间
String evaluateStartTime = LocalDateTime.now().toString();
@ -221,7 +213,7 @@ public class GameController {
if (executeImmediately) {
// 直接执行任务
BuildThread buildThread = gameService.getBuildThread(buildParams);
threadPoolTaskExecutor.execute(buildThread);
buildTaskExecutor.execute(buildThread);
response.put("ableToCreate", 1);
response.put("costTime", System.currentTimeMillis() - TimeHelper.convertTimeToMillis(evaluateStartTime));
@ -258,8 +250,6 @@ public class GameController {
@ApiParam(name = "tpmGitURL", required = true, value = "学员对应当前实训的tpm版本库地址base64编码") @RequestParam String tpmGitURL,
@ApiParam(name = "identifier", required = true, value = "push权限") @RequestParam String identifier)
throws Exception {
logger.info("tpm版本库已更新同步tpi版本库tpiID: {}, tpiGitURL: {}, tpmGitURL: {}, identifier: {}", tpiID, tpiGitURL,
tpmGitURL, identifier);
JSONObject response = new JSONObject();

View File

@ -0,0 +1,110 @@
package com.educoder.bridge.game.interceptor;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.time.Duration;
import java.time.Instant;
/**
* 对Controller以INFO级别记录每个方法的参数及耗时返回结果
* 对Service判断debug是否开启如果开启以debug级别记录每个方法参数及耗时以上两点AOP Logging实现就可以
*
* @author weishao
* @date 2019-06-03
*/
@Aspect
@Component
public class LogInterceptor {
// controller层切点
@Pointcut("execution (* com.educoder.bridge.*.controller.*.*(..))")
private void controllerPointCut() {}
// service层切点
@Pointcut("execution (* com.educoder.bridge.*.service.*.*(..))")
private void servicePointCut() {}
/**
* 统计Controller的参数响应时长和返回结果
* @param joinPoint
* @return
*/
@Around("controllerPointCut()")
public Object controllerAround(ProceedingJoinPoint joinPoint) throws Throwable {
Instant startTime = Instant.now();
Logger logger = LoggerFactory.getLogger(LogInterceptor.class);
// 获取并以info级别记录参数
Object[] args = joinPoint.getArgs();
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
String[] parameterNames = signature.getParameterNames();
StringBuffer params = new StringBuffer();
for (int i = 0; i < parameterNames.length; i++) {
params.append(parameterNames[i]).append(":").append(args[i]).append(",");
}
if (params.length() > 0) {
params.deleteCharAt(params.length() - 1);
}
logger.info("interface: {}, params: [{}]", signature.getName(), params.toString());
// 执行方法
Object obj = joinPoint.proceed(args);
// 记录耗时及结果
Duration d = Duration.between(startTime, Instant.now());
logger.info("interface: {}, time consuming: {}ms, response: [{}]", signature.getName(), d.toMillis(), obj);
return obj;
}
/**
* 统计Service的参数响应时长和返回结果
* @param joinPoint
* @return
*/
@Around("servicePointCut()")
public Object serviceAround(ProceedingJoinPoint joinPoint) throws Throwable {
Instant startTime = Instant.now();
Logger logger = LoggerFactory.getLogger(LogInterceptor.class);
// 获取并以debug级别记录参数
Object[] args = joinPoint.getArgs();
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
String[] parameterNames = signature.getParameterNames();
StringBuffer params = new StringBuffer();
for (int i = 0; i < parameterNames.length; i++) {
params.append(parameterNames[i]).append(":").append(args[i]).append(",");
}
if (params.length() > 0) {
params.deleteCharAt(params.length() - 1);
}
if (logger.isDebugEnabled()) {
logger.debug("method: {}, params: [{}]", signature.getName(), params.toString());
}
// 执行方法
Object obj = joinPoint.proceed(args);
// 记录耗时及结果
Duration d = Duration.between(startTime, Instant.now());
if (logger.isDebugEnabled()) {
logger.debug("method: {}, time consuming: {}ms, result: [{}]", signature.getName(), d.toMillis(), obj);
}
return obj;
}
}

View File

@ -18,6 +18,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
@ -54,7 +55,8 @@ public class GameService {
private K8sService k8sService;
@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
@Qualifier("buildTaskExecutor")
private ThreadPoolTaskExecutor buildTaskExecutor;
@Autowired
PodManagerService podManagerService;
@ -751,7 +753,7 @@ public class GameService {
if (task != null) {
// 获取执行线程池
BuildThread buildThread = getBuildThread(task);
threadPoolTaskExecutor.execute(buildThread);
buildTaskExecutor.execute(buildThread);
logger.debug("等待队列:队列中还剩 {} 个任务:", JedisUtil.zlen("task"));
}
} else if (waitingTaskNum > 0) {

View File

@ -34,9 +34,7 @@ public class PodManagerService {
@PostConstruct
public void init() {
new Thread(() -> {
this.doWork();
}).start();
new Thread(() -> this.doWork(), "pod-delete-thread").start();
}
private void doWork() {

View File

@ -43,7 +43,6 @@ public class ShellExecManageService {
/**
* 加入执行时间
*
* @param execWatch
* @param delayTime
* 单位秒
*/
@ -59,7 +58,7 @@ public class ShellExecManageService {
public void init() {
new Thread(() -> {
this.doWork();
}).start();
}, "shell-exec-thread").start();
}
private void doWork() {

View File

@ -5,18 +5,16 @@ import java.time.Duration;
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.*;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Scope;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONArray;
@ -33,6 +31,7 @@ import com.educoder.bridge.game.service.KubernetesService;
import io.fabric8.kubernetes.api.model.Pod;
/**
* 评测线程
*
@ -52,6 +51,9 @@ public class BuildThread extends Thread {
private AppConfig appConfig;
@Autowired
private K8sService k8sService;
@Autowired
@Qualifier("simpleTaskExecutor")
private ThreadPoolTaskExecutor simpleTaskExecutor;
private JSONObject buildParams;
@ -76,6 +78,7 @@ public class BuildThread extends Thread {
String tpiID = buildParams.getString("tpiID");
String buildID = buildParams.getString("buildID");
String tpiWorkspace = appConfig.getWorkspace() + File.separator + "myshixun_" + buildParams.getString("tpiID");
int timeLimit = buildParams.getInteger("timeLimit");
Map<String, String> podInLabels = new HashMap<>(1);
Map<String, String> podNotInLabels = new HashMap<>(1);
podInLabels.put("tpiID", tpiID);
@ -86,14 +89,14 @@ public class BuildThread extends Thread {
String out = GameHelper.getTextMsgResult("服务启动中...");
gameService.encapsulateStepByStepResult(out, buildParams, testCases);
ExecutorService executor = Executors.newFixedThreadPool(2);
// 传递MDC的内容
final Map<String, String> mdc = MDC.getCopyOfContextMap();
// git pull
Future<Boolean> pull = executor.submit(() -> {
Future<Boolean> pull = simpleTaskExecutor.submit(() -> {
Thread.currentThread().setName("pull-thread-" + tpiID);
MDC.setContextMap(mdc);
try {
MDC.setContextMap(mdc);
Instant pullStartInstant = Instant.now();
gameService.gitPull(tpiWorkspace, buildParams.getString("tpiGitURL"),
buildParams.getInteger("contentModified"));
@ -110,11 +113,11 @@ public class BuildThread extends Thread {
// 创建pod
final PodRef podRef = new PodRef();
Instant podStartInstant = Instant.now();
Future<Object> createPod = executor.submit(Executors.callable(() -> {
Future<Object> createPod = simpleTaskExecutor.submit(Executors.callable(() -> {
Thread.currentThread().setName("pod-create-thread-" + tpiID);
MDC.setContextMap(mdc);
// 设立(重设)对应pod的定时删除任务这个时间必须大于${timeLimit}确保脚本执行期间pod不会中途死 单位
int timeLimit = buildParams.getInteger("timeLimit");
gameService.reTiming(type + "-" + tpiID, timeLimit + 60);
Pod pod = k8sService.getK8sRunningPod(podInLabels, podNotInLabels);
@ -134,20 +137,28 @@ public class BuildThread extends Thread {
}));
boolean pullResult = false;
try {
pullResult = pull.get();
pullResult = pull.get(timeLimit, TimeUnit.SECONDS);
} catch (InterruptedException e) {// 不应该发生
logger.error("等待下载实训" + tpiID + "代码被中断", e);
pull.cancel(true);
} catch (ExecutionException e) {// 不应该发生
logger.error("等待下载实训" + tpiID + "代码出错", e);
pull.cancel(true);
} catch (TimeoutException e) {
logger.error("等待下载实训" + tpiID + "代码超时", e);
pull.cancel(true);
} finally {
try {
createPod.get();
createPod.get(timeLimit, TimeUnit.SECONDS);
} catch (InterruptedException e) {// 不应该发生
logger.error("创建实训pod " + tpiID + "代码被中断", e);
createPod.cancel(true);
} catch (ExecutionException e) {
logger.error("创建实训pod " + tpiID + "失败", e);
} finally {
executor.shutdown();
createPod.cancel(true);
} catch (TimeoutException e) {
logger.error("创建实训pod " + tpiID + "超时", e);
createPod.cancel(true);
}
}

View File

@ -11,8 +11,10 @@
http://www.springframework.org/schema/websocket
http://www.springframework.org/schema/websocket/spring-websocket.xsd">
<aop:aspectj-autoproxy/>
<context:component-scan base-package="com.educoder.bridge.*.*"/>
<aop:aspectj-autoproxy proxy-target-class="true"/>
<context:component-scan base-package="com.educoder.bridge.*.*">
<context:exclude-filter type="annotation" expression="org.springframework.stereotype.Controller"/>
</context:component-scan>
<!-- freemaker配置 -->
<bean id="freemarkerConfig"
@ -38,11 +40,21 @@
<bean id="websshHandler" class="com.educoder.bridge.webssh.handler.WebsshHandler"/>
<bean id="threadPoolTaskExecutor" class="com.educoder.bridge.game.thread.BridgeThreadPoolTaskExecutor">
<!-- 评测任务线程池 -->
<bean id="buildTaskExecutor" class="com.educoder.bridge.game.thread.BridgeThreadPoolTaskExecutor">
<property name="threadNamePrefix" value="build-thread-"/>
<property name="corePoolSize" value="400"/>
<property name="maxPoolSize" value="400"/>
<property name="queueCapacity" value="5000"/>
<property name="waitForTasksToCompleteOnShutdown" value="true"/>
</bean>
<!-- 小任务线程池比如pullcreatePod -->
<bean name="simpleTaskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="200"/>
<property name="maxPoolSize" value="800"/>
<property name="queueCapacity" value="5000"/>
<property name="waitForTasksToCompleteOnShutdown" value="false"/>
</bean>
</beans>

View File

@ -1,23 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<configuration scan="true" scanPeriod="1 minutes">
<property name="log_path" value="/tmp/logs/"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
      <encoder>
         <pattern>%-4relative [%thread] %-5level %logger{35} - %msg %n</pattern>
      </encoder>
   </appender>
<appender name="bridge" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!-- 输入到logstash -->
<appender name="bridge" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>10.9.81.28:9998</destination>
<encoder>
<pattern>%d{MM-dd HH:mm:ss} [%X{sec_key}] [%thread] %-5level -- %msg%n</pattern>
</encoder>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
</filter>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${log_path}bridge.%d{yyyy-MM-dd}.log</fileNamePattern>
</rollingPolicy>
</appender>
<!-- 屏蔽框架输出 -->
@ -30,7 +20,7 @@
<root>
<level value="DEBUG"/>
<appender-ref ref="STDOUT"/>
<appender-ref ref="bridge"/>
</root>
</configuration>

View File

@ -2,18 +2,8 @@
<configuration scan="true" scanPeriod="10 minutes">
<property name="log_path" value="${catalina.home}/logs/"/>
<!-- 日志按天生成 -->
<appender name="bridge" class="ch.qos.logback.core.rolling.RollingFileAppender">
<encoder>
<pattern>%d{MM-dd HH:mm:ss} [%X{sec_key}] [%thread] %-5level -- %msg%n</pattern>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${log_path}bridge.%d{yyyy-MM-dd}.log</fileNamePattern>
</rollingPolicy>
</appender>
<!-- 输入到logstash -->
<appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<appender name="bridge" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>10.9.184.140:9998</destination>
<!--<encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder" />-->
<encoder>
@ -31,8 +21,7 @@
<root>
<level value="INFO"/>
<appender-ref ref="bridge"/>
<appender-ref ref="stash" />
<appender-ref ref="bridge" />
</root>
</configuration>

View File

@ -2,13 +2,14 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc.xsd">
xmlns:mvc="http://www.springframework.org/schema/mvc" xmlns:aop="http://www.springframework.org/schema/aop"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd">
<!--指明 controller 所在包,并扫描其中的注解-->
<context:component-scan base-package="com.educoder.bridge.*.controller"/>
<aop:aspectj-autoproxy proxy-target-class="true" />
<!-- 静态资源(js、image等)的访问 -->
<mvc:default-servlet-handler/>