From 77dfbb9cebf5efb9cb4fc6d8c9af8f109a56c61d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B7=B4=E6=8B=89=E8=BF=AA=E7=BB=B4?= Date: Fri, 17 Sep 2021 13:16:29 +0800 Subject: [PATCH 1/5] final check of timezone --- middleware/services.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/middleware/services.yml b/middleware/services.yml index e7b81b8..f05935d 100644 --- a/middleware/services.yml +++ b/middleware/services.yml @@ -127,7 +127,6 @@ services: - TZ=Asia/Shanghai volumes: - ${DOCKER_DATA_PATH}/logs/:/data/logs/ - - TZ=Asia/Shanghai depends_on: - kafka1 - kafka2 From ca3226d2b088e9e8222f50debf81283366d46ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B7=B4=E6=8B=89=E8=BF=AA=E7=BB=B4?= Date: Fri, 17 Sep 2021 13:37:18 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E8=84=9A=E6=9C=AC=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、启动 docker 容器的脚本合并为一行 2、停止 docker 时对项目进行 mvn clean 操作 3、响应的 windows 脚本同步修改 --- middleware/end_docker_compose.bat | 2 +- middleware/end_docker_compose.sh | 3 +-- middleware/start_docker_compose.sh | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/middleware/end_docker_compose.bat b/middleware/end_docker_compose.bat index d8924a7..b06c53d 100644 --- a/middleware/end_docker_compose.bat +++ b/middleware/end_docker_compose.bat @@ -1 +1 @@ -docker-compose -f services.yml down && docker volume prune -f +docker-compose -f services.yml down && docker volume prune -f && mvn -f ../pom.xml clean diff --git a/middleware/end_docker_compose.sh b/middleware/end_docker_compose.sh index ae55462..51e7dcd 100755 --- a/middleware/end_docker_compose.sh +++ b/middleware/end_docker_compose.sh @@ -1,2 +1 @@ -docker-compose -f services.yml down -docker volume prune -f +docker-compose -f services.yml down && docker volume prune -f && mvn -f ../pom.xml clean diff --git a/middleware/start_docker_compose.sh b/middleware/start_docker_compose.sh index 1d22359..f5bfb1b 100755 --- a/middleware/start_docker_compose.sh +++ b/middleware/start_docker_compose.sh @@ -1,2 +1 @@ -mvn -f ../pom.xml clean package -DskipTests -docker-compose -f services.yml up --build --force-recreate \ No newline at end of file +mvn -f ../pom.xml clean package -DskipTests && docker-compose -f services.yml up --build --force-recreate \ No newline at end of file From fd965f0af4899c7953e0130f9ec0ce605198b476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B7=B4=E6=8B=89=E8=BF=AA=E7=BB=B4?= Date: Fri, 17 Sep 2021 14:32:44 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E9=83=A8=E5=88=86=E4=BB=A3=E7=A0=81=E7=BB=93=E6=9E=84=EF=BC=8C?= =?UTF-8?q?=E4=BC=98=E5=8C=96docker=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/{utils => config}/KafkaProducerConfig.java | 8 ++------ .../executor/core/config/KafkaConsumerConfig.java | 8 -------- .../notification/executor/service/email/EmailService.java | 3 --- executor/src/main/resources/application.yml.example | 8 ++------ reader/src/main/resources/application.yml.example | 1 - writer/src/main/resources/application.yml.example | 2 -- 6 files changed, 4 insertions(+), 26 deletions(-) rename common/src/main/java/cn/org/gitlink/notification/common/{utils => config}/KafkaProducerConfig.java (85%) diff --git a/common/src/main/java/cn/org/gitlink/notification/common/utils/KafkaProducerConfig.java b/common/src/main/java/cn/org/gitlink/notification/common/config/KafkaProducerConfig.java similarity index 85% rename from common/src/main/java/cn/org/gitlink/notification/common/utils/KafkaProducerConfig.java rename to common/src/main/java/cn/org/gitlink/notification/common/config/KafkaProducerConfig.java index 869d76e..78c737b 100644 --- a/common/src/main/java/cn/org/gitlink/notification/common/utils/KafkaProducerConfig.java +++ b/common/src/main/java/cn/org/gitlink/notification/common/config/KafkaProducerConfig.java @@ -1,4 +1,4 @@ -package cn.org.gitlink.notification.common.utils; +package cn.org.gitlink.notification.common.config; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; @@ -19,9 +19,6 @@ public class KafkaProducerConfig { @Value("${spring.kafka.producer.bootstrap_servers:#{null}}") private String bootstrapServers; - @Value("${spring.kafka.producer.client_id:#{null}}") - private String clientId; - @Value("${spring.kafka.producer.retries:#{null}}") private Integer retries; @@ -30,7 +27,7 @@ public class KafkaProducerConfig { @Bean public KafkaTemplate kafkaTemplate() { - return new KafkaTemplate<>(producerConfigs(),true); + return new KafkaTemplate<>(producerConfigs(), true); } @Bean @@ -39,7 +36,6 @@ public class KafkaProducerConfig { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); props.put(ProducerConfig.RETRIES_CONFIG, retries); props.put(ProducerConfig.ACKS_CONFIG, "all"); - props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId); props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); diff --git a/executor/src/main/java/cn/org/gitlink/notification/executor/core/config/KafkaConsumerConfig.java b/executor/src/main/java/cn/org/gitlink/notification/executor/core/config/KafkaConsumerConfig.java index 01db58b..6ab75aa 100644 --- a/executor/src/main/java/cn/org/gitlink/notification/executor/core/config/KafkaConsumerConfig.java +++ b/executor/src/main/java/cn/org/gitlink/notification/executor/core/config/KafkaConsumerConfig.java @@ -20,12 +20,6 @@ public class KafkaConsumerConfig { @Value("${spring.kafka.consumer.bootstrap_servers:#{null}}") private String servers; - @Value("${spring.kafka.consumer.group_id}") - private String groupId; - - @Value("${spring.kafka.consumer.client_id}") - private String clientId; - @Value("${spring.kafka.consumer.auto_offset_reset}") private String autoOffsetReset; @@ -46,9 +40,7 @@ public class KafkaConsumerConfig { @Bean public ConsumerFactory consumerConfigs() { Map props = new HashMap<>(); - props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers); - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); diff --git a/executor/src/main/java/cn/org/gitlink/notification/executor/service/email/EmailService.java b/executor/src/main/java/cn/org/gitlink/notification/executor/service/email/EmailService.java index d976f97..b356591 100644 --- a/executor/src/main/java/cn/org/gitlink/notification/executor/service/email/EmailService.java +++ b/executor/src/main/java/cn/org/gitlink/notification/executor/service/email/EmailService.java @@ -3,10 +3,8 @@ package cn.org.gitlink.notification.executor.service.email; import cn.org.gitlink.notification.common.utils.EmailUtils; import cn.org.gitlink.notification.model.dao.entity.EmailJob; import cn.org.gitlink.notification.model.dao.entity.EmailSendRecord; -import cn.org.gitlink.notification.model.dao.entity.vo.NewEmailJobVo; import cn.org.gitlink.notification.model.service.notification.EmailJobsService; import cn.org.gitlink.notification.model.service.notification.EmailSendRecordsService; -import com.alibaba.fastjson.JSONObject; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -14,7 +12,6 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import javax.mail.MessagingException; -import java.beans.Transient; import java.util.ArrayList; import java.util.Date; import java.util.List; diff --git a/executor/src/main/resources/application.yml.example b/executor/src/main/resources/application.yml.example index 1c389ec..4162dae 100644 --- a/executor/src/main/resources/application.yml.example +++ b/executor/src/main/resources/application.yml.example @@ -8,27 +8,23 @@ spring: kafka: producer: bootstrap_servers: kafka1:9092,kafka2:9092 - client_id: gitlink_producer_01 retries: 5 batch_size: 16384 replication_factor: 1 partitions: 3 topic_new_email_remind: topic-gitlink-new-email-remind - consumer: bootstrap_servers: kafka1:9092,kafka2:9092 - client_id: gitlink_consumer group_id: group-gitlink-notification group_id_email: group-gitlink-email group_id_new_email_remind: group-gitlink-new-email-remind auto_offset_reset: earliest max_poll_records: 100 + enable-auto-commit: true + auto-commit-interval: 1S topic: topic-gitlink-notification topic_email: topic-gitlink-email topic_new_email_remind: topic-gitlink-new-email-remind - enable-auto-commit: true - auto-commit-interval: 1S - listener: concurrency: 3 ack-mode: record diff --git a/reader/src/main/resources/application.yml.example b/reader/src/main/resources/application.yml.example index e828e2e..c5dc023 100644 --- a/reader/src/main/resources/application.yml.example +++ b/reader/src/main/resources/application.yml.example @@ -21,7 +21,6 @@ spring: kafka: producer: bootstrap_servers: kafka1:9092,kafka2:9092 - client_id: gitlink_producer_01 retries: 5 batch_size: 16384 diff --git a/writer/src/main/resources/application.yml.example b/writer/src/main/resources/application.yml.example index 0d6bf72..c4d1e3b 100644 --- a/writer/src/main/resources/application.yml.example +++ b/writer/src/main/resources/application.yml.example @@ -8,7 +8,6 @@ spring: kafka: producer: bootstrap_servers: kafka1:9092,kafka2:9092 - client_id: gitlink_producer_01 retries: 5 batch_size: 16384 replication_factor: 1 @@ -28,7 +27,6 @@ spring: min-idle: 0 timeout: 1000 - datasource: driver-class-name: com.mysql.jdbc.Driver url: jdbc:mysql://mysql:3306/gitlink_notification?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&serverTimezone=GMT%2B8&allowMultiQueries=true&useSSL=false From 21969e11b464404d16f8eb0ec9b1dc2c1c69d07f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B7=B4=E6=8B=89=E8=BF=AA=E7=BB=B4?= Date: Fri, 17 Sep 2021 15:04:57 +0800 Subject: [PATCH 4/5] docker scripts alter --- db/gns-notification.sql | 29 +---------------------------- middleware/services.yml | 14 +++++++++----- 2 files changed, 10 insertions(+), 33 deletions(-) diff --git a/db/gns-notification.sql b/db/gns-notification.sql index 9f94f9b..1637c77 100644 --- a/db/gns-notification.sql +++ b/db/gns-notification.sql @@ -35,31 +35,4 @@ ALTER TABLE gitlink_sys_notification ADD COLUMN (`type` TINYINT(4) NOT NULL DEFA -- 2021-09-09 新增 source 字段区分消息来源、新增 extra 字段保存额外信息 ALTER TABLE gitlink_sys_notification ADD source varchar(250) NULL COMMENT '消息来源'; -ALTER TABLE gitlink_sys_notification ADD extra TEXT NULL COMMENT '额外信息（备用字段）'; - -DROP TABLE IF EXISTS `gitlink_email_jobs`; -CREATE TABLE `gitlink_email_jobs` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `sender` int(11) NOT NULL COMMENT '发送者id', - `emails` text NOT NULL COMMENT '收件人全部邮件地址', - `subject` varchar(500) DEFAULT NULL COMMENT '邮件主题', - `content` text NOT NULL COMMENT '邮件内容', - `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `dispatched_at` datetime DEFAULT NULL COMMENT '处理时间', - `dispatched_status` int(11) DEFAULT '-1' COMMENT '发送状态：-1 未处理,1 处理成功,2 处理失败', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; - - -DROP TABLE IF EXISTS `gitlink_email_send_records`; -CREATE TABLE `gitlink_email_send_records` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `email` varchar(500) NOT NULL DEFAULT '' COMMENT '收件人邮件地址', - `job_id` int(11) NOT NULL COMMENT '邮件详情id', - `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `sent_at` datetime DEFAULT NULL COMMENT '发送时间', - `status` int(11) DEFAULT '-1' COMMENT '发送状态：-1 未发送,1 发送成功,2 发送失败', - PRIMARY KEY (`id`), - KEY `index_on_email_and_status` (`email`,`status`), - KEY `index_on_status` (`status`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; \ No newline at end of file +ALTER TABLE gitlink_sys_notification ADD extra TEXT NULL COMMENT '额外信息（备用字段）'; \ No newline at end of file diff --git a/middleware/services.yml b/middleware/services.yml index f05935d..8e59a71 100644 --- a/middleware/services.yml +++ b/middleware/services.yml @@ -61,7 +61,9 @@ services: ports: - ${KAFKA_01_LOCAL_PORT}:29092 volumes: - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}:/var/lib/kafka + - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/lib:/var/lib/kafka + - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/logs:/var/logs/kafka + - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/conf:/etc/kafka environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 @@ -82,7 +84,9 @@ services: ports: - ${KAFKA_02_LOCAL_PORT}:39092 volumes: - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}:/var/lib/kafka + - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/lib:/var/lib/kafka + - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/logs:/var/logs/kafka + - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/conf:/etc/kafka environment: KAFKA_BROKER_ID: 2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 @@ -105,7 +109,7 @@ services: environment: - TZ=Asia/Shanghai volumes: - - ${DOCKER_DATA_PATH}/logs/:/data/logs/ + - ${DOCKER_DATA_PATH}/gitlink/:/data/logs/ depends_on: - kafka1 - kafka2 @@ -126,7 +130,7 @@ services: environment: - TZ=Asia/Shanghai volumes: - - ${DOCKER_DATA_PATH}/logs/:/data/logs/ + - ${DOCKER_DATA_PATH}/gitlink/:/data/logs/ depends_on: - kafka1 - kafka2 @@ -145,7 +149,7 @@ services: networks: - gitlink_network volumes: - - ${DOCKER_DATA_PATH}/logs/:/data/logs/ + - ${DOCKER_DATA_PATH}/gitlink/:/data/logs/ environment: - TZ=Asia/Shanghai depends_on: From ca8035ffdf0fdacd205160a8a36a78d7d7bbbdcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B7=B4=E6=8B=89=E8=BF=AA=E7=BB=B4?= Date: Fri, 17 Sep 2021 15:14:58 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=A4=84=E7=90=86=20-v=20=E6=98=A0?= =?UTF-8?q?=E5=B0=84=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/services.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/middleware/services.yml b/middleware/services.yml index 8e59a71..bb1bb8f 100644 --- a/middleware/services.yml +++ b/middleware/services.yml @@ -46,8 +46,8 @@ services: ZOOKEEPER_TICK_TIME: 2000 ports: - ${ZOOKEEPER_LOCAL_PORT}:2181 - volumes: - - ${DOCKER_DATA_PATH}/zookeeper:/var/lib/zookeeper +# volumes: +# - ${DOCKER_DATA_PATH}/zookeeper:/var/lib/zookeeper networks: - gitlink_network @@ -60,10 +60,10 @@ services: - zookeeper ports: - ${KAFKA_01_LOCAL_PORT}:29092 - volumes: - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/lib:/var/lib/kafka - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/logs:/var/logs/kafka - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/conf:/etc/kafka +# volumes: +# - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/lib:/var/lib/kafka +# - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/logs:/var/logs/kafka +# - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_01_NAME}/conf:/etc/kafka environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 @@ -83,10 +83,10 @@ services: - zookeeper ports: - ${KAFKA_02_LOCAL_PORT}:39092 - volumes: - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/lib:/var/lib/kafka - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/logs:/var/logs/kafka - - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/conf:/etc/kafka +# volumes: +# - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/lib:/var/lib/kafka +# - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/logs:/var/logs/kafka +# - ${DOCKER_DATA_PATH}/kafka/${KAFKA_CONTAINER_02_NAME}/conf:/etc/kafka environment: KAFKA_BROKER_ID: 2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181