SpringBootSpringBatch实现批处理
Spring Boot & Spring Batch 实现批处理
需求领域
⾃动，复杂地处理⼤量信息，⽆需⽤户交互即可最有效地进⾏处理。这些操作通常包括基于时间的事件(例如⽉末计算，通知或通信)。
定期应⽤⾮常⼤的数据集(例如，保险利益确定或费率调整)重复处理复杂的业务规则。
从内部和外部系统接收的信息的集成，通常需要格式化，验证和以事务⽅式进⾏的处理到记录系统中。批处理每天⽤于为企业处理数⼗亿笔事务
Spring Batch介绍
核⼼流程图如下：
Spring Batch 是⼀个轻量级的，全⾯的批处理框架，旨在⽀持开发对企业系统的⽇常运⾏⾄关重要的强⼤的批处理应⽤程序，提供了可重⽤的功能，这些功能对于处理⼤量记录⾄关重要，可⽤于简单的⽤例(例如，将⽂件读⼊数据库或运⾏存储过程)，也可以⽤于复杂的⼤量⽤例(例如，在数据库之间移动⼤量数据，对其进⾏转换等)。上)。⼤量批处理作业可以以⾼度可扩展的⽅式利⽤框架来处理⼤量信息
总结起来就是：量⼤，复杂，流程化数据处理利器！
实现需求
1、通过远程接⼝访问得到告警信息的原始数据
2、对数据进⾏清洗，聚合进⾏本地化操作，如：发送MQ，保存本地库等
核⼼依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="/POM/4.0.0"xsi="/2001/XMLSchema-instance"
schemaLocation="/POM/4.0.0 /xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.1</version>
<relativePath />
</parent>
<groupId>***</groupId>
<artifactId>***</artifactId>
<version>0.0.1-SNAPSHOT</version>
<description>***</description>
<properties>
<java.version>17</java.version>
<spring-cloud.version>2020.0.3</spring-cloud.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring-cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-batch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-config</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-bus-amqp</artifactId> </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
springboot架构图<artifactId>spring-boot-starter-data-jdbc</artifactId> </dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<!--阿⾥云OSS依赖-->
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.10.2</version>
<exclusions>
<exclusion>
<groupId&dehaus.jettison</groupId>
<artifactId>jettison</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>3.4.3.1</version>
</dependency>
<dependency>
<groupId>org.apachemons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.11</version>
</dependency>
</dependencies>
</project>
Spring 配置
spring:
application:
name:***
batch:
job:
#默认⾃动执⾏定义的Job(true)，改为false，需要jobLaucher.run执⾏
enabled:false
jdbc:
#spring batch在数据库⾥⾯创建默认的数据表，如果不是always则会提⽰相关表不存在
initialize-schema: always
datasource:
driver-class-name: sql.cj.jdbc.Driver
url:***
username:***
password:***
hikari:
connection-test-query: SELECT 1
rabbitmq:
listener:
simple:
missing-queues-fatal:false
addresses:***
port:***
username:***
password:***
virtual-host:***
#--------------------mybatis配置----------------------------------
mybatis-plus:
mapper-locations:
- classpath*:mapper/*.xml
# 以下配置均有默认值，可以不设置
global-config:
db-config:
#主键类型 auto:"数据库ID⾃增" 1:"⽤户输⼊ID",2:"全局唯⼀ID (数字类型唯⼀ID)", 3:"全局唯⼀ID UUID";
id-type: auto
#字段策略 IGNORED:"忽略判断" NOT_NULL:"⾮ NULL 判断") NOT_EMPTY:"⾮空判断"
# field-strategy: NOT_EMPTY
#数据库类型
# db-type: MYSQL
configuration:
# 是否开启⾃动驼峰命名规则映射：从数据库列名到Java属性驼峰命名的类似映射
map-underscore-to-camel-case:true
# 如果查询结果中包含空值的列，则 MyBatis 在映射的时候，不会映射这个字段
call-setters-on-nulls:true
# 这个配置会将执⾏的sql打印出来，在开发或测试的时候可以⽤
# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
Bean
/**
* 运⾏超时告警原始数据VO
* @author huxiang
* @date 2022-01-05
*/
@Data
public class RunningOverVO implements Serializable {
/
** 版本号 */
private static final long serialVersionUID =1L;
private Annotations annotations;
private Date endsAt;
private String fingerprint;
private List<RunningOverVO.Receivers> receivers; private Date startsAt;
private Status status;
private Date updatedAt;
private String generatorURL;
private Labels labels;
@Data
public class Labels {
private String acpus;
private String alertname;
private String comment;
private String dc;
private String to;
private String instance;
private String job;
private String jobid;
private String jobname;
private String lc;
private String lnodes_s;
private String nnodes;
private String priority;
private String queue;
private String service;
private String severity;
private String ss;
private Date start;
private String state;
private Date submit;
private String user;
}
@Data
static class Receivers {
private String name;
}
@Data
class Annotations {
private String description;
private String summary;
}
@Data
class Status {
private List<String> inhibitedBy;
private List<String> silencedBy;
private String state;
}
}
版权声明：本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系QQ:729038198，我们将在24小时内删除。
发表评论