Spring Batch Tutorial (6) An example of spring boot implementing batch function annotations: reading files and writing them to mysql

Spring batch series of articles

Spring Batch Tutorial (1) A brief introduction and converting xml files to txt files through
springbatch
Example: read data from mysql and write text and read content from multiple texts and write to mysql
Spring Batch Tutorial (4) Example of tasklet usage: Timing tasks of spring batch Use
Spring Batch Tutorial (5) spring boot to implement batch function Annotation example: read and write text files

---

---

This article implements the reading of text data in the form of annotations, and stores them in mysql after summing.
This article uses jdk8, and higher versions of spring batch require jdk11 environment.

1. Example: read the content of the text file, write the sum to mysql after calculation

1. maven dependency

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-batch</artifactId>
			<version>2.3.12.RELEASE</version>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
			<version>1.18.2</version>
			<scope>provided</scope>
		</dependency>
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.31</version>
		</dependency>
		<dependency>
			<groupId>com.mchange</groupId>
			<artifactId>c3p0</artifactId>
			<version>0.9.5-pre8</version>
		</dependency>

2、java bean

1）、Student

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * 
 * @author alanchan
 *
 */
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Student {
    
    
	private String id;
	private int chinese;
	private int math;
	private int english;

}

2), StudentTotalScore

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * 
 * @author alanchan
 *
 */
@Data
@AllArgsConstructor
@NoArgsConstructor
public class StudentTotalScore {
    
    
	private String id;
	private int totalScore;

}

3. Configure ItemReader, ItemWriter and ItemProcessor of spring batch

import java.beans.PropertyVetoException;

import javax.sql.DataSource;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import org.springframework.jdbc.core.JdbcTemplate;

import com.mchange.v2.c3p0.ComboPooledDataSource;
import com.win.writedbbyannotation.bean.Student;
import com.win.writedbbyannotation.bean.StudentTotalScore;

/**
 * 
 * @author alanchan
 *
 */
@Configuration
@EnableBatchProcessing
public class BatchConfiguration {
    
    
	private final String inputFiles = "student-data.txt";

	@Bean
	public ItemReader<Student> reader() {
    
    
		FlatFileItemReader<Student> reader = new FlatFileItemReader<Student>();
		reader.setResource(new ClassPathResource(inputFiles));
		reader.setLineMapper(new DefaultLineMapper<Student>() {
    
    
			{
    
    
				setLineTokenizer(new DelimitedLineTokenizer() {
    
    
					{
    
    
						setNames(new String[] {
    
     "id", "chinese", "math", "english" });
					}
				});
				setFieldSetMapper(new BeanWrapperFieldSetMapper<Student>() {
    
    
					{
    
    
						setTargetType(Student.class);
					}
				});
			}
		});
		return reader;
	}

	@Bean
	public ItemWriter<StudentTotalScore> writer(DataSource dataSource) {
    
    
		JdbcBatchItemWriter<StudentTotalScore> writer = new JdbcBatchItemWriter<StudentTotalScore>();
		writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<StudentTotalScore>());
		writer.setSql("INSERT INTO studenttotalscore (id,totalscore) VALUES (:id,:totalScore)");
		writer.setDataSource(dataSource);
		return writer;
	}

	@Bean
	public ItemProcessor<Student, StudentTotalScore> processor() {
    
    
		return new StudentItemProcessor();
	}

	@Bean
	public Job createMarkSheet(JobBuilderFactory jobs, Step step) {
    
    
		return jobs.get("createMarkSheet").flow(step).end().build();
	}

	@Bean
	public Step step(StepBuilderFactory stepBuilderFactory, ItemReader<Student> reader, ItemWriter<StudentTotalScore> writer, ItemProcessor<Student, StudentTotalScore> processor) {
    
    
		return stepBuilderFactory.get("step").<Student, StudentTotalScore>chunk(5).reader(reader).processor(processor).writer(writer).build();
	}

	@Bean
	public JdbcTemplate jdbcTemplate(DataSource dataSource) {
    
    
		return new JdbcTemplate(dataSource);
	}

	@Bean
	public DataSource getDataSource() throws PropertyVetoException {
    
    
		ComboPooledDataSource dataSource = new ComboPooledDataSource();
		dataSource.setDriverClass("com.mysql.jdbc.Driver");
		dataSource.setJdbcUrl("jdbc:mysql://192.168.10.44:3306/test");
		dataSource.setUser("root");
		dataSource.setPassword("12345");
		return dataSource;
	}
}

4. Create an ItemProcessor implementation class

import org.springframework.batch.item.ItemProcessor;

import com.win.writedbbyannotation.bean.Student;
import com.win.writedbbyannotation.bean.StudentTotalScore;


/**
 * 
 * @author alanchan
 *
 */
public class StudentItemProcessor implements ItemProcessor<Student, StudentTotalScore> {
    
    

	@Override
	public StudentTotalScore process(final Student student) throws Exception {
    
    
		int totalScore = student.getChinese() + student.getMath() + student.getEnglish();
		System.out.println("student id:" + student.getId() + " and Total score:" + totalScore);

		StudentTotalScore studentTotalScore = new StudentTotalScore(student.getId(), totalScore);
		return studentTotalScore;
	}

}

5. Create a main class to run the job

import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;

import com.win.writedbbyannotation.bean.StudentTotalScore;

/**
 * 
 * @author alanchan
 *
 */

@ComponentScan
@EnableAutoConfiguration
//@SpringBootApplication
public class App {
    
    

	public static void main(String[] args) {
    
    
		ApplicationContext ctx = SpringApplication.run(App.class, args);
		List result = ctx.getBean(JdbcTemplate.class).query("select id,totalscore FROM StudentTotalScore", new RowMapper() {
    
    
			@Override
			public StudentTotalScore mapRow(ResultSet rs, int row) throws SQLException {
    
    
				return new StudentTotalScore(rs.getString(1), Integer.parseInt(rs.getString(2)));
			}
		});
		System.out.println("记录数:" + result.size());
	}

}

6. Prepare test data

Create a student-data.txt file under the resources folder in the project and write the following content

student-1,90,85,96
student-2,92,97,94
student-3,95,93,100

7. Create database and tables

SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;

-- ----------------------------
-- Table structure for studenttotalscore
-- ----------------------------
DROP TABLE IF EXISTS `studenttotalscore`;
CREATE TABLE `studenttotalscore`  (
  `id` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
  `totalScore` int(11) NULL DEFAULT NULL
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;

8. Verification

Start app.java, and then observe the changes in the application console and data in the mysql database.
The following exceptions may occur when starting the application:

Table 'XXX.BATCH_JOB_INSTANCE' doesn't exist

Because Spring Batch will enable an H2 database, in this database, Sping will configure the configuration required by Batch. If you use Spring JPA, you need Spring Batch to help you initialize the table.
Solution:
Add the following configuration to the application.properties file

spring.batch.initialize-schema=ALWAYS
# 或  Spring Boot 2.7 的版本中
spring.batch.jdbc.initialize-schema=ALWAYS

1), application output results

Console partial output

 Scheduler class: 'org.quartz.core.QuartzScheduler' - running locally.
  NOT STARTED.
  Currently in standby mode.
  Number of jobs executed: 0
  Using thread pool 'org.quartz.simpl.SimpleThreadPool' - with 10 threads.
  Using job-store 'org.quartz.simpl.RAMJobStore' - which does not support persistence. and is not clustered.

2023-07-24 11:08:33.785  INFO 100012 --- [           main] org.quartz.impl.StdSchedulerFactory      : Quartz scheduler 'quartzScheduler' initialized from an externally provided properties instance.
2023-07-24 11:08:33.785  INFO 100012 --- [           main] org.quartz.impl.StdSchedulerFactory      : Quartz scheduler version: 2.2.1
2023-07-24 11:08:33.785  INFO 100012 --- [           main] org.quartz.core.QuartzScheduler          : JobFactory set to: org.springframework.scheduling.quartz.SpringBeanJobFactory@73d60e76
2023-07-24 11:08:33.804  INFO 100012 --- [           main] o.s.s.quartz.SchedulerFactoryBean        : Starting Quartz Scheduler now
2023-07-24 11:08:33.804  INFO 100012 --- [           main] org.quartz.core.QuartzScheduler          : Scheduler quartzScheduler_$_NON_CLUSTERED started.
2023-07-24 11:08:33.812  INFO 100012 --- [           main] com.win.writedbbyannotation.App          : Started App in 1.949 seconds (JVM running for 2.226)
2023-07-24 11:08:33.814  INFO 100012 --- [           main] o.s.b.a.b.JobLauncherApplicationRunner   : Running default command line with: []
2023-07-24 11:08:33.936  INFO 100012 --- [           main] o.s.b.c.l.support.SimpleJobLauncher      : Job: [FlowJob: [name=createMarkSheet]] launched with the following parameters: [{
    
    }]
2023-07-24 11:08:33.971  INFO 100012 --- [           main] o.s.batch.core.job.SimpleStepHandler     : Step already complete or not restartable, so no action to execute: StepExecution: id=1, version=3, name=step, status=COMPLETED, exitStatus=COMPLETED, readCount=3, filterCount=0, writeCount=3 readSkipCount=0, writeSkipCount=0, processSkipCount=0, commitCount=1, rollbackCount=0, exitDescription=
2023-07-24 11:08:33.979  INFO 100012 --- [           main] o.s.b.c.l.support.SimpleJobLauncher      : Job: [FlowJob: [name=createMarkSheet]] completed with the following parameters: [{
    
    }] and the following status: [COMPLETED] in 21ms
记录数:3

2), program function verification

The data source file can be read normally, and the data of each student can be summed and stored in mysql.

Above, the content of the text file is read, summed, and stored in the mysql database. This work is done through annotations.