SpringBoot uses stream query to realize Mysql million-level data export
A JPA to achieve million-level data export
Reference blog: http://knes1.github.io/blog/2015/2015-10-19-streaming-mysql-results-using-java8-streams-and-spring-data.html
entity
public class Authors {
private Integer id;
private String firstName;
private String lastName;
private String email;
private Date birthdate;
private Date added;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getFirstName() {
return firstName;
}
public void setFirstName(String firstName) {
this.firstName = firstName == null ? null : firstName.trim();
}
public String getLastName() {
return lastName;
}
public void setLastName(String lastName) {
this.lastName = lastName == null ? null : lastName.trim();
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email == null ? null : email.trim();
}
public Date getBirthdate() {
return birthdate;
}
public void setBirthdate(Date birthdate) {
this.birthdate = birthdate;
}
public Date getAdded() {
return added;
}
public void setAdded(Date added) {
this.added = added;
}
@Override
public String toString() {
return this.id + "," + this.firstName + "," + this.lastName + "," + this.email + "," + this.birthdate + "," + this.added;
}
}
Repository
The return type of the method is defined as Stream. Integer.MIN_VALUE tells jdbc driver to return data one by one
@QueryHints(value = @QueryHint(name = HINT_FETCH_SIZE, value = "" + Integer.MIN_VALUE))
@Query(value = "select t from Authors t")
Stream<Authors> streamAll();
Controller
@RequestMapping(value = "/todos.csv", method = RequestMethod.GET)
@Transactional(readOnly = true)
public void exportTodosCSV(HttpServletResponse response) {
response.addHeader("Content-Type", "application/csv");
response.addHeader("Content-Disposition", "attachment; filename=todos.csv");
response.setCharacterEncoding("UTF-8");
try(Stream<Authors> todoStream = todoRepository.streamAll()) {
PrintWriter out = response.getWriter();
todoStream.forEach(rethrowConsumer(todo -> {
String line = todo.toString();
out.write(line);
out.write("\n");
entityManager.detach(todo);
}));
out.flush();
} catch (IOException e) {
log.info("Exception occurred " + e.getMessage(), e);
throw new RuntimeException("Exception occurred while exporting results", e);
}
}
Util tool class
import java.util.function.Consumer;
import java.util.function.Function;
/**
* Utility class to avoid try/catch boiler plate by PaoloC (http://stackoverflow.com/users/2365724/paoloc).
*
* Taken from here:
* http://stackoverflow.com/questions/27644361/how-can-i-throw-checked-exceptions-from-inside-java-8-streams/30974991#30974991
*
* Created on: 10/10/15
*/
public final class LambdaExceptionUtil {
@FunctionalInterface
public interface Consumer_WithExceptions<T, E extends Exception> {
void accept(T t) throws E;
}
@FunctionalInterface
public interface Function_WithExceptions<T, R, E extends Exception> {
R apply(T t) throws E;
}
/**
* .forEach(rethrowConsumer(name -> System.out.println(Class.forName(name))));
*/
public static <T, E extends Exception> Consumer<T> rethrowConsumer(Consumer_WithExceptions<T, E> consumer) throws E {
return t -> {
try {
consumer.accept(t);
} catch (Exception exception) {
throwActualException(exception);
}
};
}
/**
* .map(rethrowFunction(name -> Class.forName(name))) or .map(rethrowFunction(Class::forName))
*/
public static <T, R, E extends Exception> Function<T, R> rethrowFunction(Function_WithExceptions<T, R, E> function) throws E {
return t -> {
try {
return function.apply(t);
} catch (Exception exception) {
throwActualException(exception);
return null;
}
};
}
@SuppressWarnings("unchecked")
private static <E extends Exception> void throwActualException(Exception exception) throws E {
throw (E) exception;
}
}
2. Mybatis implements million-level data export
mapper
To obtain data one by one in MyBatis, you must customize ResultHandler, then add fetchSize="-2147483648" to the corresponding select statement in the mapper.xml file, and finally pass the custom ResultHandler to SqlSession to execute the query, and return the The results are processed
<select id="selectByExample" parameterType="com.alphathur.mysqlstreamingexport.domain.AuthorsExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
'false' as QUERYID,
<include refid="Base_Column_List" />
from authors
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="streamByExample" fetchSize="-2147483648" parameterType="com.alphathur.mysqlstreamingexport.domain.AuthorsExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
'false' as QUERYID,
<include refid="Base_Column_List" />
from authors
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
public interface AuthorsMapper {
List<Authors> selectByExample(AuthorsExample example);
List<Authors> streamByExample(AuthorsExample example); //以stream形式从mysql获取数据
}
service
@Service
public class AuthorsService {
private final SqlSessionTemplate sqlSessionTemplate;
private final AuthorsMapper authorsMapper;
public AuthorsService(SqlSessionTemplate sqlSessionTemplate, AuthorsMapper authorsMapper) {
this.sqlSessionTemplate = sqlSessionTemplate;
this.authorsMapper = authorsMapper;
}
/**
* stream读数据写文件方式
* @param httpServletResponse
* @throws IOException
*/
public void streamDownload(HttpServletResponse httpServletResponse)
throws IOException {
AuthorsExample authorsExample = new AuthorsExample();
authorsExample.createCriteria();
HashMap<String, Object> param = new HashMap<>();
param.put("oredCriteria", authorsExample.getOredCriteria());
param.put("orderByClause", authorsExample.getOrderByClause());
CustomResultHandler customResultHandler = new CustomResultHandler(new DownloadProcessor (httpServletResponse));
sqlSessionTemplate.select(
"com.alphathur.mysqlstreamingexport.mapper.AuthorsMapper.streamByExample", param, customResultHandler);
httpServletResponse.getWriter().flush();
httpServletResponse.getWriter().close();
}
/**
* 传统下载方式
* @param httpServletResponse
* @throws IOException
*/
public void traditionDownload(HttpServletResponse httpServletResponse)
throws IOException {
AuthorsExample authorsExample = new AuthorsExample();
authorsExample.createCriteria();
List<Authors> authors = authorsMapper.selectByExample (authorsExample);
DownloadProcessor downloadProcessor = new DownloadProcessor (httpServletResponse);
authors.forEach (downloadProcessor::processData);
httpServletResponse.getWriter().flush();
httpServletResponse.getWriter().close();
}
}
controller
@RestController
@RequestMapping("download")
public class HelloController {
private final AuthorsService authorsService;
public HelloController(AuthorsService authorsService) {
this.authorsService = authorsService;
}
@GetMapping("streamDownload")
public void streamDownload(HttpServletResponse response)
throws IOException {
authorsService.streamDownload(response);
}
@GetMapping("traditionDownload")
public void traditionDownload(HttpServletResponse response)
throws IOException {
authorsService.traditionDownload (response);
}
}
Util tool class
public class DownloadProcessor {
private final HttpServletResponse response;
public DownloadProcessor(HttpServletResponse response) {
this.response = response;
String fileName = System.currentTimeMillis() + ".csv";
this.response.addHeader("Content-Type", "application/csv");
this.response.addHeader("Content-Disposition", "attachment; filename="+fileName);
this.response.setCharacterEncoding("UTF-8");
}
public <E> void processData(E record) {
try {
response.getWriter().write(record.toString()); //如果是要写入csv,需要重写toString,属性通过","分割
response.getWriter().write("\n");
}catch (IOException e){
e.printStackTrace();
}
}
}
write file operation
Customize our ResultHandler, which is used to obtain java objects, and then pass them to the above DownloadProcessor processing class for file writing operations
public class CustomResultHandler implements ResultHandler {
private final DownloadProcessor downloadProcessor;
public CustomResultHandler(
DownloadProcessor downloadProcessor) {
super();
this.downloadProcessor = downloadProcessor;
}
@Override
public void handleResult(ResultContext resultContext) {
Authors authors = (Authors)resultContext.getResultObject();
downloadProcessor.processData(authors);
}
}