为了支持对关系型数据库的批量取数，防止内存溢出；调研了mysql、postgresql、sqlserver、oracle数据库的fetchSize设置；

数据库连接和处理的代码大同小异，首先贴一份可运行代码；

@Slf4j
public class TestFetchMysql {

    private Connection connection;

    private String driver;
    private String url;
    private String userName;
    private String password;

    public TestFetchMysql(String driver, String url, String userName, String password) {
        this.driver = driver;
        this.url = url;
        this.userName = userName;
        this.password = password;
    }

    private static void close(ResultSet rs, Statement st, PreparedStatement ps, Connection conn) {
        try {
            if (rs != null) rs.close();
            if (st != null) st.close();
            if (ps != null) ps.close();
            if (conn != null) conn.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    private synchronized Connection getConnection(){
        try {
            if(connection == null || connection.isClosed()){
                Class.forName(driver);
                connection = DriverManager.getConnection(url, userName, password);
            }
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return connection;
    }

    public ArrayNode query(String query){
        Connection connection = getConnection();
        PreparedStatement preparedStatement = null;
        ResultSet resultSet = null;
        ArrayNode result = null;
        try {
            //preparedStatement = connection.prepareStatement(query, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
            preparedStatement = connection.prepareStatement(query);
            preparedStatement.setFetchSize(Integer.MIN_VALUE);
            //preparedStatement.setFetchDirection(ResultSet.FETCH_REVERSE);
            resultSet = preparedStatement.executeQuery();
            printMem();
            result = toJsonStr(resultSet);
            printMem();
        } catch (SQLException e) {
            e.printStackTrace();
        }finally {
            close(resultSet, null, preparedStatement, null);
        }
        log.info("查找sql [{}] {} 查询结果 {}", query, System.getProperty("line.separator"), result);
        return result;
    }

    public static ArrayNode toJsonStr(ResultSet resultSet) throws SQLException {
        ObjectMapper objectMapper = new ObjectMapper();
        ArrayNode arrayNode = objectMapper.createArrayNode();

        ResultSetMetaData metaData = resultSet.getMetaData();
        int columnCount = metaData.getColumnCount();

        int count = 0;
        int sum = 0;
        while(resultSet.next()){
            count++;
            ObjectNode objectNode = objectMapper.createObjectNode();
            for(int i = 1; i <= columnCount; i++){
                String columnName  = metaData.getColumnLabel(i);
                String value = resultSet.getString(columnName);
                objectNode.put(columnName, value);
            }
            arrayNode.add(objectNode);
            if(count % 100 == 0){
                sum++;
                log.info("100个结果集 {}", sum);
                arrayNode = objectMapper.createArrayNode();
            }
        }
        return arrayNode;
    }

    public static void printMem(){
        log.info("totalMemory:{}, maxMemory:{}, freeMemory:{}", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(),  Runtime.getRuntime().totalMemory());
    }

    public static void main(String[] args) {
       
        TestFetchMysql mysqlClient = new TestFetchMysql(DbType.getDriver("MYSQL") , "jdbc:mysql://127.0.0.1:3306/test_fetch", "test", "123456");

        ArrayNode query = mysqlClient.query("select * from test");
        log.info("打印下行数：" + query.size());
    }
}

DbType配置类

public enum DbType {

    SQLSERVER("SQLSERVER", "com.microsoft.sqlserver.jdbc.SQLServerDriver"),
    POSTGRESQL("POSTGRESQL", "org.postgresql.Driver"),
    MYSQL("MYSQL", "com.mysql.jdbc.Driver"),
    ORACLE("ORACLE", "oracle.jdbc.driver.OracleDriver");

    private String name;
    private String driver;

    DbType(String name, String driver) {
        this.name = name;
        this.driver = driver;
    }

    public static String getDriver(String name){
        for(DbType dbType : DbType.values()){
            if(dbType.getName().equals(name)){
                return dbType.getDriver();
            }
        }
        return name;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getDriver() {
        return driver;
    }

    public void setDriver(String driver) {
        this.driver = driver;
    }

    public static void main(String[] args) {
      //System.out.println( getDriver("POSTGRESQL") );
    }

测试是否生效手段

①设置ide运行参数-Xms8m -Xmx8m -Dfile.encoding=UTF-8

②printMem方法打印运行内存

③注释toJsonStr方法的以下代码，不生效内存溢出的会在执行sql得到resultSet的时刻，注释掉会在拼接Json的时刻

        if(count % 100 == 0){
                sum++;
                log.info("100个结果集 {}", sum);
                arrayNode = objectMapper.createArrayNode();
            }

下面是结论，测试过程可以自己复现

1、Mysql fetchSize设置

mysql版本5.7，配置参数如下

    preparedStatement.setFetchSize(Integer.MIN_VALUE);

网上教程需要设置一下两个参数，实际发现不需要

connection.prepareStatement(query, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
preparedStatement.setFetchDirection(ResultSet.FETCH_REVERSE);

必须设置 preparedStatement.setFetchSize(Integer.MIN_VALUE); 设置setFetchSize =20 不起作用会内存溢出

2、postgresql fetchSize设置

pg版本9.2.24，配置参数如下，设置自动提交为false,不设置FetchSize的设置会无效；

fetchSize按照自己的需要设置大小，一般来说越大越快

connection.setAutoCommit(false);  
preparedStatement.setFetchSize(1000);

*     类似mysql的设置会报错 报错fetch size必须大于0 或者 操作要求可卷动的 ResultSet，但此 ResultSet 是 FORWARD_ONLY。
*     connection.prepareStatement(query, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
*     preparedStatement.setFetchDirection(ResultSet.FETCH_REVERSE);
*     preparedStatement.setFetchSize(Integer.MIN_VALUE)