Use HttpClient + Json parser crawling and data stored in the database

First, the questions asked

Description: I am here only to show the code crawling data, combined with the visual results of statistical data, real-time display the most current data will only thing I read this blog I use Echarts code and database data visualization can be combined

Second, the idea

Students thinking: most of my classmates idea is to use HttpClient + Jsoup acquire and parse the data of the target page, and then call the method Dao layers of screening and data stored in the database

My thoughts: I direct the use of a landing page a Json latest outbreak data online, and then use HttpClient to obtain the Json data format, then use Json parser

Data separation, and then call the method Dao filter layer and the data stored in the database.

Thoughts: individuals feel a bit lazy, but I feel as long as the completion of the purpose of it

Third, the source code

Dao.java

package Dao;


import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;


import DBUtil.DBUtil;
import Test.Yione;



public class Dao {

    public boolean add(int id,String provinceName,String confirmeNum,String curesNum, String deathsNum )
       {
             Connection conn = DBUtil.getConn();
             PreparedStatement pstmt = null;
             boolean f = false;
             int a=0;
             try {
                 String sql = "insert into todaydata_copy1(id,provinceName,confirmeNum,curesNum,deathsNum value(?,?,?,?,?)";
                 pstmt = conn.prepareStatement(sql);
                 pstmt.setInt(1, id);
                 pstmt.setString(2, provinceName);
                 pstmt.setString(3, confirmeNum);
                 pstmt.setString(4, curesNum);
                 pstmt.setString(5, deathsNum );
                a = pstmt.executeUpdate();
             }
             catch(SQLException e) {
                 e.printStackTrace();
             }
             finally {
                 DBUtil.close(pstmt, conn); 
             }
             if(a>0)
             f=true;
             
             return f;
       }
        
       
}

View Code

DBUtil.java (connection pooling code, everyone's a little different)

package DBUtil;

import java.sql.*;

/**
 * 数据库连接工具
 * @author Hu
 *
 */
public class DBUtil {
    
    public static String url =  "jdbc:mysql://localhost:3306/yiqing?serverTimezone=UTC";
    public static String user = "root";
    public static String password = "123";
    
    public static Connection getConn () {
        Connection conn = null;
        
        try {
            Class.forName("com.mysql.cj.jdbc.Driver");//加载驱动
            conn = DriverManager.getConnection(url, user, password);
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return conn;
    }
    
    /**
     * 关闭连接
     * @param state
     * @param conn
     */
    public static void close (PreparedStatement preparedState, Connection conn) {
        if (preparedState != null) {
            try {
                preparedState.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
    
    public static void close (ResultSet rs, PreparedStatement preparedState, Connection conn) {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        
        if (preparedState != null) {
            try {
                preparedState.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 关闭连接
     * @param state
     * @param conn
     */
    public static void close (Statement state, Connection conn) {
        if (state != null) {
            try {
                state.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
    
    public static void close (ResultSet rs, Statement state, Connection conn) {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        
        if (state != null) {
            try {
                state.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
    
    public static void main(String[] args) throws SQLException {
        Connection conn = getConn();
        PreparedStatement preparedStatement = null;
        ResultSet rs = null;
        SQL String = "SELECT * from testr" ; 
        the preparedStatement =conn.prepareStatement (SQL); 
        RS = PreparedStatement.executeQuery ();
         IF (rs.next ()) { 
            System.out.println ( "Database empty" ); 
        } 
        the else { 
            System.out.println ( "database is not empty" ); 
        } 
    } 
}

View Code

Javabean part of the code

package Test;

public class Yione {
 private int id;
 private String provinceName;
 private String confirmeNum;
 private String curesNum;
 private String deathsNum;
public int getId() {
    return id;
}
public void setId(int id) {
    this.id = id;
}
public String getProvinceName() {
    return provinceName;
}
public void setProvinceName(String provinceName) {
    this.provinceName = provinceName;
}
public String getConfirmeNum() {
    return confirmeNum;
}
public void setConfirmeNum(String confirmeNum) {
    this.confirmeNum = confirmeNum;
}
public String getCuresNum() {
    return curesNum;
}
public void setCuresNum(String curesNum) {
    this.curesNum = curesNum;
}
public String getDeathsNum() {
    return deathsNum;
}
public void setDeathsNum(String deathsNum) {
    this.deathsNum = deathsNum;
}



}

View Code

HttpClient used to acquire the data format of the code Json

package Test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class getEntity {
    public static String get(){
        //创建客户端
        CloseableHttpClient httpClient = HttpClients.createDefault();
        
        String entityr = "";
        //创建Get实例
        HttpGet httpGet = newHttpGet ( "http://www.dzyong.top:3005/yiqing/province" ); 
        
        // add information request headers to simulate browser access 
        httpGet.addHeader ( "User-Agent", "Mozilla / 5.0 (Windows NT 10.0; Win64; x64-; RV: 73.0) the Gecko / Firefox 20,100,101 / 73.0 " ); 

        the try {
             // get the Response 
            CloseableHttpResponse Response = httpClient.execute (HttpGet); 
            
            IF (response.getStatusLine () getStatusCode () == 200 is. ) {
                 // when the response status code is 200, and obtains the source page print 
                String = EntityUtils.toString Entity (response.getEntity (), "UTF-. 8" ); 
                entityr = Entity; 
            } 
            
        }catch(Exception e){
            e.printStackTrace();
        }
        return entityr;
    }

}

View Code

Parsing the data and the function code stored in the primary database

package Test;

import java.util.List;
import com.alibaba.fastjson.JSON;

import Dao.Dao;


public class Demo {

    public static void main(String[] args){
        Dao dao=new Dao();
        String str = getEntity.get();
        int index=str.indexOf("[");
        int index2=str.indexOf("]");
        String str1 = str.substring(index-1,index2+1);
        //字符串序列化成集合
        List<Yione> list= JSON.parseArray(str1,Yione.class);
        for(Yione item: list){
            System.out.println(item.getId() );
            System.out.println(item.getProvinceName());
            System.out.println(item.getConfirmeNum());
            System.out.println(item.getCuresNum());
            System.out.println(item.getDeathsNum());
           dao.add(item.getId(),item.getProvinceName(),item.getConfirmeNum(),item.getCuresNum(),item.getDeathsNum());
        }    
    }
}

View Code

Fourth, the results show

Because of this last effect is only a data table filled with data, do not show

Use HttpClient + Json parser crawling and data stored in the database

Guess you like