idea大数据爬取招聘信息存入数据库MySQL

在这里插入图片描述
dao 主要是一些接口，里面定义了一些用于增删改查的方法名
daoImpl 就是对dao的具体实现
Service 也是一些接口，主要是用来调用dao层的一些方法，所以这里定义的方法一般都定义好了参数；
serviceImp 对service的具体实现，这里会调用dao 的一些方法
servlet 调用 Service调用其方法，做一些操作
jsp 视图显示和输入数据
DAO层是主要是将数据保存到数据库中；
service是主要是处理业务的逻辑
serviceImpl 在实现类中实现这个方法
在这里插入图片描述

package com.ld.jsoup.beans;

public class Job {
    private int id;//主键
    private String title;//标题
    private String company;//公司名称
    private String jobDes;//职位描述
	//Alt+Insert-->Getter and Setter
    public int getId() {
        return id;
    }

    public void setId(int id) {
        this.id = id;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String getCompany() {
        return company;
    }

    public void setCompany(String company) {
        this.company = company;
    }

    public String getJobDes() {
        return jobDes;
    }

    public void setJobDes(String jobDes) {
        this.jobDes = jobDes;
    }
}

在这里插入图片描述

package com.ld.jsoup.dao;

import com.ld.jsoup.beans.Job;

public interface JobDao {
    //添加招聘信息
    public int insertJob(Job job);
    //查询工作描述
}

在下述做操作前先把数据库打开，建好表
1.打开任务管理器开启MySQL服务
在这里插入图片描述
2.连接到数据库，建表
注意：Auto Increment打上勾，不然运行会报错；
varchar的字符设置为utf8下面为utf8_general_ci

package com.ld.jsoup.dao;

import java.sql.*;

public class BaseDao {
    private static Connection conn;

    //获取connaction
    public static Connection getConnection() {
        String driver = "com.mysql.jdbc.Driver";
        String url = "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=utf-8";
        String user = "root";
        String password = "123";
        try {
            Class.forName(driver);
            conn = DriverManager.getConnection(url,user,password);
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return conn;
    }

    //执行查询的方法
    public static ResultSet executQury(Connection connection,String sql,Object...params) {
        ResultSet rs = null;
        PreparedStatement ps = null;
        try {
            ps = connection.prepareStatement(sql);
            for(int i = 0; i < params.length; i++) {
                ps.setObject(i+1,params[i]);
            }
            rs = ps.executeQuery();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return rs;
    }

    //删除、修改、更新
    public static int executUpdate(Connection connection,String sql,Object...params) {
        int result = 0;
        PreparedStatement ps = null;
        try {
            ps = connection.prepareStatement(sql);
            for(int i = 0; i < params.length; i++) {
                ps.setObject(i+1,params[i]);
            }
            result = ps.executeUpdate();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return result;
    }

    //关闭
    public static void closeAll(Connection connection,PreparedStatement ps,ResultSet rs) {
        if(connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        if(ps != null) {
            try {
                ps.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        if(rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
}

在这里插入图片描述

package com.ld.jsoup.dao.impl;

import com.ld.jsoup.beans.Job;
import com.ld.jsoup.dao.BaseDao;
import com.ld.jsoup.dao.JobDao;

import java.sql.Connection;

public class JobDaoImpl extends BaseDao implements JobDao{
    //实际保存数据操作
    public int insertJob(Job job) {
        Connection connection = BaseDao.getConnection();
        String sql = "insert into job(title,company,jobDes)" + "values(?,?,?)";

        int result = BaseDao.executUpdate(connection, sql, job.getTitle(), job.getCompany(), job.getJobDes());
        BaseDao.closeAll(connection,null,null);
        return result;
    }
}

在这里插入图片描述

package com.ld.jsoup.service.impl;

import com.ld.jsoup.service.JobService;
import com.ld.jsoup.beans.Job;
import com.ld.jsoup.dao.impl.JobDaoImpl;


public class JobServiceImpl implements JobService {
    private JobDaoImpl jobDao;
    //保存数据
    public int insertJob(Job job) {
        jobDao = new JobDaoImpl();
        return jobDao.insertJob(job);
    }
}

在这里插入图片描述

package com.ld.jsoup.service;

import com.ld.jsoup.beans.Job;

public interface JobService {
    //保存招聘信息
    public int insertJob(Job job);
}

在这里插入图片描述

package com.ld.jsoup.servlet;

import com.ld.jsoup.beans.Job;
import com.ld.jsoup.service.impl.JobServiceImpl;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class JobServlet extends HttpServlet {
    @Override
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        request.setCharacterEncoding("UTF-8");
        response.setContentType("java/html;charset=UTF-8");
        final JobServiceImpl jobService = new JobServiceImpl();
        //获取PrintWriter对象设置响应文本
        PrintWriter out = response.getWriter();
        //创建线程池
        ExecutorService executorService = Executors.newFixedThreadPool(5);
        for(int i = 1; i <= 10 ; i++) {
            String url = "https://search.51job.com/list/120400,000000,0000,00,9,99,java,2," + i + ".html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
            try {
                //connect访问某个网址，get获取Document对象
                final Document document = Jsoup.connect(url).get();
                executorService.execute(new Runnable() {
                    public void run() {
                        //select方法根据选择器获取元素集合elements
                        Elements elements = document.select("p.t1 span a");
                        Document doc = null;
                        for(Element element:elements) {
                            String absUrl =  element.absUrl("href");
                            Job job = new Job();
                            String title="";
                            String company="";
                            String jobDes="";
                            try {
                                URL u = new URL(absUrl);
                                doc = Jsoup.parse(u,2000);
                                //doc = Jsoup.connect(absUrl).get();
                                if(doc.selectFirst("div.research") == null){
                                    if(doc.selectFirst("div.cn h1") !=null){
                                        title = doc.selectFirst("div.cn h1").text();
                                    }
                                    if(doc.selectFirst("p.cname a")!=null){
                                        company = doc.selectFirst("p.cname a").text();
                                    }
                                    if(doc.selectFirst("div.bmsg.job_msg.inbox")!=null){
                                        jobDes = doc.selectFirst("div.bmsg.job_msg.inbox").text();
                                    }
                                    job.setCompany(company);
                                    job.setTitle(title);
                                    job.setJobDes(jobDes);
                                    jobService.insertJob(job);
                                }
                            } catch (IOException e) {
                                //e.printStackTrace();
                                continue;
                            }
                        }
                    }
                });
            } catch (IOException e) {
                e.printStackTrace();
            }
        }//for
        out.print("success");
        out.close();
    }

    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        doGet(request,response);
    }
}

index.jsp已经在上一篇博客写过了，在这里代码就不赘述了，只需要在index.jsp里面加上下面代码就可以运行了

<a href="javascript:ajaxRequest3();">爬取到数据库</a>

爬取结果如下：
在这里插入图片描述
这次新建的包和文件有点多，学的我脑袋瓜子嗡嗡的，一定要看好上面的路径，不要建错
同志们，继续加油啊！

idea大数据爬取招聘信息存入数据库MySQL

猜你喜欢