Spark操作JDBC将数据存入到MySQL并读取出来

Spark操作JDBC-API实现将数据存入到MySQL并读取出来(在小海豚里面操作)
代码:

bject mysql {
  def main(args: Array[String]): Unit = {
    //Spark支持通过Java JDBC访问关系型数据库。需要使用JdbcRDD
    //创建
    val conf = new SparkConf().setAppName("mysql").setMaster("local[*]")
    val sc = new SparkContext(conf)
    //插入数据
   val data: RDD[(String, Int)] = sc.parallelize(List(("aaa",18),("bbb",19),("ccc",20)))
    //调用foreachPartition针对每一个分区进行操作
   data.foreachPartition(saveToMySQL)
    //读取数据
    def getConn():Connection={
   DriverManager.getConnection("jdbc:mysql://localhost:3306/test?characterEncoding=UTF-8", "root", "root")
    }

    val studentRDD: JdbcRDD[(Int, String, Int)] = new JdbcRDD(sc,
      getConn,
      "select * from student where id >= ? and id <= ? ",
      4,
      6,
      2,
      rs => {
        val id: Int = rs.getInt("id")
        val name: String = rs.getString("name")
        val age: Int = rs.getInt("age")
        (id, name, age)
      }
    )
    println(studentRDD.collect().toBuffer)
  }
    def saveToMySQL(partitionDate:Iterator[(String,Int)]):Unit={
      //将数据存入到MySQL
      //获取数据连接
      val conn:Connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/test?characterEncoding=UTF-8", "root", "root")
      partitionDate.foreach(data=>{
        //将每一条数据存入到MySQL
        val sql = "INSERT INTO `student` (`id`, `name`, `age`) VALUES (NULL, ?, ?);"
        val ps: PreparedStatement = conn.prepareStatement(sql)
        ps.setString(1,data._1)
        ps.setInt(2,data._2)
        ps.execute()
      })
      conn.close()
    }
}

结果:
在这里插入图片描述

发布了213 篇原创文章 · 获赞 406 · 访问量 24万+

猜你喜欢

转载自blog.csdn.net/qq_45765882/article/details/105379366
今日推荐