Spark2.0程序中的持久化数据到数据库中foreachPartition()方法报错

版权声明：版权声明中 https://blog.csdn.net/lds_include/article/details/89285180

Spark2.0程序中的持久化数据到数据库中foreachPartition()方法报错

出错的地方是foreachPartition(data2Mysql)中的方法data2Mysql报错

• 代码

val data2Mysql = (it:Iterable[(String, Int)]) => {
    var conn: Connection = null;
    var ps: PreparedStatement = null;
    val sql = "insert into tb_location_info(location,counts,access_date) values (?,?,?)"
    val jdbc = "jdbc:mysql://localhost:3306/db_bigdataSpark?useUnicode=true&characterEncoding=utf8"
    val user = "root"
    val password = "123456"
    try {
      conn = DriverManager.getConnection(jdbc, user, password)
      it.foreach(line => {
        ps = conn.prepareStatement(sql)
        ps.setString(1, line._1)
        ps.setInt(2, line._2)
        ps.setDate(3, new Date(System.currentTimeMillis()))
        ps.executeUpdate()
      })
    } catch {
      case  e: Exception=>println(e.printStackTrace())
    } finally {
      if(ps != null) ps.close()
      if(conn != null) conn.close()
    }
  }

• 出错的地方的代码

//持久化数据到数据库中
    sumd.foreachPartition(data2Mysql)

• 经过检查发现
  
  报错所的是类型出错，spark2.0后的用户的是iterator

• 修改

val data2Mysql = (it:Iterator[(String, Int)]) => {
    var conn: Connection = null;
    var ps: PreparedStatement = null;
    val sql = "insert into tb_location_info(location,counts,access_date) values (?,?,?)"
    val jdbc = "jdbc:mysql://localhost:3306/db_bigdataSpark?useUnicode=true&characterEncoding=utf8"
    val user = "root"
    val password = "123456"
    try {
      conn = DriverManager.getConnection(jdbc, user, password)
      it.foreach(line => {
        ps = conn.prepareStatement(sql)
        ps.setString(1, line._1)
        ps.setInt(2, line._2)
        ps.setDate(3, new Date(System.currentTimeMillis()))
        ps.executeUpdate()
      })
    } catch {
      case  e: Exception=>println(e.printStackTrace())
    } finally {
      if(ps != null) ps.close()
      if(conn != null) conn.close()
    }
  }