Agregar dependencia
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.27</version>
</dependency>
Leer datos de MySQL
val conf: SparkConf = new SparkConf().setAppName(this.getClass.getName).setMaster("local[*]")
val sc = new SparkContext(conf)
val driver = "com.mysql.jdbc.Driver"
val url = "jdbc:mysql://localhost:3306/student"
val userName = "root"
val passWord = "root"
val rdd: JdbcRDD[(Int, String)] = new JdbcRDD(
sc,
() => {
Class.forName(driver)
DriverManager.getConnection(url, userName, passWord)
},
"select * from stuinfo where id>=? and id<=?",
1000,
2000,
1,
rs => {
(rs.getInt(1), rs.getString(2))
}
)
rdd.foreach(println)
sc.stop()
Insertar datos en MySQL
val conf: SparkConf = new SparkConf().setAppName(this.getClass.getName).setMaster("local[*]")
val sc = new SparkContext(conf)
val rdd: RDD[(String, String)] = sc.makeRDD(List(("Tom", "男"), ("Jerry", "男"),
("柯南", "男"), ("毛利兰", "女")
))
val driver = "com.mysql.jdbc.Driver"
val url = "jdbc:mysql://localhost:3306/student"
val userName = "root"
val passWord = "root"
rdd.foreachPartition(
datas => {
Class.forName(driver)
val conn: Connection = DriverManager.getConnection(url, userName, passWord)
val sql = "insert into stuinfo(sname,gender) values(?,?)"
val ps: PreparedStatement = conn.prepareStatement(sql)
datas.foreach {
case (name, gender) => {
ps.setString(1, name)
ps.setString(2, gender)
ps.executeUpdate()
}
}
ps.close()
conn.close()
})
sc.stop()