SparkStreaming版本的wordcount(updatestateByKey)保存在mysql

1. package sparkstreamday01.SparkStream
3. import java.sql.{Connection, DriverManager, PreparedStatement, ResultSet}
4. import org.apache.log4j.{Level, Logger}
5. import org.apache.spark.SparkConf
6. import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
7. import org.apache.spark.streaming.{Seconds, StreamingContext}
9. object UpdateKey2Mysql {
10. //设置日志级别
11. Logger.getLogger("org").setLevel(Level.WARN)
12. //设置一个函数
13. val upFunc = (newWord: Seq[Int], exsits: Option[Int]) => {
14. val i: Int = newWord.sum + exsits.getOrElse(0)
15. Some(i)
16. }
18. def main(args: Array[String]): Unit = {
19. //本地运行设置
20. val conf: SparkConf = new SparkConf()
21. .setMaster("local[5]")
22. .setAppName(this.getClass.getSimpleName)
23. //StreamingContext
24. val ssc: StreamingContext = new StreamingContext(conf, Seconds(5))
25. //设置检查点
26. ssc.checkpoint("up")
27. //创建socket 接受信息
28. val text: ReceiverInputDStream[String] = ssc.socketTextStream("Linux00", 9999)
29. val result: DStream[(String, Int)] = text.flatMap(_.split(" ")).map((_, 1)).updateStateByKey(upFunc)
30. result.print()
31. result.foreachRDD(rdd => {
32. rdd.foreachPartition(p => {
33. val conn: Connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306/wjy1?useUnicode=true&characterEncoding=utf8&rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&autoReconnect=true&failOverReadOnly=false&useSSL=true", "wjy1", "351991")
34. val statement: PreparedStatement = conn.prepareStatement("create table if not exists wordcount4 (word varchar(20),total int)")
35. statement.executeUpdate()
36. p.foreach(t => {
37. //如果插入的单词已经存在数据库中，则更新这个单词的值（当前出现的次数+数据库当前的次数）
38. val statement1: PreparedStatement = conn.prepareStatement("select * from wordcount4 where word=?")
39. statement1.setString(1, t._1)
40. val set: ResultSet = statement1.executeQuery()
41. if (set.next()) {
42. val count: Int = set.getInt("total")
43. val newcount: Int = count + t._2
44. //更新
45. val statement2: PreparedStatement = conn.prepareStatement("update wordcount4 set total=? where word=?")
46. statement2.setInt(1, newcount)
47. statement2.setString(2, t._1)
48. statement2.executeUpdate()
49. } else {
50. val statement3: PreparedStatement = conn.prepareStatement("insert into wordcount4 values(?,?)")
51. statement3.setString(1, t._1)
52. statement3.setInt(2, t._2)
53. statement3.executeUpdate()
54. }
55. })
56. conn.close()
57. })
58. })
59. ssc.start()
60. ssc.awaitTermination()
61. }
62. }