環境
モニタポート(ブログ参照して、下に最も直接的なターン):
C:\Users\Feng>nc -l -p 12345
hello world hello
world world world
hell
helll
lll
ll ll ll
ll ll oo
結果は:
-------------------------------------------
Time: 1581494862000 ms
-------------------------------------------
(ll,5)
(lll,1)
(oo,1)
(helll,1)
(hell,1)
ポンポン依存
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<spark.version>2.4.4</spark.version>
</properties>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
Scalaのコード
package org.feng.stream.window
import org.apache.spark._
import org.apache.spark.streaming._
/**
* Created by Feng on 2019/12/3 15:26
* CurrentProject's name is spark
* 时间窗口:3秒一个批次,窗口12秒,滑步6秒
*/
object WordCount {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[2]").setAppName("WordCount")
val streamingContext = new StreamingContext(sparkConf, Seconds(3))
// 设置检查点
streamingContext.checkpoint(".")
val lines = streamingContext.socketTextStream("localhost", 12345)
val words = lines.flatMap(_.split(" ")).map(x => (x, 1))
val result = words.reduceByKeyAndWindow((x:Int, y:Int) => x + y, Seconds(12), Seconds(6))
result.print()
streamingContext.start()
streamingContext.awaitTermination()
}
}
追加:総語数を計算します
package org.feng.stream
import org.apache.spark._
import org.apache.spark.streaming._
/**
* Created by Feng on 2019/12/3 16:04
* CurrentProject's name is spark
*/
object StateWordCount {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[2]").setAppName("StateWordCount")
val streamingContext = new StreamingContext(sparkConf, Seconds(3))
streamingContext.checkpoint(".")
val lines = streamingContext.socketTextStream("localhost", 12345)
val fun = (values: Seq[Int], state: Option[Int]) => {
val currentCount = values.sum
val previousCount = state.getOrElse(0)
Some(currentCount + previousCount)
}
lines.flatMap(_.split(" ")).map(word => (word, 1)).updateStateByKey(fun).print()
streamingContext.start()
streamingContext.awaitTermination()
}
}