Windows 환경에서 worldcount 스파크 실행 시간 창

환경

모니터 포트 ( 블로그 참조 , 아래로 가장 직접적인 턴) :

C:\Users\Feng>nc -l -p 12345
hello world hello
world world world
hell
helll
lll
ll ll ll
ll ll oo

결과는 보여

-------------------------------------------
Time: 1581494862000 ms
-------------------------------------------
(ll,5)
(lll,1)
(oo,1)
(helll,1)
(hell,1)

치어 의존

 <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
    <spark.version>2.4.4</spark.version>
  </properties>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.11</artifactId>
      <version>${spark.version}</version>
    </dependency>

스칼라 코드

package org.feng.stream.window
import org.apache.spark._
import org.apache.spark.streaming._

/**
  * Created by Feng on 2019/12/3 15:26
  * CurrentProject's name is spark
  * 时间窗口：3秒一个批次，窗口12秒，滑步6秒
  */
object WordCount {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[2]").setAppName("WordCount")
    val streamingContext = new StreamingContext(sparkConf, Seconds(3))
    // 设置检查点
    streamingContext.checkpoint(".")

    val lines = streamingContext.socketTextStream("localhost", 12345)
    val words = lines.flatMap(_.split(" ")).map(x => (x, 1))
    val result = words.reduceByKeyAndWindow((x:Int, y:Int) => x + y, Seconds(12), Seconds(6))

    result.print()
    streamingContext.start()
    streamingContext.awaitTermination()
  }
}

추가 : 총 단어 수를 계산

package org.feng.stream
import org.apache.spark._
import org.apache.spark.streaming._
/**
  * Created by Feng on 2019/12/3 16:04
  * CurrentProject's name is spark
  */
object StateWordCount {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[2]").setAppName("StateWordCount")

    val streamingContext = new StreamingContext(sparkConf, Seconds(3))
    streamingContext.checkpoint(".")
    val lines = streamingContext.socketTextStream("localhost", 12345)


    val fun = (values: Seq[Int], state: Option[Int]) => {
      val currentCount = values.sum
      val previousCount = state.getOrElse(0)
      Some(currentCount + previousCount)
    }

    lines.flatMap(_.split(" ")).map(word => (word, 1)).updateStateByKey(fun).print()

    streamingContext.start()
    streamingContext.awaitTermination()
  }
}