[Spark][spark_streaming]#1_QuickStart

spark-submit

bin/spark-submit --master local[2] --class org.apache.spark.example.streaming.NetworkWordCount --name NetworkWordCount /root/spark-2.4.3-bin-2.6.0-cdh5.15.1/examples/jars/spark-examples_2.11-2.4.3.jar 172.17.198.174 9999

spark-shell

bin/spark-shell --master local[2]

scala> import org.apache.spark.streaming.{Seconds,StreamingContext}
scala> val ssc = new StreamingContext(sc,Seconds(1))
scala> val lines = ssc.socketTextStream("localhost",9999)
scala> val words = lines.flatMap(_.split(" "))
scala> val wordCounts = words.map(x => (x,1)).reduceByKey(_+_)
scala> wordCounts.print
scala> ssc.start
scala> ssc.awaitTermination

---

import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}

object NetworkWC {
    def main(args: Array[String]): Unit = {
        val sparkConf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount")
        val ssc = new StreamingContext(sparkConf,Seconds(5))
        val lines = ssc.socketTextStream("localhost",9999)
        val result = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
        result.print
        ssc.start()
        ssc.awaitTermination()
    }

}

import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
//对进入文件夹的新文件触发
object FileWC {
    def main(args: Array[String]): Unit = {
        val sparkConf = new SparkConf().setMaster("local").setAppName("FileWordCount")
        val ssc = new StreamingContext(sparkConf,Seconds(5))
        val lines = ssc.textFileStream("file:///Users/eric/Desktop/l2118i/sparktrain/data")
        val result = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
        result.print()
        ssc.start()
        ssc.awaitTermination()
    }
}