A Spark word counting program
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
object Test {
def main(args: Array[String]): Unit = {
val inputFile = "file:///D:/words.txt"
// 设置属性参数,创建Spark上下文对象,设置本地运行模式,CPU为2核
val conf = new SparkConf().setAppName("WordCount").setMaster("local[2]")
val sc = new SparkContext(conf)
// 将文件读出,为RDD
val textFile = sc.textFile(inputFile)
// 将每一行用空格分隔-->增加一个属性,变为键值对,利于操作
// 根据Key进行相加,相同key的值相加--->再利用flatMap进行扁平化,变为一个
val wordCount = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
// foreach打印元素
wordCount.foreach(println)
}
}