[Spark small exercise] word counting

A Spark word counting program

import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
object Test {
    
    
  def main(args: Array[String]): Unit = {
    
    
    val inputFile = "file:///D:/words.txt"   
    // 设置属性参数,创建Spark上下文对象,设置本地运行模式,CPU为2核
    val conf = new SparkConf().setAppName("WordCount").setMaster("local[2]")    
    val sc = new SparkContext(conf)  
    // 将文件读出,为RDD
    val textFile = sc.textFile(inputFile)    
    // 将每一行用空格分隔-->增加一个属性,变为键值对,利于操作
    // 根据Key进行相加,相同key的值相加--->再利用flatMap进行扁平化,变为一个
    val wordCount = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)    
    // foreach打印元素
    wordCount.foreach(println)    
  } 
}

Guess you like

Origin blog.csdn.net/weixin_40433003/article/details/130428097