wordCount scala

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/huoyongliang/article/details/86749644

用scala写的wordCount

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.log4j.{Level, Logger}

object wordCount {
  Logger.getLogger("org").setLevel(Level.ERROR)
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("workCount")
    conf.set("spark.testing.memory", "2147480000")//后面的值大于512m即可
    val sc = new SparkContext(conf)
    val data = sc.textFile("d://dev/bigdata/wc.txt")
    data.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).collect().foreach(println)
  }
}