使用spark ,spark sql编写word count程序

Spark 版本

val conf=new SparkConf.setAppName("wc").setMaster("local[*]")
val sc=SparkConText(conf, 2)
val lines sc.textFile("hdfs://wc")
val paris=lines.flatMap(_.spilt(" "))
val word =paris.map(_,1)
val result=word.reduceByKey(_+_).SortBy(x->x._1,false)

spark sql版本2.x

val sc=SparkSession.builder().SetAppname("wc—sparkSql").master("local[*]").getOrcreat()
val lines=sc.read.textFile("hdfs://node-1:9000/words")
val word=lines.flatMap(_.spilt(" "))
import spark.implicits._
word.createTempView("v_wc")
val result=spark.sql("select value,count(*) counts from v_wc GroupBy value")
result.show()
spark.stop()