下载 spark.1.6.3-bin-hadoop.2.6.tgz
1,安装步骤
tar -zxvf spark.1.6.3-bin-hadoop.2.6.tgz -C /export/servers/apps
cd spark.1.6.3-bin-hadoop.2.6/conf
mv spark-env.sh.template spark-env.sh
vi spark-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_144
export SPARK_MASTER=spark30
export SPARK_MASTER_PORT=7077
mv salves.template slaves
vi slaves
spark31
spark32
2,启动hadoop集群,使用hdfs文件系统
3,启动集群
spark.1.6.3-bin-hadoop.2.6/sbin/start-all.sh
4,spark-shell启动
bin/spark-shell –master spark://spark30:7077 –executor-memory 1024m –total-executor-cores 2
5,例子
sc.textFile(“hdfs://spark30:9000/wordcount/input”).flatMap(_.split(” “)).map((_,1)).reduceByKey(_+_).sortBy(_._2,false).collect
sc.textFile(“hdfs://spark30:9000/wordcount/input”).flatMap(_.split(” “)).map((_,1)).reduceByKey(_+_).sortBy(_._2,false).saveAsTextFile(“hdfs://spark30:9000/wordcount/output/out2”)