Operación del comando shell de Spark (recuento de palabras)

Implementar el recuento de palabras

1. Modo local

SCALA> sc.textFile("file:///export/servers/sparkdatas/wordcount.txt").flatMap(_.split(" ")).map((_ , 1)).groupBy(_._1).mapValues(_.foldLeft(0)(_+_._2)).collect
res16: Array[(String, Int)] = Array((hello,3), (me,1), (you,1), (her,1))

//  或者
scala> sc.textFile("file:///export/servers/sparkdatas/wordcount.txt").flatMap(_.split(" ")).map( (_ , 1)).reduceByKey((_+_) .collect
res14: Array[(String, Int)] = Array((hello,3), (me,1), (you,1), (her,1))

2. Modo cliente

Error al leer el archivo local, archivo no encontrado: El ejecutor puede no estar en el nodo actual y los otros nodos no tienen este archivo.

scala> sc.textFile("hdfs://node01:8020/sparkwordcount/wordcount.txt").flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)collect
warning: there was one feature warning; re-run with -feature for details
res4: Array[(String, Int)] = Array((hello,3), (me,1), (you,1), (her,1))         

3.operación del proyecto maven: hay una diferencia entre el modo de clúster y el modo local

proyecto scala

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

  object scala_spark {
  def main(args: Array[String]): Unit = {
    //todo  本地模式
    //val sparkConf = new SparkConf().setAppName("scalaCount".setMaster("local[2]"))
    //todo  打包到打集群运行
    val sparkConf = new SparkConf().setAppName("scalaCount")
    //todo  获取sc
    val context = new SparkContext(sparkConf)
    //todo  设置打印日志级别
    context.setLogLevel("WARN")
    //todo  获取文件
    //val file: RDD[String] = context.textFile("file:///C:\\Users\\Administrator\\Documents\\tt\\wordcount.txt")
    //todo  打包集群运行
    val file: RDD[String] = context.textFile(args(0))

    //todo  进行操作
    val fileWord = file.flatMap(_.split(" ")).map((_, 1)).reduceByKey((_ + _))
    //todo  降序
    val sortBy: RDD[(String, Int)] = fileWord.sortBy(_._2,false)
    //todo  将RDD模式转出Array
    val array: Array[(String, Int)] = sortBy.collect()
    val result: mutable.Buffer[(String, Int)] = array.toBuffer
    println(result)
    //todo  保存为文件(路径)
    sortBy.saveAsTextFile(args(1))
    context.stop()
  }
}

proyecto java: demasiados problemas, usemos scala

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

public class java_spark {
    public static void main(String[] args) {
        JavaSparkContext sc = new JavaSparkContext(new SparkConf().setMaster("local[2]").setAppName("java_spark"));
        JavaRDD<String> file = sc.textFile("file:///C:\\Users\\Administrator\\Documents\\tt\\wordcount.txt");
        final JavaRDD<String> words = file.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public Iterator<String> call(String s) throws Exception {
                String[] lines = s.split("\t\n");
                Iterator<String> iterator = Arrays.asList(lines).iterator();
                return iterator;
            }
        });
        /**
         * 输入类型
         * 元组第一位类型
         * 元组第二位类型
         */
        final JavaPairRDD<String, Integer> wordAnd1 = words.mapToPair(new PairFunction<String, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(String s) throws Exception {
                /**
                 * Tuple2就是要返回的类型
                 */
                final Tuple2<String, Integer> Tuple2 = new Tuple2<>(s, 1);

                return Tuple2;
            }
        });
        final JavaPairRDD<String, Integer> reduceByKey = wordAnd1.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        });
        final List<Tuple2<String, Integer>> collect = reduceByKey.collect();
        for (Tuple2<String, Integer> stringIntegerTuple2 : collect) {
            System.out.println(stringIntegerTuple2._1 +"-------"+stringIntegerTuple2._2);

        }

    }
}