package com.bjsxt;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;
/**
* MapValues算子
* 传进来的值，传出去的格式可以自己设置，亦即可以定义
* 可以将tuple2中value的值加起来,相当于reduceByKey
* 也可以将key的个数加起来，相当于countByKey
*
* @author Administrator
*
*/
public class MapValues {
public static void main(String[] args) {
SparkConf conf=new SparkConf().setAppName("test").setMaster("local");
JavaSparkContext sc=new JavaSparkContext(conf);
List<Tuple2<String, Integer>> list = Arrays.asList(
new Tuple2<>("m1", 1),
new Tuple2<>("m2", 2),
new Tuple2<>("m1", 3),
new Tuple2<>("m2", 4),
new Tuple2<>("m1", 5),
new Tuple2<>("m2", 6),
new Tuple2<>("m1", 7)
);
JavaPairRDD<String,Integer> rdd1 = sc.parallelizePairs(list);
JavaPairRDD<String, Tuple2<Integer, Integer>> rdd2 = rdd1.mapValues(new Function<Integer, Tuple2<Integer,Integer>>() {
@Override
public Tuple2<Integer, Integer> call(Integer speed) throws Exception {
return new Tuple2<Integer, Integer>(speed, 1);
}
});
JavaPairRDD<String, Tuple2<Integer, Integer>> result = rdd2.reduceByKey(new Function2<Tuple2<Integer,Integer>, Tuple2<Integer,Integer>, Tuple2<Integer,Integer>>() {
@Override
public Tuple2<Integer, Integer> call(Tuple2<Integer, Integer> t1, Tuple2<Integer, Integer> t2)
throws Exception {
// TODO Auto-generated method stub
return new Tuple2<Integer, Integer>(t1._1+t2._1, t1._2+t2._2);
}
});
result.foreach(new VoidFunction<Tuple2<String,Tuple2<Integer,Integer>>>() {
@Override
public void call(Tuple2<String, Tuple2<Integer, Integer>> arg0) throws Exception {
System.out.println(arg0);
}
});
}
}