java实现spark常用算子之Repartitions

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.VoidFunction;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

/**
 * repartitions 算子:
 * 增加分区，使用shuffle操作
 */
public class RepartitionsOperator {

 public static void main(String[] args){
 SparkConf conf = new SparkConf().setMaster("local").setAppName("repartitions");
 JavaSparkContext sc = new JavaSparkContext(conf);
 List<String> names = Arrays.asList("w1","w2","w3","w4","w5","w6");

 JavaRDD<String> nameRdd = sc.parallelize(names,2);

 JavaRDD<String> namefristRdd = nameRdd.mapPartitionsWithIndex(new Function2<Integer, Iterator<String>, Iterator<String>>() {
 @Override
 public Iterator<String> call(Integer index, Iterator<String> iterator) throws Exception {

 List<String> list = new ArrayList<>();
 while (iterator.hasNext()){
 list.add("1["+index+"]"+iterator.next());
 }
 return list.iterator();
 }
 },true);

 //增加分区
 JavaRDD<String> temp = namefristRdd.repartition(5);

 JavaRDD<String> nameseconedRdd = temp.mapPartitionsWithIndex(new Function2<Integer, Iterator<String>, Iterator<String>>() {
 @Override
 public Iterator<String> call(Integer index, Iterator<String> iterator) throws Exception {
 List<String> list = new ArrayList<>();
 while (iterator.hasNext()){
 list.add("2["+index+"]:"+iterator.next());
 }
 return list.iterator();
 }
 },false);


 nameseconedRdd.foreach(new VoidFunction<String>() {
 @Override
 public void call(String s) throws Exception {
 System.err.println(s);
 }
 });

 }
}

微信扫描下图二维码加入博主知识星球,获取更多大数据、人工智能、算法等免费学习资料哦！