spark实现PageRank


import java.util.ArrayList;
import java.util.List;
import java.util.Iterator;
import java.util.regex.Pattern;

import scala.Tuple2;

import com.google.common.collect.Iterables;

import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.SparkSession;

/**
 * 数据格式如下:
 * URL         neighbor URL
 * URL         neighbor URL
 * URL         neighbor URL
 */
public final class JavaPageRank {

  private static final Pattern SPACES = Pattern.compile("\\s+");

/**
 *  实现元素的相加，reduceByKey使用 根据key求和
 */
  private static class Sum implements Function2<Double, Double, Double> {
    @Override
    public Double call(Double a, Double b) {
      return a + b;
    }
  }

  public static void main(String[] args) throws Exception {
    if (args.length < 2) {
      System.err.println("Usage: JavaPageRank <file> <number_of_iterations>");
      System.exit(1);
    }



    SparkSession spark = SparkSession
      .builder()
      .appName("JavaPageRank")
      .getOrCreate();


    JavaRDD<String> lines = spark.read().textFile(args[0]).javaRDD();

    /** links 是(String ,Iterable<String>)
     *  先将数据转为key value形式的RDD，然后根据key进行分组  key是某一个url  value是这个key的的所有邻居url组成的集合
      */
    JavaPairRDD<String, Iterable<String>> links = lines.mapToPair(
      new PairFunction<String, String, String>() {
        @Override
        public Tuple2<String, String> call(String s) {
          String[] parts = SPACES.split(s);
          return new Tuple2<>(parts[0], parts[1]);
        }
      }).distinct().groupByKey().cache();


      // ranks的初始形式是(String ,Double)其中double一列初始都是1
    JavaPairRDD<String, Double> ranks = links.mapValues(new Function<Iterable<String>, Double>() {
      @Override
      public Double call(Iterable<String> rs) {
        return 1.0;
      }
    });


    for (int current = 0; current < Integer.parseInt(args[1]); current++) {
      // Calculates URL contributions to the rank of other URLs.
        // links 是(String ,Iterable<String>) ranks是(String ,Double)  join之后是( String ,（Iterable<String>，Double）)
      JavaPairRDD<String, Double> contribs = links.join(ranks).values()
        .flatMapToPair(new PairFlatMapFunction<Tuple2<Iterable<String>, Double>, String, Double>() {
          @Override
          //对values进行flatMapToPair操作  ，value的格式Tuple2<Iterable<String>, Double>
          //迭代元组中第一个元素 然后权重分别设置为第二个元素除以第一个元素的里面url的个数
          //组成二元组Tuple2<String, Double>  ，因为第一个元素里面有多个url会迭代
          // 所以call方法返回多个Tuple2<String, Double> 组成的一个Iterator，
          // 最后会进行扁平化，所以flatMapToPair返回了JavaPairRDD<String, Double>
          public Iterator<Tuple2<String, Double>> call(Tuple2<Iterable<String>, Double> s) {
            int urlCount = Iterables.size(s._1);
            List<Tuple2<String, Double>> results = new ArrayList<>();
            for (String n : s._1) {
              results.add(new Tuple2<>(n, s._2() / urlCount));
            }
            return results.iterator();
          }
      });


        //contribs是JavaPairRDD<String, Double>  然后我们进行根据key操作对value求和
        //那么得到了key及其自己对应的权重值，然后得到平滑后的每个url对应的权重得到新的ranks
        //继续迭代，继续把links跟ranks进行join然后更新权重
      ranks = contribs.reduceByKey(new Sum()).mapValues(new Function<Double, Double>() {
        @Override
        public Double call(Double sum) {
          return 0.15 + sum * 0.85;
        }
      });
    }

    //进行输出ranks存放了url及其对应的权重
    List<Tuple2<String, Double>> output = ranks.collect();
    for (Tuple2<?,?> tuple : output) {
        System.out.println(tuple._1() + " has rank: " + tuple._2() + ".");
    }

    spark.stop();
  }
}
猜你喜欢
