graphx-最短路径

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/lm709409753/article/details/85017864

1.最近在总结图计算，把相关算法实现贴出来，坐下总结，作为督促。算法实现多数是graphx。

package org.apache.spark.graphx.algorithms

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.graphx.{EdgeDirection, VertexId, Graph}
import org.apache.spark.graphx.util.GraphGenerators


object Pregel_SSSP {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("lm-shotestpath").setMaster("local[*]")
    val sc = new SparkContext(conf)
    // A graph with edge attributes containing distances
    val graph: Graph[Long, Double] =
      GraphGenerators.logNormalGraph(sc, numVertices = 5).mapEdges(e => e.attr.toDouble)
    graph.vertices.foreach(println)
    graph.edges.foreach(println)
    val sourceId: VertexId = 0 // The ultimate source

    // Initialize the graph such that all vertices except the root have distance infinity.
    val initialGraph: Graph[(Double, List[VertexId]), Double] = graph.mapVertices((id, _) =>
      if (id == sourceId) (0.0, List[VertexId](sourceId))
      else (Double.PositiveInfinity, List[VertexId]()))

    val sssp = initialGraph.pregel((Double.PositiveInfinity, List[VertexId]()), Int.MaxValue, EdgeDirection.Out)(

      // Vertex Program
      (id, dist, newDist) => if (dist._1 < newDist._1) dist else newDist,

      // Send Message
      triplet => {
        if (triplet.srcAttr._1 < triplet.dstAttr._1 - triplet.attr) {
          Iterator((triplet.dstId, (triplet.srcAttr._1 + triplet.attr, triplet.srcAttr._2 :+ triplet.dstId)))
        } else {
          Iterator.empty
        }
      },
      //Merge Message
      (a, b) => if (a._1 < b._1) a else b)
    println(sssp.vertices.collect.mkString("\n"))
  }
}

2.上面这个记录了到每个顶点的路径。

3.不记录路径

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.graphx.{Graph, VertexId}
import org.apache.spark.graphx.util.GraphGenerators
// $example off$
import org.apache.spark.sql.SparkSession

object SSSPExample {
  def main(args: Array[String]): Unit = {
    //     Creates a SparkSession.
    /*val spark = SparkSession
      .builder
      .appName(s"${this.getClass.getSimpleName}")
      .getOrCreate()
    val sc = spark.sparkContext*/

    val sparkConf = new SparkConf().setAppName("SparkKMeans").setMaster("local[*]")
    val sc = new SparkContext(sparkConf)

    // $example on$
    // A graph with edge attributes containing distances
    val graph: Graph[Long, Double] =
    GraphGenerators.logNormalGraph(sc, numVertices = 20).mapEdges(e => e.attr.toDouble)
    println(graph.edges.collect().mkString("\n"))

    val sourceId: VertexId = 12 // The ultimate source
    // Initialize the graph such that all vertices except the root have distance infinity.
    val initialGraph = graph.mapVertices((id, _) =>
      if (id == sourceId) 0.0 else Double.PositiveInfinity)
    val sssp = initialGraph.pregel(Double.PositiveInfinity)(
      (id, dist, newDist) => math.min(dist, newDist), // Vertex Program
      triplet => { // Send Message
        if (triplet.srcAttr + triplet.attr < triplet.dstAttr) {
          Iterator((triplet.dstId, triplet.srcAttr + triplet.attr))
        } else {
          Iterator.empty
        }
      },
      (a, b) => math.min(a, b) // Merge Message
    )
    println(sssp.vertices.collect.mkString("\n"))
    // $example off$

    //    spark.stop()
  }
}

4.Pregel算法步骤

5.导入边的属性

val edgeFile:RDD[String] = sc.textFile("hdfs://address/edge.txt")
val vertexFile:RDD[String] = sc.textFile("hdfs://address/vertex.txt")

//edge
val edge = edgeFile.map { e =>
  val fields = e.split(" ")
  Edge(fields(0).toLong,fields(1).toLong,fields(2))
}
//vertex
val vertex = vertexFile.map{e=>
  val fields = e.split(" ")
  (fields(0).toLong,fields(1))

}

val graph = Graph(vertex,edge,"").persist()
println(graph.edges.collect.mkString("n"))