Spark序列化

• 默认没有序列化(StorageLevel.MEMORY_ONLY)

def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
     // .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    // .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }

• 如下图 95.4M
  
• 使用默认Java序列化

 def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
    //  .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    // .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }

• 如下图29.4M
  

• 使用Kryo的序列化(但是没有注册)

 def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    // .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }

• 如下图 61.9M
  
• 使用Kryo注册类之后的序列化

 def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }

• 如下图 19.0M
  结论:使用Kryo序列化后,文件更小了