1.通过是使用case class的方式,不过在scala 2.10中最大支持22个字段的case class,这点需要注意
2.是通过spark内部的StructType方式,将普通的RDD转换成DataFrame
装换成DataFrame后,就可以使用SparkSQL来进行数据筛选过滤等操作
下面直接代码说话
package spark_rdd import org.apache.spark._ import org.apache.spark.sql._ import org.apache.spark.sql.types._ object SparkRDDtoDF { //StructType and convert RDD to DataFrame def rddToDF(sparkSession : SparkSession):DataFrame = { //设置schema结构 val schema = StructType( Seq( StructField("name",StringType,true) ,StructField("age",IntegerType,true) ) ) val rowRDD = sparkSession.sparkContext .textFile("file:/E:/scala_workspace/z_spark_study/people.txt",2) .map( x => x.split(",")).map( x => Row(x(0),x(1).trim().toInt)) sparkSession.createDataFrame(rowRDD,schema) } //use case class Person case class Person(name:String,age:Int) def rddToDFCase(sparkSession : SparkSession):DataFrame = { //导入隐饰操作,否则RDD无法调用toDF方法 import sparkSession.implicits._ val peopleRDD = sparkSession.sparkContext .textFile("file:/E:/scala_workspace/z_spark_study/people.txt",2) .map( x => x.split(",")).map( x => Person(x(0),x(1).trim().toInt)).toDF() peopleRDD } def main(agrs : Array[String]):Unit = { val conf = new SparkConf().setMaster("local[2]") conf.set("spark.sql.warehouse.dir","file:/E:/scala_workspace/z_spark_study/") conf.set("spark.sql.shuffle.partitions","20") val sparkSession = SparkSession.builder().appName("RDD to DataFrame") .config(conf).getOrCreate() //通过代码的方式,设置Spark log4j的级别 sparkSession.sparkContext.setLogLevel("WARN") import sparkSession.implicits._ //use case class convert RDD to DataFrame //val peopleDF = rddToDFCase(sparkSession) //use StructType convert RDD to DataFrame val peopleDF = rddToDF(sparkSession) peopleDF.show() peopleDF.select($"name",$"age").filter($"age">20).show() } }