版权声明:本文为博主原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接和本声明。
小文件的危害完我就不在多说,请见:https://blog.csdn.net/qq_34341930/article/details/89031661
直接上代码,可以做个定时任务结合自己的业务去定时调度
import org.apache.spark.sql.{SaveMode, SparkSession}
/**
* 使用Spark SQL合并小文件
*/
object SmallFileMerger {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder()
.appName("SmallFileMerger")
.master("local")
.getOrCreate()
val inputPath = spark.conf.get("spark.small.file.merge.inputPath",
"hdfs://mycluster/user/hadoop-jrq/dw-course/streaming-etl/user-action-parquet/year=2019/month=201909/day=20190906")
val numberPartition = spark.conf.get("spark.small.file.merge.numberPartition", "2").toInt
val outputPath = spark.conf.get("spark.small.file.merge.outputPath",
"hdfs://mycluster/user/hadoop-jrq/dw-course/streaming-etl/user-action-merged/year=2019/month=201909/day=20190906")
spark.read.parquet(inputPath)
.repartition(numberPartition)
//.coalesce(numberPartition)
.write
.mode(SaveMode.Overwrite)
.parquet(outputPath)
spark.stop()
}
}