hive 小文件数据合并

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/haohaixingyun/article/details/84852615

hive 数据有时候需要进行数据合并

#!/bin/bash

hadoop jar /software/servers/bdp_tools/mergefiles-1.7.jar -u lzo -p hdfs://ns1/user/dd_edw/adm.db/table_name/dt=2018-09-01

2018/12/06 今儿谈话聊天更精进一部lovely

#!/bin/bash

end_date='2018-12-05'
beg_date='2017-01-01'


beg_s=`date -d "$beg_date" +%s`
end_s=`date -d "$end_date" +%s`

while [ $beg_s -le $end_s ];do

	cur_day=`date -d @$beg_s +"%Y-%m-%d"`;
	
	echo $cur_day
	hadoop jar /software/servers/bdp_tools/mergefiles-1.7.jar -u orc -p hdfs://ns1/user/dd_edw/adm.db/table_name/dt=$cur_day/tp='day'
    hadoop jar /software/servers/bdp_tools/mergefiles-1.7.jar -u orc -p hdfs://ns1/user/dd_edw/adm.db/table_name/dt=$cur_day/tp='week'
	
	beg_s=$((beg_s+86400));
	
done 
echo 'process merger done'