A bulk add Subdivision:
use bigdata;
alter table siebel_member
add if not exists
partition(dt='20180401') location '20180401'
partition(dt='20180402') location '20180402'
partition(dt='20180403') location '20180403'
partition(dt='20180404') location '20180404';
II. Batch delete partition
method 1:
alter table siebel_member drop if exists
partition(dt='20180401'),
partition(dt='20180402'),
partition(dt='20180403'),
partition(dt='20180404');
Method 2
alter table tb_partition drop partition (etl_dt>='20181102',etl_dt<='20181104')
Add III. A large number of partitions
Although the bulk add partitions, but all of a sudden add 2000 partition, if the 2000 partition in a written statement which, hive is not allowed.
So we had a month plus one month, try using the shell generation line = "hive -e" xxxxx "" $ {line}, but parsing errors, using python + shell solve the problem.
python code is as follows:
#!/usr/bin/env python
#encoding=utf8
import datetime
import os
import sys
def dateRange(beginDate, endDate):
dates = []
dt = datetime.datetime.strptime(beginDate, "%Y%m%d")
date = start date [:]
while date < endDate:
dates.append(date)
dt = dt + datetime.timedelta (1)
date = dt.strftime("%Y%m%d")
return dates
def genSql(datelist, table):
sql = """
hive -e "
use bigdata;
alter table """ + table + """
add if not exists """
for i in datelist:
sql = sql + """partition(dt='""" + i + """') location '""" + i + """' """
sql = sql.strip() + """;" """
os.system(sql)
if __name__ == '__main__':
if sys.argv.__len__() != 4:
print ( "parameter error")
sys.exit()
st = sys.argv[1]
and = sys.argv [2]
table = sys.argv[3]
datelist = dateRange(st,et)
genSql(datelist,table)
py shell is used to schedule
# Add a partition, add the partition month will be added to the last day of the month where et
function add_par(){
st=$1
and $ 2 =
tbname = $ 3
while [ ${et} -ge ${st} ]
do
pet=$(date --date="+1 month ${et}" +"%Y%m%d")
python a.py ${et} ${pet} ${tbname}
et=$(date --date="1 month ago ${et}" +"%Y%m%d")
done
}
