Execution log script and the script used in the article should be able to find in the annex at the end of the text.
nohup sh /tmp/discp/distcp.sh
nohup sh /tmp/distcp/distcp.sh &>/tmp/distcp/distcp.log &
hadoop distcp -update -log hdfs://master1:8020/discpLogs hdfs://slave11:8020/cdn hdfs://master1:8020/cdn
#!/bin/bash
hadoop distcp -update -log hdfs://master1:8020/discpLogs/ha hdfs://slave11:8020/ha hdfs://master1:8020/ha
#迁移HBase
hadoop distcp -log hdfs://master1:8020/discpLogs/hbase webhdfs://slave11:50070/hbase/data/default hdfs://master1:50070/hbase/data/
#snapsot方式：
hbase shell
snapshot 'myTable', 'myTableSnapshot-122112'
hdfs dfs -du -h /hbase/data/
#列出所有的HBase表：
#!/bin/sh
TMP_FILE=tmp_hbase_tables
TABLES_FILE=hbase_tables.txt
echo "list" | hbase shell > tmp_hbase_tables
sleep 2
sed '1,6d' $TMP_FILE | tac | sed '1,2d' | tac > $TABLES_FILE
sed -i '$d' $TABLES_FILE
sleep 2
#创建HBase表快照
count 'myTable', 'myTableSnapshot-122112'
vim createSnapshot.sh
#!/bin/sh
TABLES_FILE=hbase_tables.txt
snapshot=_snapshot_20190625
#echo "list" | hbase shell > tmp_hbase_tables
for table in $(cat $TABLES_FILE); do
snapshotTable=$table$snapshot
snapshotTable=${snapshotTable#*:}
echo "snapshot '$table', '$snapshotTable'" | hbase shell
sleep 5
done
#后台执行
SH /tmp/distcp/hbase/createSnapshot.sh & nohup> // tmp / invocation of DistCp is / HBase / createSnapshot.log &
# View Snapshot
HBase shell list_snapshots
vim listSnapshots.sh
#! / bin / SH
TMP_FILE = tmp_hbase_snapshots
TABLES_FILE = hbase_snapshots.txt
echo "list_snapshots" | HBase shell> tmp_hbase_snapshots
SLEEP 2
sed '1,6d' $ TMP_FILE | tac | sed '1, 2D' | tac> $ TABLES_FILE
sed -i '$ d' $ TABLES_FILE
SLEEP 2
# and then execute the script, export HBase is a snapshot
and then remove the spaces in the first row of windows, the file back back to linux (this step can save the modified shell <spaces do not write the first column write file>)
# increased HBase configuration (ban automatic majorcompaction):
<Property>
<name> hbase.hregion.majorcompaction </ name>
<value> 0 </ value>
</ Property>
# migrate a table:
-Copy-from HDFS #: // src-HBase-root-dir / HBase \
# table: CDN: AddDomainUseCDNDetail 33154 Row (S) in 4.1160 seconds The
# 2 in the cluster to perform the migration Snapshot
HBase org.apache.hadoop.hbase.snapshot .ExportSnapshot \
-SNAPSHOT AddDomainUseCDNDetail_snapshot_20190625 \
-copy-from HDFS: // slave11: 8020 / HBase \
-copy-to HDFS: // Master1: 8020 / HBase \
-mappers 20 \
-bandwidth 500
#echo 'happy Camp 2014 1 set '| Cut -d' '-f1
# write snapshots batch migration script
vim sendSnapshots.sh
# / bin / bash!
snapshotListFile = hbase_snapshots.txt
for Line in $ (CAT $ snapshotListFile | awk' {$ 1} Print ')
do
Line -copy org.apache.hadoop.hbase.snapshot.ExportSnapshot -SNAPSHOT $ HBase-from HDFS: // slave11: 8020 / HBase -copy-to HDFS: // Master1: 8020 / HBase -mappers 20 -bandwidth 500
DONE
# in the background cluster 1
nohup SH /tmp/distcp/hbase/sendSnapshots.sh &> // tmp / invocation of DistCp is / HBase / sendSnapshots.log &
# perform in a cluster 1
HBase shell
# create a namespace
create_namespace 'CDN'
# re-create the table
Create 'CDN: AddDomainUseCDNDetail', {NAME => 'D'}
# disable table
disable 'CDN: AddDomainUseCDNDetail'
# change file permissions
HDFS 777 -R & lt -chmod DFS / HBase / Archive / Data / CDN
HDFS 777 DFS -R & lt -chmod /hbase/.hbase-snapshot
# restore snapshots
restore_snapshot 'AddDomainUseCDNDetail_snapshot_20190625'
# re-enabled table
enable 'CDN: AddDomainUseCDNDetail'
# table to confirm whether the recovery is successful
COUNT 'CDN: AddDomainUseCDNDetail'
# display
33154 Row (S) in 5.3510 seconds The
HBase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles \
-Dhbase.mapreduce.bulkload.max. 1024 = hfiles.perRegion.perFamily \
hdfs: // Master1: 8020 / HBase / Archive / the Data / CDN / AddDomainUseCDNDetail
# restore cnzz snapshot
# create a namespace
create_namespace 'cnzz'
# turn (hdfs user to perform) performed in HBase shell All statements following text
cnzz_tables_create.txt
disable_cnzz_tables.txt
alter_cnzz_table_TTL.txt
# migrate Hive
# 2 in the cluster to perform
hadoop FS -mkdir / tmp / DZ
# in namenode / second namenode in:
su HDFS
vi ~ / .hiverc
use CNZZ;
# generate export scripts
hive -e "show tables" | awk '{printf "export table% s to | / tmp / dz /% s |; \ n", $ 1, $ 1}' | sed "s / | / '/ g"> ~ /export.hql
# distinguish inner and outer tables
internal table hive_native_tables.txt
# export script generated (note to be replaced newline \ n under Linux)
CAT hive_native_tables.txt | awk '{the printf "export to table S% | / tmp / DZ /% S |; \ n-",. 1 $, $. 1} '| Sed" S / | /' / G "> ~ / export_native_tables.hql
# Hive export data to HDFS (full scale operation actually when experiencing an external table is stopped, the step to distinguish internal table, this step is not)
the nohup to false Hive --hivevar hive.security.authorization.enabled = -f ~ / export.hql> ~ / export.log 2> & 1 &
# Hive internal table deriving
nohup hive --hivevar hive.security.authorization.enabled = false -f ~ / export_native_tables.hql> ~ / export_native_tables..log 2> & 1 &
# 2 of the cluster hive export files are copied to the cluster 1
directory on the # 1 Creating a cluster (cluster 1 execution)
hadoop FS -mkdir / tmp / DZ
# Create a script distcpHiveExportFiles.sh
! # / bin / bash
hadoop -update invocation of DistCp is -log HDFS: // Master1: 8020 / discpLogs / hiveExport_distCPFiles HDFS: // slave11: 8020 / tmp / DZ HDFS: // Master1: 8020 / tmp / DZ
# copy running in the background data script
nohup SH distcpHiveExportFiles.sh> /tmp/export_hive/hiveExport_distCPFiles_nohup.log 2> & 1 &
# running in the background to monitor the log
tail -100f /tmp/export_hive/hiveExport_distCPFiles_nohup.log
# after the copy is completed, the structure of import hive statement
cp export_native_tables.hql import_native_tables. HQL
Sed -i 'S / Table Export / import Table / G' import_native_tables.hql
Sed -i 'S / to / from / G' import_native_tables.hql
# settings into the new change into the default library
# cluster 1 namenode / the second namenode:
SU HDFS
VI ~ / .hiverc
use CNZZ;
# import data
Hive = hive.security.authorization.enabled to false --hivevar the nohup -f ~ / import_native_tables.hql> ~ / import_native_tables.log 2> &. 1 &
# View log
tail -100F import_native_tables.log
Grant All default to ON User Database HDFS;
# Since the Hive cluster and cluster 1 Hive 2 for the same service. So reload the Hive cluster 1 (direct manipulation interface management)
Hive # 1 Creating a cluster table cnzz
the Create Database cnzz;
# create an external table cnzz library (slave1 execute Hive)
su HDFS
mkdir / tmp / hiveImport
cd / tmp / hiveImport /
# slave1 set the default database
vi ~ / .hiverc
use CNZZ;
# hive_external_table_DDL.txt will be uploaded to the / tmp / hiveImport
# the background to create an external table HQL
nohup Hive --hivevar hive.security.authorization.enabled = false -f /tmp/hiveImport/hive_external_table_DDL.txt> hive_external_table_DDL.log 2> & 1 &
# confirm the observation log output is created
-100F hive_external_table_DDL.log tail
# verify that the external table is successfully created, the data show that successfully created
the SELECT * from `hive_to_ha_source_dis_day` limit 1;
the SELECT * from` ha_visit_page_day` limit 1;
# import_native_tables.hql will be uploaded to the / tmp / hiveImport
# background Hive introducing internal table
the nohup to false Hive --hivevar hive.security.authorization.enabled = -f /tmp/hiveImport/import_native_tables.hql> import_native_tables.log 2> &. 1 &
# observation confirmed the log output is created
tail -100F import_native_tables.log
# table confirm whether the import was successful
#hive shell executed
desc `ss_session_search_temp`;
the SELECT * from` ss_session_search_temp` limit 1;
# the following is a test, do not do
# first create a namespace
create_namespace 'CNZZ'
# then create table
create 'cnzz: HA_EXIT_COLUMN_DAY', NAME = {> 'D'}
# disable table
disable 'CNZZ: HA_EXIT_COLUMN_DAY'
# change file permissions
HDFS the DFS -chmod -R 777 / HBase / Archive / the Data / CNZZ
HDFS /hbase/.hbase-snapshot the DFS -chmod -R 777
# restore snapshots
restore_snapshot 'HA_EXIT_COLUMN_DAY_snapshot_20190625'
# re-enabled tABLE
enable 'CNZZ: HA_EXIT_COLUMN_DAY'
# confirmation table is restored successfully
COUNT 'CNZZ: HA_EXIT_COLUMN_DAY'
# display
5661445 Row (S) in 275.0210 seconds The
, {NAME => 'D'}: Create 'HA_EXIT_COLUMN_HOUR CNZZ'
attachment list:
other scripts and The results .7z
script execution and results .7z