CDH相关包的下载地址：
http://archive.cloudera.com/cm5/cm/5/
http://archive.cloudera.com/cdh5/parcels/latest/
用户：
userdel -r cloudera-scm
ps -ef | grep cloudera-scm | awk '{ print $2 }' | sudo xargs kill -9
ps -ef | grep mysql | awk '{ print $2 }' | xargs kill -9
root TPaxuv_0
scm@2018@baoneng
jboss Qhsdvf_9
然后切换到root TPaxuv_0
;M0.vHxbR>lx
/qhapp/cdh
.
├── cloudera
│ ├── parcel-cache
│ └── parcels
├── cm
│ └── cm-5.10.0
├── software
│ ├── CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel
│ ├── CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel.sha
│ ├── cloudera-manager-el6-cm5.10.0_x86_64.tar.gz
│ ├── jdk1.7.0_80.zip
│ ├── manifest.json
│ ├── mysql-advanced-5.7.9-linux-glibc2.5-x86_64.tar.gz
│ └── mysql-connector-java-5.1.40.jar
└── var
├── data
├── he
├── lib
├── log
├── run
└── tmp
mkdir -p /qhapp/cdh/var/data
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
10.60.81.157 sz280317
10.60.81.158 sz280318
10.60.81.159 sz280319
10.60.81.160 sz280320
10.60.81.161 sz280321
10.60.81.162 sz280322
10.60.81.163 sz280323
10.60.81.164 sz280324
10.60.81.165 sz280325
10.60.81.166 sz280326
10.60.81.167 sz280327
10.60.81.168 sz280328
10.60.81.169 sz280329
10.60.81.170 sz280330
10.60.81.171 sz280331
NETWORKING=yes
HOSTNAME=sz280330
server 10.60.4.12 prefer
server 10.60.4.13
server 10.50.4.12
server 10.50.4.13
driftfile /var/lib/ntp/drift
keys /etc/ntp/keys
disable monitor
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
restrict 127.0.0.1
restrict -6 ::1
tinker panic 120
root ALL=(ALL) ALL
%cloudera-scm ALL=(ALL) NOPASSWD: CDH
scp ~/.ssh/authorized_keys root@slave01:~/.ssh/
sz280325
sz280324
sz280326
sz280327
sz280328
sz280329
sz280330
sz280331
scp ~/.ssh/authorized_keys jboss@sz280317:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280318:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280319:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280320:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280321:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280322:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280323:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280324:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280325:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280326:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280327:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280328:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280329:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280330:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280331:~/.ssh/
Qhsdvf_9
rpm -ivh jdk-8u151-linux-x64.rpm
alter user 'root'@'localhost' identified by 'Hgx@123456';
flush privileges;
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'Hgx@123456' WITH GRANT OPTION;
/qhapp/cdh/cm/cm-5.15.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -pMysql@123456 --scm-host localhost scm scm scm
chown -r cloudera-scm:cloudera-scm cdh
useradd --system --home=/qhapp/cdh/cm/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm2
usermod -a -G root cloudera-scm
root TPaxuv_0
groupadd cloudera-scm
useradd -g cloudera-scm cloudera-scm
passwd cloudera-scm
scm@2018
rpm -e mysql-community-libs-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-server-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-common-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-client-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-libs-compat-5.7.20-1.el6.x86_64 --nodeps
sudo /qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-server start
sudo /qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-agent start
/qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-server stop
/qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-agent stop
chown -R cloudera-scm:cloudera-scm /qhapp/cdh
ps -ef |grep cloudera-scm |grep -v grep|cut -c 9-15|xargs kill -9
ps -ef |grep mycanal-1.0-SNAPSHOT.jar |grep -v grep| cut -c 9-15 | xargs kill -9
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
cp mysql-connector-java-8.0.11.jar /qhapp/cdh/cloudera/parcels/CDH-5.15.0-1.cdh5.15.0.p0.21/lib/hive/lib/
kill -9 $(pgrep -f supervisord)
hadoop namenode -recover
hadoop namenode -recover
./hadoop-daemon.sh start journalnode
rpm -ivh mysql-community-common-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-client-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-server-5.7.20-1.el6.x86_64.rpm
alter user 'root'@'localhost' identified by 'Mysql@123456';
set password for 'root'@'localhost' = password('Mysql@123456');
alter user 'root'@'localhost' identified with mysql_native_password by 'Mysql@123456';
flush privileges;
3. 创建同步数据用户及授权用户
3.1．授权其他机器登陆
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'Mysql@123456' WITH GRANT OPTION;
flush privileges;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
rm -rf /qhapp/cdh/cloudera/parcel-cache /qhapp/cdh/cloudera/parcels
cp mysql-connector-java-5.1.35.jar /qhapp/cdh/cm/cm-5.15.0/share/cmf/lib/
cp mysql-connector-java-5.1.35.jar /opt/cm-5.3.8/share/cmf/lib/
/qhapp/cdh/cm/cm-5.15.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -pMysql@123456 --scm-host localhost scm scm scm
/qhapp/cdh/cm/cm-5.15.0/etc/cloudera-scm-agent/config.ini
cloudera-scm
root TPaxuv_0
scm@2018
useradd --system --home=/qhapp/cdh/cm/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
!connect jdbc:hive2://localhost:10000
chown -R cloudera-scm:cloudera-scm /qhapp/cdh
/qhapp/cdh/cloudera/parcels/CDH/lib/hadoop
rm -rf /usr/share/cmf;
rm -rf /var/lib/cloudera*
rm -rf /var/cache/yum/x86_64/6/cloudera*
rm -rf /var/log/cloudera*
rm -rf /var/run/cloudera* /etc/cloudera*
grant all privileges on hue.* to 'hive'@'localhost' identified by 'hue';
grant all privileges on hue.* to 'hive'@'%' identified by 'hue';
flush privileges;
sudo -u hdfs hdfs dfs -chown hive:hive /user
sudo -u hdfs hdfs dfs -chown hue:hue /user
mkdir -p /qhapp/cdh/var/data
mkdir -p /qhapp/cdh/var/lib
mkdir -p /qhapp/cdh/var/log
mkdir -p /qhapp/cdh/var/run
mkdir -p /qhapp/cdh/var/tmp
chown -R cloudera-scm:cloudera-scm /qhapp/cdh
chown cloudera-scm:cloudera-scm /etc/profile
source /etc/profile
solrctl instancedir --generate /qhapp/cdh/var/lib/solr/demo_instancedir
/qhapp/cdh/var/lib/solr/data
ntpdate -u 10.60.81.168
./bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn lib/spark-examples-1.6.0-cdh5.15.0-hadoop2.6.0-cdh5.15.0.jar 10
{"head":{"binlog_pos":24824,"type":"DELETE","binlog_file":"mysql-bin.000005","db":"aa","table":"test_tb2"},"after":{"author":"sss","id":"666","title":"aadd","rowKey":"34555"}}
{"head":{"binlog_pos":24824,"type":"INSERT","binlog_file":"mysql-bin.000005","db":"aa","table":"test_tb2"},"after":{"author":"442","id":"666","title":"aadd","rowKey":"34555"}}
solrctl instancedir --generate /qhapp/cdh/var/lib/solr2
solrctl instancedir --create hgxtest /qhapp/cdh/var/lib/solr2
solrctl instancedir --list
solrctl collection --create hgxtest -s 2 -r 2
zookeeper.connect=10.60.81.166:2181,10.60.81.167:2181,10.60.81.168:2181
10.60.81.157:9092,10.60.81.158:9092,10.60.81.160:9092
zookeeper.connect=10.60.81.157:4183,10.60.81.158:4183,10.60.81.160:4183
./kafka-server-start.sh -daemon ../config/server.properties
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-topics.sh --create --zookeeper 10.60.81.160:4183 --replication-factor 2 --partitions 3 --topic zkKafka
#创建一个broker，发布者
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-console-producer.sh --broker-list 10.60.81.160:9092 --topic zkKafka
'''在一台服务器上创建一个订阅者'''
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-console-consumer.sh --zookeeper 10.60.81.160:4183 --topic canal2kafka2 --from-beginning
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-topics.sh --list --zookeeper 10.60.81.160:4183
/qhapp/cdh/kafka10/kafka_2.11-1.0.0/bin/kafka-topics.sh -describe --zookeeper 10.60.81.160:4183 --topic zkKafka
-describe
/qhapp/cdh/kafka/kafka_2.10-0.9.0.0/bin/kafka-topics.sh --delete --zookeeper 10.60.81.167:2181 --topic kafkae
:
rmr /brokers/topics/heguoxiu2
rmr /brokers/topics/heguoxiu921
rmr /brokers/topics/testh
rmr /brokers/topics/heguoxiu91
rmr /brokers/topics/heguoxiu7
rmr /brokers/topics/canal2kafka2
rmr /brokers/topics/heguoxiu9
rmr /brokers/topics/canal2kafka
rmr /brokers/topics/heguo
rmr /brokers/topics/canal2kafka3
zookeeper.connect=10.60.81.166:2181,10.60.81.168:2181,10.60.81.167:2181
sudo /qhapp/cdh/zookeeper/zookeeper-3.4.12/bin/zkServer.sh start /qhapp/cdh/zookeeper/zookeeper-3.4.12/conf/zoo.cfg
/qhapp/cdh/zookeeper/zookeeper-3.4.12/bin/zkServer.sh stop
/opt/zookeeper/zookeeper-3.4.6/bin/../conf/zoo.cfg
./kafka-server-start.sh -daemon ../config/server.properties
spark-submit --master yarn-client \
--num-executors 4 --driver-memory 2g --executor-memory 2g --executor-cores 4 \
--jars /qhapp/cdh/sparkproject/kafkalib/kafka_2.10-0.8.2.1.jar,/qhapp/cdh/sparkproject/kafkalib/zkclient-0.3.jar,/qhapp/cdh/sparkproject/kafkalib/metrics-core-2.2.0.jar \
--class com.bn.sparkstreaming.spark2hbase /qhapp/cdh/sparkproject/hgx/spark2hbase.jar
--jars /qhapp/cdh/sparkproject/sparklibs/zookeeper-3.4.12.jar
spark-submit --master yarn-client --num-executors 4 --driver-memory 2g --executor-memory 2g --executor-cores 2 \
--jars /qhapp/cdh/sparkproject/sparklibs/spark-examples-1.6.0-hadoop2.6.0.jar --class com.bn.sparkstreaming.spark2hbase /qhapp/cdh/sparkproject/hgx/spark2hbase.jar
{"head":{"binlog_pos":2002,"type":"INSERT","binlog_file":"mysql-bin.000001","db":"kafk2a2","table":"hgx"},"after":{"name":"dd","id":"111216","rowKey":"111216"}}
/qhapp/cdh/kafka011/kafka-logs
--jar /qhapp/cdh/cloudera/parcels/CDH/jars/hive-hbase-handler-1.1.0-cdh5.15.0.jar
show master status
show binlog events in 'mysql-bin.000001
/qhapp/cdh/cloudera/parcels/CDH/jars
mysql主库链接时起始的binlog文件
mysql主库链接时起始的binlog偏移量
mysql主库链接时起始的binlog的时间戳
/qhapp/cdh/cloudera/parcels/CDH/jars
ln -s test.log test2.log
ln -s ../../../jars/jline-2.11.jar jline-2.11.jar
231067965
1533191555000
1483200000
把文件添加到/etc/ld.so.conf,然后执行命令ldconfig 生效
/usr/lib64/mysql/libmysqlclient_r.so.16.0.0
/usr/lib64/mysql/libmysqlclient.so.16.0.0