Spark 启停应用

启动
jerry用户:

mongo:

mongod --dbpath=opt/mongodb-3.4.0/data/db/ --logpath=opt/mongodb-3.4.0/log/mongod.log

mongod --dbpath=opt/mongodb-3.4.0/data/db/ --logpath=opt/mongodb-3.4.0/log/mongod.log  --auth

hadoop用户:

zookeeper-3.4.11/bin/zkCli.sh -server 10.26.51.89:2181 ---存在内网,外网的情况下,指定内网IP进行访问
存在内外网的情况下: zookeeper.connect 如果无法指定所有的ip:port/chroot(会出现(/kafka10,10.26.51.89:2181),ip,chroot粘连的情况),可以只指定自身的ip:port/chroot

redis:
redis-server  /usr/redis/redis.conf  &

Zookeeper:
zkServer.sh start

HDFS:
start-dfs.sh

kafka:
kafka-server-start.sh /usr/kafka/config/server.properties   &
kafka-server-start.sh /usr/kafka/config/server1.properties   &

flume: 
flume-ng agent -c conf -f $FLUME_HOME/conf/hdfsAndKafka.conf -n a1 -Dflume.root.logger=INFO,console

/home/hadoop/properties/hdfs.properties 

spark:
/usr/spark2.2/sbin/start-all.sh

启动应用

 -Dspark.deploy.defaultCores  coreNums(设置启动cpu核数)
--conf  spark.cores.max=N (spark application 分配的最大核数

 spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077     --executor-memory 1G     --total-executor-cores 10   --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails  -XX:+PrintGCTimeStamps"   --conf "spark.executor.extraJavaOptions=  -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -verbose:gc  -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"   --driver-java-options  "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -verbose:gc  -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"     /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar  /home/hadoop/properties/hdfs.properties   com.sznongfu.cn.online.statistic.NginxLogStatistic

 spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077     --executor-memory 1G     --total-executor-cores 10   --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails  -XX:+PrintGCTimeStamps"   --conf "spark.executor.extraJavaOptions=  -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled   -XX:+HeapDumpOnOutOfMemoryError   -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"   --driver-java-options  "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled   -XX:+HeapDumpOnOutOfMemoryError   -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"     /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar  /home/hadoop/properties/hdfs.properties   com.sznongfu.cn.online.statistic.NginxLogStatistic

--conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails  -XX:+PrintGCTimeStamps"   --conf "spark.executor.extraJavaOptions=  -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled   -XX:+HeapDumpOnOutOfMemoryError   -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"  

--driver-java-options  "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled   -XX:+HeapDumpOnOutOfMemoryError   -XX:MaxTenuringThreshold=10 -XX:SurvivorRatio=8"

/* 备用 不可用
spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077     --executor-memory 1G     --total-executor-cores 5   --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -XX:ConcGCThread=20"    /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties   com.sznongfu.cn.online.statistic.NginxLogStatistic
备用  不可用*/

/* 可用 */

 spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077     --executor-memory 1G     --total-executor-cores 5   --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark "    /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties   com.sznongfu.cn.online.statistic.NginxLogStatistic

/* 可用 */

英特尔最佳JVM实践
-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -Xms88g -Xmx88g -XX:InitiatingHeapOccupancyPercent=35 -XX:ConcGCThread=20

kafka GC方式:
00:05:54 /home/sznongfu/opt/jdk-1.8.0/bin/java -Xmx512M -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/home/sznongfu/opt/kafka_2.11-0.10.0.0/bin/../logs -Dlog4j.configuration=file:/home/sznongfu/opt/kafka_2.11-0.10.0.0/bin/../config/tools-log4j.properties -cp

 
spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077  --executor-memory 2G    --total-executor-cores 2       --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps"    /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties    com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic

spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077    --driver-cores 5  --driver-memory 1G   --total-executor-cores 8 --executor-memory 1G --executor-cores 4    --conf "spark.executor.extraJavaOptions=-XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions"    /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties    com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic   >> trace01.log  2>&1   &

parllel GC  1010  883   902  差不多 763  759  761  405  压测:1131  785  794    二次: 843  703  784
spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077   --driver-cores 5  --driver-memory 1G  --total-executor-cores 4     --executor-memory 1G    --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps"    /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties    com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic  >> trace01Tune.log  2>&1   &

PID    USER     PR   NI    VIRT     RES     SHR   S %CPU   %MEM     TIME+                                                                                                                        
21257 hadoop    20   0     4805760  1.193g  31676 S  98.7  15.5   5:00.10  

G1 GC   所用时间:    925  939  946  差不多  831  831  1159   压测 810  810  821  二次:  752  783 805

测试环境启动
 ~/opt/spark-2.2.0-bin-hadoop2.7/bin/spark-submit     --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://IZ2zehlhwjhnr93dioubfzZ:7077   --driver-cores 1  --driver-memory 1G  --total-executor-cores 4 --executor-cores 4      --executor-memory 1G    --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"    ./libs/LogProcess-1.0-SNAPSHOT-all.jar  ./conf/la.benchmark.properties     com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic  >> ./logs/offline.log   2>&1   &

测试环境启动
 ~/opt/spark-2.2.0-bin-hadoop2.7/bin/spark-submit     --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://iZ2zehlhwjhnr93dioubfzZ:7077   --driver-cores 1  --driver-memory 1G  --total-executor-cores 4 --executor-cores 4      --executor-memory 1G    --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"    /tmp/offlinejar/LogProcess-1.0-SNAPSHOT-all.jar  ./conf/la.benchmark.properties     com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic

/tmp/offlinejar/LogProcess-1.0-SNAPSHOT-all.jar

spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077   --driver-cores 2  --driver-memory 1G  --total-executor-cores 2    --executor-memory 1G    --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -XX:ConcGCThread=20"    /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties    com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic

  --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC  -XX:ConcGCThread=20  -XX:MaxGCPauseMillis=20"

 --driver-java-options  "-XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC  -XX:ConcGCThread=20  -XX:MaxGCPauseMillis=20"
  

spark-submit     --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077   --driver-cores 1  --driver-memory 1G  --total-executor-cores 4 --executor-cores 4      --executor-memory 1G    --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"   /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties    com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic

最优实践:
 

 spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077  --executor-memory 1G   --driver-cores 2  --driver-memory 1G  --total-executor-cores 2       --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC  -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"    /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties    com.sznongfu.cn.online.statistic.NginxLogStatistic
 
 
---可行

 spark-submit    --class com.sznongfu.cn.center.StartCenter  --name  sznongfuzhanggui   --master spark://hadoop:7077  --executor-memory 1G   --driver-cores 2  --driver-memory 1G  --total-executor-cores 2           /home/jerry/racoon/nongfu.ngx.log.analyser/modules/LogProcess/out/libs/LogProcess-1.0-SNAPSHOT-all.jar   /home/hadoop/properties/hdfs.properties    com.sznongfu.cn.offline.statistic.UserTrajectoryStatistic

关闭应用 ctrl+c

/usr/spark2.2/sbin/stop-all.sh

1.kafka:
kafka-server-stop.sh /usr/kafka/config/server.properties 
kafka-server-stop.sh /usr/kafka/config/server1.properties 

2.reids:
redis-cli shutdown

3.HDFS:
stop-dfs.sh

4.Zookeeper:
zkServer.sh stop

5.mongo:
ctrl+c

zkServer.sh stop

exec $SPARK_HOME/bin/spark-submit \
--class com.bigeyedata.mort.Main \
--driver-class-path $MORT_HOME/libs/*.jar \
--master yarn-client \
--deploy-mode cluster \
--jars /appcom/mort/thirdparty_jars/customer_provided.jar \
--queue queue_0100_01 \
$MORT_HOME/mort.jar > $MORT_HOME/mort.log 2>&1

#! /bin/bash

while true
do

echo "a"
sleep 10s

done
~          

 

回退版本: 
git reset --hard  5997af6ace0769da74d5b7659f1e3c62e1c9afa0

 2018-05-21  Browser   5aa91001220f130432ba6279
 2018-05-21  App       9QZrh9wogSSdIoCS7gdQjMHA
 
 

-Xmx1G -Xms1G -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true

 -Xloggc:/home/sznongfu/opt/kafka_2.11-0.10.0.0/bin/../logs