下载并配置livy
http://livy.incubator.apache.org/download/ 并上传到服务器
wget http://apache-mirror.8birdsvideo.com/incubator/livy/0.6.0-incubating/apache-livy-0.6.0-incubating-bin.zip
修改livy配置文件
livy.conf
livy.server.host = prd-zboffline-gw1.prd.leyantech.com
livy.server.port = 8998
livy.ui.basePath = "/livy"
livy.spark.master = yarn
livy.spark.deploy-mode = client
livy.server.session.timeout-check = true
livy.server.session.timeout = 1h
livy-env.sh
JAVA_HOME=/usr/java/jdk1.8.0_191-amd64
HADOOP_CONF_DIR=/opt/hadoop/etc/hadoop
SPARK_HOME=/opt/spark
SPARK_CONF_DIR=/opt/spark/conf
log4j.properties
log4j.rootCategory=INFO, RFA, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
log4j.logger.org.eclipse.jetty=WARN
log4j.appender.RFA=org.apache.log4j.RollingFileAppender
log4j.appender.RFA.File=logs/livy.log
log4j.appender.RFA.MaxFileSize=256MB
log4j.appender.RFA.MaxBackupIndex=10
log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
# Pattern format: Date LogLevel LoggerName LogMessage
log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
启动
测试启动
~/bin/livy-server
后台启动
./livy-server start
测试livy
//创建一个sparksession
curl -X POST --data '{"kind": "spark"}' -H "Content-Type:application/json" prd-zboffline-gw1.prd.leyantech.com:8998/sessions | python -m json.tool
//删除一个session
curl prd-zboffline-gw1.prd.leyantech.com:8998/sessions/0 -X DELETE -H 'Content-Type: application/json' | python -m json.tool
//在指定session上执行一段代码
curl prd-zboffline-gw1.prd.leyantech.com:8998/sessions/0/statements -X POST -H 'Content-Type: application/json' -d '{"code":"sc.parallelize(1 to 2).count()"}' | python -m json.tool
//执行jar包
curl -H "Content-Type: application/json" -X POST -d '{"file":"hdfs:///user/kun.wan/spark-examples_2.11-2.4.3.jar","className":"org.apache.spark.examples.SparkPi" , "args": ["100"]}' prd-zboffline-gw1.prd.leyantech.com:8998/batches | python -m json.tool
// 查看提交的任务
curl prd-zboffline-gw1.prd.leyantech.com:8998/batches/0 | python -m json.tool
// 查看任务执行日志
curl prd-zboffline-gw1.prd.leyantech.com:8998/batches/0/log | python -m json.tool
// 查看回话信息
curl prd-zboffline-gw1.prd.leyantech.com:8998/sessions