Livy submit mr、spark job

1、下载 livy

wget http://mirrors.hust.edu.cn/apache/incubator/livy/0.5.0-incubating/livy-0.5.0-incubating-bin.zip

2、解压安装

依赖环境变量

export SPARK_HOME=/opt/apps/cloudera/parcels/SPARK2-2.2.0.cloudera2-1.cdh5.12.0.p0.232957/lib/spark2
export HADOOP_CONF_DIR=/etc/hadoop/conf

3、修改配置文件

{LIVY_HOME}/conf/livy.conf

livy.spark.master = yarn

4、启动

5、测试

//创建一个sparksession
curl -X POST --data '{"kind": "spark"}' -H "Content-Type:application/json" JY2TEHDP01:8998/sessions
//删除一个session
curl JY2TEHDP01:8998/sessions/0 -X DELETE -H 'Content-Type: application/json'
//在指定session上执行一段代码
curl JY2TEHDP01:8998/sessions/15/statements -X POST -H 'Content-Type: application/json' -d '{"code":"sc.parallelize(1 to 2).count()"}'
//执行jar包 spark or mr
curl -H "Content-Type: application/json" -X POST -d '{"file":"hdfs://HOST:PORT/user/application/spark-examples-1.6.0-cdh5.9.3-hadoop2.6.0-cdh5.9.3.jar","className":"org.apache.spark.examples.SparkPi" }' HOST:PORT/batches

curl -X POST -H "Content-Type: application/json"  HOST:PORT/batches --data '{"file": "hdfs://HOST:PORT/user/zouzhanshun/application/hadoop-mapreduce-examples-2.6.0-cdh5.9.0.jar", "className": "org.apache.hadoop.examples.WordCount", "name": "WordCount Example", "executorCores":1, "executorMemory":"512m", "driverCores":1, "driverMemory":"512m", "queue":"spark", "args":["/user/input","/user/output"]}'