版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/fzuzhanghao1993/article/details/79655794
版本依赖
spark.version：2.1.0
hadoop.version：2.6.5
springboot-mybatis.version：1.1.1
springboot：1.5.10
实现功能
通过HTTP提交job作业请求，并记录日志到数据库中
项目DAO部分使用mybatis实现，本文中不做记录
编码实现
pom
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.test</groupId>
<artifactId>miniSparkJobServer</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>miniSparkJobServer</name>
<description>micro-server</description>
<properties>
<!-- mybatis版本号 -->
<springboot-mybatis.version>1.1.1</springboot-mybatis.version>
<!-- log4j日志文件管理包版本 -->
<slf4j.version>1.7.7</slf4j.version>
<log4j.version>1.2.17</log4j.version>
<spark.version>2.1.0</spark.version>
<hadoop.version>2.6.5</hadoop.version>
<java.version>1.8</java.version>
</properties>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.10.RELEASE</version>
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<!-- 表示开发的时候引入，发布的时候不会加载此包 -->
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
<artifactId>javax.servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>${springboot-mybatis.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.22</version>
</dependency>
</dependencies>
<profiles>
<profile>
<!-- 开发环境 -->
<id>dev</id>
<properties>
<profiles.active>dev</profiles.active>
</properties>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
</profile>
<profile>
<!-- 测试环境 -->
<id>test</id>
<properties>
<profiles.active>test</profiles.active>
</properties>
</profile>
<profile>
<!-- 生产环境 -->
<id>product</id>
<properties>
<profiles.active>product</profiles.active>
</properties>
</profile>
</profiles>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<!-- 资源根目录排除各环境的配置，使用单独的资源目录来指定 -->
<excludes>
<exclude>test/**</exclude>
<exclude>product/**</exclude>
<exclude>dev/**</exclude>
</excludes>
</resource>
<resource>
<directory>src/main/resources/${profiles.active}</directory>
</resource>
</resources>
<finalName>miniSparkJobServer</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>框架配置application.properties
mybatis.type-aliases-package=com.test.sparkjob.entity
spring.datasource.driverClassName = com.mysql.jdbc.Driver
spring.datasource.url = jdbc:mysql://localhost:3360/spark_test?useUnicode=true&characterEncoding=utf-8
spring.datasource.username = root
spring.datasource.password = 123456SparkJobApplication
package com.test;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@MapperScan("com.test.sparkjob.dao")
@SpringBootApplication
public class SparkJobApplication {
public static void main(String[] args) throws Exception {
SpringApplication.run(SparkJobApplication.class, args);
}
}
controller
package com.tesst.sparkjob.controller;
import java.util.Date;
import javax.annotation.Resource;
import org.apache.log4j.Logger;
import org.apache.spark.deploy.SparkSubmit;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import com.test.sparkjob.entity.SparkJobLog;
import com.test.sparkjob.pojo.RetMsg;
import com.test.sparkjob.service.ISparkJobLogService;
@Controller
public class SparkJobController {
private final static Logger log = Logger
.getLogger(SparkJobController.class);
@Resource
private ISparkJobLogService sparkJobLogService;
@RequestMapping(value="/job/executeJob",method=RequestMethod.GET)
@ResponseBody
RetMsg executeSparkJob(@RequestParam("jarId") String jarId,@RequestParam("sparkUri") String sparkUri) {
RetMsg ret = new RetMsg();
StringBuffer msg = new StringBuffer(jarId+":"+sparkUri);
ret.setMsg(msg.toString());
SparkJobLog jobLog = new SparkJobLog();
jobLog.setExecTime(new Date());
String[] arg0=new String[]{
"/usr/job/"+jarId,
"--master",sparkUri,
"--name","web polling",
"--executor-memory","1G"
};
log.info("提交作业...");
try {
SparkSubmit.main(arg0);
} catch (Exception e) {
log.info("出错了！", e);
ret.setCode(1);
ret.setMsg(e.getMessage());
msg.append(e.getMessage());
}
jobLog.setMsg(msg.toString());
sparkJobLogService.insertLog(jobLog);
return ret;
}
}
