IDEA spark程序本地调试

准备:
linux spark2.1.1 集群
linux hadoop-2.7.7集群
windows hadoop-2.7.7环境
windows scala-2.11.8

WordCount.scala

package com.zuo.spark

import org.apache.spark.{SparkConf, SparkContext}
import org.slf4j.LoggerFactory

object WordCount {

  val logger = LoggerFactory.getLogger(WordCount.getClass)

  def main(args: Array[String]): Unit = {

    //创建SparkConf()并设置APP名称
    //    val conf = new SparkConf().setAppName("WC")//如果打jar放集群运行,直接这样就可以
    val conf = new SparkConf().setMaster("spark://app-dev-yunying:7077").setAppName("WC")
      .setJars(List("E:\\DEV_WORKSPACE\\spark\\target\\wordcount-jar-with-dependencies.jar"))
      .setIfMissing("spark.driver.host", "192.168.3.87")//driver的ip地址

    //创建SparkContext,该对象是提交spark APP的入口
    val sc = new SparkContext(conf)
    val file = sc.textFile("hdfs://app-dev-h5:9000/README.txt");
    val words = file.flatMap(_.split(" "))
    val wordsTuple = words.map((_, 1))
    wordsTuple.reduceByKey(_ + _).saveAsTextFile("hdfs://app-dev-h5:9000/out4")


    //使用sc创建RDD并执行相应的transformation和action
    //    sc.textFile(args(0)).flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _, 1).sortBy(_._2, false).saveAsTextFile(args(1))

    //停止sc,结束该任务
    logger.info("complete!")

    sc.stop()

  }
}

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.zuo</groupId>
    <artifactId>spark</artifactId>
    <version>1.0-SNAPSHOT</version>

<properties>
    <scala.version>2.11.8</scala.version>
    <spark.version>2.1.1</spark.version>
    <hadoop.version>2.7.7</hadoop.version>
    <slf4j.version>1.7.25</slf4j.version>
    <log4j.version>1.2.17</log4j.version>
</properties>

    <dependencies>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.version}</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
            <scope>provided</scope>
        </dependency>
        <!-- Logging -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>jcl-over-slf4j</artifactId>
            <version>${slf4j.version}</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>${slf4j.version}</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>${slf4j.version}</version>
        </dependency>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>${log4j.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.8</version>
        </dependency>
    </dependencies>
    <build>
        <finalName>wordcount</finalName>
        <plugins>
            <!-- maven编译scala插件 -->
            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>3.2.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>

            <!-- 依赖打包插件 -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.0.0</version>
                <configuration>
                    <archive>
                        <manifest>
                            <mainClass>com.zuo.spark.WordCount</mainClass>
                        </manifest>
                    </archive>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

打包命令:

mvn clean package

运行结果:

 

 

spark 本地远程调试需要注意的地方:

spark本地debug运行时是通过java -jar 运行的,而不是通过scala运行的,所以为了防止一些依赖在运行的时候找不到,

需要将pom.xml中的一些jar包的scope=provided的标签注释掉

或者在EditConfigurations中勾选

 

遇到的问题:

val =  new SparkConf()  这行报错

NoClassDefFoundError: org.apache.commons.lang3.SystemUtils

但是搜索项目中依赖是有org.apache.commons.lang3:3.5.jar的

分析:

原因未找出来,还有待探索...

解决:

在pom.xml中加org.apache.commons.lang3:3.8.jar

        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.8</version>
        </dependency>