Spark 编译

[hadoop@hadoop001 soft]$ tar -zxvf spark-2.4.2.tgz -C ~/source/


[hadoop@hadoop001 soft]$ scala -version
Scala code runner version 2.12.8 -- Copyright 2002-2018, LAMP/EPFL and Lightbend, Inc.
[hadoop@hadoop001 soft]$ java -version
java version "1.8.0_45"
Java(TM) SE Runtime Environment (build 1.8.0_45-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)
[hadoop@hadoop001 soft]$ mvn -v
Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe; 2018-06-18T02:33:14+08:00)
Maven home: /home/hadoop/app/apache-maven-3.5.4
Java version: 1.8.0_45, vendor: Oracle Corporation, runtime: /usr/java/jdk1.8.0_45/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "2.6.32-754.el6.centos.plus.x86_64", arch: "amd64", family: "unix"
[hadoop@hadoop001 soft]$ git --version
git version 1.7.1
[hadoop@hadoop001 soft]$ echo $HADOOP_HOME
/home/hadoop/app/hadoop-2.6.0-cdh5.7.0
[hadoop@hadoop001 soft]$ 



[hadoop@hadoop001 spark-2.4.2]$ vi ~/.bash_profile 
export MAVEN_OPTS="-Xmx3g -XX:ReservedCodeCacheSize=1024m"
[hadoop@hadoop001 spark-2.4.2]$ source ~/.bash_profile




pom.xml

<repository>
	<id>cloudera</id>
	<name>cloudera repository</name>
	<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>





[hadoop@hadoop001 spark-2.4.2]$ vi dev/make-distribution.sh 

VERSION=2.4.2
SCALA_VERSION=2.12
SPARK_HADOOP_VERSION=2.6.0-cdh5.7.0
SPARK_HIVE=1


export MAVEN_OPTS="${MAVEN_OPTS:--Xmx3g -XX:ReservedCodeCacheSize=1024m}"


#MVN="$SPARK_HOME/build/mvn"
MVN="${MAVEN_HOME}/bin/mvn"


./dev/make-distribution.sh --name 2.6.0-cdh5.7.0 --tgz  -Pyarn -Phive -Phive-thriftserver  -Phadoop-2.6 -Dhadoop.version=2.6.0-cdh5.7.0




[WARNING] Zinc server is not available at port 3030 - reverting to normal incremental compile
[INFO] Using incremental compilation
[WARNING] Zinc server is not available at port 3030 - reverting to normal incremental compile

。。。。。
[INFO] ------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 9.359 s (Wall Clock)
[INFO] Finished at: 2019-05-05T17:00:23+08:00
[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal net.alchim31.maven:scala-maven-plugin:3.2.2:compile (scala-compile-first) on project spark-tools_2.11: wrap: java.lang.ClassNotFoundException: xsbt.CompilerInterface -> [Help 1]
[ERROR] Failed to execute goal net.alchim31.maven:scala-maven-plugin:3.2.2:compile (scala-compile-first) on project spark-tags_2.11: wrap: java.lang.ClassNotFoundException: xsbt.CompilerInterface -> [Help 1]