本地执行Spark SQL程序：
package com.fc //import common.util.{phoenixConnectMode, timeUtil} import org.apache.spark.sql.SQLContext import org.apache.spark.sql.functions.col import org.apache.spark.{SparkConf, SparkContext} /* 每天执行 */ object costDay { def main(args: Array[String]): Unit = { val conf = new SparkConf() .setAppName("fdsf") .setMaster("local") val sc = new SparkContext(conf) val sqlContext = new SQLContext(sc) // val df = sqlContext.load( // "org.apache.phoenix.spark" // , Map("table" -> "ASSET_NORMAL" // , "zkUrl" -> "node3,node4,node5:2181") // ) val tableName = "ASSET_NORMAL" val columes = Array( "ID", "ASSET_ID", "ASSET_NAME", "ASSET_FIRST_DEGREE_ID", "ASSET_FIRST_DEGREE_NAME", "ASSET_SECOND_DEGREE_ID", "ASSET_SECOND_DEGREE_NAME", "GB_DEGREE_ID", "GB_DEGREE_NAME", "ASSET_USE_FIRST_DEGREE_ID", "ASSET_USE_FIRST_DEGREE_NAME", "ASSET_USE_SECOND_DEGREE_ID", "ASSET_USE_SECOND_DEGREE_NAME", "MANAGEMENT_TYPE_ID", "MANAGEMENT_TYPE_NAME", "ASSET_MODEL", "FACTORY_NUMBER", "ASSET_COUNTRY_ID", "ASSET_COUNTRY_NAME", "MANUFACTURER", "SUPPLIER", "SUPPLIER_TEL", "ORIGINAL_VALUE", "USE_DEPARTMENT_ID", "USE_DEPARTMENT_NAME", "USER_ID", "USER_NAME", "ASSET_LOCATION_OF_PARK_ID", "ASSET_LOCATION_OF_PARK_NAME", "ASSET_LOCATION_OF_BUILDING_ID", "ASSET_LOCATION_OF_BUILDING_NAME", "ASSET_LOCATION_OF_ROOM_ID", "ASSET_LOCATION_OF_ROOM_NUMBER", "PRODUCTION_DATE", "ACCEPTANCE_DATE", "REQUISITION_DATE", "PERFORMANCE_INDEX", "ASSET_STATE_ID", "ASSET_STATE_NAME", "INSPECTION_TYPE_ID", "INSPECTION_TYPE_NAME", "SEAL_DATE", "SEAL_CAUSE", "COST_ITEM_ID", "COST_ITEM_NAME", "ITEM_COMMENTS", "UNSEAL_DATE", "SCRAP_DATE", "PURCHASE_NUMBER", "WARRANTY_PERIOD", "DEPRECIABLE_LIVES_ID", "DEPRECIABLE_LIVES_NAME", "MEASUREMENT_UNITS_ID", "MEASUREMENT_UNITS_NAME", "ANNEX", "REMARK", "ACCOUNTING_TYPE_ID", "ACCOUNTING_TYPE_NAME", "SYSTEM_TYPE_ID", "SYSTEM_TYPE_NAME", "ASSET_ID_PARENT", "CLASSIFIED_LEVEL_ID", "CLASSIFIED_LEVEL_NAME", "ASSET_PICTURE", "MILITARY_SPECIAL_CODE", "CHECK_CYCLE_ID", "CHECK_CYCLE_NAME", "CHECK_DATE", "CHECK_EFFECTIVE_DATE", "CHECK_MODE_ID", "CHECK_MODE_NAME", "CHECK_DEPARTMENT_ID", "CHECK_DEPARTMENT_NAME", "RENT_STATUS_ID", "RENT_STATUS_NAME", "STORAGE_TIME", "UPDATE_USER", "UPDATE_TIME", "IS_ON_PROCESS", "IS_DELETED", "FIRST_DEPARTMENT_ID", "FIRST_DEPARTMENT_NAME", "SECOND_DEPARTMENT_ID", "SECOND_DEPARTMENT_NAME", "CREATE_USER", "CREATE_TIME" ) val df = phoenixConnectMode.getMode1(sqlContext, tableName, columes) .filter(col("USE_DEPARTMENT_ID") isNotNull) df.registerTempTable("asset_normal") // df.show(false) def costingWithin(originalValue: Double, years: Int): Double = (originalValue*0.95)/(years*365) sqlContext.udf.register("costingWithin", costingWithin _) def costingBeyond(originalValue: Double): Double = originalValue*0.05/365 sqlContext.udf.register("costingBeyond", costingBeyond _) def expire(acceptanceDate: String, years: Int): Boolean = timeUtil.dateStrAddYears2TimeStamp(acceptanceDate, timeUtil.SECOND_TIME_FORMAT, years) > System.currentTimeMillis() sqlContext.udf.register("expire", expire _) val costDay = sqlContext .sql( "select " + "ID" + ",USE_DEPARTMENT_ID as FIRST_DEPARTMENT_ID" + ",case when expire(ACCEPTANCE_DATE, DEPRECIABLE_LIVES_NAME) then costingWithin(ORIGINAL_VALUE, DEPRECIABLE_LIVES_NAME) else costingBeyond(ORIGINAL_VALUE) end as ACTUAL_COST" + ",ORIGINAL_VALUE" + ",current_timestamp() as GENERATION_TIME" + " from asset_normal" ) costDay.printSchema() println(costDay.count()) costDay.col("ORIGINAL_VALUE") costDay.describe("ORIGINAL_VALUE").show() costDay.show(false) // costDay.write // .format("org.apache.phoenix.spark") // .mode("overwrite") // .option("table", "ASSET_FINANCIAL_DETAIL_DAY") // .option("zkUrl", "node3,node4,node5:2181") // .save() } }
执行结果参考《Spark教程——（10）Spark SQL读取Phoenix数据本地执行计算》，如下：
19/09/24 00:52:14 INFO spark.SparkContext: Running Spark version 1.6.0 19/09/24 00:52:16 INFO spark.SecurityManager: Changing view acls to: cf_pc 19/09/24 00:52:16 INFO spark.SecurityManager: Changing modify acls to: cf_pc 19/09/24 00:52:16 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(cf_pc); users with modify permissions: Set(cf_pc) 19/09/24 00:52:16 INFO util.Utils: Successfully started service 'sparkDriver' on port 54691. 19/09/24 00:52:17 INFO slf4j.Slf4jLogger: Slf4jLogger started 19/09/24 00:52:17 INFO Remoting: Starting remoting 19/09/24 00:52:17 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://[email protected]:54708] 19/09/24 00:52:17 INFO Remoting: Remoting now listens on addresses: [akka.tcp://[email protected]:54708] 19/09/24 00:52:17 INFO util.Utils: Successfully started service 'sparkDriverActorSystem' on port 54708. 19/09/24 00:52:17 INFO spark.SparkEnv: Registering MapOutputTracker 19/09/24 00:52:17 INFO spark.SparkEnv: Registering BlockManagerMaster 19/09/24 00:52:17 INFO storage.DiskBlockManager: Created local directory at C:\Users\cf_pc\AppData\Local\Temp\blockmgr-9012d6b9-eab0-41f0-8e9e-b63c3fe7fb09 19/09/24 00:52:17 INFO storage.MemoryStore: MemoryStore started with capacity 478.2 MB 19/09/24 00:52:17 INFO spark.SparkEnv: Registering OutputCommitCoordinator 19/09/24 00:52:18 INFO server.Server: jetty-8.y.z-SNAPSHOT 19/09/24 00:52:18 INFO server.AbstractConnector: Started SelectChannelConnector@0.0.0.0:4040 19/09/24 00:52:18 INFO util.Utils: Successfully started service 'SparkUI' on port 4040. 19/09/24 00:52:18 INFO ui.SparkUI: Started SparkUI at http://10.200.74.155:4040 19/09/24 00:52:18 INFO executor.Executor: Starting executor ID driver on host localhost 19/09/24 00:52:18 INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54720. 19/09/24 00:52:18 INFO netty.NettyBlockTransferService: Server created on 54720 19/09/24 00:52:18 INFO storage.BlockManagerMaster: Trying to register BlockManager 19/09/24 00:52:18 INFO storage.BlockManagerMasterEndpoint: Registering block manager localhost:54720 with 478.2 MB RAM, BlockManagerId(driver, localhost, 54720) 19/09/24 00:52:18 INFO storage.BlockManagerMaster: Registered BlockManager 19/09/24 00:52:21 INFO storage.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 247.5 KB, free 477.9 MB) 19/09/24 00:52:22 INFO storage.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 15.4 KB, free 477.9 MB) 19/09/24 00:52:22 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:54720 (size: 15.4 KB, free: 478.2 MB) 19/09/24 00:52:22 INFO spark.SparkContext: Created broadcast 0 from newAPIHadoopRDD at PhoenixRDD.scala:49 19/09/24 00:52:22 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:22 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:23 INFO log.QueryLoggerDisruptor: Starting QueryLoggerDisruptor for with ringbufferSize=8192, waitStrategy=BlockingWaitStrategy, exceptionHandler=org.apache.phoenix.log.QueryLoggerDefaultExceptionHandler@4f169009... 19/09/24 00:52:23 INFO query.ConnectionQueryServicesImpl: An instance of ConnectionQueryServices was created. 19/09/24 00:52:23 INFO zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x4bc59b27 connecting to ZooKeeper ensemble=node3:2181 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:host.name=DESKTOP-0CDQ4PM 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.version=1.8.0_212 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.home=C:\3rd\Java\jdk1.8.0_212\jre 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.class.path=C:\3rd\Java\jdk1.8.0_212\jre\lib\charsets.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\deploy.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\access-bridge-64.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\cldrdata.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\dnsns.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\jaccess.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\jfxrt.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\localedata.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\nashorn.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunec.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunjce_provider.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunmscapi.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunpkcs11.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\zipfs.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\javaws.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jce.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jfr.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jfxswt.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jsse.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\management-agent.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\plugin.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\resources.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\rt.jar;C:\development\cf\scalapp\target\classes;C:\3rd\scala\scala-2.10.5\lib\scala-actors-migration.jar;C:\3rd\scala\scala-2.10.5\lib\scala-actors.jar;C:\3rd\scala\scala-2.10.5\lib\scala-library.jar;C:\3rd\scala\scala-2.10.5\lib\scala-reflect.jar;C:\3rd\scala\scala-2.10.5\lib\scala-swing.jar;C:\3rd\scala\scala-2.10.5\src\scala-actors-src.jar;C:\3rd\scala\scala-2.10.5\src\scala-library-src.jar;C:\3rd\scala\scala-2.10.5\src\scala-reflect-src.jar;C:\3rd\scala\scala-2.10.5\src\scala-swing-src.jar;C:\development\MavenRepository\org\apache\spark\spark-core_2.10\1.6.0-cdh5.14.2\spark-core_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\avro\avro-mapred\1.7.6-cdh5.14.2\avro-mapred-1.7.6-cdh5.14.2-hadoop2.jar;C:\development\MavenRepository\org\apache\avro\avro-ipc\1.7.6-cdh5.14.2\avro-ipc-1.7.6-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\avro\avro-ipc\1.7.6-cdh5.14.2\avro-ipc-1.7.6-cdh5.14.2-tests.jar;C:\development\MavenRepository\com\twitter\chill_2.10\0.5.0\chill_2.10-0.5.0.jar;C:\development\MavenRepository\com\esotericsoftware\kryo\kryo\2.21\kryo-2.21.jar;C:\development\MavenRepository\com\esotericsoftware\reflectasm\reflectasm\1.07\reflectasm-1.07-shaded.jar;C:\development\MavenRepository\com\esotericsoftware\minlog\minlog\1.2\minlog-1.2.jar;C:\development\MavenRepository\org\objenesis\objenesis\1.2\objenesis-1.2.jar;C:\development\MavenRepository\com\twitter\chill-java\0.5.0\chill-java-0.5.0.jar;C:\development\MavenRepository\org\apache\xbean\xbean-asm5-shaded\4.4\xbean-asm5-shaded-4.4.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-client\2.6.0-cdh5.14.2\hadoop-client-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-hdfs\2.6.0-cdh5.14.2\hadoop-hdfs-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\xerces\xercesImpl\2.9.1\xercesImpl-2.9.1.jar;C:\development\MavenRepository\xml-apis\xml-apis\1.3.04\xml-apis-1.3.04.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-app\2.6.0-cdh5.14.2\hadoop-mapreduce-client-app-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-common\2.6.0-cdh5.14.2\hadoop-mapreduce-client-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-client\2.6.0-cdh5.14.2\hadoop-yarn-client-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-server-common\2.6.0-cdh5.14.2\hadoop-yarn-server-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-shuffle\2.6.0-cdh5.14.2\hadoop-mapreduce-client-shuffle-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-api\2.6.0-cdh5.14.2\hadoop-yarn-api-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-jobclient\2.6.0-cdh5.14.2\hadoop-mapreduce-client-jobclient-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-aws\2.6.0-cdh5.14.2\hadoop-aws-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\com\amazonaws\aws-java-sdk-bundle\1.11.134\aws-java-sdk-bundle-1.11.134.jar;C:\development\MavenRepository\org\apache\spark\spark-launcher_2.10\1.6.0-cdh5.14.2\spark-launcher_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\spark\spark-network-common_2.10\1.6.0-cdh5.14.2\spark-network-common_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;C:\development\MavenRepository\com\fasterxml\jackson\core\jackson-annotations\2.2.3\jackson-annotations-2.2.3.jar;C:\development\MavenRepository\org\apache\spark\spark-network-shuffle_2.10\1.6.0-cdh5.14.2\spark-network-shuffle_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\spark\spark-unsafe_2.10\1.6.0-cdh5.14.2\spark-unsafe_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\net\java\dev\jets3t\jets3t\0.9.0\jets3t-0.9.0.jar;C:\development\MavenRepository\org\apache\httpcomponents\httpcore\4.1.2\httpcore-4.1.2.jar;C:\development\MavenRepository\com\jamesmurty\utils\java-xmlbuilder\0.4\java-xmlbuilder-0.4.jar;C:\development\MavenRepository\org\apache\curator\curator-recipes\2.7.1\curator-recipes-2.7.1.jar;C:\development\MavenRepository\org\apache\curator\curator-framework\2.7.1\curator-framework-2.7.1.jar;C:\development\MavenRepository\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;C:\development\MavenRepository\org\eclipse\jetty\orbit\javax.servlet\3.0.0.v201112011016\javax.servlet-3.0.0.v201112011016.jar;C:\development\MavenRepository\org\apache\commons\commons-lang3\3.3.2\commons-lang3-3.3.2.jar;C:\development\MavenRepository\org\apache\commons\commons-math3\3.4.1\commons-math3-3.4.1.jar;C:\development\MavenRepository\com\google\code\findbugs\jsr305\1.3.9\jsr305-1.3.9.jar;C:\development\MavenRepository\org\slf4j\slf4j-api\1.7.5\slf4j-api-1.7.5.jar;C:\development\MavenRepository\org\slf4j\jul-to-slf4j\1.7.5\jul-to-slf4j-1.7.5.jar;C:\development\MavenRepository\org\slf4j\jcl-over-slf4j\1.7.5\jcl-over-slf4j-1.7.5.jar;C:\development\MavenRepository\log4j\log4j\1.2.17\log4j-1.2.17.jar;C:\development\MavenRepository\org\slf4j\slf4j-log4j12\1.7.5\slf4j-log4j12-1.7.5.jar;C:\development\MavenRepository\com\ning\compress-lzf\1.0.3\compress-lzf-1.0.3.jar;C:\development\MavenRepository\org\xerial\snappy\snappy-java\1.0.4.1\snappy-java-1.0.4.1.jar;C:\development\MavenRepository\net\jpountz\lz4\lz4\1.3.0\lz4-1.3.0.jar;C:\development\MavenRepository\org\roaringbitmap\RoaringBitmap\0.5.11\RoaringBitmap-0.5.11.jar;C:\development\MavenRepository\commons-net\commons-net\2.2\commons-net-2.2.jar;C:\development\MavenRepository\org\spark-project\akka\akka-remote_2.10\2.2.3-shaded-protobuf\akka-remote_2.10-2.2.3-shaded-protobuf.jar;C:\development\MavenRepository\org\spark-project\akka\akka-actor_2.10\2.2.3-shaded-protobuf\akka-actor_2.10-2.2.3-shaded-protobuf.jar;C:\development\MavenRepository\com\typesafe\config\1.0.2\config-1.0.2.jar;C:\development\MavenRepository\org\spark-project\protobuf\protobuf-java\2.4.1-shaded\protobuf-java-2.4.1-shaded.jar;C:\development\MavenRepository\org\uncommons\maths\uncommons-maths\1.2.2a\uncommons-maths-1.2.2a.jar;C:\development\MavenRepository\org\spark-project\akka\akka-slf4j_2.10\2.2.3-shaded-protobuf\akka-slf4j_2.10-2.2.3-shaded-protobuf.jar;C:\development\MavenRepository\org\scala-lang\scala-library\2.10.5\scala-library-2.10.5.jar;C:\development\MavenRepository\org\json4s\json4s-jackson_2.10\3.2.10\json4s-jackson_2.10-3.2.10.jar;C:\development\MavenRepository\org\json4s\json4s-core_2.10\3.2.10\json4s-core_2.10-3.2.10.jar;C:\development\MavenRepository\org\json4s\json4s-ast_2.10\3.2.10\json4s-ast_2.10-3.2.10.jar;C:\development\MavenRepository\org\scala-lang\scalap\2.10.0\scalap-2.10.0.jar;C:\development\MavenRepository\org\scala-lang\scala-compiler\2.10.0\scala-compiler-2.10.0.jar;C:\development\MavenRepository\com\sun\jersey\jersey-server\1.9\jersey-server-1.9.jar;C:\development\MavenRepository\asm\asm\3.1\asm-3.1.jar;C:\development\MavenRepository\com\sun\jersey\jersey-core\1.9\jersey-core-1.9.jar;C:\development\MavenRepository\org\apache\mesos\mesos\0.21.1\mesos-0.21.1-shaded-protobuf.jar;C:\development\MavenRepository\io\netty\netty-all\4.0.29.Final\netty-all-4.0.29.Final.jar;C:\development\MavenRepository\com\clearspring\analytics\stream\2.7.0\stream-2.7.0.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-core\3.1.2\metrics-core-3.1.2.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-jvm\3.1.2\metrics-jvm-3.1.2.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-json\3.1.2\metrics-json-3.1.2.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-graphite\3.1.2\metrics-graphite-3.1.2.jar;C:\development\MavenRepository\com\fasterxml\jackson\core\jackson-databind\2.2.3\jackson-databind-2.2.3.jar;C:\development\MavenRepository\com\fasterxml\jackson\core\jackson-core\2.2.3\jackson-core-2.2.3.jar;C:\development\MavenRepository\com\fasterxml\jackson\module\jackson-module-scala_2.10\2.2.3\jackson-module-scala_2.10-2.2.3.jar;C:\development\MavenRepository\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;C:\development\MavenRepository\org\apache\ivy\ivy\2.4.0\ivy-2.4.0.jar;C:\development\MavenRepository\oro\oro\2.0.8\oro-2.0.8.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-client\0.8.2\tachyon-client-0.8.2.jar;C:\development\MavenRepository\commons-lang\commons-lang\2.4\commons-lang-2.4.jar;C:\development\MavenRepository\commons-io\commons-io\2.4\commons-io-2.4.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-underfs-hdfs\0.8.2\tachyon-underfs-hdfs-0.8.2.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-underfs-s3\0.8.2\tachyon-underfs-s3-0.8.2.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-underfs-local\0.8.2\tachyon-underfs-local-0.8.2.jar;C:\development\MavenRepository\net\razorvine\pyrolite\4.9\pyrolite-4.9.jar;C:\development\MavenRepository\net\sf\py4j\py4j\0.9\py4j-0.9.jar;C:\development\MavenRepository\com\intel\chimera\chimera\0.9.2\chimera-0.9.2.jar;C:\development\MavenRepository\org\spark-project\spark\unused\1.0.0\unused-1.0.0.jar;C:\development\MavenRepository\org\apache\spark\spark-sql_2.10\1.6.0-cdh5.14.2\spark-sql_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\spark\spark-catalyst_2.10\1.6.0-cdh5.14.2\spark-catalyst_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\scala-lang\scala-reflect\2.10.5\scala-reflect-2.10.5.jar;C:\development\MavenRepository\org\codehaus\janino\janino\2.7.8\janino-2.7.8.jar;C:\development\MavenRepository\org\codehaus\janino\commons-compiler\2.7.8\commons-compiler-2.7.8.jar;C:\development\MavenRepository\com\twitter\parquet-column\1.5.0-cdh5.14.2\parquet-column-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-common\1.5.0-cdh5.14.2\parquet-common-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-encoding\1.5.0-cdh5.14.2\parquet-encoding-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-hadoop\1.5.0-cdh5.14.2\parquet-hadoop-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-format\2.1.0-cdh5.14.2\parquet-format-2.1.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-jackson\1.5.0-cdh5.14.2\parquet-jackson-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-spark\1.2.0-cdh5.14.2\hbase-spark-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-common\2.6.0-cdh5.14.2\hadoop-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;C:\development\MavenRepository\commons-httpclient\commons-httpclient\3.1\commons-httpclient-3.1.jar;C:\development\MavenRepository\org\mortbay\jetty\jetty\6.1.26.cloudera.4\jetty-6.1.26.cloudera.4.jar;C:\development\MavenRepository\org\mortbay\jetty\jetty-util\6.1.26.cloudera.4\jetty-util-6.1.26.cloudera.4.jar;C:\development\MavenRepository\com\sun\jersey\jersey-json\1.9\jersey-json-1.9.jar;C:\development\MavenRepository\org\codehaus\jettison\jettison\1.1\jettison-1.1.jar;C:\development\MavenRepository\com\sun\xml\bind\jaxb-impl\2.2.3-1\jaxb-impl-2.2.3-1.jar;C:\development\MavenRepository\tomcat\jasper-compiler\5.5.23\jasper-compiler-5.5.23.jar;C:\development\MavenRepository\tomcat\jasper-runtime\5.5.23\jasper-runtime-5.5.23.jar;C:\development\MavenRepository\commons-el\commons-el\1.0\commons-el-1.0.jar;C:\development\MavenRepository\commons-configuration\commons-configuration\1.6\commons-configuration-1.6.jar;C:\development\MavenRepository\commons-digester\commons-digester\1.8\commons-digester-1.8.jar;C:\development\MavenRepository\commons-beanutils\commons-beanutils\1.7.0\commons-beanutils-1.7.0.jar;C:\development\MavenRepository\commons-beanutils\commons-beanutils-core\1.8.0\commons-beanutils-core-1.8.0.jar;C:\development\MavenRepository\com\google\code\gson\gson\2.2.4\gson-2.2.4.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-auth\2.6.0-cdh5.14.2\hadoop-auth-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\directory\server\apacheds-kerberos-codec\2.0.0-M15\apacheds-kerberos-codec-2.0.0-M15.jar;C:\development\MavenRepository\org\apache\directory\server\apacheds-i18n\2.0.0-M15\apacheds-i18n-2.0.0-M15.jar;C:\development\MavenRepository\org\apache\directory\api\api-asn1-api\1.0.0-M20\api-asn1-api-1.0.0-M20.jar;C:\development\MavenRepository\org\apache\directory\api\api-util\1.0.0-M20\api-util-1.0.0-M20.jar;C:\development\MavenRepository\com\jcraft\jsch\0.1.42\jsch-0.1.42.jar;C:\development\MavenRepository\org\apache\curator\curator-client\2.7.1\curator-client-2.7.1.jar;C:\development\MavenRepository\org\apache\htrace\htrace-core4\4.0.1-incubating\htrace-core4-4.0.1-incubating.jar;C:\development\MavenRepository\org\apache\commons\commons-compress\1.4.1\commons-compress-1.4.1.jar;C:\development\MavenRepository\org\tukaani\xz\1.0\xz-1.0.jar;C:\development\MavenRepository\org\apache\hbase\hbase-client\1.2.0-cdh5.14.2\hbase-client-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\jruby\jcodings\jcodings\1.0.8\jcodings-1.0.8.jar;C:\development\MavenRepository\com\yammer\metrics\metrics-core\2.2.0\metrics-core-2.2.0.jar;C:\development\MavenRepository\org\apache\hbase\hbase-protocol\1.2.0-cdh5.14.2\hbase-protocol-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-server\1.2.0-cdh5.14.2\hbase-server-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-procedure\1.2.0-cdh5.14.2\hbase-procedure-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-common\1.2.0-cdh5.14.2\hbase-common-1.2.0-cdh5.14.2-tests.jar;C:\development\MavenRepository\org\apache\hbase\hbase-prefix-tree\1.2.0-cdh5.14.2\hbase-prefix-tree-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\com\github\stephenc\high-scale-lib\high-scale-lib\1.1.1\high-scale-lib-1.1.1.jar;C:\development\MavenRepository\org\apache\commons\commons-math\2.1\commons-math-2.1.jar;C:\development\MavenRepository\org\mortbay\jetty\jetty-sslengine\6.1.26.cloudera.4\jetty-sslengine-6.1.26.cloudera.4.jar;C:\development\MavenRepository\org\mortbay\jetty\jsp-2.1\6.1.14\jsp-2.1-6.1.14.jar;C:\development\MavenRepository\org\mortbay\jetty\jsp-api-2.1\6.1.14\jsp-api-2.1-6.1.14.jar;C:\development\MavenRepository\org\mortbay\jetty\servlet-api-2.5\6.1.14\servlet-api-2.5-6.1.14.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-jaxrs\1.8.8\jackson-jaxrs-1.8.8.jar;C:\development\MavenRepository\org\jamon\jamon-runtime\2.4.1\jamon-runtime-2.4.1.jar;C:\development\MavenRepository\org\hamcrest\hamcrest-core\1.3\hamcrest-core-1.3.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-core\2.6.0-mr1-cdh5.14.2\hadoop-core-2.6.0-mr1-cdh5.14.2.jar;C:\development\MavenRepository\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;C:\development\MavenRepository\hsqldb\hsqldb\1.8.0.10\hsqldb-1.8.0.10.jar;C:\development\MavenRepository\org\eclipse\jdt\core\3.1.1\core-3.1.1.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-hdfs\2.6.0-cdh5.14.2\hadoop-hdfs-2.6.0-cdh5.14.2-tests.jar;C:\development\MavenRepository\commons-daemon\commons-daemon\1.0.13\commons-daemon-1.0.13.jar;C:\development\MavenRepository\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;C:\development\MavenRepository\commons-logging\commons-logging\1.2\commons-logging-1.2.jar;C:\development\MavenRepository\com\github\stephenc\findbugs\findbugs-annotations\1.3.9-1\findbugs-annotations-1.3.9-1.jar;C:\development\MavenRepository\org\apache\phoenix\phoenix-spark\4.14.0-cdh5.14.2\phoenix-spark-4.14.0-cdh5.14.2.jar;C:\development\MavenRepository\com\lmax\disruptor\3.3.8\disruptor-3.3.8.jar;C:\development\MavenRepository\org\apache\phoenix\phoenix-core\4.14.0-cdh5.14.2\phoenix-core-4.14.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\tephra\tephra-api\0.14.0-incubating\tephra-api-0.14.0-incubating.jar;C:\development\MavenRepository\org\apache\tephra\tephra-core\0.14.0-incubating\tephra-core-0.14.0-incubating.jar;C:\development\MavenRepository\com\google\inject\guice\3.0\guice-3.0.jar;C:\development\MavenRepository\javax\inject\javax.inject\1\javax.inject-1.jar;C:\development\MavenRepository\aopalliance\aopalliance\1.0\aopalliance-1.0.jar;C:\development\MavenRepository\com\google\inject\extensions\guice-assistedinject\3.0\guice-assistedinject-3.0.jar;C:\development\MavenRepository\org\apache\thrift\libthrift\0.9.0\libthrift-0.9.0.jar;C:\development\MavenRepository\it\unimi\dsi\fastutil\6.5.6\fastutil-6.5.6.jar;C:\development\MavenRepository\org\apache\twill\twill-common\0.8.0\twill-common-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-core\0.8.0\twill-core-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-api\0.8.0\twill-api-0.8.0.jar;C:\development\MavenRepository\org\ow2\asm\asm-all\5.0.2\asm-all-5.0.2.jar;C:\development\MavenRepository\org\apache\twill\twill-discovery-api\0.8.0\twill-discovery-api-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-discovery-core\0.8.0\twill-discovery-core-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-zookeeper\0.8.0\twill-zookeeper-0.8.0.jar;C:\development\MavenRepository\org\apache\tephra\tephra-hbase-compat-1.2-cdh\0.14.0-incubating\tephra-hbase-compat-1.2-cdh-0.14.0-incubating.jar;C:\development\MavenRepository\org\antlr\antlr-runtime\3.5.2\antlr-runtime-3.5.2.jar;C:\development\MavenRepository\jline\jline\2.11\jline-2.11.jar;C:\development\MavenRepository\sqlline\sqlline\1.2.0\sqlline-1.2.0.jar;C:\development\MavenRepository\com\google\guava\guava\13.0.1\guava-13.0.1.jar;C:\development\MavenRepository\joda-time\joda-time\1.6\joda-time-1.6.jar;C:\development\MavenRepository\com\github\stephenc\jcip\jcip-annotations\1.0-1\jcip-annotations-1.0-1.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-core-asl\1.8.8\jackson-core-asl-1.8.8.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-mapper-asl\1.8.8\jackson-mapper-asl-1.8.8.jar;C:\development\MavenRepository\junit\junit\4.12\junit-4.12.jar;C:\development\MavenRepository\org\apache\httpcomponents\httpclient\4.0.1\httpclient-4.0.1.jar;C:\development\MavenRepository\org\iq80\snappy\snappy\0.3\snappy-0.3.jar;C:\development\MavenRepository\org\apache\htrace\htrace-core\3.2.0-incubating\htrace-core-3.2.0-incubating.jar;C:\development\MavenRepository\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;C:\development\MavenRepository\commons-codec\commons-codec\1.7\commons-codec-1.7.jar;C:\development\MavenRepository\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar;C:\development\MavenRepository\org\apache\commons\commons-csv\1.0\commons-csv-1.0.jar;C:\development\MavenRepository\org\apache\hbase\hbase-annotations\1.2.0-cdh5.14.2\hbase-annotations-1.2.0-cdh5.14.2.jar;C:\3rd\Java\jdk1.8.0_212\lib\tools.jar;C:\development\MavenRepository\org\apache\hbase\hbase-common\1.2.0-cdh5.14.2\hbase-common-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-hadoop-compat\1.2.0-cdh5.14.2\hbase-hadoop-compat-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-hadoop2-compat\1.2.0-cdh5.14.2\hbase-hadoop2-compat-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-annotations\2.6.0-cdh5.14.2\hadoop-annotations-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\jruby\joni\joni\2.1.2\joni-2.1.2.jar;C:\development\MavenRepository\com\salesforce\i18n\i18n-util\1.0.4\i18n-util-1.0.4.jar;C:\development\MavenRepository\com\ibm\icu\icu4j\60.2\icu4j-60.2.jar;C:\development\MavenRepository\com\ibm\icu\icu4j-localespi\60.2\icu4j-localespi-60.2.jar;C:\development\MavenRepository\com\ibm\icu\icu4j-charset\60.2\icu4j-charset-60.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-core\2.6.0-cdh5.14.2\hadoop-mapreduce-client-core-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-common\2.6.0-cdh5.14.2\hadoop-yarn-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\javax\xml\bind\jaxb-api\2.2.2\jaxb-api-2.2.2.jar;C:\development\MavenRepository\javax\xml\stream\stax-api\1.0-2\stax-api-1.0-2.jar;C:\development\MavenRepository\javax\activation\activation\1.1\activation-1.1.jar;C:\development\MavenRepository\javax\servlet\servlet-api\2.5\servlet-api-2.5.jar;C:\development\MavenRepository\com\sun\jersey\jersey-client\1.9\jersey-client-1.9.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-xc\1.8.8\jackson-xc-1.8.8.jar;C:\development\MavenRepository\com\sun\jersey\contribs\jersey-guice\1.9\jersey-guice-1.9.jar;C:\development\MavenRepository\org\apache\avro\avro\1.7.6-cdh5.14.2\avro-1.7.6-cdh5.14.2.jar;C:\development\MavenRepository\com\google\inject\extensions\guice-servlet\3.0\guice-servlet-3.0.jar;C:\development\MavenRepository\io\netty\netty\3.10.5.Final\netty-3.10.5.Final.jar;C:\development\MavenRepository\mysql\mysql-connector-java\5.1.43\mysql-connector-java-5.1.43.jar;C:\3rd\JetBrains\IntelliJ IDEA 2019.1.3\lib\idea_rt.jar 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.library.path=C:\3rd\Java\jdk1.8.0_212\bin;C:\WINDOWS\Sun\Java\bin;C:\WINDOWS\system32;C:\WINDOWS;C:\3rd\Anaconda2;C:\3rd\Anaconda2\Library\mingw-w64\bin;C:\3rd\Anaconda2\Library\usr\bin;C:\3rd\Anaconda2\Library\bin;C:\3rd\Anaconda2\Scripts;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\iCLS\;C:\Program Files\Intel\Intel(R) Management Engine Components\iCLS\;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\DAL;C:\Program Files\Intel\Intel(R) Management Engine Components\DAL;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\IPT;C:\Program Files\Intel\Intel(R) Management Engine Components\IPT;C:\3rd\MATLAB\R2016a\runtime\win64;C:\3rd\MATLAB\R2016a\bin;C:\3rd\MATLAB\R2016a\polyspace\bin;C:\WINDOWS\System32\OpenSSH\;C:\Program Files\TortoiseSVN\bin;C:\Users\cf_pc\Documents\Caffe\Release;C:\3rd\scala\scala-2.10.5\bin;C:\3rd\Java\jdk1.8.0_212\bin;C:\development\apache-maven-3.6.1\bin;C:\3rd\mysql-8.0.16-winx64\bin;C:\Program Files\Intel\WiFi\bin\;C:\Program Files\Common Files\Intel\WirelessCommon\;C:\3rd\hadoop-common-2.6.0\bin;C:\Users\cf_pc\AppData\Local\Microsoft\WindowsApps;;C:\Program Files\Microsoft VS Code\bin;. 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.io.tmpdir=C:\Users\cf_pc\AppData\Local\Temp\ 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.compiler=<NA> 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.name=Windows 10 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.arch=amd64 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.version=10.0 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.name=cf_pc 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.home=C:\Users\cf_pc 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.dir=C:\development\cf\scalapp 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=node3:2181 sessionTimeout=90000 watcher=hconnection-0x4bc59b270x0, quorum=node3:2181, baseZNode=/hbase 19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Opening socket connection to server node3/10.200.101.133:2181. Will not attempt to authenticate using SASL (unknown error) 19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Socket connection established to node3/10.200.101.133:2181, initiating session 19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Session establishment complete on server node3/10.200.101.133:2181, sessionid = 0x36ca2ccfed69558, negotiated timeout = 60000 19/09/24 00:52:24 INFO query.ConnectionQueryServicesImpl: HConnection established. Stacktrace for informational purposes: hconnection-0x4bc59b27 java.lang.Thread.getStackTrace(Thread.java:1559) org.apache.phoenix.util.LogUtil.getCallerStackTrace(LogUtil.java:55) org.apache.phoenix.query.ConnectionQueryServicesImpl.openConnection(ConnectionQueryServicesImpl.java:427) org.apache.phoenix.query.ConnectionQueryServicesImpl.access$400(ConnectionQueryServicesImpl.java:267) org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2515) org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2491) org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:76) org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2491) org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:255) org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:150) org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:221) java.sql.DriverManager.getConnection(DriverManager.java:664) java.sql.DriverManager.getConnection(DriverManager.java:208) org.apache.phoenix.mapreduce.util.ConnectionUtil.getConnection(ConnectionUtil.java:113) org.apache.phoenix.mapreduce.util.ConnectionUtil.getInputConnection(ConnectionUtil.java:58) org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getSelectColumnMetadataList(PhoenixConfigurationUtil.java:354) org.apache.phoenix.spark.PhoenixRDD.toDataFrame(PhoenixRDD.scala:118) org.apache.phoenix.spark.SparkSqlContextFunctions.phoenixTableAsDataFrame(SparkSqlContextFunctions.scala:39) com.fc.phoenixConnectMode$.getMode1(phoenixConnectMode.scala:16) com.fc.costDay$.main(costDay.scala:113) com.fc.costDay.main(costDay.scala) 19/09/24 00:52:25 INFO Configuration.deprecation: hadoop.native.lib is deprecated. Instead, use io.native.lib.available 19/09/24 00:52:27 INFO mapreduce.PhoenixInputFormat: UseSelectColumns=true, selectColumnList.size()=86, selectColumnList=ID,ASSET_ID,ASSET_NAME,ASSET_FIRST_DEGREE_ID,ASSET_FIRST_DEGREE_NAME,ASSET_SECOND_DEGREE_ID,ASSET_SECOND_DEGREE_NAME,GB_DEGREE_ID,GB_DEGREE_NAME,ASSET_USE_FIRST_DEGREE_ID,ASSET_USE_FIRST_DEGREE_NAME,ASSET_USE_SECOND_DEGREE_ID,ASSET_USE_SECOND_DEGREE_NAME,MANAGEMENT_TYPE_ID,MANAGEMENT_TYPE_NAME,ASSET_MODEL,FACTORY_NUMBER,ASSET_COUNTRY_ID,ASSET_COUNTRY_NAME,MANUFACTURER,SUPPLIER,SUPPLIER_TEL,ORIGINAL_VALUE,USE_DEPARTMENT_ID,USE_DEPARTMENT_NAME,USER_ID,USER_NAME,ASSET_LOCATION_OF_PARK_ID,ASSET_LOCATION_OF_PARK_NAME,ASSET_LOCATION_OF_BUILDING_ID,ASSET_LOCATION_OF_BUILDING_NAME,ASSET_LOCATION_OF_ROOM_ID,ASSET_LOCATION_OF_ROOM_NUMBER,PRODUCTION_DATE,ACCEPTANCE_DATE,REQUISITION_DATE,PERFORMANCE_INDEX,ASSET_STATE_ID,ASSET_STATE_NAME,INSPECTION_TYPE_ID,INSPECTION_TYPE_NAME,SEAL_DATE,SEAL_CAUSE,COST_ITEM_ID,COST_ITEM_NAME,ITEM_COMMENTS,UNSEAL_DATE,SCRAP_DATE,PURCHASE_NUMBER,WARRANTY_PERIOD,DEPRECIABLE_LIVES_ID,DEPRECIABLE_LIVES_NAME,MEASUREMENT_UNITS_ID,MEASUREMENT_UNITS_NAME,ANNEX,REMARK,ACCOUNTING_TYPE_ID,ACCOUNTING_TYPE_NAME,SYSTEM_TYPE_ID,SYSTEM_TYPE_NAME,ASSET_ID_PARENT,CLASSIFIED_LEVEL_ID,CLASSIFIED_LEVEL_NAME,ASSET_PICTURE,MILITARY_SPECIAL_CODE,CHECK_CYCLE_ID,CHECK_CYCLE_NAME,CHECK_DATE,CHECK_EFFECTIVE_DATE,CHECK_MODE_ID,CHECK_MODE_NAME,CHECK_DEPARTMENT_ID,CHECK_DEPARTMENT_NAME,RENT_STATUS_ID,RENT_STATUS_NAME,STORAGE_TIME,UPDATE_USER,UPDATE_TIME,IS_ON_PROCESS,IS_DELETED,FIRST_DEPARTMENT_ID,FIRST_DEPARTMENT_NAME,SECOND_DEPARTMENT_ID,SECOND_DEPARTMENT_NAME,CREATE_USER,CREATE_TIME root |-- ID: string (nullable = true) |-- FIRST_DEPARTMENT_ID: string (nullable = true) |-- ACTUAL_COST: double (nullable = true) |-- ORIGINAL_VALUE: double (nullable = true) |-- GENERATION_TIME: timestamp (nullable = false) 19/09/24 00:52:32 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:32 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:32 INFO mapreduce.PhoenixInputFormat: UseSelectColumns=true, selectColumnList.size()=86, selectColumnList=ID,ASSET_ID,ASSET_NAME,ASSET_FIRST_DEGREE_ID,ASSET_FIRST_DEGREE_NAME,ASSET_SECOND_DEGREE_ID,ASSET_SECOND_DEGREE_NAME,GB_DEGREE_ID,GB_DEGREE_NAME,ASSET_USE_FIRST_DEGREE_ID,ASSET_USE_FIRST_DEGREE_NAME,ASSET_USE_SECOND_DEGREE_ID,ASSET_USE_SECOND_DEGREE_NAME,MANAGEMENT_TYPE_ID,MANAGEMENT_TYPE_NAME,ASSET_MODEL,FACTORY_NUMBER,ASSET_COUNTRY_ID,ASSET_COUNTRY_NAME,MANUFACTURER,SUPPLIER,SUPPLIER_TEL,ORIGINAL_VALUE,USE_DEPARTMENT_ID,USE_DEPARTMENT_NAME,USER_ID,USER_NAME,ASSET_LOCATION_OF_PARK_ID,ASSET_LOCATION_OF_PARK_NAME,ASSET_LOCATION_OF_BUILDING_ID,ASSET_LOCATION_OF_BUILDING_NAME,ASSET_LOCATION_OF_ROOM_ID,ASSET_LOCATION_OF_ROOM_NUMBER,PRODUCTION_DATE,ACCEPTANCE_DATE,REQUISITION_DATE,PERFORMANCE_INDEX,ASSET_STATE_ID,ASSET_STATE_NAME,INSPECTION_TYPE_ID,INSPECTION_TYPE_NAME,SEAL_DATE,SEAL_CAUSE,COST_ITEM_ID,COST_ITEM_NAME,ITEM_COMMENTS,UNSEAL_DATE,SCRAP_DATE,PURCHASE_NUMBER,WARRANTY_PERIOD,DEPRECIABLE_LIVES_ID,DEPRECIABLE_LIVES_NAME,MEASUREMENT_UNITS_ID,MEASUREMENT_UNITS_NAME,ANNEX,REMARK,ACCOUNTING_TYPE_ID,ACCOUNTING_TYPE_NAME,SYSTEM_TYPE_ID,SYSTEM_TYPE_NAME,ASSET_ID_PARENT,CLASSIFIED_LEVEL_ID,CLASSIFIED_LEVEL_NAME,ASSET_PICTURE,MILITARY_SPECIAL_CODE,CHECK_CYCLE_ID,CHECK_CYCLE_NAME,CHECK_DATE,CHECK_EFFECTIVE_DATE,CHECK_MODE_ID,CHECK_MODE_NAME,CHECK_DEPARTMENT_ID,CHECK_DEPARTMENT_NAME,RENT_STATUS_ID,RENT_STATUS_NAME,STORAGE_TIME,UPDATE_USER,UPDATE_TIME,IS_ON_PROCESS,IS_DELETED,FIRST_DEPARTMENT_ID,FIRST_DEPARTMENT_NAME,SECOND_DEPARTMENT_ID,SECOND_DEPARTMENT_NAME,CREATE_USER,CREATE_TIME 19/09/24 00:52:32 INFO mapreduce.PhoenixInputFormat: Select Statement: SELECT "ID","0"."ASSET_ID","0"."ASSET_NAME","0"."ASSET_FIRST_DEGREE_ID","0"."ASSET_FIRST_DEGREE_NAME","0"."ASSET_SECOND_DEGREE_ID","0"."ASSET_SECOND_DEGREE_NAME","0"."GB_DEGREE_ID","0"."GB_DEGREE_NAME","0"."ASSET_USE_FIRST_DEGREE_ID","0"."ASSET_USE_FIRST_DEGREE_NAME","0"."ASSET_USE_SECOND_DEGREE_ID","0"."ASSET_USE_SECOND_DEGREE_NAME","0"."MANAGEMENT_TYPE_ID","0"."MANAGEMENT_TYPE_NAME","0"."ASSET_MODEL","0"."FACTORY_NUMBER","0"."ASSET_COUNTRY_ID","0"."ASSET_COUNTRY_NAME","0"."MANUFACTURER","0"."SUPPLIER","0"."SUPPLIER_TEL","0"."ORIGINAL_VALUE","0"."USE_DEPARTMENT_ID","0"."USE_DEPARTMENT_NAME","0"."USER_ID","0"."USER_NAME","0"."ASSET_LOCATION_OF_PARK_ID","0"."ASSET_LOCATION_OF_PARK_NAME","0"."ASSET_LOCATION_OF_BUILDING_ID","0"."ASSET_LOCATION_OF_BUILDING_NAME","0"."ASSET_LOCATION_OF_ROOM_ID","0"."ASSET_LOCATION_OF_ROOM_NUMBER","0"."PRODUCTION_DATE","0"."ACCEPTANCE_DATE","0"."REQUISITION_DATE","0"."PERFORMANCE_INDEX","0"."ASSET_STATE_ID","0"."ASSET_STATE_NAME","0"."INSPECTION_TYPE_ID","0"."INSPECTION_TYPE_NAME","0"."SEAL_DATE","0"."SEAL_CAUSE","0"."COST_ITEM_ID","0"."COST_ITEM_NAME","0"."ITEM_COMMENTS","0"."UNSEAL_DATE","0"."SCRAP_DATE","0"."PURCHASE_NUMBER","0"."WARRANTY_PERIOD","0"."DEPRECIABLE_LIVES_ID","0"."DEPRECIABLE_LIVES_NAME","0"."MEASUREMENT_UNITS_ID","0"."MEASUREMENT_UNITS_NAME","0"."ANNEX","0"."REMARK","0"."ACCOUNTING_TYPE_ID","0"."ACCOUNTING_TYPE_NAME","0"."SYSTEM_TYPE_ID","0"."SYSTEM_TYPE_NAME","0"."ASSET_ID_PARENT","0"."CLASSIFIED_LEVEL_ID","0"."CLASSIFIED_LEVEL_NAME","0"."ASSET_PICTURE","0"."MILITARY_SPECIAL_CODE","0"."CHECK_CYCLE_ID","0"."CHECK_CYCLE_NAME","0"."CHECK_DATE","0"."CHECK_EFFECTIVE_DATE","0"."CHECK_MODE_ID","0"."CHECK_MODE_NAME","0"."CHECK_DEPARTMENT_ID","0"."CHECK_DEPARTMENT_NAME","0"."RENT_STATUS_ID","0"."RENT_STATUS_NAME","0"."STORAGE_TIME","0"."UPDATE_USER","0"."UPDATE_TIME","0"."IS_ON_PROCESS","0"."IS_DELETED","0"."FIRST_DEPARTMENT_ID","0"."FIRST_DEPARTMENT_NAME","0"."SECOND_DEPARTMENT_ID","0"."SECOND_DEPARTMENT_NAME","0"."CREATE_USER","0"."CREATE_TIME" FROM ASSET_NORMAL 19/09/24 00:52:32 INFO zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x22617270 connecting to ZooKeeper ensemble=node3:2181 19/09/24 00:52:32 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=node3:2181 sessionTimeout=90000 watcher=hconnection-0x226172700x0, quorum=node3:2181, baseZNode=/hbase 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Opening socket connection to server node3/10.200.101.133:2181. Will not attempt to authenticate using SASL (unknown error) 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Socket connection established to node3/10.200.101.133:2181, initiating session 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Session establishment complete on server node3/10.200.101.133:2181, sessionid = 0x36ca2ccfed69559, negotiated timeout = 60000 19/09/24 00:52:32 INFO util.RegionSizeCalculator: Calculating region sizes for table "IDX_ASSET_NORMAL". 19/09/24 00:52:32 INFO client.ConnectionManager$HConnectionImplementation: Closing master protocol: MasterService 19/09/24 00:52:32 INFO client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x36ca2ccfed69559 19/09/24 00:52:32 INFO zookeeper.ZooKeeper: Session: 0x36ca2ccfed69559 closed 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: EventThread shut down 19/09/24 00:52:32 INFO spark.SparkContext: Starting job: count at costDay.scala:139 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Registering RDD 7 (count at costDay.scala:139) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Got job 0 (count at costDay.scala:139) with 1 output partitions 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Final stage: ResultStage 1 (count at costDay.scala:139) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 0) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 0) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at count at costDay.scala:139), which has no missing parents 19/09/24 00:52:33 INFO storage.MemoryStore: Block broadcast_1 stored as values in memory (estimated size 24.8 KB, free 477.9 MB) 19/09/24 00:52:33 INFO storage.MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 9.5 KB, free 477.9 MB) 19/09/24 00:52:33 INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:54720 (size: 9.5 KB, free: 478.1 MB) 19/09/24 00:52:33 INFO spark.SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at count at costDay.scala:139) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:33 INFO scheduler.TaskSchedulerImpl: Adding task set 0.0 with 1 tasks 19/09/24 00:52:33 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, ANY, 2538 bytes) 19/09/24 00:52:33 INFO executor.Executor: Running task 0.0 in stage 0.0 (TID 0) 19/09/24 00:52:33 INFO rdd.NewHadoopRDD: Input split: org.apache.phoenix.mapreduce.PhoenixInputSplit@20b488 19/09/24 00:52:33 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:33 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:33 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:33 INFO codegen.GeneratePredicate: Code generated in 309.4486 ms 19/09/24 00:52:33 INFO codegen.GenerateUnsafeProjection: Code generated in 20.3535 ms 19/09/24 00:52:34 INFO codegen.GenerateMutableProjection: Code generated in 10.5156 ms 19/09/24 00:52:34 INFO codegen.GenerateMutableProjection: Code generated in 10.4614 ms 19/09/24 00:52:34 INFO codegen.GenerateUnsafeRowJoiner: Code generated in 6.8774 ms 19/09/24 00:52:34 INFO codegen.GenerateUnsafeProjection: Code generated in 6.7907 ms 19/09/24 00:52:43 INFO executor.Executor: Finished task 0.0 in stage 0.0 (TID 0). 2629 bytes result sent to driver 19/09/24 00:52:43 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 10699 ms on localhost (executor driver) (1/1) 19/09/24 00:52:43 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 19/09/24 00:52:43 INFO scheduler.DAGScheduler: ShuffleMapStage 0 (count at costDay.scala:139) finished in 10.745 s 19/09/24 00:52:43 INFO scheduler.DAGScheduler: looking for newly runnable stages 19/09/24 00:52:43 INFO scheduler.DAGScheduler: running: Set() 19/09/24 00:52:43 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 1) 19/09/24 00:52:43 INFO scheduler.DAGScheduler: failed: Set() 19/09/24 00:52:43 INFO scheduler.DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[10] at count at costDay.scala:139), which has no missing parents 19/09/24 00:52:43 INFO storage.MemoryStore: Block broadcast_2 stored as values in memory (estimated size 25.3 KB, free 477.9 MB) 19/09/24 00:52:43 INFO storage.MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 9.8 KB, free 477.8 MB) 19/09/24 00:52:43 INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:54720 (size: 9.8 KB, free: 478.1 MB) 19/09/24 00:52:43 INFO spark.SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:43 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at count at costDay.scala:139) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:43 INFO scheduler.TaskSchedulerImpl: Adding task set 1.0 with 1 tasks 19/09/24 00:52:43 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, executor driver, partition 0, NODE_LOCAL, 1999 bytes) 19/09/24 00:52:43 INFO executor.Executor: Running task 0.0 in stage 1.0 (TID 1) 19/09/24 00:52:43 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks 19/09/24 00:52:44 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 20 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 22.44 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 13.992 ms 19/09/24 00:52:44 INFO executor.Executor: Finished task 0.0 in stage 1.0 (TID 1). 1959 bytes result sent to driver 19/09/24 00:52:44 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 199 ms on localhost (executor driver) (1/1) 19/09/24 00:52:44 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 19/09/24 00:52:44 INFO scheduler.DAGScheduler: ResultStage 1 (count at costDay.scala:139) finished in 0.205 s 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Job 0 finished: count at costDay.scala:139, took 11.157661 s 50858 19/09/24 00:52:44 INFO spark.SparkContext: Starting job: describe at costDay.scala:141 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Registering RDD 14 (describe at costDay.scala:141) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Got job 1 (describe at costDay.scala:141) with 1 output partitions 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Final stage: ResultStage 3 (describe at costDay.scala:141) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 2) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[14] at describe at costDay.scala:141), which has no missing parents 19/09/24 00:52:44 INFO storage.MemoryStore: Block broadcast_3 stored as values in memory (estimated size 27.2 KB, free 477.8 MB) 19/09/24 00:52:44 INFO storage.MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 10.4 KB, free 477.8 MB) 19/09/24 00:52:44 INFO storage.BlockManagerInfo: Added broadcast_3_piece0 in memory on localhost:54720 (size: 10.4 KB, free: 478.1 MB) 19/09/24 00:52:44 INFO spark.SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[14] at describe at costDay.scala:141) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:44 INFO scheduler.TaskSchedulerImpl: Adding task set 2.0 with 1 tasks 19/09/24 00:52:44 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 2.0 (TID 2, localhost, executor driver, partition 0, ANY, 2538 bytes) 19/09/24 00:52:44 INFO executor.Executor: Running task 0.0 in stage 2.0 (TID 2) 19/09/24 00:52:44 INFO rdd.NewHadoopRDD: Input split: org.apache.phoenix.mapreduce.PhoenixInputSplit@20b488 19/09/24 00:52:44 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:44 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:44 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:44 INFO codegen.GenerateUnsafeProjection: Code generated in 11.405 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 10.5886 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 39.2201 ms 19/09/24 00:52:44 INFO codegen.GenerateUnsafeRowJoiner: Code generated in 8.3737 ms 19/09/24 00:52:44 INFO codegen.GenerateUnsafeProjection: Code generated in 22.542 ms 19/09/24 00:52:45 INFO storage.BlockManagerInfo: Removed broadcast_2_piece0 on localhost:54720 in memory (size: 9.8 KB, free: 478.1 MB) 19/09/24 00:52:53 INFO executor.Executor: Finished task 0.0 in stage 2.0 (TID 2). 2629 bytes result sent to driver 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 2.0 (TID 2) in 8824 ms on localhost (executor driver) (1/1) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 19/09/24 00:52:53 INFO scheduler.DAGScheduler: ShuffleMapStage 2 (describe at costDay.scala:141) finished in 8.825 s 19/09/24 00:52:53 INFO scheduler.DAGScheduler: looking for newly runnable stages 19/09/24 00:52:53 INFO scheduler.DAGScheduler: running: Set() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 3) 19/09/24 00:52:53 INFO scheduler.DAGScheduler: failed: Set() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting ResultStage 3 (MapPartitionsRDD[18] at describe at costDay.scala:141), which has no missing parents 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_4 stored as values in memory (estimated size 28.7 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 11.0 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.BlockManagerInfo: Added broadcast_4_piece0 in memory on localhost:54720 (size: 11.0 KB, free: 478.1 MB) 19/09/24 00:52:53 INFO spark.SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[18] at describe at costDay.scala:141) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Adding task set 3.0 with 1 tasks 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 3.0 (TID 3, localhost, executor driver, partition 0, NODE_LOCAL, 1999 bytes) 19/09/24 00:52:53 INFO executor.Executor: Running task 0.0 in stage 3.0 (TID 3) 19/09/24 00:52:53 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks 19/09/24 00:52:53 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 19/09/24 00:52:53 INFO codegen.GenerateMutableProjection: Code generated in 21.5454 ms 19/09/24 00:52:53 INFO codegen.GenerateMutableProjection: Code generated in 10.7314 ms 19/09/24 00:52:53 INFO codegen.GenerateUnsafeProjection: Code generated in 13.0969 ms 19/09/24 00:52:53 INFO codegen.GenerateSafeProjection: Code generated in 8.2397 ms 19/09/24 00:52:53 INFO executor.Executor: Finished task 0.0 in stage 3.0 (TID 3). 2138 bytes result sent to driver 19/09/24 00:52:53 INFO scheduler.DAGScheduler: ResultStage 3 (describe at costDay.scala:141) finished in 0.097 s 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Job 1 finished: describe at costDay.scala:141, took 8.948356 s 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 3.0 (TID 3) in 97 ms on localhost (executor driver) (1/1) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool +-------+--------------------+ |summary| ORIGINAL_VALUE| +-------+--------------------+ | count| 50858| | mean|2.427485546653306E12| | stddev|5.474385018305400...| | min| -7970934.0| | max|1.234567890123456...| +-------+--------------------+ 19/09/24 00:52:53 INFO spark.SparkContext: Starting job: show at costDay.scala:142 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Got job 2 (show at costDay.scala:142) with 1 output partitions 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Final stage: ResultStage 4 (show at costDay.scala:142) 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Parents of final stage: List() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Missing parents: List() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting ResultStage 4 (MapPartitionsRDD[22] at show at costDay.scala:142), which has no missing parents 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_5 stored as values in memory (estimated size 23.6 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 9.0 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.BlockManagerInfo: Added broadcast_5_piece0 in memory on localhost:54720 (size: 9.0 KB, free: 478.1 MB) 19/09/24 00:52:53 INFO spark.SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[22] at show at costDay.scala:142) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Adding task set 4.0 with 1 tasks 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 4.0 (TID 4, localhost, executor driver, partition 0, ANY, 2549 bytes) 19/09/24 00:52:53 INFO executor.Executor: Running task 0.0 in stage 4.0 (TID 4) 19/09/24 00:52:53 INFO rdd.NewHadoopRDD: Input split: org.apache.phoenix.mapreduce.PhoenixInputSplit@20b488 19/09/24 00:52:53 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:53 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:53 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:53 INFO codegen.GenerateUnsafeProjection: Code generated in 33.6563 ms 19/09/24 00:52:53 INFO codegen.GenerateSafeProjection: Code generated in 7.0589 ms 19/09/24 00:52:54 INFO executor.Executor: Finished task 0.0 in stage 4.0 (TID 4). 5544 bytes result sent to driver 19/09/24 00:52:54 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 4.0 (TID 4) in 476 ms on localhost (executor driver) (1/1) 19/09/24 00:52:54 INFO scheduler.DAGScheduler: ResultStage 4 (show at costDay.scala:142) finished in 0.466 s 19/09/24 00:52:54 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 4.0, whose tasks have all completed, from pool 19/09/24 00:52:54 INFO scheduler.DAGScheduler: Job 2 finished: show at costDay.scala:142, took 0.489113 s +--------------------------------+-------------------+---------------------+--------------+-----------------------+ |ID |FIRST_DEPARTMENT_ID|ACTUAL_COST |ORIGINAL_VALUE|GENERATION_TIME | +--------------------------------+-------------------+---------------------+--------------+-----------------------+ |d25bb550a290457382c175b0e57c0982|1149564326588321792|0.0010410958904109589|2.0 |2019-09-24 00:52:31.864| |492016e2f7ec4cd18615c164c92c6c6d|1149564326584127489|5.205479452054794E-4 |1.0 |2019-09-24 00:52:31.864| |1d138a7401bd493da12f8f8323e9dee0|1149564326588321710|0.00353972602739726 |34.0 |2019-09-24 00:52:31.864| |09718925d34b4a099e09a30a0621ded8|1149564326588321710|0.002891643835616438 |11.11 |2019-09-24 00:52:31.864| |d5cfd5e898464130b71530d74b43e9d1|1149564326584127489|3135.8751712328767 |9638690.0 |2019-09-24 00:52:31.864| |6b39ac96b8734103b2413520d3195ee6|1149564326584127489|1744.2413835616437 |6701559.0 |2019-09-24 00:52:31.864| |8d20d0abd04d49cea3e52d9ca67e39da|1149569393202696195|0.22175342465753425 |852.0 |2019-09-24 00:52:31.864| |66ae7e7c7a104cea99615358e12c03b0|1149569393202696195|1.0410958904109588 |2000.0 |2019-09-24 00:52:31.864| |d49b0324bbf14b70adefe8b1d9163db2|1149569393202696195|1.0410958904109588 |2000.0 |2019-09-24 00:52:31.864| |d4d701514a2a425e8192acf47bb57f9b|1149569393202696195|0.032534246575342464 |100.0 |2019-09-24 00:52:31.864| |d6a016c618c1455ca0e2c7d73ba947ac|1149569393202696195|0.6506849315068494 |2000.0 |2019-09-24 00:52:31.864| |5dfa3be825464ddd98764b2790720fae|1149569393202696195|147.91532534246576 |454645.0 |2019-09-24 00:52:31.864| |6e5653ef4aaa4c03bcd00fbeb1e6811d|1149569393202696195|0.1952054794520548 |600.0 |2019-09-24 00:52:31.864| |32bd2654082645cba35527d50e0d52f9|1149569393202696195|0.6506849315068494 |2000.0 |2019-09-24 00:52:31.864| |8ed4424408bc458dbe200acffe5733bf|1149564326584127488|5.205479452054794E-4 |1.0 |2019-09-24 00:52:31.864| |1b2faa31f139461488847e77eacd794a|1149564326584127488|33499.042109589034 |6.4353423E7 |2019-09-24 00:52:31.864| |f398245c9ccc4760a5eb3251db3680bf|1149564326584127488|33499.042109589034 |6.4353423E7 |2019-09-24 00:52:31.864| |2696de9733d247e5bf88573244f36ba2|1149564326584127488|0.011452054794520548 |22.0 |2019-09-24 00:52:31.864| |9c8cfad3d4334b37a7b9beb56b528c22|1149569976173203457|0.06506849315068493 |200.0 |2019-09-24 00:52:31.864| |3e2721b79e754a798d0be940ae011d72|1149569976173203457|0.004001712328767123 |12.3 |2019-09-24 00:52:31.864| +--------------------------------+-------------------+---------------------+--------------+-----------------------+ only showing top 20 rows 19/09/24 00:52:54 INFO spark.SparkContext: Invoking stop() from shutdown hook 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/static/sql,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/metrics/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/api,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/static,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs,null} 19/09/24 00:52:54 INFO ui.SparkUI: Stopped Spark web UI at http://10.200.74.155:4040 19/09/24 00:52:54 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 19/09/24 00:52:54 INFO storage.MemoryStore: MemoryStore cleared 19/09/24 00:52:54 INFO storage.BlockManager: BlockManager stopped 19/09/24 00:52:54 INFO storage.BlockManagerMaster: BlockManagerMaster stopped 19/09/24 00:52:54 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 19/09/24 00:52:54 INFO spark.SparkContext: Successfully stopped SparkContext 19/09/24 00:52:54 INFO util.ShutdownHookManager: Shutdown hook called 19/09/24 00:52:54 INFO util.ShutdownHookManager: Deleting directory C:\Users\cf_pc\AppData\Local\Temp\spark-4fcfa10d-c258-46b7-b4f5-ee977276fa00 Process finished with exit code 0
执行打包操作，返回信息：
C:\3rd\Java\jdk1.8.0_212\bin\java.exe -Dmaven.multiModuleProjectDirectory=C:\development\cf\scalapp -Dmaven.home=C:\development\apache-maven-3.6.1 -Dclassworlds.conf=C:\development\apache-maven-3.6.1\bin\m2.conf "-javaagent:C:\3rd\JetBrains\IntelliJ IDEA 2019.1.3\lib\idea_rt.jar=49792:C:\3rd\JetBrains\IntelliJ IDEA 2019.1.3\bin" -Dfile.encoding=UTF-8 -classpath C:\development\apache-maven-3.6.1\boot\plexus-classworlds-2.6.0.jar org.codehaus.classworlds.Launcher -Didea.version2019.1.3 -s C:\development\apache-maven-3.6.1\conf\settings.xml -Dmaven.repo.local=C:\development\MavenRepository package [INFO] Scanning for projects... [INFO] [INFO] ---------------------------< com.fc:scalapp >--------------------------- [INFO] Building scalapp 1.0-SNAPSHOT [INFO] --------------------------------[ jar ]--------------------------------- [INFO] [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ scalapp --- [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] Copying 0 resource [INFO] [INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ scalapp --- [INFO] Nothing to compile - all classes are up to date [INFO] [INFO] --- maven-scala-plugin:2.15.2:compile (default) @ scalapp --- [INFO] Checking for multiple versions of scala [WARNING] Expected all dependencies to require Scala version: 2.10.5 [WARNING] com.twitter:chill_2.10:0.5.0 requires scala version: 2.10.4 [WARNING] Multiple versions of scala libraries detected! [INFO] includes = [**/*.java,**/*.scala,] [INFO] excludes = [] [INFO] C:\development\cf\scalapp\src\main\scala:-1: info: compiling [INFO] Compiling 4 source files to C:\development\cf\scalapp\target\classes at 1569255082006 [WARNING] warning: there were 1 feature warning(s); re-run with -feature for details [WARNING] one warning found [INFO] prepare-compile in 0 s [INFO] compile in 7 s [INFO] [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ scalapp --- [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:\development\cf\scalapp\src\test\resources [INFO] [INFO] --- maven-compiler-plugin:3.1:testCompile (default-testCompile) @ scalapp --- [INFO] Nothing to compile - all classes are up to date [INFO] [INFO] --- maven-scala-plugin:2.15.2:testCompile (default) @ scalapp --- [INFO] Checking for multiple versions of scala [WARNING] Expected all dependencies to require Scala version: 2.10.5 [WARNING] com.twitter:chill_2.10:0.5.0 requires scala version: 2.10.4 [WARNING] Multiple versions of scala libraries detected! [INFO] includes = [**/*.java,**/*.scala,] [INFO] excludes = [] [WARNING] No source files found. [INFO] [INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ scalapp --- [INFO] [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ scalapp --- [INFO] Building jar: C:\development\cf\scalapp\target\scalapp-1.0-SNAPSHOT.jar [INFO] [INFO] --- maven-shade-plugin:3.1.0:shade (default) @ scalapp --- [INFO] Including org.apache.spark:spark-core_2.10:jar:1.6.0-cdh5.14.2 in the shaded jar. [INFO] Including org.apache.avro:avro-mapred:jar:hadoop2:1.7.6-cdh5.14.2 in the shaded jar. …… [INFO] Including mysql:mysql-connector-java:jar:5.1.43 in the shaded jar. [WARNING] commons-collections-3.2.2.jar, commons-beanutils-1.7.0.jar, commons-beanutils-core-1.8.0.jar define 10 overlapping classes: [WARNING] - org.apache.commons.collections.FastHashMap$EntrySet [WARNING] - org.apache.commons.collections.FastHashMap$KeySet [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView$CollectionViewIterator [WARNING] - org.apache.commons.collections.ArrayStack [WARNING] - org.apache.commons.collections.FastHashMap$Values [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView [WARNING] - org.apache.commons.collections.FastHashMap$1 [WARNING] - org.apache.commons.collections.Buffer [WARNING] - org.apache.commons.collections.FastHashMap [WARNING] - org.apache.commons.collections.BufferUnderflowException …… [WARNING] mvn dependency:tree -Ddetail=true and the above output. [WARNING] See http://maven.apache.org/plugins/maven-shade-plugin/ [INFO] Replacing original artifact with shaded artifact. [INFO] Replacing C:\development\cf\scalapp\target\scalapp-1.0-SNAPSHOT.jar with C:\development\cf\scalapp\target\scalapp-1.0-SNAPSHOT-shaded.jar [INFO] ------------------------------------------------------------------------ [INFO] BUILD SUCCESS [INFO] ------------------------------------------------------------------------ [INFO] Total time: 01:00 min [INFO] Finished at: 2019-09-24T00:12:08+08:00 [INFO] ------------------------------------------------------------------------ Process finished with exit code 0
上传到服务器上，执行命令：
spark-submit --class com.fc.costDay --executor-memory 500m --total-executor-cores 1 /home/cf/scalapp-1.0-SNAPSHOT.jar
报错：
java.lang.ClassNotFoundException: Class org.apache.phoenix.spark.PhoenixRecordWritable not found
参考：https://www.jianshu.com/p/f336f7e5f31b，更改执行命令为：
spark-submit --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2 --class com.fc.costDay --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar
仍然报错：
File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/……
完整错误信息如下：
[root@node1 ~]# spark-submit --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2 --class com.fc.costDay --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:01:36 INFO client.RMProxy: Connecting to ResourceManager at node1/10.200.101.131:8032 19/09/24 00:01:36 INFO yarn.Client: Requesting a new application from cluster with 3 NodeManagers 19/09/24 00:01:36 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (40874 MB per container) 19/09/24 00:01:36 INFO yarn.Client: Will allocate AM container, with 4505 MB memory including 409 MB overhead 19/09/24 00:01:36 INFO yarn.Client: Setting up container launch context for our AM 19/09/24 00:01:36 INFO yarn.Client: Setting up the launch environment for our AM container 19/09/24 00:01:37 INFO yarn.Client: Preparing resources for our AM container 19/09/24 00:01:37 INFO yarn.Client: Uploading resource file:/home/cf/scalapp-1.0-SNAPSHOT.jar -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:01:38 INFO yarn.Client: Uploading resource file:/tmp/spark-2c548285-8012-414b-ab4e-797a164e38bc/__spark_conf__8997139291240118668.zip -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip 19/09/24 00:01:38 INFO spark.SecurityManager: Changing view acls to: root 19/09/24 00:01:38 INFO spark.SecurityManager: Changing modify acls to: root 19/09/24 00:01:38 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root) 19/09/24 00:01:38 INFO yarn.Client: Submitting application 105 to ResourceManager 19/09/24 00:01:38 INFO impl.YarnClientImpl: Submitted application application_1566100765602_0105 19/09/24 00:01:39 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:39 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: N/A ApplicationMaster RPC port: -1 queue: root.users.root start time: 1569254498669 final status: UNDEFINED tracking URL: http://node1:8088/proxy/application_1566100765602_0105/ user: root 19/09/24 00:01:40 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:41 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:42 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:43 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:44 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:45 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:46 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:47 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:48 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:49 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:50 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:51 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:52 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:53 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:54 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:55 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:56 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:57 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:58 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:59 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:02:00 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:02:01 INFO yarn.Client: Application report for application_1566100765602_0105 (state: FAILED) 19/09/24 00:02:01 INFO yarn.Client: client token: N/A diagnostics: Application application_1566100765602_0105 failed 2 times due to AM Container for appattempt_1566100765602_0105_000002 exited with exitCode: -1000 For more detailed output, check application tracking page:http://node1:8088/proxy/application_1566100765602_0105/Then, click on links to logs of each attempt. Diagnostics: File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip java.io.FileNotFoundException: File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1269) at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1261) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1261) at org.apache.hadoop.yarn.util.FSDownload.copy(FSDownload.java:251) at org.apache.hadoop.yarn.util.FSDownload.access$000(FSDownload.java:61) at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:364) at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:362) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1920) at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:361) at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:60) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Failing this attempt. Failing the application. ApplicationMaster host: N/A ApplicationMaster RPC port: -1 queue: root.users.root start time: 1569254498669 final status: FAILED tracking URL: http://node1:8088/cluster/app/application_1566100765602_0105 user: root Exception in thread "main" org.apache.spark.SparkException: Application application_1566100765602_0105 finished with failed status at org.apache.spark.deploy.yarn.Client.run(Client.scala:1025) at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1072) at org.apache.spark.deploy.yarn.Client.main(Client.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:730) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 19/09/24 00:02:01 INFO util.ShutdownHookManager: Shutdown hook called 19/09/24 00:02:01 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-2c548285-8012-414b-ab4e-797a164e38bc
参考：https://blog.csdn.net/adorechen/article/details/78746363，将源代码中设定本地执行的语句注释掉：
val conf = new SparkConf() .setAppName("fdsf") // .setMaster("local") //本地执行
重新打包，执行以上语句，成功！返回如下信息：
[root@node1 ~]# spark-submit --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2 --class com.fc.costDay --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:13:58 INFO client.RMProxy: Connecting to ResourceManager at node1/10.200.101.131:8032 19/09/24 00:13:58 INFO yarn.Client: Requesting a new application from cluster with 3 NodeManagers 19/09/24 00:13:58 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (40874 MB per container) 19/09/24 00:13:58 INFO yarn.Client: Will allocate AM container, with 4505 MB memory including 409 MB overhead 19/09/24 00:13:58 INFO yarn.Client: Setting up container launch context for our AM 19/09/24 00:13:58 INFO yarn.Client: Setting up the launch environment for our AM container 19/09/24 00:13:58 INFO yarn.Client: Preparing resources for our AM container 19/09/24 00:13:59 INFO yarn.Client: Uploading resource file:/home/cf/scalapp-1.0-SNAPSHOT.jar -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0106/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:14:00 INFO yarn.Client: Uploading resource file:/tmp/spark-875aaa2a-1c5d-4c6a-95a6-12f276a80054/__spark_conf__4530217919668141816.zip -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0106/__spark_conf__4530217919668141816.zip 19/09/24 00:14:00 INFO spark.SecurityManager: Changing view acls to: root 19/09/24 00:14:00 INFO spark.SecurityManager: Changing modify acls to: root 19/09/24 00:14:00 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root) 19/09/24 00:14:00 INFO yarn.Client: Submitting application 106 to ResourceManager 19/09/24 00:14:00 INFO impl.YarnClientImpl: Submitted application application_1566100765602_0106 19/09/24 00:14:01 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:01 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: N/A ApplicationMaster RPC port: -1 queue: root.users.root start time: 1569255240698 final status: UNDEFINED tracking URL: http://node1:8088/proxy/application_1566100765602_0106/ user: root 19/09/24 00:14:02 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:03 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:04 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:05 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:06 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:06 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: 10.200.101.135 ApplicationMaster RPC port: 0 queue: root.users.root start time: 1569255240698 final status: UNDEFINED tracking URL: http://node1:8088/proxy/application_1566100765602_0106/ user: root 19/09/24 00:14:07 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:08 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:09 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:10 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:11 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:12 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:13 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:14 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:15 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:16 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:17 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:18 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:19 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:20 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:21 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:22 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:23 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:24 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:25 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:26 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:27 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:28 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:29 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:30 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:31 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:32 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:33 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:34 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:35 INFO yarn.Client: Application report for application_1566100765602_0106 (state: FINISHED) 19/09/24 00:14:35 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: 10.200.101.135 ApplicationMaster RPC port: 0 queue: root.users.root start time: 1569255240698 final status: SUCCEEDED tracking URL: http://node1:8088/proxy/application_1566100765602_0106/ user: root 19/09/24 00:14:35 INFO util.ShutdownHookManager: Shutdown hook called 19/09/24 00:14:35 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-875aaa2a-1c5d-4c6a-95a6-12f276a80054
编排Spark任务：
选择jar包：
添加jar包：
填写jar包和主类等信息：
填写执行模式信息：
保存任务：
执行，返回执行信息： 
错误详细信息为：
2019-09-24 01:30:02,556 ERROR org.apache.oozie.command.wf.SignalXCommand: SERVER[node1] USER[admin] GROUP[-] TOKEN[] APP[Cf0924] JOB[0000000-190922201507304-oozie-oozi-W] ACTION[0000000-190922201507304-oozie-oozi-W@:start:] Workflow action failed : E0700: XML error, For input string: "" org.apache.oozie.workflow.WorkflowException: E0700: XML error, For input string: "" at org.apache.oozie.service.LiteWorkflowStoreService.getUserRetryMax(LiteWorkflowStoreService.java:171) at org.apache.oozie.service.LiteWorkflowStoreService.liteExecute(LiteWorkflowStoreService.java:122) at org.apache.oozie.service.LiteWorkflowStoreService$LiteActionHandler.start(LiteWorkflowStoreService.java:252) at org.apache.oozie.workflow.lite.ActionNodeHandler.enter(ActionNodeHandler.java:33) at org.apache.oozie.workflow.lite.LiteWorkflowInstance.signal(LiteWorkflowInstance.java:214) at org.apache.oozie.workflow.lite.LiteWorkflowInstance.signal(LiteWorkflowInstance.java:290) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:212) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:82) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.command.XCommand.call(XCommand.java:356) at org.apache.oozie.command.wf.ActionEndXCommand.execute(ActionEndXCommand.java:280) at org.apache.oozie.command.wf.ActionEndXCommand.execute(ActionEndXCommand.java:61) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.command.XCommand.call(XCommand.java:356) at org.apache.oozie.command.wf.ActionStartXCommand.callActionEnd(ActionStartXCommand.java:340) at org.apache.oozie.command.wf.ActionStartXCommand.execute(ActionStartXCommand.java:326) at org.apache.oozie.command.wf.ActionStartXCommand.execute(ActionStartXCommand.java:63) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.command.XCommand.call(XCommand.java:356) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:459) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:82) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.DagEngine.start(DagEngine.java:202) at org.apache.oozie.servlet.V1JobServlet.startWorkflowJob(V1JobServlet.java:332) at org.apache.oozie.servlet.V1JobServlet.startJob(V1JobServlet.java:76) at org.apache.oozie.servlet.BaseJobServlet.doPut(BaseJobServlet.java:80) at javax.servlet.http.HttpServlet.service(HttpServlet.java:730) at org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:289) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.oozie.servlet.AuthFilter$2.doFilter(AuthFilter.java:171) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:631) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:579) at org.apache.oozie.servlet.AuthFilter.doFilter(AuthFilter.java:176) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.oozie.servlet.HostnameFilter.doFilter(HostnameFilter.java:86) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:610) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:503) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.NumberFormatException: For input string: "" at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65) at java.lang.Integer.parseInt(Integer.java:592) at java.lang.Integer.parseInt(Integer.java:615) at org.apache.oozie.service.LiteWorkflowStoreService.getUserRetryMax(LiteWorkflowStoreService.java:164) ... 49 more
关联到oozie任务信息：
点开链接，查看任务详情：
附依赖文件：
phoenixConnectMode.scala ：
package com.fc import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.phoenix.spark._ object phoenixConnectMode { private val zookeeper = "node3:2181" def getMode1(sqlContext: SQLContext, tableName: String, columns: Array[String]): DataFrame = { val configuration = new Configuration() configuration.set("phoenix.schema.isNamespaceMappingEnabled", "true") configuration.set("phoenix.schema.mapSystemTablesToNamespace", "true") configuration.set("hbase.zookeeper.quorum", zookeeper) val df = sqlContext.phoenixTableAsDataFrame(tableName, columns, conf = configuration) df } }
其中 configuration.set("phoenix.schema.isNamespaceMappingEnabled", "true") 需要根据具体环境修改为 true 或者 false
timeUtil.scala ：
package com.fc import java.text.SimpleDateFormat import java.util.Calendar import org.joda.time.DateTime object timeUtil { final val ONE_DAY_Millis = 3600*24*1000 final val SECOND_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss" final val MILLISECOND_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS" final val DAY_DATE_FORMAT_ONE = "yyyy-MM-dd" final val DAY_DATE_FORMAT_TWO = "yyyyMMdd" final val MONTH_DATE_FORMAT = "yyyy-MM" /** * 时间字符串to时间戳 * @param dateStr 时间字符串 * @param pattern 时间格式 * @return */ def convertDateStr2TimeStamp(dateStr: String, pattern: String): Long = { new SimpleDateFormat(pattern).parse(dateStr).getTime } /** *时间戳to字符串 * @param timestamp 时间戳 * @param pattern 字符串 * @return */ def convertTimeStamp2DateStr(timestamp: Long, pattern: String): String = { new DateTime(timestamp).toString(pattern) } /** * 时间字符串+天数to时间戳 * @param dateStr 时间字符串 * @param pattern 时间格式 * @param days 增加天数 * @return */ def dateStrAddDays2TimeStamp(dateStr: String, pattern: String, days: Int): Long = { convertDateStr2Date(dateStr, pattern).plusDays(days).toDate.getTime } /** * 时间字符串+年数to时间戳 * @param dateStr 时间字符串 * @param pattern 时间格式 * @param years 增加年数 * @return */ def dateStrAddYears2TimeStamp(dateStr: String, pattern: String, years: Int): Long = { convertDateStr2Date(dateStr, pattern).plusYears(years).toDate.getTime } def dateStrAddYears2Str(dateStr: String, pattern: String, years: Int): String = { val t = convertDateStr2Date(dateStr, pattern).plusYears(years).toDate.getTime convertTimeStamp2DateStr(t, pattern) } /** * 时间字符串to日期 * @param dateStr 时间字符串 * @param pattern 时间格式 * @return */ def convertDateStr2Date(dateStr: String, pattern: String): DateTime = { new DateTime(new SimpleDateFormat(pattern).parse(dateStr)) } /** * 月差 * @param stDate 开始时间 * @param endDate 结束时间 * @return */ def getMonthSpace(stDate: String, endDate: String): Int = { val c1 = Calendar.getInstance() val c2 = Calendar.getInstance() c1.setTime(new SimpleDateFormat(MONTH_DATE_FORMAT).parse(stDate)) c2.setTime(new SimpleDateFormat(MONTH_DATE_FORMAT).parse(endDate)) val month1 = c2.get(Calendar.MONTH) - c1.get(Calendar.MONTH) val month2 = (c2.get(Calendar.YEAR) - c1.get(Calendar.YEAR)) * 12 Math.abs(month1 + month2) } /** * 日差 * @param stDate 开始时间 * @param endDate 结束时间 * @return */ def getDaySpace(stDate: String, endDate: String): Long = { val c1 = Calendar.getInstance() val c2 = Calendar.getInstance() c1.setTime(new SimpleDateFormat(SECOND_TIME_FORMAT).parse(stDate)) c2.setTime(new SimpleDateFormat(SECOND_TIME_FORMAT).parse(endDate)) val difference=c2.getTimeInMillis -c1.getTimeInMillis val days = difference/ONE_DAY_Millis Math.abs(days) } def getLastYearDateStr(dateStr: String): String = { val e = timeUtil.dateStrAddYears2TimeStamp(dateStr, "yyyy", -1) val f = timeUtil.convertTimeStamp2DateStr(e, "yyyy") f+"-12-31 23:59:59.999" } def getCurDateStr: String = { convertTimeStamp2DateStr(System.currentTimeMillis(), timeUtil.SECOND_TIME_FORMAT) } def lastYearEnd: String = { timeUtil.getLastYearDateStr(getCurDateStr) } def getCurMonthDays: Int = { val a = Calendar.getInstance() a.set(Calendar.DATE, 1) a.roll(Calendar.DATE, -1) val days = a.get(Calendar.DATE) days } }
pom.xml ：
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.fc</groupId> <artifactId>scalapp</artifactId> <version>1.0-SNAPSHOT</version> <dependencies> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> <version>1.6.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.10</artifactId> <version>1.6.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-spark</artifactId> <version>1.2.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-spark</artifactId> <version>4.14.0-cdh5.14.2</version> </dependency> <dependency> <groupId>com.lmax</groupId> <artifactId>disruptor</artifactId> <version>3.3.8</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId> <version>4.14.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>2.6.0-cdh5.14.2</version> </dependency> <!--MySQL驱动--> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.43</version> </dependency> </dependencies> <repositories> <repository> <id>mvnrepository</id> <name>mvnrepository</name> <url>http://mvnrepository.com/</url> <layout>default</layout> <releases> <enabled>true</enabled> </releases> <snapshots> <enabled>false</enabled> </snapshots> </repository> <repository> <id>cloudera</id> <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> </repository> <repository> <id>hortonworks</id> <name>hortonworks</name> <url>http://repo.hortonworks.com/content/repositories/releases/</url> </repository> </repositories> <pluginRepositories> <pluginRepository> <id>hortonworks</id> <name>hortonworks</name> <url>http://repo.hortonworks.com/content/repositories/releases/</url> </pluginRepository> </pluginRepositories> <build> <sourceDirectory>src/main/scala</sourceDirectory> <plugins> <plugin> <groupId>org.scala-tools</groupId> <artifactId>maven-scala-plugin</artifactId> <version>2.15.2</version> <executions> <execution> <goals> <goal>compile</goal> <goal>testCompile</goal> </goals> </execution> </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <version>3.1.0</version> <executions> <execution> <phase>package</phase> <goals> <goal>shade</goal> </goals> <configuration> <createDependencyReducedPom>false</createDependencyReducedPom> <filters> <filter> <artifact>*:*</artifact> <excludes> <exclude>META-INF/*.SF</exclude> <exclude>META-INF/*.DSA</exclude> <exclude>META-INF/*.RSA</exclude> </excludes> </filter> </filters> </configuration> </execution> </executions> </plugin> </plugins> </build> </project>
参考：