Spring Boot 配置Hadoop

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/eieiei438/article/details/82116527

Spring Boot 配置Hadoop

核心

• FsShell自动注入

项目工程

• Maven工程

Maven的setting.xml文件配置

 

。。。
<mirror>
    <id>nexus</id>
    <mirrorOf>*,!cloudera</mirrorOf>
    <url>http://repo1.maven.org/maven2/</url>
</mirror>
。。。

pom配置

 

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.peng</groupId>
    <artifactId>hdfstest</artifactId>
    <version>1.0-SNAPSHOT</version>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>


    <!--配置下载的仓库-->
    <repositories>
        <repository>
            <id>repo</id>
            <url>http://repo1.maven.org/maven2/</url>
        </repository>
        <repository>
            <id>cloudera</id>
            <url>https://repository.cloudera.com/content/repositories/releases/</url>
        </repository>
    </repositories>


    <!--配置依赖【注：源pom文件有冲突的依赖配置进行手动删除】-->
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.6.0-cdh5.7.0</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.10</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.mortbay.jetty</groupId>
            <artifactId>jetty</artifactId>
            <version>6.1.26</version>
        </dependency>

        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.25</version>
            <scope>test</scope>
        </dependency>

        <!--spring boot hadoop start-->
        <dependency>
            <groupId>org.springframework.data</groupId>
            <artifactId>spring-data-hadoop-boot</artifactId>
            <version>2.5.0.RELEASE</version>
        </dependency>
        <!--spring boot hadoop end-->

    </dependencies>

</project>

配置文件【resources下】

• application.properties【配置文件，映射键值对】

   

  spring.hadoop.fsUri=hdfs://centos00:8020
  

测试程序

• SpringBootHadoopTest.java

   

  package com.peng;
  
  import org.apache.hadoop.fs.FileStatus;
  import org.springframework.beans.factory.annotation.Autowired;
  import org.springframework.boot.CommandLineRunner;
  import org.springframework.boot.SpringApplication;
  import org.springframework.boot.autoconfigure.SpringBootApplication;
  import org.springframework.data.hadoop.fs.FsShell;
  
  @SpringBootApplication
  public class SpringBootHadoopTest implements CommandLineRunner {
  
      @Autowired
      FsShell fsShell;
  
      @Override
      public void run(String... strings) throws Exception {
          System.out.println("=========run start============");
          for (FileStatus fileStatus : fsShell.lsr("/data")) {
              System.out.println(">" + fileStatus.getPath());
          }
          System.out.println("===========run end===========");
      }
  
      public static void main(String[] args) {
          System.out.println("===========main start===========");
          SpringApplication.run(SpringBootHadoopTest.class, args);
          System.out.println("===========main end===========");
      }
  }
  

执行结果

 

===========main start===========
log4j:WARN No appenders could be found for logger (org.springframework.core.env.StandardEnvironment).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.

  .   ____          _            __ _ _
 /\\ / ___'_ __ _ _(_)_ __  __ _ \ \ \ \
( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \
 \\/  ___)| |_)| | | | | || (_| |  ) ) ) )
  '  |____| .__|_| |_|_| |_\__, | / / / /
 =========|_|==============|___/=/_/_/_/
 :: Spring Boot ::        (v1.4.7.RELEASE)

=========run start============
>hdfs://centos00:8020/data
>hdfs://centos00:8020/data/10000_access.log
>hdfs://centos00:8020/data/8_access.log
>hdfs://centos00:8020/data/hello.txt
===========run end===========
===========main end===========

Process finished with exit code 0