HDFS客户端Java开发

1. 配置HADOOP_HOME环境
   百度下载需要的文件。
   新建环境变量
   
   修改Path
   

2. 创建maven工程

3. 修改pom文件

   <?xml version="1.0" encoding="UTF-8"?>
   <project xmlns="http://maven.apache.org/POM/4.0.0"
            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
            xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <modelVersion>4.0.0</modelVersion>
   
       <groupId>com.chen</groupId>
       <artifactId>hadoop-hdfs-test</artifactId>
       <version>1.0-SNAPSHOT</version>
   
       <dependencies>
           <dependency>
               <groupId>junit</groupId>
               <artifactId>junit</artifactId>
               <version>RELEASE</version>
           </dependency>
   
           <dependency>
               <groupId>org.apache.logging.log4j</groupId>
               <artifactId>log4j-core</artifactId>
               <version>2.11.2</version>
           </dependency>
   
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-common</artifactId>
               <version>2.7.2</version>
           </dependency>
   
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-common</artifactId>
               <version>2.7.2</version>
           </dependency>
   
           <dependency>
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-hdfs</artifactId>
               <version>2.7.2</version>
           </dependency>
   
           <dependency>
               <groupId>jdk.tools</groupId>
               <artifactId>jdk.tools</artifactId>
               <version>1.8</version>
               <scope>system</scope>
               <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
           </dependency>
       </dependencies>
   
   </project>
   

4. 测试创建hadoop文件夹

       @Test
       public void testMkdirs() throws Exception{
           Configuration configuration = new Configuration();
   
           URI uri = new URI("hdfs://chen1:9000");
           FileSystem fs = FileSystem.get(uri,configuration,"root");
   
           boolean result = fs.mkdirs(new Path("/chen/java/f1"));
           System.out.println(result);
   
           fs.close();
       }
   

5. 测试下载文件

       @Test
       public void testDoenload() throws Exception{
           Configuration configuration = new Configuration();
   
           URI uri = new URI("hdfs://chen1:9000");
           FileSystem fs = FileSystem.get(uri,configuration,"root");
   
           fs.copyToLocalFile(false,
                   new Path("/data/hadoop/input/core-site.xml"),
                   new Path("d:/"),
                   true);
   
           fs.close();
       }
   

6. 测试上传文件

       @Test
       public void testUpload() throws Exception {
           Configuration configuration = new Configuration();
   
           URI uri = new URI("hdfs://chen1:9000");
           FileSystem fs = FileSystem.get(uri,configuration,"root");
   
           FileInputStream fileInputStream = new FileInputStream("C:\\Users\\Mr-CHEN\\Pictures\\1.jpg");
           FSDataOutputStream fsDataOutputStream = fs.create(new Path("/chen/java/f1/1.png"));
   
           IOUtils.copyBytes(fileInputStream,fsDataOutputStream,configuration);
   
           IOUtils.closeStream(fsDataOutputStream);
           IOUtils.closeStream(fileInputStream);
           fs.close();
       }