AngularJS + SpringMVC 多文件 HDFS 上传和下载

搭建 hadoop 3.1.2 独立模式，单节点和多节点伪分布式安装与使用

在 AngularJS + SpringMVC 多文件 本地 上传和下载 上增加hdfs的上传和下载

添加依赖

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>3.1.2</version>
</dependency>

application.yml 新增hdfs地址和所需上传的文件夹

hdfs:
  serverUrl: hdfs://hadoop-master:9000
  uploadDir: /upload/

hdfs上传下载工具类

@Component
public class HdfsUtil {
	private static final Logger logger = LoggerFactory.getLogger(HdfsUtil.class);

	private static String hdfsServerUrl;

	private static String hdfsUploadDir;

	@Value("${hdfs.serverUrl:''}")
	public void setHdfsServerUrl(String serverUrl) {
		hdfsServerUrl = serverUrl;
	}

	@Value("${hdfs.uploadDir:''}")
	public void setHdfsUploadDir(String uploadDir) {
		hdfsUploadDir = uploadDir;
	}

	static Configuration conf;
	static FileSystem fs;

	static{
		try {
			conf = new Configuration();
			fs = FileSystem.newInstance(conf);
		} catch (IOException e) {
			logger.error("初始化 FileSystem 发生异常:{}", e.getMessage());
		}

		Runtime.getRuntime().addShutdownHook(new Thread(){
			@Override
			public void run(){
				try{
					fs.close();
				}catch (IOException e) {
					logger.error("关闭 FileSystem 发生异常:{}", e.getMessage());
				}
			}
		});
	}

	/**
	 * 上传文件
	 * @param file
	 * @return
	 */
	public static String uploadFile(MultipartFile file){
		String fileName = file.getOriginalFilename();
		String filePath = hdfsUploadDir + fileName;
		try {
			InputStream inputStream = file.getInputStream();
			Path path = new Path(filePath);
			OutputStream outputStream = fs.create(path);
			IOUtils.copyBytes(inputStream,outputStream,4096,true);
		}catch (IOException e){
			logger.error("文件上传发生异常:{}", e.getMessage());
		}
		return hdfsServerUrl + filePath;
	}

	/**
	 * 下载文件
	 * @param filePath
	 * @param response
	 */
	public static void downloadFile(String filePath, HttpServletResponse response){
		String fileName = filePath.substring(filePath.lastIndexOf('/') + 1);
		String hdfsFilePath = filePath.replace(hdfsServerUrl,"");
		response.setCharacterEncoding ("utf-8");
		//二进制流
		response.setContentType("application/octet-stream");
		try(
				InputStream inputStream = fs.open(new Path(hdfsFilePath));
				ServletOutputStream servletOutputStream = response.getOutputStream();
		){
			response.setHeader("Content-Disposition", "attachment;filename=" + new String(fileName.getBytes("GB2312"), "ISO8859-1"));
			int len;
			byte[] buffer = new byte[1024];
			while ((len = inputStream.read(buffer)) > 0){
				servletOutputStream.write(buffer,0, len);
			}
		}catch (IOException e){
			logger.error ("文件下载发生异常:{}", e.getMessage());
		}
	}
}

1. 上传

新增hdfs上传按钮

<button class="btn" type="submit" ng-click="fileUploadHdfs()">上传Hdfs</button>

上传hdfs点击事件

//上传hdfs
$scope.fileUploadHdfs = function () {
    var form = new FormData();
    var selectFiles = $scope.repeatFiles;
    for (var i=0; i<selectFiles.length; i++) {
        form.append("file", selectFiles[i]);
    }
    var url = 'file/uploadHdfs';
    upload(url,form);
};

function upload(url,form) {
    $http({
        method: 'POST',
        url: Config.getApiRemoteUrl() + url,
        data: form,
        headers: {
            'Content-Type':undefined
        },
        transformRequest: angular.identity
    }).then(function successCallback(response) {
        if(response.status === 200){
            layer.msg("success");
        }else{
            layer.msg("fail");
        }
    }, function errorCallback(response) {
        layer.msg("error");
    });
}

后端上传接口，调用hdfs工具类上传

@PostMapping("/uploadHdfs")
public void uploadHdfs(@RequestParam(value = "file") MultipartFile[]  files){
    if(files != null && files.length > 0){
        for(MultipartFile file : files){
            HdfsUtil.uploadFile(file);
        }
    }
}

效果

2. 下载

新增下载hdfs的a标签

<a target="_self" id="downloadHdfs" ng-click="fileDownloadHdfs()">下载Hdfs</a>

替换a标签的超链接url，并没有把hdfs文件路径当参数放进去

//下载hdfs
$scope.fileDownloadHdfs = function () {
    $('#downloadHdfs').attr('href', './api/file/downloadHdfs');
};

后端下载接口，调用hdfs工具类下载

 @GetMapping("/downloadHdfs")
public void downloadHdfs(HttpServletResponse response) {
    String filePath = "hdfs://hadoop-master:9000/upload/upload1.txt";
    HdfsUtil.downloadFile(filePath,response);
}

效果