Background: When files of tens or hundreds of megabytes are read into memory at one time and then passed to users, the server explodes.
Solution principle: read a little and pass a little.
Solution: Use streams to read and write in a loop.
Use HttpURLConnection and bufferedInputStream to cache the stream to obtain the downloaded file. When reading the InputStream input stream, the size of each read is 5M. If you do not read it all at once, you can avoid memory overflow.
/**
* BufferedInputStream 缓存流下载文件
* @param downloadUrl
* @param path
*/
public static void downloadFile(String downloadUrl, String path){
InputStream inputStream = null;
OutputStream outputStream = null;
try {
URL url = new URL(downloadUrl);
//这里没有使用 封装后的ResponseEntity 就是也是因为这里不适合一次性的拿到结果，放不下content,会造成内存溢出
HttpURLConnection connection =(HttpURLConnection) url.openConnection();
//使用bufferedInputStream 缓存流的方式来获取下载文件，不然大文件会出现内存溢出的情况
inputStream = new BufferedInputStream(connection.getInputStream());
File file = new File(path);
if (file.exists()) {
file.delete();
}
outputStream = new FileOutputStream(file);
//这里也很关键每次读取的大小为5M 不一次性读取完
byte[] buffer = new byte[1024 * 1024 * 5];// 5MB
int len = 0;
while ((len = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, len);
}
connection.disconnect();
}catch (Exception e){
e.printStackTrace();
}finally {
IOUtils.closeQuietly(outputStream);
IOUtils.closeQuietly(inputStream);
}
}
To use RestTemplate streaming to download large files, you need to first use RequestCallback to define the reception type of the request header application/octet-stream, and then when restTemplate makes a download request, stream the response instead of loading it all into memory.
/**
* 文件下载示例:使用restTemplate请求第三方文件服务下载文件
*
* @author Bruce.CH
* @since 2023年9月2日
*/
@RestController
public class FileController {
private static final Logger LOGGER = LoggerFactory.getLogger(FileController.class);
/**
* 第三方文件服务下载接口
*/
private static final String DOWNLOAD_URL = "http://127.0.0.1:8080/v1/file/server/download/{fileId}";
/**
* 注入restTemplate对象
*/
@Resource
private RestTemplate restTemplate;
/**
* 【方式3】
* 请求第三方文件服务下载接口下载文件id指定的文件：字节流直接绑定到响应的输出流中
*
* @param fileId 第三方文件id
* @param response 客户端响应
*/
@GetMapping("/v1/file/download3/{fileId}")
public void download3(@PathVariable("fileId") String fileId, HttpServletResponse response) {
LOGGER.info("download file:{}", fileId);
Map<String, String> uriVariables = new HashMap<>();
uriVariables.put("fileId", fileId);
ResponseExtractor<Boolean> responseExtractor = clientHttpResponse -> {
// 设置响应头，直接用第三方文件服务的响应头
HttpHeaders headers = clientHttpResponse.getHeaders();
headers.forEach((key, value) -> response.setHeader(key, value.get(0)));
// 收到响应输入流即时拷贝写出到响应输出流中: inputStream -> outputStream
StreamUtils.copy(clientHttpResponse.getBody(), response.getOutputStream());
return true;
};
Boolean execute = restTemplate.execute(DOWNLOAD_URL, HttpMethod.GET, null, responseExtractor, uriVariables);
LOGGER.info("download file success?{}", execute);
}
}
Stream processing
// GET请求
public static void downLargeFileByStream(String url, String savePath){
// 对响应进行流式处理而不是将其全部加载到内存中
restTemplate.execute(url, HttpMethod.GET, null, response -> {
Files.copy(response.getBody(), Paths.get(savePath));
return null;
}, httpEntity);
}
// POST请求
public static void downLargeFileByStream(String url, Map headers, MultiValueMap<String,String> body, String savePath){
headers.put("Content-Type","application/x-www-form-urlencoded");
HttpHeaders header = new HttpHeaders();
header.setAll(headers);
HttpEntity<Object> httpEntity = new HttpEntity(body,header);
//定义请求头的接收类型，无请求信息时可以设置为 null
RequestCallback requestCallback = restTemplate.httpEntityCallback(httpEntity, null);
// RequestCallback requestCallback = request -> request.getHeaders()
// .setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL));
// 对响应进行流式处理而不是将其全部加载到内存中
restTemplate.execute(url,HttpMethod.POST,requestCallback, response -> {
Files.copy(response.getBody(), Paths.get(savePath));
return null;
}, httpEntity);
}
Stream processing testing
/**
* 下载文件
*
* @return
*/
@GetMapping("/test/downFile")
@ResponseBody
public HttpEntity<InputStreamResource> downFile() {
//将文件流封装为InputStreamResource对象
InputStream inputStream = this.getClass().getResourceAsStream("/1.txt");
InputStreamResource inputStreamResource = new InputStreamResource(inputStream);
//设置header
MultiValueMap<String, String> headers = new HttpHeaders();
headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=1.txt");
HttpEntity<InputStreamResource> httpEntity = new HttpEntity<>(inputStreamResource);
return httpEntity;
}
/**
* 调用上面的接口，测试获取文件
*/
@Test
public void test7() {
RestTemplate restTemplate = new RestTemplate();
String url = "http://localhost:8080/chat16/test/downFile";
/**
* 文件比较大的时候，比如好几个G，就不能返回字节数组了，会把内存撑爆，导致OOM
* 需要这么玩：
* 需要使用execute方法了，这个方法中有个ResponseExtractor类型的参数，
* restTemplate拿到结果之后，会回调{@link ResponseExtractor#extractData}这个方法，
* 在这个方法中可以拿到响应流，然后进行处理，这个过程就是变读边处理，不会导致内存溢出
*/
String result = restTemplate.execute(url,
HttpMethod.GET,
null,
new ResponseExtractor<String>() {
@Override
public String extractData(ClientHttpResponse response) throws IOException {
System.out.println("状态："+response.getStatusCode());
System.out.println("头："+response.getHeaders());
//获取响应体流
InputStream body = response.getBody();
//处理响应体流
String content = IOUtils.toString(body, "UTF-8");
return content;
}
}, new HashMap<>());
System.out.println(result);
}3 ways to implement RestTemplate download files_resttemplate download files-CSDN Blog
RestTemplate download file - Nuggets
Example of requesting file streams (normal files and compressed files) through httpClient