How does the server segment a large video file and respond to the client in segments so that the browser can play it progressively.
Spring-Boot implements HTTP fragment download breakpoint resume, so as to realize the problem of large video playback on H5 pages, realize progressive playback, and only play the content that needs to be played each time, without loading the entire file into memory.
File resuming, multi-threaded concurrent downloading of files (this is how Thunder works), etc.
Code
package com.example.insurance.controller;
import com.example.insurance.common.MediaContentUtil;
import cn.hutool.core.io.IORuntimeException;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.io.NioUtil;
import cn.hutool.core.io.StreamProgress;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.catalina.connector.ClientAbortException;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpRange;
import org.springframework.http.HttpStatus;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StopWatch;
import org.springframework.util.unit.DataSize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.List;
/**
* 内容资源控制器
*/
@SuppressWarnings("unused")
@Slf4j
@RestController("resourceController")
@RequestMapping(path = "/resource", produces = MediaType.APPLICATION_JSON_VALUE)
public class ResourceController {
/**
* 获取文件内容
*
* @param fileName 内容文件名称
* @param response 响应对象
* @see MediaContentConstant#MEDIA
*/
@GetMapping("/media/{fileName}")
public void getMedia(@PathVariable String fileName, HttpServletRequest request, HttpServletResponse response,
@RequestHeader HttpHeaders headers) {
// printRequestInfo(fileName, request, headers);
String filePath = MediaContentUtil.filePath();
try {
this.download(fileName, filePath, request, response, headers);
} catch (Exception e) {
log.error("getMedia error, fileName={}", fileName, e);
}
}
/**
* 获取封面内容
*
* @param fileName 内容封面名称
* @param response 响应对象
* @see MediaContentConstant#COVER
*/
@GetMapping("/cover/{fileName}")
public void getCover(@PathVariable String fileName, HttpServletRequest request, HttpServletResponse response,
@RequestHeader HttpHeaders headers) {
// printRequestInfo(fileName, request, headers);
String filePath = MediaContentUtil.filePath();
try {
this.download(fileName, filePath, request, response, headers);
} catch (Exception e) {
log.error("getCover error, fileName={}", fileName, e);
}
}
// ======= internal =======
private static void printRequestInfo(String fileName, HttpServletRequest request, HttpHeaders headers) {
String requestUri = request.getRequestURI();
String queryString = request.getQueryString();
log.debug("file={}, url={}?{}", fileName, requestUri, queryString);
log.info("headers={}", headers);
}
/**
* 缓冲区大小 16KB
*
* @see NioUtil#DEFAULT_BUFFER_SIZE
* @see NioUtil#DEFAULT_LARGE_BUFFER_SIZE
*/
// private static final int BUFFER_SIZE = NioUtil.DEFAULT_MIDDLE_BUFFER_SIZE;
private static final int BUFFER_SIZE = (int) DataSize.ofKilobytes(16L).toBytes();
private static final String BYTES_STRING = "bytes";
/**
* 设置请求响应状态、头信息、内容类型与长度 等。
* <pre>
* <a href="https://www.rfc-editor.org/rfc/rfc7233">
* HTTP/1.1 Range Requests</a>
* 2. Range Units
* 4. Responses to a Range Request
*
* <a href="https://www.rfc-editor.org/rfc/rfc2616.html">
* HTTP/1.1</a>
* 10.2.7 206 Partial Content
* 14.5 Accept-Ranges
* 14.13 Content-Length
* 14.16 Content-Range
* 14.17 Content-Type
* 19.5.1 Content-Disposition
* 15.5 Content-Disposition Issues
*
* <a href="https://www.rfc-editor.org/rfc/rfc2183">
* Content-Disposition</a>
* 2. The Content-Disposition Header Field
* 2.1 The Inline Disposition Type
* 2.3 The Filename Parameter
* </pre>
*
* @param response 请求响应对象
* @param fileName 请求的文件名称
* @param contentType 内容类型
* @param contentRange 内容范围对象
*/
private static void setResponse(
HttpServletResponse response, String fileName, String contentType,
ContentRange contentRange) {
// http状态码要为206：表示获取部分内容
response.setStatus(HttpStatus.PARTIAL_CONTENT.value());
// 支持断点续传，获取部分字节内容
// Accept-Ranges：bytes，表示支持Range请求
response.setHeader(HttpHeaders.ACCEPT_RANGES, BYTES_STRING);
// inline表示浏览器直接使用，attachment表示下载，fileName表示下载的文件名
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"inline;filename=" + MediaContentUtil.encode(fileName));
// Content-Range，格式为：[要下载的开始位置]-[结束位置]/[文件总大小]
// Content-Range: bytes 0-10/3103，格式为bytes 开始-结束/全部
response.setHeader(HttpHeaders.CONTENT_RANGE, toContentRange(contentRange));
response.setContentType(contentType);
// Content-Length: 11，本次内容的大小
response.setContentLengthLong(applyAsContentLength(contentRange));
}
/**
* 组装内容范围的响应头。
* <pre>
* <a href="https://www.rfc-editor.org/rfc/rfc7233#section-4.2">
* 4.2. Content-Range - HTTP/1.1 Range Requests</a>
* Content-Range: "bytes" first-byte-pos "-" last-byte-pos "/" complete-length
*
* For example:
* Content-Range: bytes 0-499/1234
* </pre>
*
* @param range 内容范围对象
* @return 内容范围的响应头
*/
private static String toContentRange(ContentRange range) {
return BYTES_STRING + ' ' + range.start + '-' + range.end + '/' + range.length;
// return "bytes " + range.start + "-" + range.end + "/" + range.length;
}
/**
* 计算内容完整的长度/总长度。
*
* @param range 内容范围对象
* @return 内容完整的长度/总长度
*/
private static long applyAsContentLength(ContentRange range) {
return range.end - range.start + 1;
}
/**
* <a href="https://www.jianshu.com/p/08db5ba3bc95">
* Spring Boot 处理 HTTP Headers</a>
*/
private void download(
String fileName, String path, HttpServletRequest request, HttpServletResponse response,
HttpHeaders headers)
throws IOException {
Path filePath = Paths.get(path + fileName);
if (!Files.exists(filePath)) {
log.warn("file not exist, filePath={}", filePath);
return;
}
long fileLength = Files.size(filePath);
// long fileLength2 = filePath.toFile().length() - 1;
// // fileLength=1184856, fileLength2=1184855
// log.info("fileLength={}, fileLength2={}", fileLength, fileLength2);
// 内容范围
ContentRange contentRange = applyAsContentRange(headers, fileLength, request);
// 要下载的长度
long contentLength = applyAsContentLength(contentRange);
log.debug("contentRange={}, contentLength={}", contentRange, contentLength);
// 文件类型
String contentType = request.getServletContext().getMimeType(fileName);
// mimeType=video/mp4, CONTENT_TYPE=null
log.debug("mimeType={}, CONTENT_TYPE={}", contentType, request.getContentType());
setResponse(response, fileName, contentType, contentRange);
// 耗时指标统计
StopWatch stopWatch = new StopWatch("downloadFile");
stopWatch.start(fileName);
try {
// case-1.参考网上他人的实现
// if (fileLength >= Integer.MAX_VALUE) {
// copy(filePath, response, contentRange);
// } else {
// copyByChannelAndBuffer(filePath, response, contentRange);
// }
// case-2.使用现成API
copyByBio(filePath, response, contentRange);
// copyByNio(filePath, response, contentRange);
// case-3.视频分段渐进式播放
// if (contentType.startsWith("video")) {
// copyForBufferSize(filePath, response, contentRange);
// } else {
// // 图片、PDF等文件
// copyByBio(filePath, response, contentRange);
// }
} finally {
stopWatch.stop();
log.info("download file, fileName={}, time={} ms", fileName, stopWatch.getTotalTimeMillis());
}
}
private static ContentRange applyAsContentRange(
HttpHeaders headers, long fileLength, HttpServletRequest request) {
/*
* 3.1. Range - HTTP/1.1 Range Requests
* https://www.rfc-editor.org/rfc/rfc7233#section-3.1
* Range: "bytes" "=" first-byte-pos "-" [ last-byte-pos ]
*
* For example:
* bytes=0-
* bytes=0-499
*/
// Range：告知服务端，客户端下载该文件想要从指定的位置开始下载
List<HttpRange> httpRanges = headers.getRange();
String range = request.getHeader(HttpHeaders.RANGE);
// httpRanges=[], range=null
// httpRanges=[448135688-], range=bytes=448135688-
log.debug("httpRanges={}, range={}", httpRanges, range);
// 开始下载位置
long firstBytePos;
// 结束下载位置
long lastBytePos;
if (CollectionUtils.isEmpty(httpRanges)) {
firstBytePos = 0;
lastBytePos = fileLength - 1;
} else {
HttpRange httpRange = httpRanges.get(0);
firstBytePos = httpRange.getRangeStart(fileLength);
lastBytePos = httpRange.getRangeEnd(fileLength);
}
return new ContentRange(firstBytePos, lastBytePos, fileLength);
}
/**
* <pre>
* <a href="https://blog.csdn.net/qq_32099833/article/details/109703883">
* Java后端实现视频分段渐进式播放</a>
* 服务端如何将一个大的视频文件做切分，分段响应给客户端，让浏览器可以渐进式地播放。
* 文件的断点续传、文件多线程并发下载（迅雷就是这么玩的）等。
*
* <a href="https://blog.csdn.net/qq_32099833/article/details/109630499">
* 大文件分片上传前后端实现</a>
* </pre>
*/
private static void copyForBufferSize(Path filePath, HttpServletResponse response, ContentRange contentRange) {
String fileName = filePath.getFileName().toString();
RandomAccessFile randomAccessFile = null;
OutputStream outputStream = null;
try {
// 随机读文件
randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
// 移动访问指针到指定位置
randomAccessFile.seek(contentRange.start);
// 注意：缓冲区大小 2MB，视频加载正常；1MB时有部分视频加载失败
int bufferSize = BUFFER_SIZE;
//获取响应的输出流
outputStream = new BufferedOutputStream(response.getOutputStream(), bufferSize);
// 每次请求只返回1MB的视频流
byte[] buffer = new byte[bufferSize];
int len = randomAccessFile.read(buffer);
//设置此次相应返回的数据长度
response.setContentLength(len);
// 将这1MB的视频流响应给客户端
outputStream.write(buffer, 0, len);
log.info("file download complete, fileName={}, contentRange={}", fileName, toContentRange(contentRange));
} catch (ClientAbortException | IORuntimeException e) {
// 捕获此异常表示用户停止下载
log.warn("client stop file download, fileName={}", fileName);
} catch (Exception e) {
log.error("file download error, fileName={}", fileName, e);
} finally {
IoUtil.close(outputStream);
IoUtil.close(randomAccessFile);
}
}
/**
* 拷贝流，拷贝后关闭流。
*
* @param filePath 源文件路径
* @param response 请求响应
* @param contentRange 内容范围
*/
private static void copyByBio(Path filePath, HttpServletResponse response, ContentRange contentRange) {
String fileName = filePath.getFileName().toString();
InputStream inputStream = null;
OutputStream outputStream = null;
try {
RandomAccessFile randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
randomAccessFile.seek(contentRange.start);
inputStream = Channels.newInputStream(randomAccessFile.getChannel());
outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);
StreamProgress streamProgress = new StreamProgressImpl(fileName);
long transmitted = IoUtil.copy(inputStream, outputStream, BUFFER_SIZE, streamProgress);
log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
} catch (ClientAbortException | IORuntimeException e) {
// 捕获此异常表示用户停止下载
log.warn("client stop file download, fileName={}", fileName);
} catch (Exception e) {
log.error("file download error, fileName={}", fileName, e);
} finally {
IoUtil.close(outputStream);
IoUtil.close(inputStream);
}
}
/**
* 拷贝流，拷贝后关闭流。
* <pre>
* <a href="https://www.cnblogs.com/czwbig/p/10035631.html">
* Java NIO 学习笔记（一）----概述，Channel/Buffer</a>
* </pre>
*
* @param filePath 源文件路径
* @param response 请求响应
* @param contentRange 内容范围
*/
private static void copyByNio(Path filePath, HttpServletResponse response, ContentRange contentRange) {
String fileName = filePath.getFileName().toString();
InputStream inputStream = null;
OutputStream outputStream = null;
try {
RandomAccessFile randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
randomAccessFile.seek(contentRange.start);
inputStream = Channels.newInputStream(randomAccessFile.getChannel());
outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);
StreamProgress streamProgress = new StreamProgressImpl(fileName);
long transmitted = NioUtil.copyByNIO(inputStream, outputStream, BUFFER_SIZE, streamProgress);
log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
} catch (ClientAbortException | IORuntimeException e) {
// 捕获此异常表示用户停止下载
log.warn("client stop file download, fileName={}", fileName);
} catch (Exception e) {
log.error("file download error, fileName={}", fileName, e);
} finally {
IoUtil.close(outputStream);
IoUtil.close(inputStream);
}
}
/**
* <pre>
* <a href="https://blog.csdn.net/lovequanquqn/article/details/104562945">
* SpringBoot Java实现Http方式分片下载断点续传+实现H5大视频渐进式播放</a>
* SpringBoot 实现Http分片下载断点续传，从而实现H5页面的大视频播放问题，实现渐进式播放，每次只播放需要播放的内容就可以了，不需要加载整个文件到内存中。
* 二、Http分片下载断点续传实现
* 四、缓存文件定时删除任务
* </pre>
*/
private static void copy(Path filePath, HttpServletResponse response, ContentRange contentRange) {
String fileName = filePath.getFileName().toString();
// 要下载的长度
long contentLength = applyAsContentLength(contentRange);
BufferedOutputStream outputStream = null;
RandomAccessFile randomAccessFile = null;
// 已传送数据大小
long transmitted = 0;
try {
randomAccessFile = new RandomAccessFile(filePath.toFile(), "r");
randomAccessFile.seek(contentRange.start);
outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);
// 把数据读取到缓冲区中
byte[] buffer = new byte[BUFFER_SIZE];
int len = BUFFER_SIZE;
//warning：判断是否到了最后不足4096（buffer的length）个byte这个逻辑（(transmitted + len) <= contentLength）要放前面
//不然会会先读取randomAccessFile，造成后面读取位置出错;
while ((transmitted + len) <= contentLength && (len = randomAccessFile.read(buffer)) != -1) {
outputStream.write(buffer, 0, len);
transmitted += len;
log.info("fileName={}, transmitted={}", fileName, transmitted);
}
//处理不足buffer.length部分
if (transmitted < contentLength) {
len = randomAccessFile.read(buffer, 0, (int) (contentLength - transmitted));
outputStream.write(buffer, 0, len);
transmitted += len;
log.info("fileName={}, transmitted={}", fileName, transmitted);
}
log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
} catch (ClientAbortException e) {
// 捕获此异常表示用户停止下载
log.warn("client stop file download, fileName={}, transmitted={}", fileName, transmitted);
} catch (Exception e) {
log.error("file download error, fileName={}, transmitted={}", fileName, transmitted, e);
} finally {
IoUtil.close(outputStream);
IoUtil.close(randomAccessFile);
}
}
/**
* 通过数据传输通道和缓冲区读取文件数据。
* <pre>
* 当文件长度超过{@link Integer#MAX_VALUE}时，
* 使用{@link FileChannel#map(FileChannel.MapMode, long, long)}报如下异常。
* java.lang.IllegalArgumentException: Size exceeds Integer.MAX_VALUE
* at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:863)
* at com.example.insurance.controller.ResourceController.download(ResourceController.java:200)
* </pre>
*
* @param filePath 源文件路径
* @param response 请求响应
* @param contentRange 内容范围
*/
private static void copyByChannelAndBuffer(
Path filePath, HttpServletResponse response, ContentRange contentRange) {
String fileName = filePath.getFileName().toString();
// 要下载的长度
long contentLength = applyAsContentLength(contentRange);
BufferedOutputStream outputStream = null;
FileChannel inChannel = null;
// 已传送数据大小
long transmitted = 0;
long firstBytePos = contentRange.start;
long fileLength = contentRange.length;
try {
inChannel = FileChannel.open(filePath, StandardOpenOption.READ, StandardOpenOption.WRITE);
// 建立直接缓冲区
MappedByteBuffer inMap = inChannel.map(FileChannel.MapMode.READ_ONLY, firstBytePos, fileLength);
outputStream = new BufferedOutputStream(response.getOutputStream(), BUFFER_SIZE);
// 把数据读取到缓冲区中
byte[] buffer = new byte[BUFFER_SIZE];
int len = BUFFER_SIZE;
// warning：判断是否到了最后不足4096（buffer的length）个byte这个逻辑（(transmitted + len) <= contentLength）要放前面
// 不然会会先读取file，造成后面读取位置出错
while ((transmitted + len) <= contentLength) {
inMap.get(buffer);
outputStream.write(buffer, 0, len);
transmitted += len;
log.info("fileName={}, transmitted={}", fileName, transmitted);
}
// 处理不足buffer.length部分
if (transmitted < contentLength) {
len = (int) (contentLength - transmitted);
buffer = new byte[len];
inMap.get(buffer);
outputStream.write(buffer, 0, len);
transmitted += len;
log.info("fileName={}, transmitted={}", fileName, transmitted);
}
log.info("file download complete, fileName={}, transmitted={}", fileName, transmitted);
} catch (ClientAbortException e) {
// 捕获此异常表示用户停止下载
log.warn("client stop file download, fileName={}, transmitted={}", fileName, transmitted);
} catch (Exception e) {
log.error("file download error, fileName={}, transmitted={}", fileName, transmitted, e);
} finally {
IoUtil.close(outputStream);
IoUtil.close(inChannel);
}
}
/**
* 内容范围对象
* <pre>
* <a href="https://www.rfc-editor.org/rfc/rfc7233#section-4.2">
* 4.2. Content-Range - HTTP/1.1 Range Requests</a>
* Content-Range: "bytes" first-byte-pos "-" last-byte-pos "/" complete-length
*
* For example:
* Content-Range: bytes 0-499/1234
* </pre>
*
* @see org.apache.catalina.servlets.DefaultServlet.Range
*/
@AllArgsConstructor
private static class ContentRange {
/**
* 第一个字节的位置
*/
private final long start;
/**
* 最后一个字节的位置
*/
private long end;
/**
* 内容完整的长度/总长度
*/
private final long length;
/**
* Validate range.
*
* @return true if the range is valid, otherwise false
*/
public boolean validate() {
if (end >= length) {
end = length - 1;
}
return (start >= 0) && (end >= 0) && (start <= end) && (length > 0);
}
@Override
public String toString() {
return "firstBytePos=" + start +
", lastBytePos=" + end +
", fileLength=" + length;
}
}
/**
* 数据流进度条
*/
private static class StreamProgressImpl implements StreamProgress {
private final String fileName;
public StreamProgressImpl(String fileName) {
this.fileName = fileName;
}
@Override
public void start() {
log.info("start progress {}", fileName);
}
@Override
public void progress(long progressSize) {
log.debug("progress {}, progressSize={}", fileName, progressSize);
}
@Override
public void finish() {
log.info("finish progress {}", fileName);
}
}
}
package com.example.insurance.common;
import java.nio.charset.StandardCharsets;
import cn.hutool.core.net.URLDecoder;
import cn.hutool.core.net.URLEncoder;
/**
* 文件内容辅助方法集
*/
public final class MediaContentUtil {
public static String filePath() {
String osName = System.getProperty("os.name");
String filePath = "/data/files/";
if (osName.startsWith("Windows")) {
filePath = "D:\\" + filePath;
}
// else if (osName.startsWith("Linux")) {
// filePath = MediaContentConstant.FILE_PATH;
// }
else if (osName.startsWith("Mac") || osName.startsWith("Linux")) {
filePath = "/home/admin" + filePath;
}
return filePath;
}
public static String encode(String fileName) {
return URLEncoder.DEFAULT.encode(fileName, StandardCharsets.UTF_8);
}
public static String decode(String fileName) {
return URLDecoder.decode(fileName, StandardCharsets.UTF_8);
}
}