1. 前言
oss是阿里巴巴研发的分布式存储中间件，可以存储各种文件：图片，文档等；今天博主就oss多线程分片上传作个分享。
2. 分片代码
2.1 pom依赖
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.11.0</version>
</dependency>
2.2 multipart 扩容配置类
@Configuration
public class MultipartResolverConfig {
@Bean(name = MultipartFilter.DEFAULT_MULTIPART_RESOLVER_BEAN_NAME)
protected MultipartResolver getMultipartResolver() {
CommonsMultipartResolver multipartResolver = new CommonsMultipartResolver();
multipartResolver.setMaxUploadSize(314572800);
multipartResolver.setMaxInMemorySize(314572800);
return multipartResolver;
}
}
2.3 主线程代码
private static final long EACH_PART_SIZE = 2 * 1024 * 1024;
// 创建InitiateMultipartUploadRequest对象
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
// 初始化分片
InitiateMultipartUploadResult uploadResult = ossClient.initiateMultipartUpload(request);
// 返回uploadId，它是分片上传事件的唯一标识，您可以根据这个uploadId发起相关的操作，如取消分片上传、查询分片上传等
String uploadId = uploadResult.getUploadId();
// 计算文件分片数量
long fileLength = file.length();
int partCount = (int) (fileLength / EACH_PART_SIZE);
if (fileLength % EACH_PART_SIZE != 0) {
partCount++;
}
final CountDownLatch countDownLatch = new CountDownLatch(partCount);
ExecutorService pool = ThreadPoolUtil.createDefaultThreadPool(partCount);
List<Future<PartETag>> futures = new ArrayList<>(partCount);
// 遍历分片上传
for (int i = 0; i < partCount; i++) {
long startPos = i * EACH_PART_SIZE;
long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : EACH_PART_SIZE;
try {
futures.add(pool.submit(new SyncOssPartConsumer(file, bucketName, key, uploadId, i, startPos, curPartSize, countDownLatch)));
} catch (Exception ex) {
LoggerUtil.error(logger, ex,"partUpload futures.add err, index=", i);
}
}
try {
countDownLatch.await(1, TimeUnit.MINUTES);
} catch (InterruptedException ex) {
LoggerUtil.error(logger, ex,"partUpload countDownLatch.await, err=", ex);
}
// partETags是PartETag的集合。PartETag由分片的ETag和分片号组成
List<PartETag> partETags = new ArrayList<>();
for (Future<PartETag> future : futures) {
try {
PartETag partETag = future.get(1, TimeUnit.MINUTES);
partETags.add(partETag);
} catch (Exception ex) {
LoggerUtil.error(logger, ex,"partUpload future get timeout, err=", ex);
} finally {
future.cancel(true);
}
}
pool.shutdown();
// 在执行完成分片上传操作时，需要提供所有有效的partETags。OSS收到提交的partETags后，会逐一验证每个分片的有效性。当所有的数据分片验证通过后，OSS将把这些分片组合成一个完整的文件。
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
// 完成上传
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
2.4 子线程代码
/**
* 多线程异步消费
*/
class SyncOssPartConsumer implements Callable<PartETag> {
File file;
String bucketName;
String key;
String uploadId;
int index;
long startPos;
long curPartSize;
CountDownLatch countDownLatch;
public SyncOssPartConsumer(File file, String bucketName, String key, String uploadId, int index, long startPos, long curPartSize, CountDownLatch countDownLatch) {
this.file = file;
this.bucketName = bucketName;
this.key = key;
this.uploadId = uploadId;
this.index = index;
this.startPos = startPos;
this.curPartSize = curPartSize;
this.countDownLatch = countDownLatch;
}
@Override
public PartETag call() throws Exception {
InputStream in = null;
try {
in = new FileInputStream(file);
// 跳过已经上传的分片
in.skip(startPos);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setUploadId(uploadId);
uploadPartRequest.setInputStream(in);
// 设置分片大小。除了最后一个分片没有大小限制，其他的分片最小为100KB
uploadPartRequest.setPartSize(curPartSize);
// 设置分片号。每一个上传的分片都有一个分片号，取值范围是1~10000，如果超出这个范围，OSS将返回InvalidArgument的错误码
uploadPartRequest.setPartNumber(index + 1);
// 每个分片不需要按顺序上传，甚至可以在不同客户端上传，OSS会按照分片号排序组成完整的文件。
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
if (Objects.nonNull(uploadPartResult)) {
return uploadPartResult.getPartETag();
}
} catch (Exception ex) {
LoggerUtil.error(logger, ex, "[SyncOssPartConsumer] part upload err");
} finally {
countDownLatch.countDown();
if (null != in) {
try {
in.close();
}catch (Exception ex) {
LoggerUtil.error(logger, ex, "[SyncOssPartConsumer] in.close err");
}
}
}
return null;
}
}
2.5 线程工具类
public class ThreadPoolUtil {
/**
* 默认的最大线程池数量
*/
private static int THREAD_MAX = Integer.MAX_VALUE;
/**
* 默认的最小线程池数量
*/
private static int THREAD_MIN = 1;
/**
* 默认的队列数量
*/
private static int DEFAULT_QUEUE_SIZE = 1024;
/**
* 扩容的线程休眠时的存活时间
*/
private static int DEFAULT_ALIVE_TIME = 30;
/**
* 存活时间的单位
*/
private static TimeUnit DEFAULT_TIME_UNIT = TimeUnit.SECONDS;
//拒绝策略
/**
* 默认的，放不进去会舍弃并抛出异常
*/
private static RejectedExecutionHandler default_AbortPolicy(){
return new ThreadPoolExecutor.AbortPolicy();
}
/**
* 放不进去会舍去，不会抛出异常
*/
private static RejectedExecutionHandler default_DiscardPolicy(){
return new ThreadPoolExecutor.DiscardPolicy();
}
/**
* 放不进去的会移除最早添加的，然后重试
*/
private static RejectedExecutionHandler default_DiscardOldestPolicy(){
return new ThreadPoolExecutor.DiscardOldestPolicy();
}
/**
* 放不进去的由调用线程来执行
*/
private static RejectedExecutionHandler default_CallerRunsPolicy(){
return new ThreadPoolExecutor.CallerRunsPolicy();
}
//阻塞队列
//有界--------》需要定义队列的大小
/**
* 一个由数组结构组成的有界阻塞队列。
* @param size 队列大小
*/
private static BlockingQueue ArrayBlockingQueue(int size){
return new ArrayBlockingQueue(size);
}
/**
* 一个由链表结构组成的有界阻塞队列。
* @param size 队列大小
*/
private static BlockingQueue LinkedBlockingQueue(int size){
return new LinkedBlockingQueue(size);
}
/**
* 一个由链表结构组成的双向阻塞队列。
* @param size 队列大小
*/
private static BlockingQueue LinkedBlockingDeque(int size){
return new LinkedBlockingDeque(size);
}
//无界--------》队列无限大，慎用
/**
* 一个支持优先级排序的无界阻塞队列。
*/
private static BlockingQueue PriorityBlockingQueue(){
return new PriorityBlockingQueue();
}
/**
* 一个使用优先级队列实现的无界阻塞队列。
*/
private static BlockingQueue DelayQueue(){
return new DelayQueue();
}
/**
* 一个由链表结构组成的无界阻塞队列。
*/
private static BlockingQueue LinkedTransferQueue(){
return new LinkedTransferQueue();
}
//特殊队列，不存放任何元素，用来转接
/**
* 一个不存储元素的阻塞队列。
*/
private static BlockingQueue SynchronousQueue(){
return new SynchronousQueue();
}
/**
* 创建默认的线程池，此线程会抛弃放不进去的任务，并抛出RejectedExecutionException异常
* 该方法创建的线程池参数是：
* 最大线程池数量是核心线程池数量左移一位的大小
* 超时时间是默认30秒
* 队列是默认的LinkedBlockingQueue（是一个用链表实现的有界阻塞队列。此队列的默认和最大长度为）
* 队列大小是，默认为1024，如果核心线程池数量大于了这个默认队列数量，就使用最大线程池数量。否则就用默认的
* @param coreSize 核心线程池大小
* @return
*/
public static ThreadPoolExecutor createDefaultThreadPool(int coreSize) {
coreSize = getCoreSize(coreSize);
int maxSize = getMaxSize(coreSize);
return new ThreadPoolExecutor(coreSize, maxSize, DEFAULT_ALIVE_TIME, DEFAULT_TIME_UNIT, new LinkedBlockingQueue(DEFAULT_QUEUE_SIZE < coreSize ? maxSize : DEFAULT_QUEUE_SIZE));
}
/**
*
* @param taskSize
* @param callableList
* @param <T>
* @return
*/
public static <T> List<Future<T>> getFutureList(int taskSize, List<Callable<T>> callableList) {
List<Future<T>> futures = new ArrayList<>(taskSize);
ThreadPoolExecutor executor = createDefaultThreadPool(taskSize);
for (int i = 0; i < taskSize; i++) {
if (null != callableList.get(i)) {
futures.add(executor.submit(callableList.get(i)));
}
}
return futures;
}
/**
* 创建默认的线程池，此线程池会丢弃放不进去的任务，但不会抛出异常
* 该方法创建的线程池参数是：
* 最大线程池数量是核心线程池数量左移一位的大小
* 超时时间是默认30秒
* 队列是默认的LinkedBlockingQueue（是一个用链表实现的有界阻塞队列。此队列的默认和最大长度为）
* 队列大小是，默认为1024，如果核心线程池数量大于了这个默认队列数量，就使用最大线程池数量。否则就用默认的
* @param coreSize 核心线程池大小
* @return
*/
public static ThreadPoolExecutor createDefaultThreadPoolNoEx(int coreSize) {
coreSize = getCoreSize(coreSize);
int maxSize = getMaxSize(coreSize);
return new ThreadPoolExecutor(coreSize, maxSize, DEFAULT_ALIVE_TIME, DEFAULT_TIME_UNIT, new LinkedBlockingQueue(DEFAULT_QUEUE_SIZE < coreSize ? maxSize : DEFAULT_QUEUE_SIZE), default_DiscardPolicy());
}
/**
* 创建自定义的线程池
* @param coreSize 核心线程池大小
* @param maxSize 最大线程池大小
* @param keepAliveTime 扩容线程池存活时间
* @param timeUnit 时间单位
* @param threadQueue 阻塞队列
*/
public static ThreadPoolExecutor createThreadPool(int coreSize, int maxSize, int keepAliveTime, TimeUnit timeUnit, BlockingDeque threadQueue) {
coreSize = getCoreSize(coreSize);
maxSize = getMaxSize(maxSize);
return new ThreadPoolExecutor(coreSize, maxSize, keepAliveTime, timeUnit, threadQueue);
}
/**
* 创建自定义的线程池
* @param coreSize 核心线程池大小
* @param maxSize 最大线程池大小
* @param keepAliveTime 扩容线程池存活时间
* @param timeUnit 时间单位
* @param threadQueue 阻塞队列
* @param rejectedExecutionHandler 拒绝策略
*/
public static ThreadPoolExecutor createThreadPool(int coreSize, int maxSize, int keepAliveTime, TimeUnit timeUnit, BlockingDeque threadQueue, RejectedExecutionHandler rejectedExecutionHandler) {
coreSize = getCoreSize(coreSize);
maxSize = getMaxSize(maxSize);
return new ThreadPoolExecutor(coreSize, maxSize, keepAliveTime, timeUnit, threadQueue, rejectedExecutionHandler);
}
/**
* 校验核心线程池数量
*/
private static int getCoreSize(int coreSize) {
return Math.max(coreSize, THREAD_MIN);
}
/**
* 校验最大线程池数量
*/
public static int getMaxSize(int coreSize) {
int maxSize = coreSize << 1;
return Math.max(maxSize, THREAD_MAX);
}
/**
* 校验最大线程池数量
*/
public static int getMaxSize(int coreSize, int maxSize) {
return Math.max(maxSize, coreSize);
}
}