Building file services with Minio

1. Minio service deployment

Minio installation directory: /usr/local/minio

Minio data storage directory: /usr/local/minio/data/minio

Minio service startup script:

./minio server --address :9966 --console-address :9967 /usr/local/minio/data/minio

• 9966 is the API port for Minio service
• 9967 is the port of the Minio service platform

Minio management platform address: http://172.40.240.162:9967/buckets, username and password: minio/minio123

2. Integrate Minio service instructions

2.1 File Md5 value calculation

Each file must generate an Md5 value, which is used for merging and comparing files after fragmentation of large files.

/**
* 分块计算文件的md5值
* @param file 文件
* @param chunkSize 分片大小
* @returns Promise
*/
function calculateFileMd5(file, chunkSize) {
    
    
    return new Promise((resolve, reject) => {
    
    
        let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
        let chunks = Math.ceil(file.size / chunkSize);
        let currentChunk = 0;
        let spark = new SparkMD5.ArrayBuffer();
        let fileReader = new FileReader();

        fileReader.onload = function (e) {
    
    
            spark.append(e.target.result);
            currentChunk++;
            if (currentChunk < chunks) {
    
    
                loadNext();
            } else {
    
    
                let md5 = spark.end();
                console.log("...md5...",md5)
                resolve(md5);
            }
        };

        fileReader.onerror = function (e) {
    
    
            reject(e);
        };

        function loadNext() {
    
    
            let start = currentChunk * chunkSize;
            let end = start + chunkSize;
            if (end > file.size) {
    
    
                end = file.size;
            }
            fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
        }

        loadNext();
    });
}

The SparkMD5.js file privately gives

2.2 Upload file naming

• Single file upload, the file name is: fileId + file extension. eg: /test/44fd7a29130511ed9fe5005056b2b395.jpg
• When uploading large files, name the fragmented files: file md5 value/segment index. eg: /test/19d66e11606ef41bd6e447156f572969/1
• After the large files are merged on the server side, the file names are the same as the single file.

test is the bucket name

Single file test page: http://localhost:19081/portal-osp/osp-file/client/file/home/upload

Large file test page: http://localhost:19081/portal-osp/osp-file/client/file/home/chunk/upload

2.3 Thumbnails

For image resources, a 0.25 times thumbnail is automatically generated with the name: fileId_thumb. suffix. eg:

• The original file name is: 44fd7a29130511ed9fe5005056b2b395.jpg
• The thumbnail name is: 44fd7a29130511ed9fe5005056b2b395_thumb.jpg

2.4 Fragmentation of Large Files

The minimum size of a fragmented file is 5M, and Minio stipulates that if the operation of merging files is required, the minimum size of each fragmented file is 5M.

Based on the principle of sharding and merging, both the frontend and the backend can perform file merging operations. If large files are merged on the server side, service resources will be affected, but a file merging interface is provided.

Front-end file fragmentation code:

/**
 * 执行分片上传
 * @param file 上传的文件
 * @param i 第几分片，从0开始
 * @param md5 文件的md5值
 */
function PostFile(file, i, md5) {
    
    
    resultDiv.innerHTML += '上传文件，当前分片为：' + i + '<br/>'
    let name = file.name,                           // 文件名
        size = file.size,                           // 总大小
        segSize = 4 * 1024 * 1024,                // 以5MB为一个分片,每个分片的大小
        segTotal = Math.ceil(size / segSize);   //总片数
    if (i >= segTotal) {
    
    
        return;
    }

    let start = i * segSize;
    let end = start + segSize;
    let packet = file.slice(start, end);  //将文件进行切片
    /*  构建form表单进行提交  */
    let form = new FormData();
    form.append("md5", md5);// 前端生成uuid作为标识符传个后台每个文件都是一个uuid防止文件串了
    form.append("file", packet); //slice方法用于切出文件的一部分
    form.append("name", name);
    form.append("fileSize", size);
    form.append("segTotal", segTotal); //总片数
    form.append("segCurrent", i + 1); //当前是第几片
    $.ajax({
    
    
        url: baseUrl + "/client/file/chunk/upload",
        type: "POST",
        data: form,
        //timeout:"10000",  //超时10秒
        async: true, //异步
        dataType: "json",
        processData: false, //很重要，告诉jquery不要对form进行处理
        contentType: false, //很重要，指定为false才能形成正确的Content-Type
        success: function (msg) {
    
    
            console.log(msg);
            /*  表示上一块文件上传成功，继续下一次  */
            if (msg.status === 20001) {
    
    
                form = '';
                i++;
                PostFile(file, i, md5);
            } else if (msg.status === 50000) {
    
    
                form = '';
                resultDiv.innerHTML += '请求状态码:' + msg.status + '，' + msg.message + '<br/>'
                // setInterval(function () {
    
    
                //     PostFile(file, i, md5)
                // }, 2000);
            } else if (msg.status === 20002) {
    
    
                // merge(segTotal, name, md5, getFileType(file.name), file.size)
                console.log("上传成功");
                resultDiv.innerHTML += '请求状态码:' + msg.status + '，' + msg.message + '<br/>'
                resultDiv.innerHTML += '<br/>---------------------------上传文件结束--------------------------------------------------------<br/>'
            } else {
    
    
                console.log('未知错误');
            }
        }
    })
}

Front-end file merge code:

 document.getElementById("mergedFile").addEventListener("change", function () {
    
    
     let file1 = this.files[0];
     let file2 = this.files[1];
     console.log('file1',file1);
     console.log('file2',file2);
     let arrayBlobs = [];
     let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
     arrayBlobs.push(blobSlice.call(file1, 0, file1.fileSize));
     arrayBlobs.push(blobSlice.call(file2, 0, file2.fileSize));
     let fileData = new Blob(arrayBlobs);
     downloadFileByBlob(fileData,"test.docx");
 });

3. Interface description

minio-dependency

<dependency>
    <groupId>io.minio</groupId>
    <artifactId>minio</artifactId>
    <version>8.4.0</version>
</dependency>

3.1 Check the file Md5 interface

/client/file/check/{md5}

• GET request

/**
 * 每个文件都有一个md5值
 * 根据文件的md5校验文件是否存在
 * 实现秒传接口
 *
 * @param md5 文件的md5
 * @return 操作是否成功
 */
@GetMapping(value = "/check/{md5}")
public CommonResponse checkFileExists(@PathVariable("md5") String md5) {
    
    

    if (ObjectUtils.isEmpty(md5)) {
    
    
        return CommonResponse.ok(StatusCode.PARAM_ERROR_MD5.getCode())
            .message(StatusCode.PARAM_ERROR_MD5.getMessage());
    }
    // 从数据库中查询该MD5是否存在
    FileInfo fileInfo = fileInfoService.selectByMd5(md5);

    // 文件不存在
    if (fileInfo == null) {
    
    
        return CommonResponse.ok(StatusCode.NOT_FOUND.getCode())
            .message(StatusCode.NOT_FOUND.getMessage());
    }

    return CommonResponse.ok(StatusCode.EXIST_FILE_SUCCESS.getCode())
        .message(StatusCode.EXIST_FILE_SUCCESS.getMessage())
        .data("fileInfo",fileInfo);
}

3.2 Single file upload interface

/client/file/upload

• POST request
• parameter:
  • md5: file Md5 value
  • file: upload file

/**
 * 单个文件上传
 * @param requestParams
 * @param file
 * @return
 */
@PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public CommonResponse upload(
    @RequestParam Map<String, Object> requestParams,
    @RequestParam("file") MultipartFile file) {
    
    

    /**
         *  md5: 5ec5cec2b522c7a647e1fa4e7c6a08c7
         */
    FileRequest request = new JSONObject(requestParams).toJavaObject(FileRequest.class);
    log.info("上传文件信息：{}", request);
    if (ObjectUtils.isEmpty(request.getMd5())) {
    
    
        return CommonResponse.ok(StatusCode.PARAM_ERROR_MD5.getCode())
            .message(StatusCode.PARAM_ERROR_MD5.getMessage());
    }
    // 检查文件是否上传过
    FileInfo fileInfo = fileInfoService.selectByMd5(request.getMd5());
    if (fileInfo != null) {
    
    
        return CommonResponse.ok(StatusCode.EXIST_FILE_SUCCESS.getCode())
            .message(StatusCode.EXIST_FILE_SUCCESS.getMessage());
    }
    // 上传过程中出现异常，状态码设置为50000
    if (file == null) {
    
    
        return CommonResponse.error(StatusCode.RECEIVE_FILE_FAILURE.getCode())
            .message(StatusCode.RECEIVE_FILE_FAILURE.getMessage());
    }
    request.setFile(file);

    // 不需要分片的文件
    try {
    
    
        // 上传文件
        FileInfo result = minoFileService.putObject(bucketName,request);
        // 设置上传分片的状态
        return CommonResponse.ok(StatusCode.SUCCESS.getCode())
            .message(StatusCode.SUCCESS.getMessage())
            .data("fileInfo",result);
    } catch (Exception e) {
    
    
        e.printStackTrace();
        return CommonResponse.ok(StatusCode.FAILURE.getCode())
            .message(StatusCode.FAILURE.getMessage());
    }
}

3.3 Large file upload interface in pieces

/client/file/chunk/upload

• POST request
• parameter:
  • md5: file MD5 value
  • file: part of the file
  • name: file name. The file name cannot be obtained for the fragmented file
  • fileSize: total file size
  • segTotal: the total number of fragments of the file
  • segCurrent: the current segment of the file

/**
 * 文件上传，适用大文件，分片上传
 * 上传文件：
 *      如果不需要分片，则用uuid生成文件名
 *      如果需要分片且不合并，则用md5值作为文件夹名，文件夹下为分片数据
 *      如果合并分片文件，则移除分片信息，并删除md5作为的文件夹名，合并成一个文件
 * @param requestParams
 * @param file
 * @return
 */
@PostMapping(value = "/chunk/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public CommonResponse chunkUpload(
    @RequestParam Map<String, Object> requestParams,
    @RequestParam("file") MultipartFile file) {
    
    

    /**
         * 分片文件上传，传的是blob，获取不到文件名
         *  md5: 5ec5cec2b522c7a647e1fa4e7c6a08c7
         *  name: 服务维护-2022-0712.docx
         *  fileSize: 7227540
         *  segTotal: 2
         *  segCurrent: 1
         */
    FileRequest request = new JSONObject(requestParams).toJavaObject(FileRequest.class);
    log.info("上传文件信息：{}", request);
    if (ObjectUtils.isEmpty(request.getMd5())) {
    
    
        return CommonResponse.error(StatusCode.PARAM_ERROR_MD5.getCode())
            .message(StatusCode.PARAM_ERROR_MD5.getMessage());
    }
    // 检查文件是否上传过
    FileInfo fileInfo = fileInfoService.selectByMd5(request.getMd5());
    if (fileInfo != null && fileInfo.getSegCurrent().intValue() == fileInfo.getSegTotal().intValue()) {
    
    
        return CommonResponse.ok(StatusCode.EXIST_FILE_SUCCESS.getCode())
            .message(StatusCode.EXIST_FILE_SUCCESS.getMessage());
    }

    // 上传过程中出现异常，状态码设置为50000
    if (file == null) {
    
    
        return CommonResponse.error(StatusCode.RECEIVE_FILE_FAILURE.getCode())
            .message(StatusCode.RECEIVE_FILE_FAILURE.getMessage());
    }

    request.setFile(file);
    String fileId = fileInfo == null ? UUIDUtil.timeUuid():fileInfo.getFileId();
    request.setFileId(fileId);

    FileInfo result = null;

    // 当不是最后一片时，上传返回的状态码为20001
    if (request.getSegCurrent() < request.getSegTotal()) {
    
    
        try {
    
    
            // 上传文件
            result = minoFileService.putChunkObject(request);
            log.info("segment file upload success {}", fileInfo);
            // 设置上传分片的状态
            return CommonResponse.ok(StatusCode.ALONE_CHUNK_UPLOAD_SUCCESS.getCode())
                .message(StatusCode.ALONE_CHUNK_UPLOAD_SUCCESS.getMessage())
                .data("fileInfo",result);
        } catch (Exception e) {
    
    
            e.printStackTrace();
            return CommonResponse.error(StatusCode.FAILURE.getCode())
                .message(StatusCode.FAILURE.getMessage());
        }
    } else {
    
    
        // 为分片文件的最后一片时状态码为20002
        try {
    
    
            // 上传文件
            result = minoFileService.putChunkObject(request);
            // 设置上传分片的状态
            return CommonResponse.ok(StatusCode.ALL_CHUNK_UPLOAD_SUCCESS.getCode())
                .message(StatusCode.ALL_CHUNK_UPLOAD_SUCCESS.getMessage())
                .data("fileInfo",result);
        } catch (Exception e) {
    
    
            e.printStackTrace();
            return CommonResponse.error(StatusCode.FAILURE.getCode())
                .message(StatusCode.FAILURE.getMessage());
        }
    }
}

3.4 Fragmented file merge interface

/client/file/merge

• POST request
• Parameters: (fileId and Md5 only need to pass one value)
  • fileId: fileId
  • md5: file Md5

/**
 * 文件合并
 * @return 分片合并的状态
 */
@PostMapping(value = "/merge")
public CommonResponse merge(
    @RequestParam Map<String, Object> requestParams) {
    
    

    /**
         * 参数：
         * fileId:
         * md5: 5ec5cec2b522c7a647e1fa4e7c6a08c7
         */
    FileRequest request = new JSONObject(requestParams).toJavaObject(FileRequest.class);
    FileInfo fileInfo = minoFileService.getFileInfo(request);
    // 检查文件是否存在
    if(fileInfo == null){
    
    
        return CommonResponse.ok(StatusCode.NOT_FOUND.getCode()).message(StatusCode.NOT_FOUND.getMessage());
    }

    if(fileInfo.getIsMerged() > 0 || fileInfo.getSegTotal() <= 1){
    
    
        // 文件不需要合并
        return CommonResponse.ok(StatusCode.NOT_SEGMENT_FILE_FAILURE.getCode()).message(StatusCode.NOT_SEGMENT_FILE_FAILURE.getMessage());
    }

    // path : /test/5ec5cec2b522c7a647e1fa4e7c6a08c7/1
    List<FileSegment> fileSegments = fileSegmentService.selectByMD5(fileInfo.getMd5());

    List<String> objectNameList = new ArrayList<>();
    for (FileSegment item : fileSegments) {
    
    
        String fileName = item.getPath().replace(bucketName,"").substring(1);
        objectNameList.add(fileName);
    }

    try {
    
    
        // 查询片数据
        if (fileInfo.getSegTotal() == fileSegments.size()) {
    
    
            // 开始合并请求
            String targetBucketName = bucketName;
            String filenameExtension = StringUtils.getFilenameExtension(fileInfo.getName());
            // 要合并成的文件名
            String objectName = fileSegments.get(0).getFileId() + "." + filenameExtension;
            // 返回合并后的新文件的路径
            String newPath = minoFileService.composeObject(objectNameList, bucketName, fileInfo.getMd5(), targetBucketName, objectName);

            log.info("桶：{} 中的分片文件，已经在桶：{},文件 {} 合并成功", fileInfo.getMd5(), targetBucketName, newPath);
            // 计算文件的md5
            String fileMd5 = null;
            try (InputStream inputStream = minoFileService.getObject(targetBucketName, objectName)) {
    
    
                fileMd5 = Md5Util.calculateMd5(inputStream);
            } catch (IOException e) {
    
    
                log.error("", e);
            }

            // 计算文件真实的类型
            List<String> typeList = new ArrayList<>();
            try (InputStream inputStreamCopy = minoFileService.getObject(targetBucketName, objectName)) {
    
    
                typeList.addAll(FileTypeUtil.getFileRealTypeList(inputStreamCopy, fileInfo.getName(), fileInfo.getFileSize()));
            } catch (IOException e) {
    
    
                log.error("", e);
            }

            // 并和前台的md5进行对比
            if (!ObjectUtils.isEmpty(fileMd5)
                && !ObjectUtils.isEmpty(typeList)
                && fileMd5.equalsIgnoreCase(fileInfo.getMd5())
                && typeList.contains(fileInfo.getFileType().toLowerCase(Locale.ENGLISH))) {
    
    
                // 表示是同一个文件, 且文件后缀名没有被修改过, 合并成功之后删除对应的分块文件
                minoFileService.removeObjects(bucketName,objectNameList);
                log.info("删除文件 {} 成功", fileInfo.getMd5());
                // 可以优化 todo
                FileInfo oldFileInfo = fileInfoService.selectByMd5(fileMd5);
                oldFileInfo.setPath(newPath);
                oldFileInfo.setIsMerged(StatusCode.FILE_MEGERD.getCode());
                fileInfo.setMtime(new Date());

                // 更新数据库的文件路径及合并状态
                fileInfoService.updateFileInfo(oldFileInfo);
                // 删除文件分块信息
                fileSegmentService.deleteFileByMD5(fileInfo.getMd5());
                // 成功，返回合并后的文件
                FileInfo result = fileInfoService.selectByMd5(oldFileInfo.getMd5());
                return CommonResponse.ok(StatusCode.SUCCESS.getCode())
                    .message(StatusCode.SUCCESS.getMessage())
                    .data("fileInfo",result);
            } else {
    
    
                log.info("非法的文件信息: 分片数量:{}, 文件名称:{}, 文件fileMd5:{}, 文件真实类型:{}, 文件大小:{}",fileInfo.getSegTotal(), fileInfo.getName(), fileMd5, typeList, fileInfo.getFileSize());

                // 文件md5对比失败，则要删除已经合并的文件
                minoFileService.removeObject(targetBucketName, objectName);
                return CommonResponse.ok(StatusCode.MERGED_FILE_FAILURE.getCode())
                    .message(StatusCode.MERGED_FILE_FAILURE.getMessage());
            }
        } else {
    
    
            // 失败，文件分片数不一致
            return CommonResponse.error(StatusCode.SEGMENT_COUNT_FILE_FAILURE.getCode())
                .message(StatusCode.SEGMENT_COUNT_FILE_FAILURE.getMessage());
        }
    } catch (Exception e) {
    
    
        log.error("", e);
        // 失败
        return CommonResponse.error(StatusCode.FAILURE.getCode())
            .message(StatusCode.FAILURE.getMessage());
    }
}

3.5 File delete interface

/client/file/delete

• POST request
• Parameters: (fileId and Md5 only need to pass one value)
  • fileId: fileId
  • md5: file Md5

@PostMapping(value = "/delete")
public CommonResponse delete(
    @RequestParam Map<String, Object> requestParams) {
    
    

    /**
         * 参数：
         * md5: 5ec5cec2b522c7a647e1fa4e7c6a08c7
         * fileId :
         */
    FileRequest request = new JSONObject(requestParams).toJavaObject(FileRequest.class);
    FileInfo fileInfo = minoFileService.getFileInfo(request);
    minoFileService.removeObjectAll(bucketName,fileInfo);
    return CommonResponse.ok(StatusCode.SUCCESS.getCode()).message(StatusCode.SUCCESS.getMessage());
}

3.6 Single file download interface

/client/file/download/{fileId}

• POST request
• parameter:
  • fileId: file id

3.7 Fragmented file download interface

/client/file/download/{fileId}/{index}

• POST request
• parameter:
  • fileId: file id
  • inde: fragmented file index, the first few fragments

@GetMapping(value = {
    
    "/download/{fileId}","/download/{fileId}/{index}"})
public void download(
    HttpServletResponse response,
    @PathVariable("fileId") String fileId,
    @PathVariable(value= "index",  required = false) String index,
    @RequestParam(value = "offset", required = false) Long offset,
    @RequestParam(value = "length", required = false) Long offLength) {
    
    
    /**
         * 参数：
         * fileId: 文件id
         * index: 分片索引
         * offset: 断点下载，指定位置
         * offLength: 读取指定长度
         * 备注：
         * path:
         *      非分块文件路径：/test/46951928120b11edadf4005056b2b395.txt
         *      分块文件路径： /bucketName/md5值/分片索引
         */

    InputStream inputStream = null;
    OutputStream outputStream = null;
    try{
    
    
        log.info("请求文件id为：{}",fileId);
        FileInfo fileInfo = fileInfoService.selectByFileId(fileId);
        String path = null;
        if(fileInfo.getIsMerged() == 0){
    
    
            // 分片文件下载路径
            path = "/" + fileInfo.getMd5() + "/" + index;
            response.setHeader("Content-Disposition", "attachment;filename="+ URLEncoder.encode(index,"UTF-8"));
        }else{
    
    
            path = fileInfo.getPath().replace(bucketName,"").substring(1);
            response.setHeader("Content-Disposition", "attachment;filename="+ URLEncoder.encode(fileInfo.getName(),"UTF-8"));
        }

        log.info("请求文件id为：{}，下载路径为：{}",fileId,path);
        response.setHeader("content-type", "application/octet-stream");
        response.setContentType("application/octet-stream");
        response.setCharacterEncoding("UTF-8");

        if(!ObjectUtils.isEmpty(offset)  && !ObjectUtils.isEmpty(offLength)){
    
    
            inputStream = minoFileService.getObject(bucketName, path, offset, offLength);
        } else {
    
    
            inputStream = minoFileService.getObject(bucketName, path);
        }

        outputStream = response.getOutputStream();
        int length = 0;
        byte[] buffer = new byte[1024];
        while((length = inputStream.read(buffer)) != -1){
    
    
            outputStream.write(buffer,0,length);
        }
        inputStream.close();
        outputStream.flush();
        outputStream.close();
    }catch (Exception e){
    
    
        e.printStackTrace();
    }finally {
    
    
        try {
    
    
            if(inputStream != null){
    
    
                inputStream.close();
            }
            if (outputStream != null){
    
    
                outputStream.close();
            }
        } catch (IOException e) {
    
    
            e.printStackTrace();
        }
    }
}

3.8 Query file information interface

/client/file//info

• POST request
• Parameters: (fileId and Md5 only need to pass one value)
  • fileId: fileId
  • md5: file Md5

@PostMapping(value = "/info")
public CommonResponse info(
    @RequestParam Map<String, Object> requestParams) {
    
    

    /** 参数：
     * md5: 5ec5cec2b522c7a647e1fa4e7c6a08c7
     * fileId :
     */
    FileRequest request = new JSONObject(requestParams).toJavaObject(FileRequest.class);
    FileInfo fileInfo = minoFileService.getFileInfo(request);
    List<FileSegment> chunkInfo = null;
    if(fileInfo.getIsMerged() == 0){
    
    
        chunkInfo = fileSegmentService.selectByFileId(fileInfo.getFileId());
        return CommonResponse.ok(StatusCode.SUCCESS.getCode())
            .message(StatusCode.SUCCESS.getMessage())
            .data("fileInfo",fileInfo)
            .data("chunkInfo",chunkInfo);
    }
    return CommonResponse.ok(StatusCode.SUCCESS.getCode())
        .message(StatusCode.SUCCESS.getMessage())
        .data("fileInfo",fileInfo);
}

4. Service Description

@Slf4j
@Service
public class MinioFileService {
    
    

    @Value("${oss.defaultBucket:default}")
    String bucketName;

    @Autowired
    MinioTemplate minioTemplate;

    @Autowired
    FileInfoService fileInfoService;

    @Autowired
    FileSegmentService fileSegmentService;

    /**
     * 查询所有Bucket
     * @return
     */
    @SneakyThrows
    public List<BucketDTO> listBuckets(){
    
    
        List<BucketDTO> result = new ArrayList<>();
        BucketDTO bucketDTO = null;
        List<Bucket> buckets = minioTemplate.listBuckets();
        for (Bucket bucket: buckets){
    
    
            bucketDTO = new BucketDTO();
            bucketDTO.setName(bucket.name());
            bucketDTO.setCreationDate(bucket.creationDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
            result.add(bucketDTO);
        }
        return result;
    }

    /**
     * 获取bucket下的所有object
     * @param bucketName
     * @return
     */
    @SneakyThrows
    public List<ObjectDTO> listObjectsFromOSS(String bucketName){
    
    
        boolean found = minioTemplate.bucketExists(bucketName);
        List<ObjectDTO> list = new ArrayList<>();
        ObjectDTO objectDTO = null;
        if(found){
    
    
            // 获取bucket下的所有object
            Iterable<Result<Item>> result = minioTemplate.listObjects(bucketName,true);
            for(Result<Item> object: result){
    
    
                objectDTO = new ObjectDTO();
                Item item = object.get();
                BeanUtils.copyProperties(item,objectDTO);
                objectDTO.setLastModified(item.lastModified().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
                objectDTO.setDir(item.isDir());
                objectDTO.setOwnerId(item.owner().id());
                objectDTO.setOwnerName(item.owner().displayName());
                list.add(objectDTO);
            }
            return list;
        }
        log.info(bucketName + "does not exist.");
        return null;
    }

    /**
     * 查询桶中所有的对象名
     * @param bucketName 桶名
     * @return objectNames
     */
    @SneakyThrows
    public List<String> listObjectNames(String bucketName) {
    
    
        List<String> objectNameList = new ArrayList<>();
        if (minioTemplate.bucketExists(bucketName)) {
    
    
            Iterable<Result<Item>> results = minioTemplate.listObjects(bucketName, true);
            for (Result<Item> result : results) {
    
    
                String objectName = result.get().objectName();
                objectNameList.add(objectName);
            }
        }
        return objectNameList;
    }


    /**
     * 获取DB里bucket中的所有记录
     * @param bucketName
     * @return
     */
    public List<FileInfo> listObjectsFromDB(String bucketName) {
    
    
        return fileInfoService.selectByGroup(bucketName);
    }

    @SneakyThrows
    public FileInfo putObject(String bucketName, FileRequest request) {
    
    

        MultipartFile file = request.getFile();
        String fileId = UUIDUtil.timeUuid();
        String fileOriginalName = file.getOriginalFilename(); // cat.jpg
        String fileType = fileOriginalName.substring(fileOriginalName.lastIndexOf(".")+1);
        String fileName = fileId + "." +fileType;
        request.setFileId(fileId);
        request.setFileSize(file.getSize());
        request.setName(fileOriginalName);
        request.setFileExt(request.getFileExtName());
        request.setFileType(fileType);
        request.setGroup(bucketName);
        request.setIsMerged(StatusCode.FILE_MEGERD.getCode());
        request.setSegTotal(1);
        request.setSegCurrent(1);

        // 检查是否为为图片，为图片生成缩略图  todo

        InputStream stream = null;
        try {
    
    
            stream = file.getInputStream();
            minioTemplate.putObject(bucketName,fileName,stream,file.getContentType());
            // 此处stream用完会被关闭，所有要再次获取
            String fileRealMimeType = FileTypeUtil.getFileMimeType(file.getInputStream()); // image/jpeg
            if(fileRealMimeType != null && fileRealMimeType.contains("image")){
    
    
                // 图片
                generateThumb(bucketName,fileId,fileType,file);
            }
        } catch (Exception e) {
    
    
            e.printStackTrace();
            log.info("putObject exception:{}",e.getMessage());
        }finally {
    
    
            stream = null;
        }

        // 文件路径
        String path =  "/" + bucketName + "/" + fileName;
        request.setPath(path);
        // 将文件信息存入数据库中
        fileInfoService.insertOrUpdateFileInfo(request);
        return request.toFileInfo();
    }

    /**
     * 上传文件，分片
     * @param request  请求信息
     * @return FileInfo
     */
    @SneakyThrows
    public FileInfo putChunkObject(FileRequest request) {
    
    
        /**
         *  md5: 5ec5cec2b522c7a647e1fa4e7c6a08c7
         *  name: 移动采录-配网服务维护-2022-0712.docx
         *  fileSize: 7227540
         *  segTotal: 2
         *  segCurrent: 1
         */
        MultipartFile file = request.getFile();
        InputStream inputStream = file.getInputStream();
        try {
    
    
            String fileOriginalName = request.getName(); // cat.jpg
            String fileType = fileOriginalName.substring(fileOriginalName.lastIndexOf(".")+1);
            // 文件服务存储的目录为：/md5值/分片段名。eg：/721fc54dbdd35fa648c4f19e00a4c0ff/1
            String objectName  = "/" + request.getMd5() + "/" + request.getSegCurrent(); // 存入文件服务器中的文件名称
            String savePath = objectName;
            request.setFileType(fileType);
            request.setFileExt(request.getFileExtName());
            request.setGroup(bucketName);

            // 存入文件服务
            minioTemplate.putChunkObject(inputStream,bucketName,objectName);
            // 文件存储的完整路径要加上bucketName
            String path =  "/" + bucketName + savePath;
            request.setPath(path);
            // 将文件信息存入数据库中
            fileInfoService.insertOrUpdateFileInfo(request);
            // 分片信息入库
            FileSegment fileSegment = new FileSegment();
            fileSegment.setFileId(request.getFileId());
            fileSegment.setMd5(request.getMd5());
            fileSegment.setSegSize(file.getSize());
            fileSegment.setPath(path);
            fileSegment.setSegIndex(request.getSegCurrent());
            log.info("文件分片信息为：{}",fileSegment);
            fileSegmentService.insertFileSegment(fileSegment);
            return  fileInfoService.selectByFileId(request.getFileId());
        } catch (Exception e){
    
    
          e.printStackTrace();
        } finally {
    
    
            if (inputStream != null) {
    
    
                inputStream.close();
            }
        }
        return  null;
    }

    /**
     * 上传文件，分片
     * @param request  请求信息
     * @return FileInfo
     */
    @SneakyThrows
    public FileInfo putUnionObject(FileRequest request,FileInfo fileInfo) {
    
    
        /**
         *  md5: 5ec5cec2b522c7a647e1fa4e7c6a08c7
         *  name: 移动采录-配网服务维护-2022-0712.docx
         *  fileSize: 7227540
         *  segTotal: 2
         *  segCurrent: 1
         */
        MultipartFile file = request.getFile();
        InputStream inputStream = file.getInputStream();
        try {
    
    
            String fileOriginalName = request.getName(); // cat.jpg
            String fileType = fileOriginalName.substring(fileOriginalName.lastIndexOf("."));
            // 根据md5查询数据库信息
            String fileId = fileInfo == null ? UUIDUtil.timeUuid():fileInfo.getFileId();

            String objectName = null; // 存入文件服务器中的文件名称
            String savePath = null;

            // 如果文件分片，文件存储的目录结构要改变
            if(request.getSegCurrent() <= request.getSegTotal() && request.getSegTotal() > 1){
    
    
                // 文件服务存储的目录为：/md5值/分片段名。eg：/721fc54dbdd35fa648c4f19e00a4c0ff/1
                // 业务服务存储的是文件目录：/md5值
                savePath = "/" + request.getMd5();
                objectName = savePath + "/" + request.getSegCurrent();
            }else {
    
    
                // 如果文件不分段，则用uuid生成文件的文件名，但不包含文件后缀，需补充。eg：e287a46c0d4811eda62f005056b2b395.jpg
                objectName = fileId + fileType;
                savePath = objectName;
                // 不分片文件需要把分片标识设为已合并状态。分片文件的合并状态需要手动调用合并接口
                request.setIsMerged(StatusCode.FILE_MEGERD.getCode());
            }
            request.setFileId(fileId);
            request.setName(fileOriginalName);
            request.setFileType(fileType);
            request.setFileExt(request.getFileExtName());
            request.setGroup(bucketName);

            // 存入文件服务
            minioTemplate.putObject(bucketName,objectName,inputStream,fileType);
            // 文件存储的完整路径
            String path =  "/" + bucketName + savePath;
            request.setPath(path);
            // 将文件信息存入数据库中
            fileInfoService.insertOrUpdateFileInfo(request);
            // 如果是分片文件，将分片文件信息存到对应分片表中
            if(request.getSegTotal() > 1){
    
    
                FileSegment fileSegment = new FileSegment();
                BeanUtils.copyProperties(request,fileSegment);
                fileSegment.setSegSize(file.getSize());
                fileSegment.setPath(path + "/" +request.getSegCurrent());
                fileSegment.setSegIndex(request.getSegCurrent());
                log.info("文件分片信息为：{}",fileSegment);
                fileSegmentService.insertFileSegment(fileSegment);
            }
            return  fileInfoService.selectByFileId(fileId);
        } finally {
    
    
            if (inputStream != null) {
    
    
                inputStream.close();
            }
        }
    }


    /**
     * GetObject接口用于获取某个文件（Object），此操作需要对此Object具有读权限
     * @param bucketName 桶名
     * @param objectName 文件路径
     */
    @SneakyThrows
    public InputStream getObject(String bucketName, String objectName) {
    
    
        return minioTemplate.getObject(bucketName,objectName);
    }


    @SneakyThrows
    public InputStream getObject(String bucketName, String objectName, long offset, long length) {
    
    
        return minioTemplate.getObject(bucketName,objectName,offset,length);
    }

    /**
     * 文件删除
     * @param bucketName
     * @param fileId
     */
    @SneakyThrows
    public void removeObject(String bucketName, String fileId) {
    
    
        FileInfo fileInfo = fileInfoService.selectByFileId(fileId);
        String fileName = fileId + "." + fileInfo.getFileType();
        minioTemplate.removeObject(bucketName,fileName);
        fileInfoService.deleteFile(fileId);
    }

    /**
     * 文件删除，根据Md5或者fileId都可以
     * @param bucketName
     * @param fileInfo
     */
    @SneakyThrows
    @Transactional
    public void removeObjectAll(String bucketName, FileInfo fileInfo) {
    
    
        // 是否删除的分片文件
        if(fileInfo.getIsMerged() == 1){
    
    
            String objectName = fileInfo.getPath().replace(bucketName,"").substring(1);
            minioTemplate.removeObject(bucketName,objectName);
        } else {
    
    
            // 删除分片文件
            List<FileSegment> fileSegments = fileSegmentService.selectByMD5(fileInfo.getMd5());
            List<String> objectNameList = new ArrayList<>();
            for (FileSegment item : fileSegments) {
    
    
                String fileName = item.getPath().replace(bucketName,"").substring(1);
                objectNameList.add(fileName);
            }
            // 表示是同一个文件, 且文件后缀名没有被修改过, 合并成功之后删除对应的分块文件
            minioTemplate.removeObjects(bucketName,objectNameList);
            fileSegmentService.deleteFile(fileInfo.getFileId());
        }
        fileInfoService.deleteFile(fileInfo.getFileId());
    }


    /**
     * 批量文件删除
     * @param bucketName
     * @param objectNameList 文件名称集合
     */
    @SneakyThrows
    public void removeObjects(String bucketName, List<String> objectNameList) {
    
    
       minioTemplate.removeObjects(bucketName,objectNameList);
    }

    /**
     * 文件合并，将分块文件组成一个新的文件
     * @param objectNameList 分片名称
     * @param bucketName 分块文件所在的桶
     * @param targetBucketName 合并文件生成文件所在的桶
     * @param objectName       存储于桶中的对象名
     * @return OssFile
     *
     * 注意：minio规定，每个分片文件最小是5M，要不然合并文件会报错
     */
    @SneakyThrows
    public String composeObject(List<String> objectNameList, String bucketName, String md5, String targetBucketName, String objectName) {
    
    

        if (ObjectUtils.isEmpty(objectNameList)) {
    
    
            throw new IllegalArgumentException(bucketName + "桶中没有文件，请检查");
        }
        List<ComposeSource> composeSourceList = new ArrayList<>(objectNameList.size());
        // 在合并文件时，需要对分片的文件进行升序排序
        for (String object : objectNameList) {
    
    
            composeSourceList.add(ComposeSource.builder()
                    .bucket(bucketName)
                    .object(object)
                    .build());
        }
        return minioTemplate.composeObject(composeSourceList, targetBucketName, objectName);
    }

    public FileInfo getFileInfo(FileRequest request) {
    
    
        FileInfo fileInfo = null;
        if(!ObjectUtils.isEmpty(request.getMd5())){
    
    
            fileInfo = fileInfoService.selectByMd5(request.getMd5());
        } else if(!ObjectUtils.isEmpty(request.getFileId())){
    
    
            fileInfo = fileInfoService.selectByFileId(request.getFileId());
        }
        return fileInfo;
    }

    /**
     * 为图片生成缩略图
     * @param bucketName
     * @param fileId
     * @param fileType
     * @param file
     */
    private void generateThumb(String bucketName, String fileId, String fileType, MultipartFile file) {
    
    

        // 存放缩略图的临时目录
        String tempDir = "." + File.separator + bucketName + File.separator;
//        String tempDir = "e:" + File.separator + bucketName + File.separator;
        // 生成文件名
        String fileName = fileId + "_thumb" + "." +fileType;
        // 文件完整路径
        String path = tempDir + fileName;
        // 创建目录
        File dir = new File(tempDir);
        if(!dir.exists()){
    
    
            dir.setWritable(true,false);
            dir.setReadable(true,false);
            dir.setExecutable(true,false);
            dir.mkdir();
        }

        try {
    
    
            File toFile = new File(path);
            // 生成缩略图
            Thumbnails.of(file.getInputStream()).scale(0.25f).toFile(path);
            minioTemplate.uploadObject(bucketName,fileName,path);
            // 缩略图是否要入库  todo
        } catch (IOException e) {
    
    
            e.printStackTrace();
        } finally {
    
    
            if(dir != null){
    
    
                File[] files = dir.listFiles();
                for(File item : files){
    
    
                    item.delete();
                }
            }
        }
    }
}

configuration file

oss:
  enabled: true
  type: minio
  endPoint: http://172.40.240.162:9966
  accessKey: minio
  secretKey: minio123
  defaultBucket: test

database script

SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;

-- ----------------------------
-- Table structure for t_osp_file_segment
-- ----------------------------
DROP TABLE IF EXISTS `t_osp_file_segment`;
CREATE TABLE `t_osp_file_segment`  (
  `fileId` varchar(32) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT '文件ID',
  `md5` varchar(32) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT '文件md5值，唯一',
  `path` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '文件存储路径',
  `segSize` bigint(255) NULL DEFAULT NULL COMMENT '分片文件大小',
  `segIndex` int(255) NULL DEFAULT NULL COMMENT '分片的顺序'
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;

SET FOREIGN_KEY_CHECKS = 1;

SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;

-- ----------------------------
-- Table structure for t_osp_fileinfo
-- ----------------------------
DROP TABLE IF EXISTS `t_osp_fileinfo`;
CREATE TABLE `t_osp_fileinfo`  (
  `fileId` varchar(32) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'default' COMMENT '文件ID',
  `md5` varchar(32) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '文件md5值',
  `parentId` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '虚拟文件标识',
  `name` varchar(1024) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '文件名',
  `path` varchar(1024) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT 'FASTDFS路径',
  `fileType` varchar(32) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '文件类型',
  `fileSize` bigint(255) NULL DEFAULT NULL COMMENT '文件总大小',
  `segCurrent` int(255) NULL DEFAULT NULL COMMENT '已上传的分片',
  `segTotal` int(255) NULL DEFAULT NULL COMMENT '总分片数',
  `isMerged` bit(1) NULL DEFAULT b'0' COMMENT '是否合并，1表示合并，0表示未合并',
  `fileExt` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '扩展参数',
  `mtime` timestamp(6) NOT NULL ON UPDATE CURRENT_TIMESTAMP(6) COMMENT '文件修改时间',
  `groupName` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT 'default' COMMENT 'FASTDFS分组',
  PRIMARY KEY (`fileId`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;

SET FOREIGN_KEY_CHECKS = 1;