Golang implements file upload in chunks

The main logic of multipart upload is as follows:

1. Calculate the total file size

2. Define the size of each block file

3. Calculate how many blocks there are in total

4. Read each file and upload in batches

5. When the upload fails, add a retry upload mechanism

The code involved is as follows:

package utils

import (
	"NativeAndroidProxyQS865V2/logconfig"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"mime/multipart"
	"net/http"
	"os"
)

var fileId int32 = 0
var fileUrl string

/*
*
分块上传文件
*/
func UploadFile(fileName string, fileDir string, uploadUrl string) (string, bool) {

	logconfig.SugarLogger.Infof("begin UploadFile,fileName=%s,fileDir=%s,uploadUrl=%s", fileName, fileDir, uploadUrl)
	//const fileChunk = 4 << 20 // 4MB

	fileChunk := 10 * 1024 * 1024 //10MB
	uploadCount := 1 //上传次数

	//打开文件句柄操作
	file, err := os.Open(fileDir + fileName)
	if err != nil {
		fmt.Println("error opening file")
		logconfig.SugarLogger.Errorf("PostFile() opening file err:%v", err)
		return "", false
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		logconfig.SugarLogger.Errorf("PostFile() file.Stat err:%v", err)
		return "", false
	}

	//文件块数
	numChunks := int(math.Ceil(float64(fi.Size()) / float64(fileChunk)))

	//计算上传文件md5
	getMd5Cmd := fmt.Sprintf("md5sum %s | awk '{print $1}'", fileDir+fileName)
	fileMd5Value, _ := LocalCommand(getMd5Cmd, "PostFile")
	logconfig.SugarLogger.Infof("PostFile() file=%s,md5=%s", fileDir+fileName, fileMd5Value)

	//文件切割
	for i := 0; i < numChunks; i++ {
		uploadCount = i + 1
		bodyBuf := &bytes.Buffer{}
		bodyWriter := multipart.NewWriter(bodyBuf)

		chunkSize := int(math.Min(float64(fileChunk), float64(fi.Size()-int64(i*fileChunk))))

		//计算单次要上传的文件
		buf := make([]byte, chunkSize)
		_, err := file.ReadAt(buf, int64(i*fileChunk))
		if err != nil && err != io.EOF {
			panic(err)
		}

		//关键的一步操作,这里写入每次要上传的文件
		fileWriter, err := bodyWriter.CreateFormFile("file", fileName)
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--file  err:%v", err)
			return "", false
		}

		_, err = io.Copy(fileWriter, bytes.NewReader(buf))
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--file Copy err:%v", err)
			return "", false
		}

		currentBlockWriter, err := bodyWriter.CreateFormField("currentBlock")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--currentBlock  err:%v", err)
			return "", false
		}
		currentBlockValue, _ := json.Marshal(i + 1)
		io.Copy(currentBlockWriter, bytes.NewReader(currentBlockValue))

		totalBlockWriter, err := bodyWriter.CreateFormField("totalBlock")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--totalBlock  err:%v", err)
			return "", false
		}
		totalBlockValue, _ := json.Marshal(numChunks)
		io.Copy(totalBlockWriter, bytes.NewReader(totalBlockValue))

		totalSizeWriter, err := bodyWriter.CreateFormField("totalSize")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--totalSize  err:%v", err)
			return "", false
		}
		totalSizeValue, _ := json.Marshal(fi.Size())
		io.Copy(totalSizeWriter, bytes.NewReader(totalSizeValue))

		//上传类型 1：应用 2：文件
		typeWriter, err := bodyWriter.CreateFormField("type")
		if err != nil {
			logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--type  err:%v", err)
			return "", false
		}
		typeValue, _ := json.Marshal(2)
		io.Copy(typeWriter, bytes.NewReader(typeValue))

		if fileId != 0 {
			idWriter, err := bodyWriter.CreateFormField("id")
			if err != nil {
				logconfig.SugarLogger.Errorf("PostFile() CreateFormFile--id  err:%v", err)
				return "", false
			}
			idValue, _ := json.Marshal(fileId)
			io.Copy(idWriter, bytes.NewReader(idValue))
		}

		bodyWriter.WriteField("fileName", fileName)

		bodyWriter.WriteField("md5", fileMd5Value)

		contentType := bodyWriter.FormDataContentType()
		bodyWriter.Close()

		logconfig.SugarLogger.Infof("PostFile() 开始上传第%d段", i+1)

        //上传失败重传机制
		isUploadSuccess, postErr := postFileForm(uploadUrl, contentType, bodyBuf)
		if isUploadSuccess == false || postErr != nil {
			for num := 1; num < 4; num++ {
				logconfig.SugarLogger.Errorf("PostFile() 开始上传第%d段--fail,第%d次重试", i+1, num)
				isUploadSuccess, postErr := postFileForm(uploadUrl, contentType, bodyBuf)
				if isUploadSuccess == true && postErr == nil {
					break
				} else {
					if num == 3 {
						return fileUrl, numChunks == uploadCount
					}
				}
			}
		}
	}

	return fileUrl, numChunks == uploadCount
}

//post上传文件

func postFileForm(uploadUrl string, contentType string, bodyBuf *bytes.Buffer) (bool, error) {
	resp, err := http.Post(uploadUrl, contentType, bodyBuf)
	if err != nil {
		logconfig.SugarLogger.Errorf("postFileForm() Post Err:%v", err)
		return false, err
	}

	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logconfig.SugarLogger.Errorf("postFileForm() ReadAll Err:%v", err)
		return false, err
	}

	logconfig.SugarLogger.Infof("resp.Status:%v,,,消息返回体%s", resp.Status, string(respBody))
    //解析返回的消息
	uploadFileCallback := &UploadFileCallback{}
	parseErr := json.Unmarshal(respBody, uploadFileCallback)
	if parseErr != nil {
		logconfig.SugarLogger.Errorf("postFileForm() parseErr:%v", parseErr)
		return false, parseErr
	}

	//0：成功、1：失败
	if uploadFileCallback.Status == 0 {
		fileId = uploadFileCallback.ResponseData.Id
		fileUrl = uploadFileCallback.ResponseData.Url
		logconfig.SugarLogger.Infof("postFileForm() fileId:%d", fileId)
		return true, nil
	} else {
		logconfig.SugarLogger.Errorf("postFileForm() uploadFileCallback.Status callback fail,status!=0")
		return false, nil
	}
}

type UploadFileCallback struct {
	Status       int16        `json:"status"`
	Msg          string       `json:"msg"`
	RetCode      int16        `json:"retCode"`
	ResponseData ResponseData `json:"data"`
}

type ResponseData struct {
	Id  int32  `json:"id"`
	Url string `json:"url"`
}

Note: When the post form upload involves multiple types of parameters, WriteField() can only pass string type values, and other types of values ​​use CreateFormField(), and the usage method refers to the above code