Why use multipart upload
This is an old story. The main reason is that the file is relatively large. If the network is interrupted during one-time upload, the client will have to upload it again, and there is no way to supplement the upload.
Slice upload process
Client:
There is a large file. Slice the file and split it according to the actual business. To put it bluntly, each file slice is a []byte
to ensure the consistency of the file:
each slice needs to be md5 processed to generate this block. md5
submission content:
Parameters:
file id - This is the sequence of the current file upload, ensuring which file is uploaded in the entire upload.
File key - the current sliced file md5. The server will perform a secondary verification to ensure the current file is sliced. Consistent
file keys - md5 file slice content of all slices of the main file
- current slice content []byte
file name - this is the file name used by the backend to determine
Server side:
You can use json to receive directly and roughly, because it is the received file slice content, and the file is actually a []byte.
Then determine whether the file key is consistent with the md5 processing of the received file. If it is inconsistent, an exception will be thrown.
Write the current slice data to the temporary file.
Determine whether the file keys have been written to the temporary file. If not, directly return the save success. If
the file keys have been written to the temporary file, then read the file keys in sequence, read the temporary file, and then write Just import the new file
Depending on the actual business, you can delete the original temporary files or you can directly upload the temporary files to your file storage server for merging.
Source code: Pure Go language to write client and server.
First we need a request structure
Author: Miajio
Link: https://www.jianshu.com/p/6b2ab6d0a082
Source: Jianshu
Copyright belongs to the author. For commercial reprinting, please contact the author for authorization. For non-commercial reprinting, please indicate the source.
package web
import (
"crypto/md5"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"github.com/gin-gonic/gin"
)
type ChunkFileRequest struct {
FileId string `json:"fileId"` // client create uuid
FileName string `json:"fileName"` // file name
FileKeys []string `json:"fileKeys"` // file slice all key md5
FileKey string `json:"fileKey"` // file now key to md5 - if server read the slice to md5 eq key not eq then fail
File []byte `json:"file"` // now file
ctx *gin.Context // ctx
}
func (cf *ChunkFileRequest) BindingForm(c *gin.Context) error {
if err := c.ShouldBind(cf); err != nil {
return err
}
cf.ctx = c
return cf.md5()
}
func (cf *ChunkFileRequest) md5() error {
fmt.Println(cf.FileKey)
hash := fmt.Sprintf("%x", md5.Sum(cf.File))
fmt.Println(hash)
if hash != cf.FileKey {
return errors.New("current file slice key error")
}
return nil
}
func (cf *ChunkFileRequest) SaveUploadedFile(tempPath, path string) (string, error) {
tempFolder := filepath.Join(tempPath, cf.FileId)
_, err := os.Stat(tempFolder)
if os.IsNotExist(err) {
err := os.MkdirAll(tempFolder, os.ModePerm)
if err != nil {
return "", err
}
}
out, err := os.Create(filepath.Join(tempFolder, cf.FileKey))
if err != nil {
return "", err
}
defer out.Close()
if _, err := out.Write(cf.File); err != nil {
return "", err
}
for _, fileKey := range cf.FileKeys {
tempFile := filepath.Join(tempFolder, fileKey)
if _, err := os.Stat(tempFile); err != nil {
return "", nil
}
}
base := filepath.Dir(path)
if _, err := os.Stat(base); err != nil {
if os.IsNotExist(err) {
err := os.MkdirAll(base, os.ModePerm)
if err != nil {
return "", err
}
}
}
file, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0664)
if err != nil {
return "", err
}
defer file.Close()
for _, fileKey := range cf.FileKeys {
tempFile := filepath.Join(tempFolder, fileKey)
bt, err := os.ReadFile(tempFile)
if err != nil {
return "", err
}
file.Write(bt)
}
return tempFolder, nil
}
// param: fileId
// param: fileName
// param: fileKeys the file slice all file key md5
// param: fileKey now file slice key md5
// param: file now slice file
func ChunkFile(c *gin.Context) {
var cf ChunkFileRequest
if err := cf.BindingForm(c); err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"code": "400", "msg": "bad file param", "err": err.Error()})
return
}
tempFolder, err := cf.SaveUploadedFile("./temp", "./uploads/"+cf.FileName)
if err != nil {
c.JSON(http.StatusServiceUnavailable, gin.H{
"code": "503", "msg": "bad save upload file", "err": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{
"code": "200", "msg": "success"})
if tempFolder != "" {
defer func(tempFolder string) {
os.RemoveAll(tempFolder)
}(tempFolder)
}
}
Server and client test code:
func TestChunkFileUploadServer(t *testing.T) {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
w := gin.Default()
w.POST("/chunkFile", web.ChunkFile)
srv := &http.Server{
Addr: ":8080",
Handler: w,
}
go func() {
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
}()
<-ctx.Done()
stop()
log.Println("shutting down gracefully, press Ctrl+C again to force")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
log.Fatal("Server forced to shutdown: ", err)
}
log.Println("Server exiting")
}
func TestChunkFileUploadClient(t *testing.T) {
// your client file path
filePath := ""
fileName := filepath.Base(filePath)
fileInfo, err := os.Stat(filePath)
if err != nil {
log.Fatalf("file stat fail: %v\n", err)
return
}
const chunkSize = 1 << (10 * 2) * 30
num := math.Ceil(float64(fileInfo.Size()) / float64(chunkSize))
fi, err := os.OpenFile(filePath, os.O_RDONLY, os.ModePerm)
if err != nil {
log.Fatalf("open file fail: %v\n", err)
return
}
fileKeyMap := make(map[string][]byte, 0)
fileKeys := make([]string, 0)
for i := 1; i <= int(num); i++ {
file := make([]byte, chunkSize)
fi.Seek((int64(i)-1)*chunkSize, 0)
if len(file) > int(fileInfo.Size()-(int64(i)-1)*chunkSize) {
file = make([]byte, fileInfo.Size()-(int64(i)-1)*chunkSize)
}
fi.Read(file)
key := fmt.Sprintf("%x", md5.Sum(file))
fileKeyMap[key] = file
fileKeys = append(fileKeys, key)
}
fileId := uuid.NewString()
for _, key := range fileKeys {
req := web.ChunkFileRequest{
FileId: fileId,
FileName: fileName,
FileKey: key,
FileKeys: fileKeys,
File: fileKeyMap[key],
}
body, _ := json.Marshal(req)
res, err := http.Post("http://127.0.0.1:8080/chunkFile", "application/json", bytes.NewBuffer(body))
if err != nil {
log.Fatalf("http post fail: %v", err)
return
}
defer res.Body.Close()
msg, _ := io.ReadAll(res.Body)
fmt.Println(string(msg))
}
}