go Download large files from url (support breakpoint resume)

package main

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/go-emix/utils"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"
)

func down(ur, dir, fn string, to time.Duration, onErr func(err error)) {
	if !utils.FileIsExist(dir) {
		err := os.MkdirAll(dir, os.ModePerm)
		if err != nil {
			fmt.Println("mkdir "+dir+" err:", err.Error())
			onErr(err)
			return
		}
	}
	dfn := dir + "/" + fn
	var file *os.File
	var size int64
	if utils.FileIsExist(dfn) {
		fi, err := os.OpenFile(dfn, os.O_RDWR, os.ModePerm)
		if err != nil {
			fmt.Println(fn, "open err:", err)
			onErr(err)
			return
		}
		stat, _ := fi.Stat()
		size = stat.Size()
		sk, err := fi.Seek(size, 0)
		if err != nil {
			fmt.Println(fn, "seek err:", err)
			_ = fi.Close()
			onErr(err)
			return
		}
		if sk != size {
			fmt.Printf("%s seek length not equal file size,"+
				"seek=%d,size=%d\n", fn, sk, size)
			_ = fi.Close()
			onErr(errors.New("seek length not equal file size"))
			return
		}
		file = fi
	} else {
		create, err := os.Create(dfn)
		if err != nil {
			fmt.Println(fn, "create err:", err)
			onErr(err)
			return
		}
		file = create
	}
	client := &http.Client{}
	client.Timeout = to
	request := http.Request{}
	request.Method = http.MethodGet
	if size != 0 {
		header := http.Header{}
		header.Set("Range", "bytes="+strconv.FormatInt(size, 10)+"-")
		request.Header = header
	}
	parse, err := url.Parse(ur)
	if err != nil {
		fmt.Println(ur, "url err:", err)
		onErr(err)
		return
	}
	request.URL = parse
	get, err := client.Do(&request)
	if err != nil {
		fmt.Println(ur, "get err:", err)
		onErr(err)
		return
	}
	defer func() {
		err := get.Body.Close()
		if err != nil {
			fmt.Println(fn, "body close:", err.Error())
			onErr(err)
		}
		err = file.Close()
		if err != nil {
			fmt.Println(fn, "file close:", err.Error())
			onErr(err)
		}
	}()
	if get.ContentLength == 0 {
		fmt.Println(fn, "already downloaded")
		return
	}
	body := get.Body
	writer := bufio.NewWriter(file)
	bs := make([]byte, 10*1024*1024) //每次读取的最大字节数,不可为0
	for {
		read, err := body.Read(bs)
		if err != nil {
			if err != io.EOF {
				fmt.Println(fn, "read err:"+err.Error())
			}
			onErr(err)
			return
		}
		_, err = writer.Write(bs[:read])
		if err != nil {
			fmt.Println(fn, "write err:"+err.Error())
			onErr(err)
			return
		}
	}
	err = writer.Flush()
	if err != nil {
		fmt.Println(fn, "writer flush:", err.Error())
		onErr(err)
		return
	}
	fmt.Println(fn, "download success")
}

ps: No need to create a directory first, it will be created automatically; onErr is an error callback function

Guess you like

Origin blog.csdn.net/takujo/article/details/108896214