GO 爬虫图片相关

1. 正则爬取图片并同步下载

 

效果:

代码:

 1 package main
 2 
 3 import (
 4     "fmt"
 5     "io/ioutil"
 6     "net/http"
 7     "regexp"
 8     "strconv"
 9     "time"
10 )
11 
12 var (
13     //<img src="https://7799520.oss-cn-hangzhou.aliyuncs.com/v2/img/woman3.jpg">   
14     reImg = `<img[\s\S]+?src="([\s\S]+?)"`
15 )
16 
17 //获取html全部内容
18 func GetHtml(url string) string {
19     resp, _ := http.Get(url)
20     defer resp.Body.Close()
21 
22     bytes, _ := ioutil.ReadAll(resp.Body)
23     html := string(bytes)
24     return html
25 }
26 
27 //获取页面上的全部图片链接
28 func GetPageImgurls(url string) []string {
29     html := GetHtml(url)
30     //fmt.Println(html)
31 
32     re := regexp.MustCompile(reImg)
33     rets := re.FindAllStringSubmatch(html, -1)
34     fmt.Println("捕获图片张数:", len(rets))
35 
36     imgUrls := make([]string, 0)
37 
38     for _, ret := range rets {
39         imgUrl := ret[1]
40         imgUrls = append(imgUrls, imgUrl)
41     }
42     return imgUrls
43 }
44 
45 //同步下载图片
46 func DownloadImg(url string) {
47     resp, _ := http.Get(url)
48     defer resp.Body.Close()
49     bytes, _ := ioutil.ReadAll(resp.Body)
50 
51     filename := `C:\Users\surface\DesktAop\图片\` + strconv.Itoa(int(time.Now().UnixNano())) + ".jpg"
52 
53     err := ioutil.WriteFile(filename, bytes, 0644)
54     if err != nil {
55         fmt.Println(filename, "下载失败:", err)
56     } else {
57         fmt.Println(filename, "下载成功！")
58     }
59 }
60 
61 func main() {
62     imgUrls := GetPageImgurls("http://www.7799520.com/")
63     for _, iu := range imgUrls {
64         //fmt.Println(iu)
65         DownloadImg(iu) //同步下载
66     }
67 }

 

1. 正则爬取图片并异步下载