AI Studio 学习 Go 豆瓣电影爬取

分析

1.首先获得每个主页面的内容
豆瓣电影 Top 250 URL:https://movie.douban.com/top250
第一页URL：https://movie.douban.com/top250?start=0&filter=
第二页URL：https://movie.douban.com/top250?start=25&filter=
第三页URL：https://movie.douban.com/top250?start=50&filter=
第四页URL：https://movie.douban.com/top250?start=75&filter=
……
第X页：https://movie.douban.com/top250?start=(X-1)*25&filter=

2.提取每个主页面中的有价值信息
获取电影名称：< img width=“100” alt="(.?)"
获取电影评分：< span class=“rating_num” property=“v:average”>(.?)< /span>
获取评价人数：< span>(.?)人评价< /span>
获取电影总结：< span class=“inq”>(.?)< /span>
获取电影详情页面URL：< a href="(.*?)/" class="">

3.打开电影详情页面获取有价值信息
导演：rel=“v:directedBy”>(.?)< /a>
类型：< span property=“v:genre”>(.?)< /span>
制片国家：地区:< /span>(.?)< br/>
语言：语言:< /span>(.?)< br/>
上映时间：< span property=“v:initialReleaseDate” content="(.?)"
片长：< span property=“v:runtime” content="(.?)"
简介：< span property=“v:summary” class="">(?s:(.*?))< /span>

package main

import(
    "io"
    "fmt"
    "regexp"
    "strings"
    "strconv"
    "net/http"
    "github.com/360EntSecGroup-Skylar/excelize"
)


//定义一个方法, 获得页面全部内容
func HttpGet(url string) (result string, err error){
    resp, err1 := http.Get(url)

    if err1 != nil{
        err = err1
        return
    }
    defer resp.Body.Close()

    buf := make([]byte, 4096)
    for{
        num, err2 := resp.Body.Read(buf)
        if num == 0{
            break;
        }
        if err2 != nil && err2 != io.EOF{
            err = err2
            return
        }
        result += string(buf[:num])
    }
    return
}

func EditingRegularExpressions(Expression, result string)([][]string){
    return regexp.MustCompile(Expression).FindAllStringSubmatch(result, -1)
}

func SpiderDetail(url string) (Director, Type, Country, Language, Time, Duration [][]string, content string){
    result, _ := HttpGet(url)

    Director = EditingRegularExpressions(`rel="v:directedBy">(.*?)</a>`,result)
    Type = EditingRegularExpressions(`<span property="v:genre">(.*?)</span>`,result)
    Country = EditingRegularExpressions(`地区:</span>(.*?)<br/>`,result)
    Language = EditingRegularExpressions(`语言:</span>(.*?)<br/>`,result)
    Time = EditingRegularExpressions(`<span property="v:initialReleaseDate" content="(.*?)"`,result)
    Duration = EditingRegularExpressions(`<span property="v:runtime" content="(.*?)"`,result)
    Introduction := EditingRegularExpressions(`<span property="v:summary"(?s:(.*?))</span>`,result)

    for _, temp := range Introduction{
        content = temp[1]
		content = strings.Replace(content, ">", "", -1)
        content = strings.Replace(content, " ", "", -1)
        content = strings.Replace(content, "\n", "", -1)
        content = strings.Replace(content, "\t", "", -1)
        content = strings.Replace(content, "<br/", "", -1)
        content = strings.Replace(content, "&nbsp; ", "", -1)
        content = strings.Replace(content, `class=""`, "", -1)
    }

    return
}

func SpiderPage(index int){
    fmt.Println("Climbing page ", index, ".....")

    f, err := excelize.OpenFile("./Film.xlsx")
    if err != nil{
        fmt.Println("open file err: ", err)
        return
    }
    rows, _ := f.GetRows("Sheet1")
    lens := len(rows)

    url := "https://movie.douban.com/top250?start=" + strconv.Itoa((index-1)*25) + "&filter="
    result, err := HttpGet(url)

    if err != nil{
        fmt.Println("HttpGet err: ", err)
        return
    }

    MovieTitle := EditingRegularExpressions(`<img width="100" alt="(.*?)"`, result)
    FilmRating := EditingRegularExpressions(`<span class="rating_num" property="v:average">(.*?)</span>`, result)
    NumberOfPeopleAssessed := EditingRegularExpressions(`<span>(.*?)人评价</span>`, result)
    FilmSummary := EditingRegularExpressions(`<span class="inq">(.*?)</span>`, result)
    DetailsPageURL := EditingRegularExpressions(`<a href="(.*?)/" class="">`, result)

    for i := 0; i < len(DetailsPageURL); i++{
        Director, Type, Country, Language, Time, Duration, Introduction := SpiderDetail(DetailsPageURL[i][1])
        content := []string{MovieTitle[i][1],FilmRating[i][1],NumberOfPeopleAssessed[i][1],FilmSummary[i][1],DetailsPageURL[i][1],
                            Director[0][1],Type[0][1],Country[0][1],Language[0][1],Time[0][1],Duration[0][1],Introduction}

        for j := 0; j < 12; j++{
            coor := fmt.Sprintf("%c",j + 65)
            coor = coor + strconv.Itoa(lens + i +1)
            f.SetCellValue("Sheet1", coor, content[j])
        }
    }

    err = f.SaveAs("./Film.xlsx")
    if err != nil {
        fmt.Println(err)
    }
}

func Spider(start, end int){
    var cont = []string{"电影名","电影评分","评分人数","电影总结","电影详情网页","导演","类型","制片国家","语言","上映时间","电影时长","简介"}
    file := excelize.NewFile()
    for i := 0; i < len(cont); i++{
        col := fmt.Sprintf("%c1",i + 65)
        file.SetCellValue("Sheet1", col, cont[i])
    }

    err := file.SaveAs("./Film.xlsx")
    if err != nil {
        fmt.Println(err)
    }

    fmt.Printf("Crawl from page %d to page %d \n", start, end)

    for i := start; i < end + 1; i++{
        SpiderPage(i)
    }
}

func main(){
    var start, end int

    fmt.Printf("Please enter the crawl start page(>=1):")
    fmt.Scan(&start)
    fmt.Printf("Please enter the crawled termination page(>=start):")
    fmt.Scan(&end)

    Spider(start, end)
}

定义一个方法, 获得页面全部内容

func (*Client) Get

Get issues a GET to the specified URL. If the response is one of the following redirect codes, Get follows the redirect after calling the Client’s CheckRedirect function:
get发出到指定url的get。如果响应是以下重定向代码之一，则get在调用客户端的checkredirect函数后遵循重定向：

301 (Moved Permanently)
301（永久移动）
302 (Found)
302（发现）
303 (See Other)
303（见其他）
307 (Temporary Redirect)
307（临时重定向）
308 (Permanent Redirect)
308（永久重定向）

An error is returned if the Client’s CheckRedirect function fails or if there was an HTTP protocol error. A non-2xx response doesn’t cause an error. Any returned error will be of type url.Error. The url.Error value’s Timeout method will report true if request timed out or was canceled.
如果客户端的检查重定向函数失败或存在HTTP协议错误，则返回错误。非2xx响应不会导致错误。任何返回的错误都将是url.error类型。如果请求超时或被取消，URL.Error值的超时方法将报告TRUE。

When err is nil, resp always contains a non-nil resp.Body. Caller should close resp.Body when done reading from it.
当err为nil时，resp始终包含非nil resp.body。调用方完成读取后应关闭响应体。

To make a request with custom headers, use NewRequest and Client.Do.
要使用自定义头发出请求，请使用newrequest和client.do。

func (re *Regexp) FindAllStringSubmatch(s string, n int) [][]string

FindAllStringSubmatch is the ‘All’ version of FindStringSubmatch; it returns a slice of all successive matches of the expression, as defined by the ‘All’ description in the package comment. A return value of nil indicates no match.
FindAllStringSubmatch是FindStringSubmatch的“all”版本；它返回表达式所有连续匹配项的一部分，如包注释中的“all”描述所定义。返回值nil表示不匹配。

遇到的问题

1.正则表达式的返回结果为空
排查后发现是获得网页信息有问题：

检测到有异常请求从你的 IP 发出

貌似是IP被封了啊。

大概是请求过多了吧。

这是在服务器上做的，没办法，只能把文件下载到Windows上run了。