爬取豆瓣网电影的数据信息

爬取豆瓣网电影的数据信息

# -*- coding: utf-8 -*-
import requests
import json
from bs4 import BeautifulSoup

def get_page():
    # 1. url
    url= 'https://movie.douban.com/cinema/nowplaying/kaifeng/'
    # 2. 请求时应该发送什么数据
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36',
    }
    # 3. GET/POST请求
    #采用GET
    #4. 发送数据
    response = requests.get(url,headers=headers)
    return response.text

def parse_page(text):
    movies = []
    soup = BeautifulSoup(text,'lxml')
    liList = soup.find_all('li',attrs={"data-category":"nowplaying"})
    for li in liList:
        movie = {}
        title = li['data-title']
        score = li['data-score']
        release = li['data-release']
        duration = li['data-duration']
        region = li['data-region']
        director = li['data-director']
        actors = li['data-actors']
        img = li.find('img')#他只有一个img
        thumbnail = img['src']
        movie['title'] = title
        movie['score'] = score
        movie['release'] = release
        movie['duration'] = duration
        movie['region'] = region
        movie['director'] = director
        movie['actors'] = actors
        movie['thumbnail'] = thumbnail
        movies.append(movie)
        # print(movie)
    return movies
def save_date(data):
    with open('douban.json','w',encoding='utf-8') as fp:
        #将字典，列表dump成满足json格式的字符串
        json.dump(data,fp,ensure_ascii=False)

if __name__=='__main__':
    text = get_page()
    movies = parse_page(text)
    save_date(movies)