import requests import re from requests.exceptions import RequestException import json from multiprocessing import Pool def get_one_page(url): headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36' } try: response=requests.get(url,headers=headers) if response.status_code==200: return response.text return None except RequestException: return None def parse_one_page(html): pattern=re.compile('<dd>.*?board-index.*?>(\d*)</i>.*?data-src="(.*?)".*?name"><a' +'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>' +'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S) items=re.findall(pattern,html) for item in items: yield { 'index':item[0], 'image':item[1], 'title':item[2], 'actor':item[3].strip()[3:], 'time':item[4].strip()[5:], 'score':item[5]+item[6] } def write_to_file(content): with open('result.txt','a',encoding='utf-8') as f: f.write(json.dumps(content,ensure_ascii=False)+'\n') f.close() def main(offset): url='https://maoyan.com/board/4?offset='+str(offset) html=get_one_page(url) for item in parse_one_page(html): print(item) write_to_file(item) if __name__=='__main__': pool=Pool() pool.map(main,[i*10 for i in range(10)])
[Python爬虫]猫眼电影榜单爬取
猜你喜欢
转载自www.cnblogs.com/lightmonster/p/11529647.html
今日推荐
周排行