python爬虫练习--爬取所有微博

Ajax，全称为Asynchronous JavaScript and XML，即异步的JavaScript和XML。它不是一门编程语言，而是利用JavaScript在保证页面不被刷新、页面链接不改变的情况下与服务器交换数据并更新部分网页的技术。

对于传统的网页，如果想更新其内容，那么必须要刷新整个页面，但有了Ajax，便可以在页面不被全部刷新的情况下更新其内容。在这个过程中，页面实际上是在后台与服务器进行了数据交互，获取到数据之后，再利用JavaScript改变网页，这样网页内容就会更新了。

可以到W3School上体验几个示例来感受一下：http://www.w3school.com.cn/ajax/ajax_xmlhttprequest_send.asp

代码如下：

 1 #! /usr/bin/env python
 2 # coding: utf-8
 3 
 4 import requests
 5 from pyquery import PyQuery as pq
 6 import pymysql
 7 from pymongo import MongoClient
 8 import time
 9 
10 '''
11 抓取所有微博信息，是通过Ajax进行分析抓取
12 '''
13 
14 
15 headers = {
16     'Host': 'm.weibo.cn',
17     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
18     'X-Requested-With': 'XMLHttpRequest',
19     'Referer': 'https://m.weibo.cn/u/2830678474'
20 }
21 
22 def get_page(page):
23     '''
24     请求url，并获取内容
25     :param page:
26     :return:
27     '''
28     params = {
29         'type':'uid',
30         'value':'2830678474',
31         'containerid':'1076032830678474',
32         'page':page
33 
34     }
35     url = 'https://m.weibo.cn/api/container/getIndex'
36     try:
37         res = requests.get(url,headers=headers,params=params)
38         if res.status_code == 200:
39             return res.json()
40     except requests.ConnectionError as e:
41         print('Error',e.args)
42 
43 
44 
45 def parse_page(json,page):
46     if json:
47         items = json.get('data').get('cards')
48         for index, item in enumerate(items):
49             if page == 1 and index == 1:
50                 continue
51             else:
52                 item = item.get('mblog')
53                 weibo = {}
54                 weibo['id'] = item.get('id')
55                 weibo['text'] = pq(item.get('text')).text()
56                 weibo['attitudes'] = item.get('attitudes_count')
57                 weibo['comments'] = item.get('comments_count')
58                 weibo['reposts'] = item.get('reposts_count')
59                 yield weibo
60 
61 
62 #往MySQL中存储
63 def save_mysql(result):
64 
65     table = 'weibo'
66     keys = ', '.join(result.keys())
67     values = ', '.join(['%s'] * len(result))
68     db = pymysql.connect(host='127.0.0.1', user='root', password='123456', port=3306,db='spiders',charset="utf8")
69     cursor = db.cursor()
70     sql = 'INSERT INTO {table}({keys}) VALUES ({values})'.format(table=table, keys=keys, values=values)
71     try:
72         if cursor.execute(sql, tuple(result.values())):
73             print('Successful')
74             db.commit()
75     except Exception as e:
76         print('Failed',e.args)
77         db.rollback()
78     db.close()
79 
80 
81 #往mangoDB中存储
82 def save_to_mongo(result):
83     client = MongoClient()
84     db = client['weibo']
85     collection = db['weibo']
86     if collection.insert(result):
87         print('Saved to Mongo')
88 
89 
90 
91 if __name__ == '__main__':
92     for page in range(1,11):
93         json = get_page(page)
94         results = parse_page(json,page)
95         for result in results:
96             print(result)
97             save_mysql(result)
98             time.sleep(1)