Python四线程爬取西刺代理

  1 import requests
  2 from bs4 import BeautifulSoup 
  3 import lxml
  4 import telnetlib #验证代理的可用性
  5 import pymysql.cursors
  6 import random
  7 import threading
  8 
  9 
 10 
 11 
 12 BASEURL = 'http://www.xicidaili.com/' #西刺首页
 13 urls = [BASEURL+ 'nn/',BASEURL+'nt/',BASEURL+'wn/',BASEURL+'wt/']#西刺分组(more)的ip信息链接列表
 14 
 15 #请求头信息，必须有User-Agent
 16 headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
 17 
 18 #proxies = {'https': 'http://123.57.85.224:80', 'http': 'http://123.57.85.224:80'}
 19 
 20 #获得与数据库的连接和游标
 21 def get_cc():
 22     # 连接MySQL数据库
 23     connection = pymysql.connect(host='127.0.0.1', port=3306, user='root', password='root', db='iptables', 
 24                                  charset='utf8', cursorclass=pymysql.cursors.DictCursor)
 25     # 通过cursor创建游标
 26     cursor = connection.cursor()
 27     return connection,cursor
 28 
 29 
 30 
 31 #保存ip_port到数据库
 32 def save_ip_port(ip_port):
 33     connection,cursor = get_cc()
 34     try:
 35         sql = 'insert into iptable(ip_port) values("'+ip_port+'")'
 36         cursor.execute(sql)
 37     except:
 38         print('保存'+ip_port+'失败!!!!!')
 39     else:
 40         connection.commit()
 41         connection.close()
 42 
 43 
 44 
 45 #从数据库获得ip_port
 46 def get_ip_port():
 47     connection,cursor = get_cc()
 48     sql_get_id = 'select id,ip_port from iptable'
 49     cursor.execute(sql_get_id)
 50     #fetchone()是查询一条数据
 51     id_list = cursor.fetchall()#得到所有的id的字典列表
 52     i = random.randint(0,len(id_list)-1)
 53     id_num = id_list[i]['id']
 54     ip_port = id_list[i]['ip_port'] #获得所有可用的代理
 55     
 56     return id_num,ip_port#返回id和ip_port：192.168.1.2:8080
 57 
 58 #删除被封的ip_port
 59 def del_ip_port(id_num):
 60     connection,cursor = get_cc()
 61     try:
 62         sql = 'delete from iptable where id = ' + str(id_num)
 63         cursor.execute(sql)
 64     except:
 65         print('删除'+ip_port+'失败!!!!!')
 66     else:
 67         connection.commit()
 68         connection.close()
 69 
 70 
 71 
 72 #获得代理
 73 def get_proxies(ip_port):#ip_port = '192.168.2.45:8088'
 74     proxy_ip = 'http://' + ip_port
 75     proxy_ips = 'https://' + ip_port
 76     proxies = {'https': proxy_ips, 'http': proxy_ip}
 77     return proxies
 78 
 79 
 80 #获得对应url分类的最大页码
 81 def get_max_pagenum(url): #url是more(分类)的链接，/nn,/nt....
 82     
 83     response = requests.get(url,headers = headers)
 84     status_code = response.status_code
 85     soup = BeautifulSoup(response.content,'lxml')
 86     max_pagenum = soup.find('div',attrs = {'class':'pagination'}).find_all('a')[-2].string
 87     max_pagenum = int(max_pagenum)
 88     return max_pagenum
 89 
 90 #验证代理是否有用,ip_port = '192.168.2.45:8088'
 91 #每得到一个ip_port都要进行验证，如果可用则保存，否则抛弃
 92 def verifyProxyList(ip_port):
 93     url = 'http://www.baidu.com'
 94     # proxies = { "http": "http://"+ ip_port }
 95     host ,port = ip_port.split(':')
 96     try:
 97         # res = requests.get(url,headers = headers,proxies = proxies,timeout = 5.0)
 98         telnetlib.Telnet(host, port=port, timeout=5)
 99     except:
100         print('---Failur:' + ip_port)
101     else:
102         #ips.append(ip_port)#这里应该存储到Redis等数据库中
103         save_ip_port(ip_port)
104         
105         
106 
107 def main(url,proxies):#这里是more的链接,/nn/1,/nn/2....
108     
109     try:
110         response = requests.get(url,headers = headers,proxies = proxies,timeout = 5.0)
111         status_code = response.status_code #503说明ip被封
112 
113         if(status_code != requests.codes.ok):#响应的不是正常状态
114             #删除旧的代理ip_port,这里还需要验证是否有bug
115             old_ip_port = proxies['http'][7:]
116             del_ip_port(old_ip_port)
117             #修改代理，重新请求
118             id_num,ip_port = get_ip_port()
119             proxies = get_proxies(ip_port)
120             print(str(proxies))
121             return
122                       
123         soup = BeautifulSoup(response.content,'lxml')
124             
125         results = soup.find_all('tr')#遍历所有的tr
126          
127         for result in results[1:]:#这里第一个tr子标签是th，所以会报错
128             tdlist = result.find_all('td')
129             ip_port = tdlist[1].string+':'+tdlist[2].string
130             verifyProxyList(ip_port)
131     except:
132        print('请求异常......')
133 
134 class myThread(threading.Thread):
135     def __init__(self, threadID, name, url):
136         threading.Thread.__init__(self)
137         self.threadID = threadID
138         self.name = name
139         self.url = url
140     
141     
142     
143     def run(self):
144         print('正在执行线程：'+self.name)#没有验证这一行的可行性
145         id_num,ip_port = get_ip_port()
146         proxies = get_proxies(ip_port)
147         max_pagenum = get_max_pagenum(self.url)
148         #print(max_pagenum)
149         for i in range(1,max_pagenum):
150             url = self.url + '/' + str(i)
151             main(url,proxies)
152         
153 #4线程爬取西刺的ip代理池
154 if __name__ == '__main__':
155    
156     t1 = myThread(1,"Thread-1",urls[0])
157     t2 = myThread(2,"Thread-2",urls[1])
158     t3 = myThread(3,"Thread-3",urls[2])
159     t4 = myThread(4,"Thread-4",urls[3])
160     t1.start()
161     t2.start()
162     t3.start()
163     t4.start()
164     t1.join()
165     t2.join()
166     t3.join()
167     t4.join()
168     
169