Flask development series Flask + redis achieve IP agent pool

Flask development series Flask + redis achieve IP agent pool

 

No. 6.11-6.15 perfect ...

A simple and easy version

import requests
import re
import time
import redis
from bloom_filter import BloomFilter
import ast

pool = redis.ConnectionPool(host='localhost',password='xxx', port=6379, decode_responses=True)
r = redis.Redis(connection_pool=pool)
bloombloom = BloomFilter(max_elements=10000, error_rate=0.1)
bloombloom.add(str({'http': '117.91.232.53:9999'}))


def get_ip(i):
    ip_list=[]
    url = 'https://www.kuaidaili.com/free/inha/'
    url = url + str(i + 1)
    html = requests.get(url=url, ).text
    regip = '<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>.*?<td.*?>(\d{1,5})</td>'
    matcher = re.compile(regip, re.S)
    ipstr = re.findall(matcher, html)
    time.sleep(1)
    for j in ipstr:
        ip_list.append(j[0] + ':' + j[1])
    print('共收集到%d个代理ip' % len(ip_list))
    print(ip_list)
    return ip_list



def valVer(proxys):
    global badNum,goodNum,good_list
    good = []
    for proxy in proxys:
        try:
            proxy_host = proxy
            protocol = 'https' if 'https' in proxy_host else 'http'
            proxies = {protocol: proxy_host}
            print('现在正在测试的IP：', proxies)
            response = requests.get('http://www.baidu.com', proxies=proxies, timeout=2)
            if response.status_code != 200:
                badNum += 1
                print(proxy_host, 'bad proxy')
            else:
                goodNum += 1
                good.append(proxies)
                good_list.append(proxies)
                print(proxy_host, 'success proxy')
        except Exception as e:
            print(e)
            # print proxy_host, 'bad proxy'
            badNum += 1
            continue
    print('success proxy num : ', goodNum)
    print('bad proxy num : ', badNum)
    print("这次：",good)
    print("此时全部：",good_list)
    return good


def time_valVer(proxys):
    good = []
    for proxy in proxys:
        try:
            print('现在正在定时测试的IP：',proxy)
            proxy = ast.literal_eval(proxy)
            response = requests.get('http://www.baidu.com', proxies=proxy, timeout=2)
            if response.status_code != 200:
                r.lrem("ip_list", proxy, 1)
                print(proxy, 'bad proxy')
            else:
                good.append(proxy)
                good_list.append(proxy)
                print(proxy, 'success proxy')
        except Exception as e:
            print(e)
            continue

def stone(good):
    for IP in good:
        if str(IP) in bloombloom:
            print("%s不能存储，有相同的IP",IP)
            continue
        else:
            print("存储的IP：", IP)
            bloombloom.add(str(IP))
            r.rpush("ip_list", str(IP))

if __name__ == '__main__':

    badNum = 0
    goodNum = 0
    good_list = []
    for i in range(0,10):
        if i%10 == 0 and i!=0:
            proxy_list = []
            for i in range(0, r.llen("ip_list")):
                proxy_list.append(r.lindex("ip_list", i))
            time_valVer(proxy_list)
        else:
            ip_list = get_ip(i)
            good = valVer(ip_list)
            stone(good)

 

 

from Flask Import the Flask
 Import redis    # introduced redis module, can be operated by a python database cache operation directly redis redis host server 


R & lt = redis.Redis (Host = ' localhost ' , Port = 6379, password = ' XXX ' , decode_responses = True) 
App = the Flask ( the __name__ ) 
@ app.route ( ' / IP / <int: index> ' )
 DEF reponse (index):
     Print (index)
     Print (r.lindex ( " ip_list " , index))
     return R & lt .lindex ( "ip_list", index)
if __name__ == '__main__':
    app.run(debug=True)

 

 Get ip: