python连接redis并插入url

from bs4 import BeautifulSoup
import requests
from lxml import etree
import redis


pool = redis.ConnectionPool(host='127.0.0.1', port=6379)
r = redis.Redis(connection_pool=pool)
# r = Redis.from_url("redis://127.0.0.1:6379", decode_responses=True)

def get_urls(url):
    result = requests.get(url)
    selector = etree.HTML(result.text)
    links = selector.xpath(r'//*[@id="archive"]/div/div[2]/p[1]/a[1]/@href')
    for link in links:
        r.sadd("first_urlsss", link)
    next_url = extract_next_url(result.text)
    if next_url:
        get_urls(next_url)


def extract_next_url(html):

    soup = BeautifulSoup(html, "lxml")
    next_url = soup.select('a[class="next page-numbers"]')
    for url in next_url:

        url = str(url)
        soup = BeautifulSoup(url, "lxml")
        next_url = soup.a["href"]
    return next_url


if __name__ == '__main__':
    url = "http://python.jobbole.com/all-posts/"
    get_urls(url)

猜你喜欢

转载自www.cnblogs.com/z-x-y/p/9377014.html
今日推荐