from bs4 import BeautifulSoup import requests from lxml import etree import redis pool = redis.ConnectionPool(host='127.0.0.1', port=6379) r = redis.Redis(connection_pool=pool) # r = Redis.from_url("redis://127.0.0.1:6379", decode_responses=True) def get_urls(url): result = requests.get(url) selector = etree.HTML(result.text) links = selector.xpath(r'//*[@id="archive"]/div/div[2]/p[1]/a[1]/@href') for link in links: r.sadd("first_urlsss", link) next_url = extract_next_url(result.text) if next_url: get_urls(next_url) def extract_next_url(html): soup = BeautifulSoup(html, "lxml") next_url = soup.select('a[class="next page-numbers"]') for url in next_url: url = str(url) soup = BeautifulSoup(url, "lxml") next_url = soup.a["href"] return next_url if __name__ == '__main__': url = "http://python.jobbole.com/all-posts/" get_urls(url)
python连接redis并插入url
猜你喜欢
转载自www.cnblogs.com/z-x-y/p/9377014.html
今日推荐
周排行