python3学习（3）：ID 遍历爬虫

从python3学习（2）中可知所有爬取的网站URL只有在结尾处有区别，因此，可以利用该弱点来遍历访问所有URL。

 

### 二、 ID 遍历爬虫，利用网站结构的弱点，轻松访问所有内容。
# Downloading: http://example.webscraping.com/places/default/view/Afghanistan-1
# Downloading: http://example.webscraping.com/places/default/view/Aland-Islands-2
# Downloading: http://example.webscraping.com/places/default/view/Albania-3
# Downloading: http://example.webscraping.com/places/default/view/Algeria-4
# Downloading: http://example.webscraping.com/places/default/view/American-Samoa-5
# Downloading: http://example.webscraping.com/places/default/view/Andorra-6
# Downloading: http://example.webscraping.com/places/default/view/Angola-7
## 由上可知，这些 URL 只有结尾处有区别。
import urllib.request  ## -- written by LiSongbo
def Rocky_dnload(url,user_agent='wswp',num_retries = 2):
    print('Downloading:',url)
    LiSongbo_he={'User-agent':user_agent}
    request = urllib.request.Request(url, headers=LiSongbo_he)
    try:  ## -- written by LiSongbo
html = urllib.request.urlopen(request).read()
    except urllib.request.URLError as e:  ## -- written by LiSongbo
print('Download error:',e.reason)
        html = None
        if num_retries > 0:  ## -- written by LiSongbo
if hasattr(e,'code') and 500 <= e.code < 600:
                return Rocky_dnload(url,user_agent,num_retries-1) ## retry 5xx HTTP errors
return html

import re  ## -- written by LiSongbo
def Rocky_crawl_sitemap(url):  ## -- written by LiSongbo
sitemap = Rocky_dnload(url)  ## download the sitmap file
sitemap = sitemap.decode('utf-8')
    links = re.findall('<loc>(.*?)</loc>', sitemap)  ## extract the sitemap links from flag loc
for link in links:  ## download each link
html = Rocky_dnload(link)  ## crape html here
import itertools   ## -- written by LiSongbo
max_errors = 5
n_errors = 0
for page in itertools.count(1):   ## -- written by LiSongbo
url = 'http://example.webscraping.com/view/-%d' % page
    html = Rocky_dnload(url)
    if html is None:   ## -- written by LiSongbo
n_errors += 1
        if n_errors==max_errors:
            break
        else:
            n_errors = 0

运行结果如下：

Downloading: http://example.webscraping.com/view/-1
Downloading: http://example.webscraping.com/view/-2
Downloading: http://example.webscraping.com/view/-3
Downloading: http://example.webscraping.com/view/-4
Downloading: http://example.webscraping.com/view/-5
Downloading: http://example.webscraping.com/view/-6
Downloading: http://example.webscraping.com/view/-7
Downloading: http://example.webscraping.com/view/-8

Downloading: http://example.webscraping.com/view/-9

……

 

## -- written by LiSongbo