python crawling entire file to a local novel

headers={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36"}
url = "https://www.quanben.net/274/274596/"
rps = requests.get(url,headers=headers)  # 获取url
rps.encoding = "utf-8"
#print(rps.text)2401
dom= etree.HTML(rps.text)
for i in dom.xpath('//*[@id="main"]/div[@class="inner"]'):
    for j in range(10,2402,1):
        #爬取目录
        picName0 = i.xpath("dl[@class='chapterlist']/dd["+str(j)+"]/a/text()")
        picurl=i.xpath("dl[@class='chapterlist']/dd["+str(j)+"]/a/@href")
        piculr1="https://www.quanben.net"+str(picurl).replace("['","").replace("']","")

        #爬取内容
        #https://www.quanben.net/274/274596/31096087.html
        print(picName0,str(piculr1))
        url = piculr1
        rps = requests.get(url, headers=headers)  # 获取url
        rps.encoding = "utf-8"
        # print(rps.text)2401
        dom = etree.HTML(rps.text)
        #//*[@id="BookText"]
        #for n in dom.xpath('//*[@id="BookText"]'):
        content=str(dom.xpath('//*[@id="BookText"]/text()')).replace("\xa0","")
        print(content)
Published 133 original articles · won praise 53 · views 20000 +

Guess you like

Origin blog.csdn.net/weixin_43599377/article/details/103723219