import re
import requests
def parse_url(url):
headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"}
response=requests.get(url,headers=headers)
text=response.text
end=[]
#titles=re.findall(r'<div\sclass="ycd">.*?<h2>.*?<a.*?>(.*?)</a>',text,re.DOTALL)
titles=re.findall(r'<div\sclass="cont">.*?<b>(.*?)</b>',text,re.DOTALL)
dynesties=re.findall(r'<p\sclass="source">.*?<a.*?>(.*?)</a>',text,re.DOTALL)
authors=re.findall(r'<p\sclass="source">.*?<a.*?><a.*?>(.*?)</a>',text,re.DOTALL)
poems=re.findall(r'<div class="contson" .*?>(.*?)</div>',text,re.DOTALL)
content=[]
for poem in poems:
x=re.sub(r'<.*?>'," ",poem)
content.append(x.strip())
ends=[]
for value in zip(titles,dynesties,authors,content):
titles, dynesties, authors, content=value
poe={
"title":titles,
"dynasty":dynesties,
"author":authors,
"content":content
}
ends.append(poe)
for end in ends:
print(end)
#print(authors)
def main():
for i in range(10):
url="https://www.gushiwen.org/default_%s.aspx" %i
#url = "https://gushiwen.com/type/n/xianqin/n/1.html"
parse_url(url)
main()
爬虫实战——利用正则表达式爬取古诗文网
猜你喜欢
转载自blog.csdn.net/devilangel2/article/details/105441537
今日推荐
周排行