Getting started with crawlers-4

Crawling Tencent News

This is the last task of this event

import time
from  selenium import webdriver
from bs4 import BeautifulSoup

driver=webdriver.Chrome(executable_path="F:/NewDesktop/spider/task3/chromedriver.exe")
driver.get("https://news.qq.com")
#了解ajax加载
for i in range(1,100):
    time.sleep(2)
    driver.execute_script("window.scrollTo(window.scrollX, %d);"%(i*200))

html=driver.page_source
bsObj=BeautifulSoup(html,"lxml")

jxtits=bsObj.find_all("div",{
    
    "class":"jx-tit"})[0].find_next_sibling().find_all("li")

print("index",",","title",",","url")

for i,jxtit in enumerate(jxtits):    
    try:
        text=jxtit.find_all("img")[0]["alt"]
    except:
        text=jxtit.find_all("div",{
    
    "class":"lazyload-placeholder"})[0].text
    try:
        url=jxtit.find_all("a")[0]["href"]
    except:
        print(jxtit)
    print(i+1,",",text,",",url) 

Insert picture description here

Guess you like

Origin blog.csdn.net/DZZ18803835618/article/details/105800964