day 03 selenium与Beautifulsoup4的原理与使用

#爬取京东商品数据
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
def get_good(driver):
    num=1
    try:
        time.sleep(5)
        # 下拉滑动5000px
        js_code='''
            window.scrollTo(0,5000)
        '''
        driver.execute_script(js_code)
        # 等待5秒，待商品数据加载
        time.sleep(5)
        good_list = driver.find_elements_by_class_name('gl-item')
        for good in good_list:
            # print(good)
            # 商品名称
            good_name = good.find_element_by_css_selector('.p-name em').text
            # print(good_name)
            good_url = good.find_element_by_css_selector('.p-name a').get_attribute('href')
            # print(good_url)
            good_price = good.find_element_by_class_name('p-price').text
            # print(good_price)

            # 商品评价
            good_commit = good.find_element_by_class_name('p-commit').text
            good_content = f'''
               商品名称:{good_name}
               商品链接:{good_url}
               商品价格:{good_price}
               商品评价:{good_commit}
               \n
               '''
            print(good_content)
            with open('jd.txt', 'a', encoding='utf-8')as f:
                f.write(good_content)
            num+=1
        print('商品信息写入成功！')
        # 找到下一页并点击
        next_tag=driver.find_element_by_class_name('pn-next')
        next_tag.click()
        time.sleep(5)
        # 递归调用函数本身
        get_good(driver)
    finally:
        driver.close()

if __name__=='__main__':
    driver=webdriver.Chrome()
    try:
        driver.implicitly_wait(10)
        # 往京东发送请求
        driver.get('http://www.jd.com/')
        # 往京东主页输入墨菲定律，按回车键
        input_tag=driver.find_element_by_id('key')
        input_tag.send_keys('墨菲定律')
        input_tag.send_keys(Keys.ENTER)

        # 调取商品信息函数
        get_good(driver)
    finally:
        driver.close()
Beautifulsoup4的原理与使用

html_doc='''
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="sister"><b>$37</b></p>

<p class="story" id="p">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" >Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
'''
from bs4 import BeautifulSoup
# python自带的解析库
# soup=BeautifulSoup(html_doc,'html.parser')

# 利用bs4得到一个soup对象
soup=BeautifulSoup(html_doc,'lxml')
# bs4对象
# print(soup)
# bs4类型
# print(type(soup))
# 美化功能
# html=soup.prettify()
# print(html)


# 1、直接选择标签（返回的是一个对象）   *****
print(soup.a)  # 获取第一个a标签
print(soup.p)  # 获取第一个p标签
print(type(soup.a))  # <class 'bs4.element.Tag'>

# 2、获取标签的名称
print(soup.a.name)  # 获取a标签的名字

# 3、获取标签的属性     *****
print(soup.a.attrs)  # 获取a标签内所有的属性
print(soup.a.attrs['href'])  # 获取a标签内的href属性

# 4、获取标签的文本内容   *****
print(soup.p.text)      #  $37
# 5、嵌套选择标签
print(soup.p.b)  # 获取第一个p标签内的b标签
print(soup.p.b.text)  # 打印b标签内的文本

# 6、子节点、子孙节点
# 获取子节点
print(soup.p.children)  # 获取第一个p标签所有的子节点，返回的是一个迭代器
print(list(soup.p.children))  # list转成列表
# 7、父节点，祖先节点
print(soup.b.parent)
print(soup.b.parents)
print(list(soup.b.parents))

day 03 selenium与Beautifulsoup4的原理与使用

猜你喜欢