自学python爬虫（六）PyQuery使用

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_38736612/article/details/81456115

一、概念

PyQuery库也是很强大的网页解析库，适合熟悉Jquery的人使用，因为PyQuery和Jquery的API几乎一模一样。

二、详细讲解

1、初始化
1.1字符串初始化

# 字符串初始化
html = """
<div>
    <ul>
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
# CSS选择器
print(doc('li'))

1.2URL初始化

# URL初始化
from pyquery import PyQuery as pq
doc = pq(url='http://www.baidu.com')
print(doc('head'))

1.3文件初始化

# 文件初始化
from pyquery import PyQuery as pq
doc = pq(filename='demo.html')
print(doc('li'))

2、基本的CSS选择器

# 基本CSS选择器
html = """
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
#  container 、list、li之间只要有层级关系就好，不一定非是直接得父子关系
print(doc('#container .list li'))
print(doc('#container .list a'))

3、查找元素
3.1子元素

html = """
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc(' .list')
print(type(items))
print(items)
lis = items.find('li')
print(type(lis))
print(lis)

lis = items.children()
print(type(lis))
print(lis)

lis = items.children(' .active')
print(lis)

3.2父元素

# 父元素
html = """
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc(' .list')
container = items.parent()
print(type(container))
print(container)

# 父元素
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc(' .list')
parents = items.parents()
print(type(parents))
print(parents)

parent = items.parents('.wrap')
print(parent)

3.3兄弟元素

# 兄弟元素
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
# 两个选择同时满足
li = doc(' .list .item-0.active')
print(li.siblings())
print(li.siblings(' .active'))

4、遍历
4.1获取单个元素

# 获取单个元素
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc(' .item-0.active')
print(li)

4.2获取多个元素

# 获取多个元素
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
list = doc('li').items()
print(type(list))
for li in list:
    print(li)

5、获取信息
5.1获取属性

# 获取属性
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc(' .item-0.active a')
print(a)
print(a.attr('href'))
print(a.attr.href)

5.2获取文本

#   获取文本
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc(' .item-0.active a')
print(a)
print(a.text())

5.3获取HTML

# 获取HTML
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc(' .item-0.active')
print(a)
print(a.html())

6、DOM操作
6.1 addClass removeClass

# addClass removeClass
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc(' .item-0.active')
print(li)
li.removeClass('active')
print(li)
li.addClass('active')
print(li)

6.2 attr css

# attr css
html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc(' .item-0.active')
print(li)
# 增加一个属性name，如果name属性存在，则把值设为link
li.attr('name','link')
print(li)
# 设置css样式
li.css('font-size','13px')
print(li)

6.3 remove

# remove
html="""
<div class="wrap">
    Hello world
    <p>this is a paragraph</p>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
wrap = doc(' .wrap')
print(wrap.text())
wrap.find('p').remove()
print(wrap.text())

7、伪类选择器

html = """
<div class="wrap">
<div id="container"> 
    <ul class="list">
        <li class="item-0">first item</li>
        <li class="item-1"><a href="#">second item</a></li>
        <li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
        <li class="item-1 active"><a href="#">fourth item</a></li>
        <li class="item-0"><a href="#">fifth item</a></li>
    </ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
# first child
li = doc('li:first-child')
print(li)
# last child
li = doc('li:last-child')
print(li)
# second,2可以替换掉
li = doc('li:nth-child(2)')
print(li)
#great than
li = doc('li:gt(2)')
print(li)
# 2n,偶数
li = doc('li:nth-child(2n)')
print(li)
# 包含 second字眼的li
li = doc('li:contains(second)')
print(li)