网页解析--BeautifulSoup练习

  1 # coding = utf-8
  2 # BeautifulSoup 主要功能是解析提取HTML数据
  3 # re lxml bs4
  4 
  5 # pip install Beautifulsoup4
  6 
  7 # from bs4 import BeautifulSoup
  8 
  9 html = '''
 10 <html><head><title>The Dormouse's story</title></head>
 11 
 12 <p class="title"><b>The Dormouse's story</b></p>
 13 
 14 <p class="story">Once upon a time there were three little sisters; and their names were
 15 <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
 16 <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
 17 <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
 18 and they lived at the bottom of a well.</p>
 19 
 20 <p class="story">...</p>
 21 
 22 '''
 23 ############################################################################
 24 # BeautifulSoup部分
 25 #############################################################################
 26 
 27 # soup = BeautifulSoup(html, 'lxml')
 28 
 29 # 四大对象种类：Tag NavigableString Beautifulsoup Comment
 30 
 31 # print(soup.a)  # 获取a标签
 32 # print(soup.a.get('href'))   # 取a标签的属性,获得超链接
 33 # print(soup.a.text)  # 获取a标签下的文本，若a下有子标签，可能获取不到
 34 # print(soup.a.string)  # 获取a标签（包含a下的子标签）下的文本
 35 
 36 
 37 # 搜索文档：find  find_all 按照一定的过滤条件进行匹配
 38 
 39 # 字符串
 40 # print(soup.find_all('a'))  # 匹配整个文档中的a标签
 41 # print(soup.find_all(attrs={'class': 'title'})) # 匹配class为title的标签
 42 
 43 # #正则表达式
 44 # import re
 45 # print(soup.find_all(re.compile('^p')))  # 匹配以p开头的标签
 46 # print(soup.find_all(re.compile('y$')))  # 匹配以y结尾的标签
 47 # print(soup.find_all(re.compile('t')))  # 匹配包含t的标签
 48 
 49 # 列表
 50 # for tag in soup.find_all(['a', 'b']):  # 匹配a标签，b标签
 51 #     print(tag)
 52 
 53 # for tag in soup.find_all('p', class_='story'):  # 匹配class=story的p标签
 54 #     print(tag)
 55 
 56 # # 方法  给find_all传入一个方法作为过滤条件
 57 # def has_class_but_no_id(tag):
 58 #     """
 59 #     定义一个判断有class属性但是没有id属性的方法，作为过滤条件
 60 #     """
 61 #     return tag.has_attr('class') and not tag.has_attr('id')
 62 #
 63 # for tag in soup.find_all(has_class_but_no_id):
 64 #     print(tag)
 65 
 66 
 67 # css选择器
 68 # print(soup.select('title'))   # 通过标签名查找
 69 # print(soup.select('.sister')) # 通过class名查找
 70 # print(soup.select('#link1'))  # 通过id名查找
 71 # print(soup.select('p #link2'))  # 组合查找，id为link2的p标签
 72 
 73 # > 只能够一级一级向下查找
 74 # print(soup.select('body > p .sister')) # 查找body下类名为sister的p
 75 
 76 
 77 # 百度搜索python，对返回页面进行属性查找
 78 # import requests
 79 # url = 'http://www.baidu.com/s?wd=python'
 80 # response = requests.get(url)  # 获取的数据是网页源代码，未经过js渲染
 81 #
 82 # soup = BeautifulSoup(response.text, 'lxml')
 83 
 84 # 查找返回页面搜索到的结果
 85 # items = soup.find_all('div', class_='result c-container ')
 86 
 87 # 打印搜索结果
 88 # for item in items:
 89 #     print(item.select('h3 > a')[0].get('href')  # 取a标签
 90 #     print(item.select('h3 > a')[0].get_text())
 91 
 92 #################################################################################
 93 # xpath 部分
 94 # 通配符  /  //  @  # .  ..
 95 #   /表示从当前节点匹配  //整个文档匹配  @选取属性 *
 96 ########################################################################################
 97 html = '''
 98 <html><head><title>The Dormouse's story</title></head>
 99 <p class="title"><b>The Dormouse's story</b></p>
100 <p class="story">Once upon a time there were three little sisters; and their names were
101 <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
102 <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
103 <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
104 and they lived at the bottom of a well.</p>
105 <p class="story">...</p>
106 '''
107 # from lxml import etree
108 # e = etree.HTML(html)
109 # for i in e.xpath('//p'):  # 整个文档中搜索p标签
110 #     # print(i.xpath('string(.)'))  # 获取当前标签下所有文本（标签下套标签），包括下面子标签的文本
111 #     print(i.text) # 匹配当前标签下的文本内容，不包含子标签
112 
113 """
114 # for i in e.xpath('//p/@class'):  # 选取p的class属性
115 # for i in e.xpath('//p[@class=title]'):  # 搜索class=title的p标签
116 //title[@*]  匹配所有有属性的title标签
117 """
118 # 百度搜索python，用xpath查找
119 import requests
120 from lxml import etree
121 
122 url = 'http://www.baidu.com/s?wd=python'
123 response = requests.get(url)  # 获取的数据是网页源代码
124 tree = etree.HTML(response.text)
125 
126 # 查找返回页面搜索到的结果
127 items = tree.xpath('//div[@class="result c-container "]')
128 for item in items:
129     # print(item.xpath('h3/a/@href'))
130     print(item.xpath('h3/a')[0].xpath('string(.)'))
网页解析--BeautifulSoup练习

猜你喜欢
