最近学习了一下python 写了一小端代码 练习一下
一、获取网页数据（html）
url：网址 headers：请求头部信息 （见下图）
找了一个 常用的 User-Agent:
headers = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)',
]
一般我们通过
'''
获取html页面
'''
def get_html(url,headers):
req = urllib2.Request(url)
header = random.choice(headers)
req.add_header('User-Agent', header)
req.add_header('Sec-Fetch-User', '?1')
req.add_header('Sec-Fetch-Site', 'none')
req.add_header('Sec-Fetch-Mode', 'navigate')
html = urllib2.urlopen(req).read()
return html二、将获取的html转化为可用的数据 （使用的是 xpath）
google xpath插件可以安装一下 方便你使用xpath
soup.xpath('//div[@class="witkey-item-top "]') 里面是你要截取的数据 这个根据你的需要改变哦！
'''
获取html页面内数据
'''
def get_page_data(html):
soup = etree.HTML(html)
div_list = soup.xpath('//div[@class="class"]')
with open('C:\Users\Administrator\Desktop\onelinux\os\\ces.csv', 'a')as f:
for div in div_list:
title = div.xpath('.//div[@class="class"]/text()')[0]
f.write('{}\n'.format(title));
'''三、创建csv文件 一定要先创建csv文件再去写入数据
'''
生成csv
'''
def creat_csv():
csv_headers = ['标题']
with open('C:\Users\Administrator\Desktop\onelinux\os\\ces.csv', 'wb') as f:
f_csv = csv.writer(f)
f_csv.writerow(csv_headers)
# f_csv.writerows()
f.close()三、记得引入用的模块
import urllib2
import random
import csv
import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )
from lxml import etree四、还是把完整代码写一下吧 里面的参数需要替换成你自己的哦
# -*- coding:utf-8 -*-
import urllib2
import random
import csv
import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )
from lxml import etree
headers = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)',
]
'''
获取html页面
'''
def get_html(url,headers):
req = urllib2.Request(url)
header = random.choice(headers)
req.add_header('User-Agent', header)
req.add_header('Sec-Fetch-User', '?1')
req.add_header('Sec-Fetch-Site', 'none')
req.add_header('Sec-Fetch-Mode', 'navigate')
html = urllib2.urlopen(req).read()
return html
'''
生成csv
'''
def creat_csv():
csv_headers = ['名称', '网址']
with open('C:\Users\Administrator\Desktop\onelinux\os\\ces.csv', 'wb') as f:
f_csv = csv.writer(f)
f_csv.writerow(csv_headers)
# f_csv.writerows()
f.close()
'''
获取html页面内数据
'''
def get_page_data(html):
soup = etree.HTML(html)
div_list = soup.xpath('//div[@class="你的类属性"]')
with open('C:\Users\Administrator\Desktop\onelinux\os\\ces.csv', 'a')as f:
for div in div_list:
title = div.xpath('.//div[@class="你的类属性"]/a/text()')[0]
link= div.xpath('.//div[@class="你的类属性"]/a/@herf')[0]
f.write('{},{}\n'.format(title, link));
'''
主函数
'''
def main():
num = input('请输入你要爬取的页数');
keyword = raw_input('请输入你要爬取关键词');
keyword = urllib2.quote(keyword)
for i in range(int(num)):
page = (i-1)*5 + i*65
if page < 0:
page = 0
url = '你的地址?page={}&key={}'.format(page,keyword)
html = get_html(url,headers)
get_page_data(html)
creat_csv() #创建csv
main() #主函数五、如果有好的方法、逻辑 ,欢迎留言 沟通 指教
