1，打开淘宝页面，搜索iPad,获得Ajax请求。可以发现商品接口的参数很多
Request URL: https://s.taobao.com/api?_ksTS=1526438463462_212&callback=jsonp213&ajax=true&m=customized&sourceId=tb.index&q=ipad&spm=a21bo.2017.201856-taobao-item.1&s=36&imgfile=&initiative_id=tbindexz_20170306&bcoffset=-1&commend=all&ie=utf8&rn=4e6b2a857715657b15d4ba9dc61a8fb4&ssid=s5-e&search_type=item
2，用Selenium进行抓取
def index_page(page):
"""
抓取索引页
:param page: 页码
"""
print('正在爬取第', page, '页')
try:
url = 'https://s.taobao.com/search?q=' + quote(KEYWORD)
browser.get(url)
if page > 1:
input = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager div.form > input')))
submit = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager div.form > span.btn.J_Submit')))
input.clear()
input.send_keys(page)
submit.click()
wait.until(
EC.text_to_be_present_in_element((By.CSS_SELECTOR, '#mainsrp-pager li.item.active > span'), str(page)))
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.m-itemlist .items .item')))
get_products()
except TimeoutException:
index_page(page)
3 ,解析商品列表
from pyquery import PyQuery as pq
def get_products():
"""
提取商品数据
"""
html = browser.page_source
doc = pq(html)
items = doc('#mainsrp-itemlist .items .item').items()
for item in items:
product = {
'image': item.find('.pic .img').attr('data-src'),
'price': item.find('.price').text(),
'deal': item.find('.deal-cnt').text(),
'title': item.find('.title').text(),
'shop': item.find('.shop').text(),
'location': item.find('.location').text()
}
print(product)
4，运行
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from urllib.parse import quote
browser = webdriver.Chrome()
wait = WebDriverWait(browser, 10)
KEYWORD = 'iPad'
MAX_PAGE = 100
def main():
"""
遍历每一页
"""
for i in range(1, MAX_PAGE + 1):
index_page(i)
main()
5, 结果
正在爬取第 1 页
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/imgextra/i1/13022581/TB27gZ1l5qAXuNjy1XdXXaYcVXa_!!0-saturn_solar.jpg', 'price': '¥\n2699.00', 'deal': '167人付款', 'title': '12期免息【2年保修】Apple/苹果\niPad\nmini 4 7.9英寸平板电脑', 'shop': '绿森数码官方旗舰店', 'loaction': '浙江 杭州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i1/1669409267/TB1VGsnfBjTBKNjSZFwXXcG4XXa_!!0-item_pic.jpg', 'price': '¥\n2298.00', 'deal': '6466人付款', 'title': '[12期分期]Apple/苹果\niPad\n2018款 9.7英寸wifi新款平板电脑128G', 'shop': '卓辰数码旗舰店', 'loaction': '浙江 杭州'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i1/268451883/TB1Oxo8jiOYBuNjSsD4XXbSkFXa_!!0-item_pic.jpg', 'price': '¥\n2288.00', 'deal': '2092人付款', 'title': '【12期分期】Apple/苹果\niPad\n2018款 9.7英寸wifi新款 平板电脑', 'shop': '三际数码官方旗舰店', 'loaction': '山东 济南'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i2/898225/TB25bFmleuSBuNjSsziXXbq8pXa_!!898225.jpg', 'price': '¥\n1888.00', 'deal': '475人付款', 'title': 'Apple/苹果\niPad\n2018款\niPad\nair3 平板电脑\nipad\n2018新款9.7', 'shop': 'ygsd', 'loaction': '北京'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i4/2271900793/TB1SL7Hi7OWBuNjSsppXXXPgpXa_!!2-item_pic.png', 'price': '¥\n239.00', 'deal': '478人付款', 'title': '美洲野牛手包男真皮大容量手拿包牛皮青年手抓包\nipad\n信封包男夹包', 'shop': '温欣箱包专营店', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/2670850710/TB1QrYSduuSBuNjSsziXXbq8pXa_!!0-item_pic.jpg', 'price': '¥\n38.00', 'deal': '2659人付款', 'title': '汽车车载手机平板电脑车用\nipad\n懒人后座支架车内后枕后排头枕mini', 'shop': 'pzoz数码旗舰店', 'loaction': '湖北 武汉'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i1/2200274095/TB1zgroikOWBuNjSsppXXXPgpXa_!!0-item_pic.jpg', 'price': '¥\n329.00', 'deal': '400人付款', 'title': '卓梵 阿玛尼男包手拿包商务\nIPAD\n包信封包手包真皮软皮牛皮大容量', 'shop': 'josephamani黛言专卖店', 'loaction': '广东 广州'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i3/673208397/TB2rhroddHO8KJjSZFtXXchfXXa_!!673208397.jpg', 'price': '¥\n23.80', 'deal': '719人付款', 'title': '帆布包男包单肩包斜挎包男韩版时尚时尚户外休闲商务小包\nIPAD\n背包', 'shop': '炫彩包袋', 'loaction': '河北 保定'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i1/TB1YvQgLXXXXXbhXpXXYXGcGpXX_M2.SS2', 'price': '¥\n35.00', 'deal': '392人付款', 'title': '手包街头软皮手包新款韩版手拿包潮流铆钉手包休闲单肩斜跨\niPad\n包', 'shop': 'sedl5565850', 'loaction': '浙江 杭州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i7/TB1iH.IKVXXXXc5XpXXYXGcGpXX_M2.SS2', 'price': '¥\n25.00', 'deal': '333人付款', 'title': '韩版潮男手拿包 男士折叠手抓包 潮流手拿包男休闲商务手包\nIPAD\n包', 'shop': 'sedl5565850', 'loaction': '浙江 杭州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/1812529380/TB1RjyFpDlYBeNjSszcXXbwhFXa_!!0-item_pic.jpg', 'price': '¥\n138.00', 'deal': '632人付款', 'title': '七匹狼男包单肩包男士防水牛津布斜挎包\nipad\n平板包包旅行休闲布包', 'shop': 'septwolves翔熙专卖店', 'loaction': '湖南 株洲'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i1/893352404/TB2XkDpcC8YBeNkSnb4XXaevFXa_!!893352404.jpg', 'price': '¥\n39.00', 'deal': '38人付款', 'title': '袋鼠男包单肩包男斜挎包男士皮包包竖款商务包挂包休闲背包\niPad\n包', 'shop': '晨曦feixun', 'loaction': '河北 保定'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i1/TB1KtkpHVXXXXbcXFXXXXXXXXXX_!!0-item_pic.jpg', 'price': '¥\n29.00', 'deal': '65人付款', 'title': '信封大包女夏拼接手拿包文件\nipad\n资料包韩版时尚大容量手抓晚宴包', 'shop': '青年女包11', 'loaction': '浙江 杭州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/1693601207/TB2ZCTVrVXXXXbsXXXXXXXXXXXX_!!1693601207.jpg', 'price': '¥\n49.00', 'deal': '9人付款', 'title': '欧美时尚潮男手包小怪兽防水牛津布手挽包男士信封包小恶魔\nipad\n包', 'shop': '时尚一百分之一', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i3/2618438013/TB2FdWLrTJYBeNjy1zeXXahzVXa_!!2618438013.jpg', 'price': '¥\n45.00', 'deal': '3人付款', 'title': '重生epiphqny韩版休闲迷你学院风豹纹双肩包\nipad\n背包女包51011', 'shop': 'epiphqny折扣店', 'loaction': '福建 厦门'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i2/53520749/TB2O73nXMDD8KJjy0FdXXcjvXXa_!!53520749.jpg', 'price': '¥\n38.90', 'deal': '3人付款', 'title': '新款男包时尚铆钉单肩包邮差包休闲潮流斜挎包\nipad\n包书包学生包', 'shop': 'liangjick', 'loaction': '浙江 杭州'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i2/TB1IQuyKFXXXXcRXVXXXXXXXXXX_!!0-item_pic.jpg', 'price': '¥\n49.80', 'deal': '4人付款', 'title': '男包单肩包 防水牛津布中性斜挎包背包 时尚潮男休闲包男士\niPad\n包', 'shop': '时尚一百分之一', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/3333027949/TB21tSjXsrHK1JjSspfXXbsrXXa_!!3333027949.jpg', 'price': '¥\n78.00', 'deal': '868人付款', 'title': '浴缸架浴室伸缩置物架板多功能浴缸隔板卫生间泡澡\niPad\n手机支架竹', 'shop': '全品竹木家私', 'loaction': '浙江 金华'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i1/2108728798/TB1ElRsfsj_B1NjSZFHXXaDWpXa_!!0-item_pic.jpg', 'price': '¥\n680.00', 'deal': '128人付款', 'title': '新款编织风格bv手包男商务男士手拿包大容量男包真皮信封包\nipad\n包', 'shop': '巴黎生活2014', 'loaction': '广东 广州'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i3/3309133710/TB1AMhJk4TI8KJjSspiXXbM4FXa_!!0-item_pic.jpg', 'price': '¥\n309.00', 'deal': '141人付款', 'title': '牛皮单肩斜挎手提包男士竖款真皮背包软皮欧美时尚\nipad\n平板电脑包', 'shop': '艾利斯斯普林斯旗舰店', 'loaction': '广东 佛山'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i2/835893472/TB1b7GaXbsTMeJjSszgXXacpFXa_!!0-item_pic.jpg', 'price': '¥\n49.00', 'deal': '1145人付款', 'title': '倍思车载支架汽车上后排头枕后座椅\nipad\n平板手机多功能支撑架用品', 'shop': 'baseus倍思旗舰店', 'loaction': '浙江 杭州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/2742967129/TB2O_TcoJRopuFjSZFtXXcanpXa_!!2742967129.jpg', 'price': '¥\n158.00', 'deal': '180人付款', 'title': '汽车用品卡通座椅收纳袋车载多功能椅背置物袋杂物储物袋挂袋\nIPAD', 'shop': '创意_汽车用品', 'loaction': '上海'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i3/2382101516/TB2GUghc0nJ8KJjSszdXXaxuFXa_!!2382101516.jpg', 'price': '¥\n148.00', 'deal': '220人付款', 'title': '2017新款单肩包 男包斜挎包休闲时尚韩版包包薄款\nipad\n背包潮挎包', 'shop': '新天地电商中心', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i2/2742967129/TB2xvIWtXXXXXboXpXXXXXXXXXX_!!2742967129.jpg', 'price': '¥\n108.00', 'deal': '188人付款', 'title': '汽车用品多功能车载后背杂物挂袋汽车座椅收纳袋椅背袋置物袋\nIPAD', 'shop': '创意_汽车用品', 'loaction': '上海'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i1/1026909821/TB2JsRCf2uSBuNkHFqDXXXfhVXa_!!1026909821-0-item_pic.jpg', 'price': '¥\n129.00', 'deal': '356人付款', 'title': '巴朗胸包男士大容量斜挎包休闲运动背包三角水滴包\nipad\n单肩包男潮', 'shop': '巴朗成都专卖店', 'loaction': '广东 广州'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i4/2200274095/TB16ktojZLJ8KJjy0FnXXcFDpXa_!!0-item_pic.jpg', 'price': '¥\n298.00', 'deal': '164人付款', 'title': '卓梵 阿玛尼男士手包软皮大容量商务\nIPAD\n包手机包信封包真皮牛皮', 'shop': 'josephamani黛言专卖店', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i1/77852282/TB1SRbifGmgSKJjSspiXXXyJFXa_!!0-item_pic.jpg', 'price': '¥\n158.00', 'deal': '181人付款', 'title': '小怪兽手拿包女大容量恶魔眼睛牛皮老佛爷休闲信封青年\nipad\n包潮男', 'shop': '情趣衣店', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i3/1737111535/TB1aOdFreSSBuNjy0FlXXbBpVXa_!!0-item_pic.jpg', 'price': '¥\n358.00', 'deal': '107人付款', 'title': 'Samsonite/新秀丽I81单肩包专柜同款小挎包竖式休闲斜挎包\nIPAD\n包', 'shop': '恒艺箱包专营店', 'loaction': '江苏 苏州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i3/1026909821/TB27h0IluySBuNjy1zdXXXPxFXa_!!1026909821-0-item_pic.jpg', 'price': '¥\n139.00', 'deal': '344人付款', 'title': '巴朗胸包男商务双肩包背包单肩两用斜挎包\nIpad\n笔记本电脑休闲男包', 'shop': '巴朗成都专卖店', 'loaction': '广东 广州'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i4/731456220/TB1c7ccbHGYBuNjy0FoXXciBFXa_!!0-item_pic.jpg', 'price': '¥\n99.00', 'deal': '346人付款', 'title': '2018新款手包信封包韩版潮流男女通用放\nIPAD\n大容量夹包男包手拿包', 'shop': '杰西维尔旗舰店', 'loaction': '北京'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/3403941459/TB1UblhrFuWBuNjSszbXXcS7FXa_!!0-item_pic.jpg', 'price': '¥\n299.00', 'deal': '59人付款', 'title': '香港代购日本潮牌单肩包斜挎包真皮死飞邮差包男包旅行包\nipad\n背包', 'shop': '昌盛世纪贸易', 'loaction': '上海'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i1/1916349919/TB2eNB7gr1YBuNjSszeXXablFXa_!!1916349919.jpg', 'price': '¥\n129.00', 'deal': '401人付款', 'title': '埋不烂原创 复古油蜡帆布包男士斜挎包夏季小包单肩包\nipad\n包B108', 'shop': 'mybrand2009', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i1/2258674414/TB1S76vqkSWBuNjSszdXXbeSpXa_!!0-item_pic.jpg', 'price': '¥\n129.00', 'deal': '230人付款', 'title': '七匹狼男包单肩包休闲男士帆布包牛津布斜挎包男生挎包男背包\nipad', 'shop': 'septwolves隆丰专卖店', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/TB1az0yIVXXXXagaFXXXXXXXXXX_!!0-item_pic.jpg', 'price': '¥\n268.00', 'deal': '102人付款', 'title': '男士疯马牛皮手拿包真皮单肩信封包A4文件包\nIPAD\n包大容量潮流夹包', 'shop': 'guoyan0788', 'loaction': '安徽 淮北'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i2/2659463963/TB2N5v_lXXXXXXeXXXXXXXXXXXX_!!2659463963.jpg', 'price': '¥\n38.00', 'deal': '1432人付款', 'title': '特价吉田潮男包防水单肩包休闲斜挎包学生邮差包复古潮版包\niPad\n包', 'shop': '吉田porter打工者', 'loaction': '上海'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i3/521312444/TB1tK6UcsjI8KJjSsppXXXbyVXa_!!0-item_pic.jpg', 'price': '¥\n36.00', 'deal': '1017人付款', 'title': '艾乖男士手包韩版时尚磨砂PU男包商务手抓信封包\nIPAD\n文件包手拿包', 'shop': '艾乖旗舰店', 'loaction': '天津'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i1/2640980397/TB2Q2kBfXXXXXXHXpXXXXXXXXXX_!!2640980397.jpg', 'price': '¥\n25.90', 'deal': '1人付款', 'title': '日韩男包单肩包商务男包皮包竖款斜挎包休闲背包潮流\nipad\n男士包包', 'shop': '广州箱包189', 'loaction': '广东 广州'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i1/2270441695/TB1PN6ncQOWBuNjSsppXXXPgpXa_!!0-item_pic.jpg', 'price': '¥\n398.00', 'deal': '35人付款', 'title': '车载笔记本电脑桌多功能汽车小桌板折叠车内椅背后座餐桌\nipad\n支架', 'shop': '博缘堂旗舰店', 'loaction': '浙江 杭州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i3/1128925972/TB2CwTbaAfb_uJjSsD4XXaqiFXa_!!1128925972.jpg', 'price': '¥\n188.00', 'deal': '134人付款', 'title': '原创手工单肩包时尚休闲真皮男包头层牛皮斜跨多功能\nipad\n潮包竖款', 'shop': '霸必图babitu', 'loaction': '广东 佛山'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i1/TB1Wi1iKFXXXXaqXXXXYXGcGpXX_M2.SS2', 'price': '¥\n188.00', 'deal': '88人付款', 'title': 'SPA浴缸置物架多功能浴缸支架 卫生间浴室隔板泡澡\nipad\n手机架', 'shop': '式微式微00', 'loaction': '浙江 杭州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i2/180534398/TB2Va9ukVXXXXXPXpXXXXXXXXXX_!!180534398.gif', 'price': '¥\n128.00', 'deal': '135人付款', 'title': '男包单肩包竖款男士斜挎包\niPad\n牛皮包商务休闲背包精品真皮男包包', 'shop': 'tongxin179979', 'loaction': '广东 广州'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i2/1916349919/TB2xDvKdf6H8KJjSspmXXb2WXXa_!!1916349919.jpg', 'price': '¥\n379.00', 'deal': '58人付款', 'title': '埋不烂 头层牛皮男士胸包复古单肩真皮斜挎包\nipad\n迷你潮包XB20', 'shop': 'mybrand2009', 'loaction': '广东 广州'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i3/2382101516/TB2BVi.cYsrBKNjSZFpXXcXhFXa_!!2382101516.jpg', 'price': '¥\n168.00', 'deal': '90人付款', 'title': '新款男包单肩包男士斜挎包商务背包休闲韩版包包学生日韩\nipad\n包包', 'shop': '新天地电商中心', 'loaction': '广东 广州'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i1/57832598/TB2YGGEdBDH8KJjSszcXXbDTFXa_!!57832598.jpg', 'price': '¥\n899.00', 'deal': '12人付款', 'title': 'MRKOO米斯酷车载\niPad\n/mini导航改装内嵌中控一体车机超越CarPlay', 'shop': 'i故政', 'loaction': '广东 东莞'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i4/773384983/TB2gDE0bNvzQeBjSZPfXXbWGFXa_!!773384983.jpg', 'price': '¥\n159.00', 'deal': '119人付款', 'title': '头层牛皮休闲背包油蜡皮男包黑色真皮单肩斜挎小包商务软皮\nIpad\n包', 'shop': 'guoyue5558', 'loaction': '江西 吉安'}
{'image': '//g-search2.alicdn.com/img/bao/uploaded/i4/i3/1688593879/TB1MgqNXL9TBuNjy1zbXXXpepXa_!!0-item_pic.jpg', 'price': '¥\n199.00', 'deal': '67人付款', 'title': '七匹狼单肩包时尚男士斜挎包帆布包男包韩版背包潮流休闲\nipad\n包', 'shop': 'septwolves七匹狼旗舰店', 'loaction': '浙江 杭州'}
{'image': '//g-search1.alicdn.com/img/bao/uploaded/i4/i3/3838015761/TB1NbxbrXOWBuNjy0FiXXXFxVXa_!!0-item_pic.jpg', 'price': '¥\n580.00', 'deal': '32人付款', 'title': '2018新款bv编织风格手包男真皮钱包牛皮编织信封包\nipad\n包休闲男包', 'shop': '包包红发', 'loaction': '广东 广州'}
{'image': '//g-search3.alicdn.com/img/bao/uploaded/i4/i2/136180673/TB2.HcfhAKWBuNjy1zjXXcOypXa_!!136180673.jpg', 'price': '¥\n259.00', 'deal': '40人付款', 'title': '韩国正品大号手拿包\nIPAD\n包手拿包平板电脑包多功能包信封钱包', 'shop': '我的运动鞋批发', 'loaction': '上海'}
正在爬取第 2 页