金融数据分析（六）亚马逊商品信息自定义获取（requests库）

案例（二）爬虫预热

项目二：亚马逊商品信息自定义获取，可自定商品名称和爬取页数

# -*- coding: utf-8 -*-
"""
Created on Thur Sept 17 15:56:36 2020

@author: mly
"""
import requests
import re
import pandas as pd

ilt = []
iltl = []


def getHTMLText(url):
    try:
        kv = {
    
    'user-agent': 'Mozilla/5.0',
              'Cookie': 'x-wl-uid=1+EeiKz9a/J/y3g6XfXTnSbHAItJEus3oQ6Gz+T/haur7dZfkNIgoxzMGwviB+42iWIyk9LR+iHQ=;'
                        ' session-id=457-2693740-8878563; ubid-acbcn=459-5133849-3255047; lc-acbcn=zh_CN; i18n-prefs=CNY; '
                        'session-token="8n/Oi/dUCiI9zc/0zDLjB9FQRC6sce2+Tl7F0oXncOcIYDK4SEJ7eek/Vs3UfwsRchW459OZni0AFjMW+'
                        '9xMMBPSLM8MxLNDPP1/13unryj8aiRIZAE1WAn6GaeAgauNsijuBKKUwwLh8Dba7hYEjwlI1J6xlW0LKkkyVuApjRXnOsvdYr'
                        'X8IURVpOxDBnuAF9r7O71d/NPkIQsHy7YCCw=="; session-id-time=2082787201l;'
                        ' csm-hit=tb:s-85XYJNXFEJ5NBKR0JE6H|1566558845671&t:1566558845672&adb:adblk_no'}
        r = requests.get(url, headers=kv, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""


def parsePage(ilt, html):
    try:
        plt = re.findall('<span class="a-offscreen">￥(.*?)</span>', html)
        #print(plt)
        tlt = re.findall('<span class="a-size-base-plus a-color-base a-text-normal" dir="auto">(.*?)</span>', html)
        #print(tlt)
        for i in range(len(tlt)):
            ilt.append([plt[i], tlt[i]])
    except:
        return ""


def printGoodsList(ilt):
    column = ["序号", "价格", "商品名称"]
    count = 0
    for g in ilt:
        count = count + 1
        iltl.append([count, g[0], g[1]])
    test = pd.DataFrame(columns=column, data=iltl)
    test.to_csv('finance.csv', encoding='utf_8_sig', index=False)


def main():
    goods = input("请输入商品名称：")
    depth = int(input("请输入想查看到的页码："))
    start_url = 'https://www.amazon.cn/s?k=' + goods
    infoList = []
    for i in range(depth):
        try:
            url = start_url + '&page=' + str(i + 1)
            html = getHTMLText(url)
            parsePage(infoList, html)
        except:
            continue
    printGoodsList(infoList)

main()

运行时可自定义商品名称和爬取页数：

爬取结果：