爬虫之Buff平台CSGO饰品市场数据

代码运行结果：

各字段分别为[each["id"], each["name"],each["sell_num"],
                           each["goods_info"]["steam_price"],each["sell_min_price"],
                                         float(each["sell_min_price"])/float(each["goods_info"]["steam_price"]),]

即：饰品id、饰品名称、数量、steam价格、buff饰品最低价格、buff饰品最低价格/steam价格

代码： 

注意：①请设置Cookie！！②特别注意爬取频率，否则Buff会永久封禁饰品市场访问接口

import requests
import csv
import time
def getDataJson(goodName,pageNum):
    my_headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
        "Cookie":"设置你的Cookie"
        "Host": "buff.163.com",
        "Referer": "https://buff.163.com/market/?game=csgo",
        "X-Requested-With": "XMLHttpRequest"
    }

    url_init = "https://buff.163.com/api/market/goods?game=csgo&page_num={}&category_group={}".format(pageNum,goodName)
    try:
        res = requests.get(url_init, headers=my_headers, timeout=10)
        res.raise_for_status()
        res.encoding = res.apparent_encoding
        if res.status_code == 200:
            return res.json()["data"]["items"]
    except Exception as e:
        print(e)

if __name__ == '__main__':
    goodInfo = {
        "pistol":116, #手枪
        "rifle":128,#步枪
        "smg":79,#冲锋枪
        "shotgun":45,#散弹枪
        "machinegun":15,
        "sticker":187
    }
    for key,value in goodInfo.items():
        print(key,value)
        for index in range(1,value): #115
            dataJson = getDataJson(key,index)
            print(dataJson)
            try:
                with open("csgoData.csv", "a",encoding="utf-8") as csvfile:
                    writer = csv.writer((csvfile))
                    for each in dataJson:
                        writer.writerow([each["id"], each["name"],each["sell_num"],
                                         each["goods_info"]["steam_price"],each["sell_min_price"],
                                         float(each["sell_min_price"])/float(each["goods_info"]["steam_price"]),
                                         ])
                    csvfile.close()
                    print("page {} is ok!".format(index))
                    time.sleep(5)
            except Exception as e:
                print("Exception in page:",index)
                print(e)