python爬虫 爬取淘宝网商品信息

import json
import re
from time import sleep

import requests

# 指定要搜索的商品并把商品的名字作为参数加入到url中
input_name = input('输入你要查找商品的名字：')


def get_taobao(url):

    html = requests.get(url).text

    # 利用正则提取想要的数据
    reg = r'g_page_config = .*?g_srp_loadCss'
    reg = re.compile(reg, re.S)
    result = re.findall(reg, html)[0]
    result = re.findall(r'{.*}', result)
    # 把文件处理成字典类型
    result = json.loads(result[0])
    # 提取想要的内容
    auctions = (result['mods']['itemlist']['data']['auctions'])
    # 把内容生成字典
    for auction in auctions:
        title = auction['title'].replace('</span>', '').replace('<span class=H>', '')
        image = 'https:' + auction['pic_url']
        price = auction['view_price']
        adress = auction['item_loc']
        store = auction['nick']
        dict1 = {
            'title': title,
            'image': image,
            'price': price,
            'adress': adress,
            'store': store
        }
        sleep(2)
        return dict1

# 保存文件
def save_dict(dict1):
    with open('content.txt', 'a', encoding='gbk') as f:
        f.write(json.dumps(dict1, ensure_ascii=False))


def main():
# 利用循环生成淘宝网商品每一页的url
    for i in range(10):
        url = 'https://s.taobao.com/search?q={}&imgfile=&' \
              'commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&' \
              'spm=a21bo.1000386.201856-taobao-item.1&ie=utf8&' \
              'initiative_id=tbindexz_20170306&bcoffset=3&ntoffset=0&' \
              'p4ppushleft=1%2C48&s=0&data-key=s&data-value={}'.format(input_name, i*44)
        sleep(2)
        dict1 = get_taobao(url)
        save_dict(dict1)


if __name__ == '__main__':
    main()