python爬虫-京东全网搜索

# 搜索京东商品  使用动态数据抓取
import requests
from bs4 import BeautifulSoup
import re

class JD:
    def __init__(self):
        self.headers = {'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}  ##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）

    def req(self,url):
        # 解析京东搜索首页地址
        soup1=self.requests_utf(url)
        allcount=soup1.find('span',id='J_resCount').get_text()
        print('共有%s件商品'%allcount)
        print('**************************************************************************')
        # 查询搜索的商品总页数
        page=int(soup1.find('span',class_='fp-text').i.get_text())

        for i in range(1,page*2,2):
            url_star=url[:-2]+str(i)
            # 根据商品页数解析搜索地址
            soup=self.requests_utf(url_star)

            # 定位商品信息
            li_all = soup.find_all('li', class_='gl-item')

            for i in li_all:
                # 商品标题
                title = i.a['title']

                # 商品实际地址
                href = i.a['href']
                if href[:4]=='http':
                    pass
                else:
                    href='https:'+ href

                # 价格
                price = float(i.i.get_text())

                # 解析商品实际地址
                soup_href=self.requests_gbk(href)
                real_href=soup_href.find('link',rel="canonical")['href']
                real_href='https:'+real_href

                # 定位商品名称并去空格
                sku_name=soup_href.find('div',class_='sku-name')
                product_name=str(sku_name.get_text()).strip()

                # 产品Id
                search_ID=re.search('\d+',real_href)
                product_ID=search_ID.group()

                # 解析产品评价js，返回数据
                summary=self.product_summary(product_ID)
                goodRateShow=''.join(summary[0])
                goodCountStr=''.join(summary[1])

                if price <= 10000:
                    print('商品ID：',product_ID)
                    print('商品：',product_name)
                    print('商品标题：',title)
                    print('好评累计：',goodCountStr)
                    print('好评率：',goodRateShow +'%')
                    print('链接地址：',real_href)
                    print('价格：',price)
                    print('*************************************************************************')
                else:
                    continue
                # print('可用优惠券：',quan)
                # print('优惠券信息：',quan_item)


    # 产品评价
    def product_summary(self,product_ID):
        url='https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv5292&productId='+product_ID+'&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1'
        pro_req=self.requests_gbk(url)
        # 好评率
        pattern0 = re.compile('goodRateShow":(.*?),"poorRateShow')
        # 好评累计
        pattern1 = re.compile('"goodCountStr":"(.*?)",')
        # 部分评价
        pattern_ping0=re.compile('("content":"(.*?){2}","creationTime)')
        search0 = re.findall(pattern0, str(pro_req))
        search1 = re.findall(pattern1, str(pro_req))
        return search0,search1

    # 解析网页utf-8
    def requests_utf(self,url):
        try:
            content = requests.get(url, headers=self.headers)
            content.encoding = 'utf-8'
            soup = BeautifulSoup(content.text, 'lxml')
            return soup
        except:
            print('网页解析发生错误！！！！')

    # 解析网页gbk
    def requests_gbk(self,url):
        try:
            content = requests.get(url, headers=self.headers)
            content.encoding = 'gbk'
            soup = BeautifulSoup(content.text, 'lxml')
            return soup
        except:
            print('网页解析发生错误！！！！')


jd=JD()
search_word='女士长款钱包'
url='https://search.jd.com/Search?keyword=' + search_word +'&enc=utf-8'
jd.req(url)