Get the data from Amazon America, then convert the English data into Chinese, and finally write it into the excle table
# -*- coding: utf-8 -*-
from urllib import request, parse
import requests, time, re, xlrd, json, pymssql, datetime
from bs4 import BeautifulSoup
from lxml import etree
from xlutils.copy import copy
from datetime import timedelta
from gevent import monkey
class Amazon_books(object):
def __init__(self):
self.url = "https://www.amazon.com/gp/bestsellers/boost/12035215011/ref=pd_zg_hrsr_boost_1_3_last"
self.base_url = "http://fy.iciba.com/ajax.php?a=fy" # 翻译url
self.index = 1 # 记录图书序号
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
}
self.workbook = xlrd.open_workbook(r'D:\PycharmProjects\reptile\亚马逊\自拍杆美国市场分析表 .xls') # 文件路径
self.xlsx = copy(self.workbook)
self.shtc = self.xlsx.get_sheet(0)
self.num = 1
def get_page(self):
"""获取前20个商品信息"""
reqeons = requests.get(self.url, headers=self.headers).text
soup = BeautifulSoup(reqeons, "lxml")
li_list = soup.select('li[class="zg-item-immersion"]')
for li in li_list[:20]:
commodity_url = li.select('a')[0].attrs['href'] # 商品url
commodity_url = "https://www.amazon.com" + commodity_url
self.shtc.write(self.num, 2, commodity_url)
self.xlsx.save('自拍杆美国市场分析表 .xls')
self.handle_page(commodity_url)
img_url = li.select('img')[0].attrs['src'] # 图片url
self.shtc.write(self.num, 1, img_url)
self.xlsx.save('自拍杆美国市场分析表 .xls')
price = li.select('span[class="p13n-sc-price"]')[0].get_text() # 商品价格
self.shtc.write(self.num, 5, price)
self.xlsx.save('自拍杆美国市场分析表 .xls')
try:
grade = li.select('span[class="a-icon-alt"]')[0].get_text() # 评分
grade = "".join(re.findall(r"\d.\d", grade))
self.shtc.write(self.num, 8, grade)
self.xlsx.save('自拍杆美国市场分析表 .xls')
except Exception as e:
grade = 0
self.shtc.write(self.num, 8, grade)
self.xlsx.save('自拍杆美国市场分析表 .xls')
try:
comment_amounts = li.select('a[class="a-size-small a-link-normal"]')[0].get_text() # 总评论数
self.shtc.write(self.num, 6, comment_amounts)
self.xlsx.save('自拍杆美国市场分析表 .xls')
except Exception as e:
comment_amounts = 0
self.shtc.write(self.num, 6, comment_amounts)
self.xlsx.save('自拍杆美国市场分析表 .xls')
try:
comment_url1 = li.select('a[class="a-size-small a-link-normal"]')[0].attrs['href'] # 评论url
comment_url = "https://www.amazon.com" + comment_url1 + "&sortBy=recent&pageNumber=1&reviewerType=avp_only_reviews&filterByStar=one_star" # 一星评论url
self.one_star_comment(comment_url)
comment_amount_url = "https://www.amazon.com" + comment_url1 + "&sortBy=recent&pageNumber=1&reviewerType=avp_only_reviews&filterByStar=positive" # 实际评论数
response = requests.get(comment_amount_url, headers=self.headers).text
soup = etree.HTML(response)
time.sleep(1)
comment_amount = "".join(soup.xpath('//*[@id="filter-info-section"]/span[1]/text()'))
comment_amount1 = re.findall(r"of (\d,\d+)", comment_amount)
if not comment_amount1:
comment_amount = re.findall(r"of (\d+)", comment_amount)
self.shtc.write(self.num, 7, comment_amount)
self.xlsx.save('自拍杆美国市场分析表 .xls')
self.num += 1
else:
print(comment_amount1)
self.shtc.write(self.num, 7, comment_amount1)
self.xlsx.save('自拍杆美国市场分析表 .xls')
self.num += 1
except Exception as e:
comment_amount = 0
self.shtc.write(self.num, 7, comment_amount)
self.xlsx.save('自拍杆美国市场分析表 .xls')
self.num += 1
def handle_page(self, commodity_url):
"""解析商品详情"""
response = requests.get(commodity_url, headers=self.headers).text
soup = etree.HTML(response)
trademark = "".join(soup.xpath('//*[@id="bylineInfo"]/text()')) # 品牌
self.shtc.write(self.num, 3, trademark)
self.xlsx.save('自拍杆美国市场分析表 .xls')
trait = soup.xpath('//*[@id="feature-bullets"]/ul/li[position()>1]//span/text()') # 卖点
num = 9
for i in trait:
i = i.replace("\t", "")
i = i.replace("\n", "")
self.translate(i, num)
num += 1
soup1 = BeautifulSoup(response, "lxml")
li_list = soup1.find(attrs={"id": "productDetails_detailBullets_sections1"})
try:
li_list = li_list.find_all("tr")
data = {}
for i in li_list:
key = i.find(attrs={"class": "a-color-secondary a-size-base prodDetSectionEntry"})
key = key.text
try:
key = key.replace("\n", "")
key = key.replace("\r", "")
key = key.replace(" ", "")
key = key.replace("\t", "")
except Exception as e:
pass
try:
value = i.find('td', class_="a-size-base").get_text()
except Exception as e:
value = None
try:
value = value.replace("\n", "")
value = value.replace("\r", "")
value = value.replace("\t", "")
value = value.replace(" ", "")
except Exception as e:
pass
data.update({key: value})
items = data.get('DatefirstlistedonAmazon')
if items is None:
items = data.get('DateFirstAvailable')
num = 4
self.translate(items, num)
else:
num = 4
self.translate(items, num)
except Exception as e:
num = 4
current_content = "2008/08/08"
self.shtc.write(self.num, num, current_content)
self.xlsx.save('自拍杆美国市场分析表 .xls')
def page_img(self, img_url):
"""下载图片"""
img_url1 = requests.get(img_url, headers=self.headers).content
with open(r'./图片/' + str(self.index) + ".jpg", 'wb') as file:
file.write(img_url1)
self.index += 1
# with open('图片.txt', 'a', encoding='utf8') as f:
# # for data in img_url:
# json.dump(img_url, f, ensure_ascii=False)
# f.write('\n')
def one_star_comment(self, comment_url):
"""解析一星评论"""
response = requests.get(comment_url, headers=self.headers).text
soup = etree.HTML(response)
one_comment = soup.xpath('//*[@data-hook="review"]//span[@data-hook="review-body"]//span/text()')
num = 14
for i in one_comment[:5]:
try:
self.translate(i, num)
num += 1
except Exception as e:
pass
def translate(self, i, num):
"""翻译"""
data = {
"f": "auto",
"t": "auto",
"w": i
}
data_str = parse.urlencode(data)
# 封装
response = request.Request(url=self.base_url, headers=self.headers, data=bytes(data_str, encoding="utf-8"))
req = json.loads(request.urlopen(response).read().decode("utf-8"))
current_state = req["status"]
# 在这里判断中译英
if current_state == 1:
current_content = req["content"]
current_content = current_content["out"]
# 否则则汉译英
else:
current_content = req["content"]
current_content = current_content["word_mean"]
try:
current_content = current_content.replace("年", "/")
current_content = current_content.replace("月", "/")
current_content = current_content.replace("日", "")
current_content = current_content.replace("。", "")
except Exception as e:
pass
self.shtc.write(self.num, num, current_content)
self.xlsx.save('自拍杆美国市场分析表 .xls')
def run(self):
self.get_page()
time.sleep(50)
if __name__ == '__main__':
while True:
now = datetime.datetime.now()
print(now.hour, now.minute)
amazon = Amazon_books()
amazon.run()
# 每隔60秒检测一次
time.sleep(50)