A recent study reptiles, and read the teacher's day Song video, see Taobao price crawling when encountered a problem and video code is as follows:
import requests
import re
def getHTMLText(url):
try:
r = requests.get(url, timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return ""
def parsePage(ilt, html):
try:
plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)
tlt = re.findall(r'\"raw_title\"\:\".*?\"',html)
for i in range(len(plt)):
price = eval(plt[i].split(':')[1])
title = eval(tlt[i].split(':')[1])
ilt.append([price , title])
except:
print("")
def printGoodsList(ilt):
tplt = "{:4}\t{:8}\t{:16}"
print(tplt.format("序号", "价格", "商品名称"))
count = 0
for g in ilt:
count = count + 1
print(tplt.format(count, g[0], g[1]))
def main():
goods = '书包'
depth = 3
start_url = 'https://s.taobao.com/search?q=' + goods
infoList = []
for i in range(depth):
try:
url = start_url + '&s=' + str(44*i)
html = getHTMLText(url)
parsePage(infoList, html)
except:
continue
printGoodsList(infoList)
main()
But in the climbing process, the discovery list is empty acquired, acquired html pages for the search, suspect may be Taobao anti-climb mechanism to require login, this time to see an article: coping mechanism Taobao anti-climb
obtain a solution , i.e., user-agent was added, and cookie in the code:
import requests
import re
def getHTMLText(url):
kv = {'cookie' : 'miid=1392388859950663781; thw=cn; cna=s9C2FVFUVw8CAXyg1bT0oJif; tracknick=yszadzw; tg=0; enc=LKTwWx5cRa7pcIw0aew7%2BMFpza2orP5NQdySiUvThCWjHfWOeIJGXUDclX3aFAaphV78tK9BrfKjvgcClh0qWw%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0; t=c200d632588fdde99c22fb5bc089e475; cookie2=75920c1bc5b4b48811c80e267d2ecb58; v=0; _tb_token_=517a771ee35e7; alitrackid=www.taobao.com; lastalitrackid=www.taobao.com; _samesite_flag_=true; unb=1950093010; uc3=id2=UojRYJe7AiUMDw%3D%3D&lg2=UtASsssmOIJ0bQ%3D%3D&nk2=GhjzM3VPjg%3D%3D&vt3=F8dBxdz0zON4NRHYUT8%3D; csg=8419b888; lgc=yszadzw; cookie17=UojRYJe7AiUMDw%3D%3D; dnk=yszadzw; skt=56a1565345d25a12; existShop=MTU4MTg0NTE0OQ%3D%3D; uc4=nk4=0%40GJS%2B%2FX1w68SeUc7Nu69OmqIh&id4=0%40UOBVB5%2BYP0rAfnAHrBQdIeoWae71; _cc_=WqG3DMC9EA%3D%3D; _l_g_=Ug%3D%3D; sg=w0c; _nk_=yszadzw; cookie1=U7OWPoqTbqinHZBsRIHLTyByecisOOh%2F7RDTSGvTsmQ%3D; mt=ci=63_1; JSESSIONID=F2D466FB38FDCCA69750D23492E2EA97; l=cBjbzcUuqbr-UEiSBOCgZuI8LO7tMIRA_uPRwCVXi_5pJ6Y1e8bOo5N4JFv6cjWd9K8B4NSLzt29-etksi4pJA--g3fP.; isg=BIiIZbCeQuJ1dK1vwsVoq3QkWfaaMew7CYOlP0I5oYP2HSiH6kONyx0XlfVtKqQT; uc1=cookie16=URm48syIJ1yk0MX2J7mAAEhTuw%3D%3D&cookie21=UIHiLt3xThH8t7YQoFNq&cookie15=W5iHLLyFOGW7aA%3D%3D&existShop=false&pas=0&cookie14=UoTUO8hSRNy6Aw%3D%3D&tag=8&lng=zh_CN',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
try:
r = requests.get(url, timeout=30, headers = kv)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return ""
def parsePage(ilt, html):
try:
plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
tlt = re.findall(r'\"raw_title\"\:\".*?\"', html)
for i in range(len(plt)):
price = eval(plt[i].split(':')[1])
title = eval(tlt[i].split(':')[1])
ilt.append([price, title])
except:
print("")
def printGoodsList(ilt):
tplt = "{:4}\t{:8}\t{:16}"
print(tplt.format("序号", "价格", "商品名称"))
count = 0
for g in ilt:
count = count + 1
print(tplt.format(count, g[0], g[1]))
if __name__ == '__main__':
goods = '书包'
depth = 3
start_url = 'https://s.taobao.com/search?q=' + goods
infoList = []
for i in range(depth):
try:
url = start_url + '&s=' + str(44 * i)
html = getHTMLText(url)
parsePage(infoList, html)
except:
continue
printGoodsList(infoList)
cookie and user-agent acquisition method in the link there, namely F12, as shown in Taobao after login:
