Python网络爬虫与信息提取之单元一requests库入门代码实战

@TOCPython网络爬虫与信息提取之单元一requests库入门代码实战

爬取网页的通用代码框架

import requests

def getHTMLText(url):
    try:
        r = requests.get(url,timeout=30)
        r.raise_for_status() #如果状态不是200，引发HTTPError异常
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"

    if _name_ == "_main_":
        url = "http://www.hao123.com"
        print(getHTMLText(url))

小编纯手工敲打
部分有错，还望小伙伴们多多指点一二

>>> import requests
>>> r = requests.get("http://www.baidu.com")
>>> print(r.status_code)
200
>>> r.text

>>> import requests
>>> r = requests.get("http://www.baidu.com")
>>> print(r.status_code)
200
>>> type(r)
<class 'requests.models.Response'>
>>> r.headers
{'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'keep-alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Date': 'Thu, 25 Jun 2020 09:38:10 GMT', 'Last-Modified': 'Mon, 23 Jan 2017 13:28:12 GMT', 'Pragma': 'no-cache', 'Server': 'bfe/1.0.8.18', 'Set-Cookie': 'BDORZ=27315; max-age=86400; domain=.baidu.com; path=/', 'Transfer-Encoding': 'chunked'}
>>> r.text

>>> r = requests.head('http://httpbin.org/get')
r
>>> .headers
SyntaxError: invalid syntax
>>> r.headers
{'Date': 'Thu, 25 Jun 2020 10:02:28 GMT', 'Content-Type': 'application/json', 'Content-Length': '308', 'Connection': 'keep-alive', 'Server': 'gunicorn/19.9.0', 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Credentials': 'true'}
>>> r.text
''
>>> payload = {'key1':'value1','key2':'value2'}
>>> r = requests.post('http://httpbin.org/post',data = payload)
>>> print(r.text)
{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "key1": "value1", 
    "key2": "value2"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Content-Length": "23", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.22.0", 
    "X-Amzn-Trace-Id": "Root=1-5ef476c8-7815704ec7aedd86f72fc668"
  }, 
  "json": null, 
  "origin": "221.178.125.112", 
  "url": "http://httpbin.org/post"
}

>>> r = requests.post('http://httpbin.org/post',data = 'ABC')
print()
>>> print(r.text)
{
  "args": {}, 
  "data": "ABC", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Content-Length": "3", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.22.0", 
    "X-Amzn-Trace-Id": "Root=1-5ef47729-52c8f35e3c6ab34886542646"
  }, 
  "json": null, 
  "origin": "221.178.125.112", 
  "url": "http://httpbin.org/post"
}

>>> payload = {'key1':'value1','key2':'value2'}
>>> r = requests.put('http://httpbin.org/put',data = payload)
>>> print(r.text)
{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "key1": "value1", 
    "key2": "value2"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Content-Length": "23", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.22.0", 
    "X-Amzn-Trace-Id": "Root=1-5ef4779e-e65f231aa1fd3b3cf778caba"
  }, 
  "json": null, 
  "origin": "221.178.125.112", 
  "url": "http://httpbin.org/put"
}

>>> kv = {'key1':'value1','key2':'value2'}
>>> r = requests.request('GET','http://python123.io/ws',params=kv)
>>> print(r.url)
https://python123.io/ws?key1=value1&key2=value2
>>> kv = {'key1':'value1','key2':'value2'}
>>> r = requests.request('POST','http://python123.io/ws',data=kv)
>>> body = '主体内容'
>>> r = requests.request('POST','http://python123.io/ws',data=body)

>>> kv = {'key1':'value1'}
>>> r = requests.request('POST','http://python123.io/ws',json=kv)
>>> hd = {'user-agent':'Chrome/10'}
>>> r = requests.request('POST','http://python123.io/ws',headers=hd)
>>> print(r.url)
http://python123.io/ws
>>> fs = {'file':open('data.xls','rb')}
Traceback (most recent call last):
  File "<pyshell#43>", line 1, in <module>
    fs = {'file':open('data.xls','rb')}
FileNotFoundError: [Errno 2] No such file or directory: 'data.xls'
>>> r = requests.request('POST','http://python123.io/ws',files=fs)
Traceback (most recent call last):
  File "<pyshell#44>", line 1, in <module>
    r = requests.request('POST','http://python123.io/ws',files=fs)
NameError: name 'fs' is not defined
>>> r = requests.request('GET','http://www.baidu.com',timeout=10)
>>> pxs = {'':''}
>>> pxs = {'http':'http://user:[email protected]:1234''https':'https://10.10.10.1.4321'}
SyntaxError: invalid syntax
>>> pxs = {'http':'http://user:[email protected]:1234'      'https':'https://10.10.10.1.4321'}
SyntaxError: invalid syntax
>>> r = requests.request('GET','http://www.baidu.com',proxies=pxs)