Python 爬虫基础部分

GET请求

    import urllib.request
    import urllib.parse
    
    url = "http://www.baidu.com/s?"
    
    keyword = input("请输入你要搜索的关键字：")
    
    # 带上请求头，为了模拟访问网站是浏览器运行的，而不是脚本运行
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3902.4 Safari/537.36"
    }
    
    # url需要带的参数
    data = {
        "ie": "utf-8",
        "wd": keyword
    }
    # 进行解析
    query_string = urllib.parse.urlencode(data)
    # ie=utf-8&wd=php
    
    # 参数拼接到url，形成一条完整的url
    url += query_string
    # http://www.baidu.com/s?ie=utf-8&wd=java
    
    # 构造请求对象
    request = urllib.request.Request(url=url, headers=headers)
    
    content = urllib.request.urlopen(request)
    
    file_name = keyword + '.html'
    with open(file_name, 'w', encoding="utf-8") as fp:
        fp.write(content.read().decode('utf8'))
        

POST请求

    import urllib.request
    import urllib.parse
    
    # json字符串转成json格式
    import json
    
    url = "https://fanyi.baidu.com/sug"
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3902.4 Safari/537.36',
    }
    keyword = input('请输入要翻译的单词：')
    form_data = {
        'kw': keyword
    }
    
    request = urllib.request.Request(url=url, headers=headers)
    
    form_data = urllib.parse.urlencode(form_data).encode()
    
    response = urllib.request.urlopen(request, form_data)
    
    # unicode转义编码
    result = json.loads(response.read().decode('unicode_escape'))
    
    # 通过字典key获取值，不用result['data']是因为如果没有data这个key那么就会报错, 如果用result.get('data')没有key就会返回None,不会报错
    for item in result.get('data'):
        print(item['k'] + '---' + item['v'])