python micro-letter domain name or link batch testing

Long time no write up a blog, straight into the subject.

We often use google search, how to extract it links to search results

google search results url extract, F12, came to console end; paste the following statement, Enter.

 

var tag=document.getElementsByClassName('r');

 for (var i=0;i<tag.length;i++){
        var a=tag[i].getElementsByTagName("a");
        console.log(a[0].href)
 }

Extracted, saved to url.txt. Url be detected and domain names, one per line, to go through to re-blank line

import io
import shutil
readPath='oldurl.txt'
writePath='url.txt'
lines_seen=set()
outfiile=io.open(writePath,'a+',encoding='utf-8')
f=io.open(readPath,'r',encoding='utf-8')
for line in f:
    if not len(line):
        continue
    if line not in lines_seen:
        outfiile.write(line)
        lines_seen.add(line)

Then batch testing

ok.txt normal domain name

red.txt has shielded domain names and links

#! /usr/bin/env python
#coding:utf-8
import os,urllib,linecache
import sys
import time
import requests

result = list()
strxx = '"Code":"102"'
html = ''
for y in linecache.updatecache(r'url.txt'):
    try:
       headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
       #response = urllib.urlopen(x)        
       #html = response.read()
       x = 'http://wx.rrbay.com/pro/wxUrlCheck.ashx?url=' +  y
       response = requests.get(x,headers=headers)
       html = response.text

       time.sleep(3)
       #print x,a
    except Exception,e:
        html = ''
        print e
    if strxx in html:
        print 'ok:'
        print x
        with open ('ok.txt','a') as f:  
            f.write(y)
    else:
        print 'error:'        
        print y
        html = ''
        with open ('red.txt','a') as f:  
            f.write(y)