Knowledge points:
1. Analysis of request header encryption parameters
2. Multiple implementation methods of JS base64 encryption
Target website: aHR0cHM6Ly93d3cub2tsaW5rLmNvbS96aC1jbi9idGMvdHgtbGlzdD9saW1pdD0yMCZwYWdlTnVtPTE=
By capturing packets and analyzing the request, it is not difficult to find that there is a parameter x-apiKey in the request headers of the website.
Search for x-apiKey, and find the following results in the index.js file:
Search for getApiKey in the index.js file, and it is not difficult to find that getApiKey processes the two parameters e and t by the comb function, and the comb function performs base64 encoding on the spliced parameters (as shown in the box btoa).
In this way, as long as the source of the two parameters is found, the problem will be solved.
It is not difficult to find from the above figure that the generation position of the two parameters:
By searching the two function names of encryptApiKey and encryptTime, it is not difficult to find out the principle of their generation.
Extract the above js code, create a new js file okyunlian.js, and write a function entry get_x_apiKey() that generates x-apiKey parameters . The js file is as follows:
// const CryptoJS =require('crypto-js')
const CryptoJS=require('D:\\nodejs\\node_modules\\crypto-js\\crypto-js')
function encryptApiKey() {
API_KEY = "a2c903cc-b31e-4547-9299-b6d07b7631ab"
var t = API_KEY
, e = t.split("")
, r = e.splice(0, 8);
return e.concat(r).join("")
}
function encryptTime(t) {
o = 1111111111111
var e = (1 * t + o).toString().split("")
, r = parseInt(10 * Math.random(), 10)
, n = parseInt(10 * Math.random(), 10)
, i = parseInt(10 * Math.random(), 10);
return e.concat([r, n, i]).join("")
}
function comb(t, e) {
var r = "".concat(t, "|").concat(e);
base64 = Buffer.from(r, 'utf-8').toString('base64') //base64 加密
// console.log(base64)
// 另一种方法加密
var str=CryptoJS.enc.Utf8.parse(r);
var base64=CryptoJS.enc.Base64.stringify(str)
// console.log(base64)
return base64
}
function get_x_apiKey(){
t=(new Date).getTime()
t=encryptTime(t)
// console.log(t)
e = encryptApiKey()
x_apiKey=comb(e,t)
console.log(x_apiKey)
return x_apiKey
}
// console.log(get_x_apiKey())
// console.log(module.paths)
The corresponding python code is as follows:
import requests,json,csv,time,execjs
x_apiKey=execjs.compile(open('./okyunlian.js','r',encoding='utf-8').read()).call('get_x_apiKey')
# print(x_apiKey)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
'x-apiKey': x_apiKey
}
url='https://www.oklink.com/api/explorer/v1/btc/transactionsNoRestrict'
def main():
for page in range(1,4):
# print(page)
params = {
't': str(int(time.time()*1000)),
'limit': '20',
'offset': (page-1)*20,
}
res=requests.get(url=url,params=params,headers=headers,timeout=2)
print(res.status_code)
json_data=res.json()['data']['hits']
print(json_data)
# json_data=json.loads(res.text)
# print(json_data)
if __name__=='__main__':
main()
The crawling results are as follows:
