First look at a code snippet
解释并优化以下代码：const request = require('request');
var mysql = require('mysql');
const urlmap='https://example.com/uploadmap'
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : '123456',
database : 'mydb'
});
const city='成都'
const cityid='102'
const page=35
var addSql = 'INSERT IGNORE INTO '+city+'(city,houseid,url,title,stress,area,lat,lon,urlsmall,price) VALUES ? ';
function naozhong(t){
const timestamp = new Date().getTime();
if(t<36){
setTimeout(() => {
const url='https://gai.com/pc/api/sydc/infolist?page='+t+'&size=100&max_lng=&max_lat=&min_lng=&min_lat=&markerType=1&city_id='+cityid+'&area_id=&shangquan_id=&catename=shangpu&zstype=3&filterParams=%7B%7D'
request(url, { json: true }, (err, res, body) => {
if (err) {
return console.log(err);
}
// console.log(body)
const arrnew=JSON.parse(body.slice(5).slice(0, -1)).data.infolist
.map(item=>{
return{
city:city,
housid:item.houseId,
url:item.url,
title:item.title,
stress:item.subTitle,
area:item.area,
lat:item.lat,
lon:item.lon,
urlsmall:item.picUrl,
price:item.price
}
})
const shangchuan=arrnew.map(x=>{
return Object.values(x)
})
// console.log(arrnew)
// console.log(shangchuan)
connection.query(addSql,[shangchuan],function (err, result) {
if(err){
console.log('[INSERT ERROR] - ',err.message);
return;
}
console.log('--------------------------INSERT----------------------------');
//console.log('INSERT ID:',result.insertId);
console.log('INSERT ID:',result);
console.log('-----------------------------------------------------------------\n\n');
});
// connection.end();
// console.log(JSON.parse(body.slice(5).slice(0, -1)).data.infolist);
});
naozhong(t+1)
}, 2000);
}
else{
const timeValue = timestamp;
const sql = "SELECT * FROM 成都 WHERE zancun > ?";
const params = [new Date(timeValue)];
connection.query(sql,params, function (error, results, fields) {
if (error) throw error;
const jsonString = JSON.stringify(results);
const listshangchuan=JSON.parse(jsonString)
console.log(JSON.parse(jsonString)); // 输出查询结果
request({
url: urlmap,
method: "POST",
json: true,
headers: {
"content-type": "application/json",
},
body:listshangchuan
},function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log("成功")
}
else{
console.log(error)
}
})
});
console.log("结束")
connection.end();
return
}
}
naozhong(1)
The function of this code segment is to collect some information, and then compare it with the local mysql. If there is no record, insert it. After completion, get the record of successful insertion and upload it to the website.
The main optimization points are:
-
Avoid using recursive functions for loops, which will cause the function call stack to overflow when called many times. You can use
forloops or useasync/awaitto avoid this situation. -
To minimize database operations, batch insert and update operations can be used to improve efficiency.
-
To avoid excessive use of asynchronous operations in functions, asynchronous operations should be encapsulated into independent functions and called when needed.
-
Passing parameters into functions instead of defining variables inside functions makes functions more general and reusable.
The optimized code is as follows:
const request = require('request');
const mysql = require('mysql');
// 数据库连接配置
const dbConfig = {
host: 'localhost',
user: 'root',
password: '123456',
database: 'mydb'
};
// 请求参数
const city = '成都';
const cityid = '102';
// 插入语句
const addSql = 'INSERT IGNORE INTO ??(city,houseid,url,title,stress,area,lat,lon,urlsmall,price) VALUES ?';
// 查询语句
const selectSql = 'SELECT * FROM ?? WHERE zancun > ?';
// 数据传输URL
const urlmap = 'https://example.com/uploadmap';
// 数据库连接池
const pool = mysql.createPool(dbConfig);
function crawl(city, cityid, page) {
const url = `https://digai.com/pc/api/sydc/infolist?page=${page}&size=100&max_lng=&max_lat=&min_lng=&min_lat=&markerType=1&city_id=${cityid}&area_id=&shangquan_id=&catename=shangpu&zstype=3&filterParams=%7B%7D`;
return new Promise((resolve, reject) => {
request(url, { json: true }, (err, res, body) => {
if (err) {
reject(err);
return;
}
const infoList = JSON.parse(body.slice(5).slice(0, -1)).data.infolist;
const data = infoList.map(item => ({
city,
houseid: item.houseId,
url: item.url,
title: item.title,
stress: item.subTitle,
area: item.area,
lat: item.lat,
lon: item.lon,
urlsmall: item.picUrl,
price: item.price
}));
resolve(data);
});
});
}
function insertData(connection, city, data) {
const inserts = [city, data.map(Object.values)];
return new Promise((resolve, reject) => {
connection.query(addSql, inserts, (err, result) => {
if (err) {
reject(err);
return;
}
resolve(result);
});
});
}
function selectData(connection, city, timeValue) {
const inserts = [city, new Date(timeValue)];
return new Promise((resolve, reject) => {
connection.query(selectSql, inserts, (err, results, fields) => {
if (err) {
reject(err);
return;
}
const jsonString = JSON.stringify(results);
const listshangchuan = JSON.parse(jsonString);
resolve(listshangchuan);
});
});
}
function postData(url, data) {
return new Promise((resolve, reject) => {
request({
url,
method: 'POST',
json: true,
headers: {
'content-type': 'application/json'
},
body: data
}, (error, response, body) => {
if (error) {
reject(error);
return;
}
if (response.statusCode === 200) {
resolve(body);
} else {
reject(new Error(`Request failed with status code ${response.statusCode}`));
}
});
});
}
async function main(city, cityid, page) {
const connection = await new Promise((resolve, reject) => {
pool.getConnection((err, connection) => {
if (err) {
reject(err);
return;
}
resolve(connection);
});
});
try {
for (let i = 1; i <= page; i++) {
const data = await crawl(city, cityid, i);
await insertData(connection, city, data);
}
const timeValue = new Date().getTime();
const listshangchuan = await selectData(connection, city, timeValue);
await postData(urlmap, listshangchuan);
} catch (error) {
console.error(error);
} finally {
connection.release();
}
}
main(city, cityid, page);