一，原理
说得简单些，就是获取到页面的html,然后从中提取想要的数据，整理成自己的接口。
所以说，理论上，所有的后端语言，只要能发送请求的，都可以。
二，koa配合axios
koa是node的一个轻量化后台框架。axios是http请求库，于是就可以利用koa搭建后台服务器，然后axios发送请求，获取到页面的html,然后处理数据，形成后端接口。
三，处理数据的工具
我们爬取的数据，还需要进行处理，我这边获取到的主要是html形式的数据，这个库就可以：
npm install cheerio
这是它的使用方法：
https://www.jianshu.com/p/629a81b4e013
四，我的代码
在koa的router文件中：
const Router = require('koa-router');
const cheerio = require('cheerio');//jq处理dom元素的库
const axios = require("axios");
let router = new Router();
router.get('/api/search',async(ctx) => {
//定义一个数组，用来放爬回的数据
const books = [];
const res=await axios.get("https://s.weibo.com/top/summary")
if(res.data){
res.data=res.data.slice(res.data.indexOf("<section"),res.data.indexOf("</section>")+10);
}
const $ = cheerio.load(res.data);//这里是选择页面结构的标签，根据具体情况按需选择
console.log("返回的数据",res.data)
// //jquery的each遍历的回调函数的第一参数是表示索引,这个返回的是所有满足条件的dom对象组成的数组
$(".list .list_a li").each(function (index) {
let book = {
};
//给每个数据一个id
book.id = index;
//跳转详情路径
book.url = "https://s.weibo.com"+$(this).find("a").attr("href");
book.num= $(this).find("a span").text()
//把对象放进数组中
books.push(book);
});
console.log("测试返回的数据",books)
let json = {
success:true,
data:books
};
ctx.response.body = json;
})
module.exports = router;
于是接口的返回：
具体的：
测试返回的数据 [
{
id: 0,
url: 'https://s.weibo.com/weibo?q=%23%E6%9C%80%E9%AB%98%E6%A3%80%E5%8F%91%E5%B8%835%E8%B5%B7%E6%B6%89%E7%96%AB%E7%8A%AF%E7%BD%AA%E5%85%B8%E5%9E%8B%E6%A1%88%E4%BE%8B%23&Refer=new_time',
num: '最高检发布5起涉疫犯罪典型案例'
},
{
id: 1,
url: 'https://s.weibo.com/weibo?q=%23%E4%BA%8E%E6%9C%88%E4%BB%99%E4%B8%88%E5%A4%AB%E5%8F%91%E8%AE%A3%E5%91%8A%23&Refer=top',
num: '于月仙丈夫发讣告2351913 '
},
{
id: 2,
url: 'https://s.weibo.com/weibo?q=%23%E5%9B%9B%E5%B7%9D10%E9%A2%97%E4%BD%9B%E5%A4%B4%E8%A2%AB%E7%9B%974%E5%B9%B4%E7%BB%88%E5%9B%9E%E5%AE%B6%23&Refer=top',
num: '四川10颗佛头被盗4年终回家1506561 '
},
{
id: 3,
url: 'https://s.weibo.com/weibo?q=%23%E9%83%91%E5%B7%9E%E6%9A%82%E5%81%9C%E5%85%A8%E5%B8%82%E6%96%B0%E5%86%A0%E7%97%85%E6%AF%92%E7%96%AB%E8%8B%97%E6%8E%A5%E7%A7%8D%23&Refer=top',
num: '郑州暂停全市新冠病毒疫苗接种1213761 '
},
{
id: 4,
url: 'https://s.weibo.com/weibo?q=%23%E8%A2%AB%E9%97%AF%E6%88%BF%E9%97%B4%E5%A5%B3%E5%AD%90%E5%90%91%E5%85%A8%E5%AD%A3%E9%85%92%E5%BA%97%E7%B4%A2%E8%B5%941%E5%85%83%23&Refer=top',
num: '被闯房间女子向全季酒店索赔1元1189174 '
},
{
id: 5,
url: 'https://s.weibo.com/weibo?q=%23%E5%AD%90%E5%A5%B3%E5%8E%BB%E4%B8%96%E8%A6%81%E7%9E%92%E4%BD%8F%E8%80%81%E4%BA%BA%E5%90%97%23&Refer=top',
num: '子女去世要瞒住老人吗1165816 '
},
{
id: 6,
url: 'https://s.weibo.com/weibo?q=%E4%BD%A0%E6%98%AF%E6%88%91%E7%9A%84%E8%8D%A3%E8%80%80&Refer=top',
num: '你是我的荣耀930110 '
},
{
id: 7,
url: 'https://s.weibo.com/weibo?q=%23%E5%8F%99%E5%88%A9%E4%BA%9A%E4%B9%92%E4%B9%93%E5%A5%B3%E5%AD%A9%E8%AF%B4%E5%92%8C%E5%81%B6%E5%83%8F%E9%A9%AC%E9%BE%99%E6%89%93%E6%8B%9B%E5%91%BC%E4%BA%86%23&Refer=top',
num: '叙利亚乒乓女孩说和偶像马龙打招呼了832515 '
},
{
id: 8,
url: 'https://s.weibo.com/weibo?q=%23%E6%B9%96%E5%8D%97%E4%B8%80%E7%94%B7%E5%AD%90%E8%A2%AB%E9%9B%B7%E5%87%BB%E8%BA%AB%E4%BA%A1%23&Refer=top',
num: '湖南一男子被雷击身亡806698 '
},
{
id: 9,
url: 'https://s.weibo.com/weibo?q=%23%E6%9D%A8%E5%80%A9%E5%85%A8%E7%BA%A2%E5%A9%B5%E7%AD%89%E5%A5%A5%E8%BF%90%E5%81%A5%E5%84%BF%E8%8E%B7%E4%B8%AD%E5%9B%BD%E9%9D%92%E5%B9%B4%E4%BA%94%E5%9B%9B%E5%A5%96%E7%AB%A0%23&Refer=top',
num: '杨倩全红婵等奥运健儿获中国青年五四奖章693290 '
},
{
id: 10,
url: 'https://s.weibo.com/weibo?q=%23%E4%BA%BA%E6%B0%91%E6%97%A5%E6%8A%A5%E8%AF%84%E9%98%BF%E9%87%8C%E5%91%98%E5%B7%A5%E8%A2%AB%E6%80%A7%E4%BE%B5%E4%BA%8B%E4%BB%B6%23&Refer=top',
num: '人民日报评阿里员工被性侵事件675183 '
},
{
id: 11,
url: 'https://s.weibo.com/weibo?q=%23%E7%A7%A6%E7%9A%87%E5%B2%9B%E8%90%BD%E6%B0%B4%E8%A2%AB%E6%95%91%E5%A5%B3%E5%AD%A9%E5%AE%B6%E5%B1%9E%E9%81%93%E6%AD%89%23&Refer=top',
num: '秦皇岛落水被救女孩家属道歉659300 '
},
{
id: 12,
url: 'https://s.weibo.com/weibo?q=%2329%E5%B2%81%E5%BA%9F%E5%93%81%E6%94%B6%E8%B4%AD%E4%B8%AA%E4%BD%93%E6%88%B7%E5%A4%B1%E8%81%94%E5%B7%B24%E4%B8%AA%E6%9C%88%23&Refer=top',
num: '29岁废品收购个体户失联已4个月593510 '
},
{
id: 13,
url: 'https://s.weibo.com/weibo?q=%23%E4%B9%A1%E6%9D%91%E7%88%B1%E6%83%85%E5%AE%98%E6%96%B9%E5%BE%AE%E5%8D%9A%E6%82%BC%E5%BF%B5%E4%BA%8E%E6%9C%88%E4%BB%99%23&Refer=top',
num: '乡村爱情官方微博悼念于月仙518462 '
},
{
id: 14,
url: 'https://s.weibo.com/weibo?q=%23%E8%B5%B5%E4%B8%BD%E9%A2%96%E6%BC%94%E7%9A%84%E5%90%B4%E6%81%AC%E9%9D%99%E5%A5%BD%E7%BE%8E%23&Refer=top',
num: '赵丽颖演的吴恬静好美506017 '
},
{
id: 15,
url: 'https://s.weibo.com/weibo?q=%23%E6%9D%8E%E5%85%89%E6%B4%99%E8%B0%88%E4%B8%8E%E6%9D%8E%E5%85%88%E5%BD%AC%E6%81%8B%E6%83%85%23&Refer=top',
num: '李光洙谈与李先彬恋情447768 '
},
{
id: 16,
url: 'https://s.weibo.com/weibo?q=%23%E6%9D%A8%E6%B4%8B%E7%BF%BB%E7%89%8C%23&Refer=top',
num: '杨洋翻牌446911 '
},
{
id: 17,
url: 'https://s.weibo.com/weibo?q=%23%E4%BA%8E%E9%80%94%E5%AF%B9%E6%99%B6%E6%99%B6%E8%AF%B4%E4%BA%86%E4%B8%89%E9%81%8D%E5%AF%B9%E4%B8%8D%E8%B5%B7%23&Refer=top',
num: '于途对晶晶说了三遍对不起446102 '
},
{
id: 18,
url: 'https://s.weibo.com/weibo?q=%23%E8%8D%A3%E8%80%80%E5%A4%AB%E5%A6%87%E6%88%90%E4%BA%BA%E5%BC%8F%E6%81%8B%E7%88%B1%23&Refer=top',
num: '荣耀夫妇成人式恋爱445593 '
},
{
id: 19,
url: 'https://s.weibo.com/weibo?q=%23%E4%B8%AD%E5%9B%BD%E8%8F%9C%E9%A2%9C%E5%80%BC%E5%A4%A9%E8%8A%B1%E6%9D%BF%23&Refer=top',
num: '中国菜颜值天花板444866 '
},
{
id: 20,
url: 'https://s.weibo.com/weibo?q=%23%E5%A4%AE%E8%A7%86%E6%B7%B1%E5%BA%A6%E8%A7%A3%E6%9E%90%E7%BE%8E%E5%9B%BD%E6%8A%97%E7%96%AB%E7%9C%9F%E7%9B%B8%23&Refer=top',
num: '央视深度解析美国抗疫真相438615 '
},
{
id: 21,
url: 'https://s.weibo.com/weibo?q=%23%E5%BC%A0%E6%9D%B0%E8%B5%B5%E4%B8%BD%E9%A2%96%E9%AD%8F%E5%A4%A7%E5%8B%8B%E5%AF%B9%E6%89%8B%E6%88%8F%23&Refer=top',
num: '张杰赵丽颖魏大勋对手戏437918 '
},
{
id: 22,
url: 'https://s.weibo.com/weibo?q=%23%E9%82%93%E8%B6%85%E6%99%92%E5%B0%8F%E8%8A%B1%E7%AD%89%E7%AD%89%E7%9A%84%E8%A1%A8%E7%99%BD%23&Refer=top',
num: '邓超晒小花等等的表白437611 '
},
{
id: 23,
url: 'https://s.weibo.com/weibo?q=%23%E8%BF%99%E6%89%8D%E6%98%AF%E6%9C%80%E4%B8%87%E8%83%BD%E7%9A%84%E5%9B%9E%E5%A4%8D%23&Refer=top',
num: '这才是最万能的回复435621 '
},
{
id: 24,
url: 'https://s.weibo.com/weibo?q=%E4%B8%80%E4%BA%BA%E4%B9%8B%E4%B8%8B&Refer=top',
num: '一人之下373297 '
},
{
id: 25,
url: 'https://s.weibo.com/weibo?q=%23%E6%B0%B4%E5%9F%8E%E5%A8%81%E5%B0%BC%E6%96%AF%E9%81%AD%E6%B0%B4%E6%B7%B9%E6%B0%91%E4%BC%97%E6%B7%A1%E5%AE%9A%E9%81%9B%E7%8B%97%23&Refer=top',
num: '水城威尼斯遭水淹民众淡定遛狗369508 '
},
{
id: 26,
url: 'https://s.weibo.com/weibo?q=%23%E5%85%A8%E7%BA%A2%E5%A9%B5%E7%AB%A5%E5%B9%B4%E7%85%A7%23&Refer=top',
num: '全红婵童年照359829 '
},
{
id: 27,
url: 'https://s.weibo.com/weibo?q=%23%E5%A5%B3%E5%AD%90%E5%B0%8699000%E5%85%83%E7%8E%B0%E9%87%91%E5%BD%93%E5%9E%83%E5%9C%BE%E4%B8%A2%E6%8E%89%23&Refer=top',
num: '女子将99000元现金当垃圾丢掉355299 '
},
{
id: 28,
url: 'https://s.weibo.com/weibo?q=%23%E4%BD%A0%E6%98%AF%E6%88%91%E7%9A%84%E8%8D%A3%E8%80%80%E5%90%BB%E6%88%8F%23&Refer=top',
num: '你是我的荣耀吻戏348768 '
},
{
id: 29,
url: 'https://s.weibo.com/weibo?q=%23%E4%BA%8E%E6%9C%88%E4%BB%99%E4%B9%98%E5%9D%90%E8%BD%A6%E8%BE%86%E6%92%9E%E4%B8%8A%E9%AA%86%E9%A9%BC%E8%87%B4%E8%BD%A6%E7%A5%B8%23&Refer=top',
num: '于月仙乘坐车辆撞上骆驼致车祸318432 '
},
{
id: 30,
url: 'https://s.weibo.com/weibo?q=%23%E6%B5%8E%E5%8D%97%E5%8D%8E%E8%81%94%E8%B6%85%E5%B8%82%E5%9B%9E%E5%BA%94%E9%98%BF%E9%87%8C%E5%91%98%E5%B7%A5%E8%A2%AB%E4%BE%B5%E5%AE%B3%E4%BA%8B%E4%BB%B6%23&Refer=top',
num: '济南华联超市回应阿里员工被侵害事件310028 '
},
{
id: 31,
url: 'https://s.weibo.com/weibo?q=%23%E4%B9%94%E6%99%B6%E6%99%B6%E6%9C%89%E7%82%B9%E6%84%BF%E6%84%8F%E4%BD%86%E5%8F%88%E4%B8%8D%E5%BC%80%E5%BF%83%23&Refer=top',
num: '乔晶晶有点愿意但又不开心306117 '
},
{
id: 32,
url: 'https://s.weibo.com/weibo?q=%23%E7%AB%A0%E5%AD%90%E6%80%A1%E6%82%BC%E5%BF%B5%E4%BA%8E%E6%9C%88%E4%BB%99%23&Refer=top',
num: '章子怡悼念于月仙284821 '
},
{
id: 33,
url: 'https://s.weibo.com/weibo?q=%2316%E5%B2%81%E5%AD%A6%E7%94%9F%E5%8F%82%E5%8A%A0%E6%B2%99%E6%BC%A0%E6%8E%A2%E9%99%A9%E8%BA%AB%E4%BA%A1%23&Refer=top',
num: '16岁学生参加沙漠探险身亡283635 '
},
{
id: 34,
url: 'https://s.weibo.com/weibo?q=%23%E4%BD%9B%E5%B1%B124%E5%B2%81%E8%84%8A%E6%A4%8E%E7%95%B8%E5%BD%A2%E5%A5%B3%E5%AD%A9%E5%87%BA%E5%BE%81%E6%AE%8B%E5%A5%A5%E4%BC%9A%23&Refer=top',
num: '佛山24岁脊椎畸形女孩出征残奥会266319 '
},
{
id: 35,
url: 'https://s.weibo.com/weibo?q=%23%E7%99%BD%E9%B9%BF%E7%BB%99%E8%BE%A3%E7%9B%AE%E6%B4%8B%E5%AD%90%E7%82%B9%E5%A4%96%E5%8D%96%23&Refer=top',
num: '白鹿给辣目洋子点外卖264043 '
},
{
id: 36,
url: 'https://s.weibo.com/weibo?q=%23%E5%BC%A0%E8%89%BA%E5%85%B4%E6%BC%94%E6%9E%97%E8%AD%A6%E5%AE%98%23&Refer=top',
num: '张艺兴演林警官248328 '
},
{
id: 37,
url: 'https://s.weibo.com/weibo?q=%E6%89%AB%E9%BB%91%E9%A3%8E%E6%9A%B4&Refer=top',
num: '扫黑风暴237753 '
},
{
id: 38,
url: 'https://s.weibo.com/weibo?q=%23%E5%8D%8E%E5%B0%91%E8%AF%B4%E8%AE%B8%E6%98%95%E6%98%AF%E5%9B%BD%E4%B9%92%E5%A5%BD%E5%A3%B0%E9%9F%B3%23&Refer=top',
num: '华少说许昕是国乒好声音219751 '
},
{
id: 39,
url: 'https://s.weibo.com/weibo?q=%23%E8%85%BE%E8%AE%AF%E5%AD%A6%E7%94%9F%E8%BA%AB%E4%BB%BD%E4%BF%A1%E6%81%AF%E5%A4%84%E7%90%86%E4%B8%93%E5%88%A9%E8%8E%B7%E6%8E%88%E6%9D%83%23&Refer=top',
num: '腾讯学生身份信息处理专利获授权196349 '
},
{
id: 40,
url: 'https://s.weibo.com/weibo?q=%23CGTN%E6%99%BA%E5%BA%93%E6%B0%91%E8%B0%83%E8%B6%8580%25%E7%BD%91%E5%8F%8B%E6%94%AF%E6%8C%81%E5%AF%B9%E7%BE%8E%E8%B0%83%E6%9F%A5%23&Refer=top',
num: 'CGTN智库民调超80%网友支持对美调查192383 '
},
{
id: 41,
url: 'https://s.weibo.com/weibo?q=%23%E5%A4%AE%E8%A7%86%E6%9B%9D%E5%85%89%E5%85%8D%E8%B4%B9%E9%A2%86%E7%BA%B8%E5%B7%BE%E9%AA%97%E5%B1%80%23&Refer=top',
num: '央视曝光免费领纸巾骗局191081 '
},
{
id: 42,
url: 'https://s.weibo.com/weibo?q=%23%E4%BD%A0%E6%98%AF%E6%88%91%E7%9A%84%E8%8D%A3%E8%80%80%E8%B6%85%E5%89%8D%E7%82%B9%E6%92%AD%23&Refer=top',
num: '你是我的荣耀超前点播190777 '
},
{
id: 43,
url: 'https://s.weibo.com/weibo?q=%23%E7%9C%8B%E7%94%84%E5%AC%9B%E4%BC%A0%E7%9A%84%E6%88%91%E4%BB%BF%E4%BD%9B%E5%9C%A8%E7%85%A7%E9%95%9C%E5%AD%90%23&Refer=top',
num: '看甄嬛传的我仿佛在照镜子185837 '
},
{
id: 44,
url: 'https://s.weibo.com/weibo?q=%23%E7%AB%A0%E9%B1%BC%E5%B0%8F%E4%B8%B8%E5%AD%90%E5%8A%A8%E5%9B%BE%23&Refer=top',
num: '章鱼小丸子动图180897 '
},
{
id: 45,
url: 'https://s.weibo.com/weibo?q=%23%E6%9D%A8%E7%9A%93%E7%84%B6%E7%9A%84%E9%9A%94%E7%A6%BBvlog%23&Refer=top',
num: '杨皓然的隔离vlog180176 '
},
{
id: 46,
url: 'https://s.weibo.com/weibo?q=%23%E6%9D%91%E6%B0%91%E8%B0%88%E6%89%AC%E5%B7%9E1%E4%BA%BA%E4%BC%A0%E5%A4%9A%E4%BA%BA%E6%A3%80%E6%B5%8B%E7%82%B9%23&Refer=top',
num: '村民谈扬州1人传多人检测点167653 '
},
{
id: 47,
url: 'https://s.weibo.com/weibo?q=%23%E9%87%8D%E5%BA%86%E5%A7%90%E5%BC%9F%E5%9D%A0%E4%BA%A1%E6%A1%88%E6%AF%8D%E4%BA%B2%E7%A7%B0%E8%A2%AB%E5%91%8A%E5%A4%9A%E6%AC%A1%E7%8B%A1%E8%BE%A9%23&Refer=top',
num: '重庆姐弟坠亡案母亲称被告多次狡辩164989 '
},
{
id: 48,
url: 'https://s.weibo.com/weibo?q=%23%E8%88%AA%E5%A4%A9%E5%91%98%E7%9A%84%E6%B1%97%E8%A6%81%E6%90%9C%E9%9B%86%E8%B5%B7%E6%9D%A5%E5%88%B6%E6%B0%A7%23&Refer=top',
num: '航天员的汗要搜集起来制氧161364 '
url: 'https://s.weibo.com/weibo?q=%23%E6%9D%A8%E7%9A%93%E7%84%B6%E7%9A%84%E9%9A%94%E7%A6%BBvlog%23&Refer=top',
num: '杨皓然的隔离vlog180176 '
},
{
id: 46,
url: 'https://s.weibo.com/weibo?q=%23%E6%9D%91%E6%B0%91%E8%B0%88%E6%89%AC%E5%B7%9E1%E4%BA%BA%E4%BC%A0%E5%A4%9A%E4%BA%BA%E6%A3%80%E6%B5%8B%E7%82%B9%23&Refer=top',
num: '村民谈扬州1人传多人检测点167653 '
},
{
id: 47,
url: 'https://s.weibo.com/weibo?q=%23%E9%87%8D%E5%BA%86%E5%A7%90%E5%BC%9F%E5%9D%A0%E4%BA%A1%E6%A1%88%E6%AF%8D%E4%BA%B2%E7%A7%B0%E8%A2%AB%E5%91%8A%E5%A4%9A%E6%AC%A1%E7%8B%A1%E8%BE%A9%23&Refer=top',
num: '重庆姐弟坠亡案母亲称被告多次狡辩164989 '
},
{
id: 48,
url: 'https://s.weibo.com/weibo?q=%23%E8%88%AA%E5%A4%A9%E5%91%98%E7%9A%84%E6%B1%97%E8%A6%81%E6%90%9C%E9%9B%86%E8%B5%B7%E6%9D%A5%E5%88%B6%E6%B0%A7%23&Refer=top',
num: '航天员的汗要搜集起来制氧161364 '
},
{
id: 49,
url: 'https://s.weibo.com/weibo?q=%23%E5%A6%82%E4%BD%95%E7%9C%8B%E5%BE%85%E9%AB%98%E6%95%8F%E8%AF%B4%E8%A6%81%E5%86%B7%E9%9D%99%E7%9C%8B%E5%BE%85%E5%85%A8%E7%BA%A2%E5%A9%B5%E7%9A%84%E6%88%90%E5%8A%9F%23&Refer=top',
num: '如何看待高敏说要冷静看待全红婵的成功161043 '
},
{
id: 50,
url: 'https://s.weibo.com/weibo?q=%23%E5%8C%97%E4%BA%AC%E8%8D%AF%E5%BA%97%E5%AF%B9%E5%9B%9B%E7%B1%BB%E8%8D%AF%E5%93%81%E5%AE%9E%E5%90%8D%E7%99%BB%E8%AE%B0%23&Refer=top',
num: '北京药店对四类药品实名登记160117 '
}
]