超简单的node爬虫小案例

同前端爬取参数一样，输入三个参数进行爬取

注意点也一样：
注意分页的字段需要在代码里面定制化修改，根据你爬取的接口，他的业务规则改代码中的字段。比如我这里总条数叫total，人家的不一定。返回的数据我这里是data.rows，看看人家的是叫什么字段，改改代码。再比如我这里的分页叫pageNum，人家的可能叫pageNo

在这里插入图片描述
data目录手动建立上哦，要放爬下来的数据

依赖

{"dependencies": {"axios": "^1.6.5","colors": "^1.4.0","fs": "^0.0.1-security","readline": "^1.3.0"}
}

代码

const readline = require("readline");
const axios = require('axios');
const fs = require("fs");
const colors = require('colors');// 创建询问实例
let RL = readline.createInterface({input: process.stdin,output: process.stdout
})// 封装异步询问
function question(question) {return new Promise((resolve, reject) => {RL.question(`${question}\t`, function (value) {return resolve(value);})})
}var total = 0;
var pageNum = 1;
var pageSize = 30;
var api = "";
var headers = "";
var params = "";// 循环异步方法，执行同步结果
(async function () {const questions = ["请输入接口：", "请输入请求头：", "请输入参数："];for (let i = 0; i < questions.length; i++) {const value = await question(questions[i]);if (i === 0) {api = value;} else if (i === 1) {headers = value;} else {params = value;}}RL.close();
})()// 监听readline关闭，结束终端输入
RL.on("close", function () {console.log(`<<<------------------------- 开始爬取 ------------------------->>>\n`.blue);// console.log(api, headers, params);crawling();
})// 封装接口请求
async function getData() {const response = await axios({url: api,method: "post",headers: {"Content-Type": "application/json",...JSON.parse(headers)},data: JSON.stringify({...JSON.parse(params),"pageSize": pageSize,"pageNum": pageNum})})return response.data;
}// 爬取执行入口
async function crawling() {const data = await getData();console.log(data);if (data.code !== 0) {console.log('================= 数据读取失败 ================='.red);process.exit(0);}console.log('================= 数据读取成功 ================='.green);total = data.total;const page = Math.ceil(total / pageSize);console.log(`共${page}页`);saveFile(data.rows, `第1页`);loading();
}// 持续执行爬取
async function loading() {const page = Math.ceil(total / pageSize);for (let i = 1; i < page; i++) {pageNum++;const data = await getData();saveFile(data.rows, `第${i + 1}页`);}console.log(`<<<------------------------- 爬取完毕，已下载数据 ------------------------->>>\n`.bgGreen);total = 0;pageNum = 1;process.exit(0);
}// 下载json文件
function saveFile(res, name) {console.log(`<<<------------------------- 开始写入 ------------------------->>>\n`.blue);console.log(name);const writerStream = fs.createWriteStream("data/" + name + ".json");writerStream.write(JSON.stringify(res));writerStream.end();writerStream.on('finish', function () {console.log(`<<<------------------------- 写入完成 ------------------------->>>\n`.green);});writerStream.on('error', function (err) {console.log(err);console.log(`<<<------------------------- 写入错误 ------------------------->>>\n`.red);process.exit(0);});
}