熟练使用爬虫

1. node 原生api 的使用

const http = require('http');
var md5 = require('md5');
var querystring = require('querystring');
var url = 'xxxx/login';

var listurl = `xxx/api/list`;
var cookie = '';

function LoginApi(params) {
    const postData = querystring.stringify({
        name: 'hp',
        password: md5('hp')
    });
    return new Promise((reslove, reject) => {
        const req = http.request(url, {
            headers: {
                'Content-Type': 'application/x-www-form-urlencoded'
            },
            method: 'POST'
        }, (res) => {
            // console.log(res);
            // console.log(`响应头: ${JSON.stringify(res.headers)}`);
            res.setEncoding('utf8');
            res.on('data', (chunk) => {
                console.log(`响应主体: ${chunk}`);
            });
            res.on('end', () => {
                // console.log('响应中已无数据');
                reslove(res.headers['set-cookie']);
            });
        });
        // 将数据写入请求主体。
        req.write(postData);
        req.end();
    });
}

async function main() {
    try {
        cookie = await LoginApi();
        await list(cookie);
    } catch (error) {
        console.log('error', error);
    }
}

function list(cookie) {
    const postData = querystring.stringify({
        page: 1,
        pageSize: 40,
        auditStatus: -1
    });
    return new Promise((reslove, reject) => {
        console.log(cookie);
        const req = http.request(listurl, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/x-www-form-urlencoded',
                Cookie: cookie
            }
        }, (res) => {
            console.log(`状态码: ${res.statusCode}`);
            res.setEncoding('utf8');
            res.on('data', (chunk) => {
                console.log(`响应主体: ${chunk}`);
            });
            res.on('end', () => {
                console.log('响应中已无数据');
            });
        });
        // 将数据写入请求主体。
        // req.setHeader('Cookie', ['JSESSIONID=af793a55-0672-40a4-a1d1-0fcba8329ca2']);
        // console.log(req.getHeader('Cookie'));
        req.write(postData);
        // console.log(req);

        req.end();
    });
}
main();

axios 的使用

const axios = require('axios');
var md5 = require('md5');
var fs = require('fs');
var querystring = require('querystring');
var url = 'xxxx/login';

var listurl = `xxx/api/list`;

function LoginApi(params) {
    return axios({
        url,
        headers: {
            'Content-Type': 'application/x-www-form-urlencoded'
        },
        data: querystring.stringify({
            name: 'admin',
            password: md5('admin')
        }),
        method: 'POST'
    });
}

async function main() {
    try {
        var res = await LoginApi();
        var result = await list(res.headers['set-cookie']);
        console.log(result.data.data);
    } catch (error) {
        console.log('error', error);
    }
}

function list(cookie) {
    return axios({
        url: listurl,
        headers: {
            'Content-Type': 'application/x-www-form-urlencoded',
            Cookie: cookie
        },
        data: querystring.stringify({
            page: 1,
            pageSize: 40,
            auditStatus: -1
        }),
        method: 'POST'
    });
}
main();