redis & node

const cluster = require('cluster');

const cpuNums = require('os').cpus().length;

const redisLock = require('./redis');

// const crypto = require('crypto');

const spider = require('./action');

cluster.setupMaster({

    exec: 'work.js',

    args: ['--use', 'http']

});

function analyzeArr(arr) {

    let obj = {};

    arr.forEach((id, idx, arr) => {

        obj['work' + id] = obj['work' + id] !== void 0 ? obj['work' + id] + 1 : 1;

    });

    return obj;

}

const pageLenth= 100;

let pageNum = 0;

const startTime = new Date().getTime();

let widArr = [];

// (async () => {

//    for (let i =0; i< pageLenth;i++) {

//        await spider(i)

//    }

//    console.log(`-----------------Time:${new Date().getTime() - startTime}-------------------\n`)

// })()

redisLock.set('page', '0')

for (let i = 0; i < cpuNums; ++i) {

    let work = cluster.fork();

    work.send([i,cpuNums,pageLenth])

    console.log(`[master] : fork worker ${i + 1}\n`)

}

cluster.on('message',(worker,msg)=>{

    ++pageNum;

    // 收集进程

    widArr.push(worker.id);

    if(pageNum === pageLenth){

        console.log(`-----------------Time:${new Date().getTime() - startTime}-------------------\n`)

        console.log(analyzeArr(widArr));

        setTimeout(() => {

            cluster.disconnect();

        },1000)

    }

});

const redis = require('redis')

/**

* redis set 命令

* key value  nx(存在时才操作) px time(锁时间)

*

*/

const lockScript = 'return redis.call("set", KEYS[1], ARGV[1], "NX", "PX", ARGV[2])';

const unlockScript = 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';

// 创建redis 默认6379本地连接

redisClient = redis.createClient();

redisClient.on("error", function (err) {

    console.log("Error " + err);

});

/**

* 键 值 时间

*

* @param {any} resourse

* @param {any} uniqueStr

* @param {any} ttl

* @returns

*/

async function loop(resourse, uniqueStr, ttl) {

    return new Promise((resolve, reject) => {

        setTimeout(async () => {

            let result = await lock(resourse, uniqueStr, ttl)

            resolve(!!result)

        }, 0)

    })

}

/**

* 递归抢锁

*

* 键 值 时间

*

* @param {any} resourse

* @param {any} uniqueStr

* @param {any} ttl

* @returns

*/

async function attemplock(resourse, uniqueStr, ttl) {

    let result = await loop(resourse, uniqueStr, ttl)

    if (result) {

        return true

    } else {

        return attemplock(resourse, uniqueStr, ttl)

    }

}

/**

* 键 值 时间

*

* redis 对键上锁

*

* @param {any} resourse

* @param {any} uniqueStr

* @param {any} ttl

* @returns

*/

async function lock(resourse, uniqueStr, ttl) {

    ttl = ttl ? ttl : "30000"

    return new Promise((resolve, reject) => {

        redisClient.eval(lockScript, 1, resourse, uniqueStr, ttl, (err, reply) => {

            if (err) {

                console.error(err)

                reject(err)

            }

            resolve(reply)

        })

    })

}

/**

*  键 值 时间

*

* redis 解锁 即删除键值

*

* @param {any} resourse

* @param {any} uniqueStr

* @returns

*/

async function unlock(resourse, uniqueStr) {

    return new Promise((resolve, reject) => {

        redisClient.eval(unlockScript, 1, resourse, uniqueStr, (err, reply) => {

            if (err) {

                console.error(err)

                reject(err)

            }

            resolve(reply)

        })

    })

}

async function get(key) {

    return new Promise((resolve, reject) => {

        redisClient.get(key, (err, reply) => {

            if (err) {

                console.error(err);

                reject(err)

            }

            resolve(reply)

        })

    })

}

async function set(key, value) {

    return new Promise((resolve, reject) => {

        redisClient.set(key, value, (err, reply) => {

            if (err) {

                console.error(err)

                reject(err)

            }

            resolve(reply)

        })

    })

}

module.exports = {

    redisClient,

    attemplock,

    set,

    get,

    unlock,

    lock,

    loop

}

const spider = require('./action');

const crypto = require('crypto');

const redisLock = require('./redis');

console.log(process.pid)

process.on('message', async (param) => {

    let resourse = "lock:page"

    // 进行redis加锁，多进程操作同一值

    let uniqueStr = crypto.randomBytes(10).toString('hex')

    await redisLock.attemplock(resourse, uniqueStr)

    let nowCityNum = Number(await redisLock.get('page'))

    let flag = false;

    // 当小于最大值 进行循环

    while(nowCityNum < param[2]) {

        if(flag){

            uniqueStr = crypto.randomBytes(10).toString('hex')

            await redisLock.attemplock(resourse, uniqueStr)

            nowCityNum = Number(await redisLock.get('page'))

        }else{

            flag=true

        }

        if (nowCityNum < param[2]) {

            // 执行爬取操作

            await spider(nowCityNum);

            process.send(nowCityNum);

            nowCityNum++

            // 页码重新赋值

            await redisLock.set('page', String(nowCityNum))

            // redis删锁

            await redisLock.unlock(resourse, uniqueStr)

        }

    }

    // redis删锁

    await redisLock.unlock(resourse, uniqueStr)

    //正常退出进程

    process.exit(0)

})

// const superagent = require('superagent');

const axios = require('axios')

const api = 'http://10.35.93.10:801/document/picture/list';

module.exports = (pageStart) => {

    return new Promise((resolve, reject) => {

        axios({

            method: 'get',

            params: {

                pageStart

            },

            url: api,

            contentType: "application/json; charset=UTF-8"

        }).then(data => {

            console.log(`-------------子进程 ${process.pid} 成功爬去第${pageStart + 1}页数据-------------\n`)

            resolve(data.data.data.data);

        }).catch(err => {

            console.log(err)

            reject(err)

        })

    });

};