Mongodb数据库自动备份并打包上传到七牛云

目的：
防止数据库丢失造成的损失我们应该对自己的数据定期备份

目标

技术要求

js,nodejs
基本 shell 脚本编写能力
工具依赖
- shelljs：js 编写 shell 脚本
- qiniu：上传七牛云
- node-schedule： nodejs 定时

思路

mongodb 数据库导出mongodump
shell 导出打包压缩(可直接编写 shell 脚本,也可用 shelljs 编写,本文以shelljs为例)
nodejs 上传云存储(本文以七牛云为例)

上传文件到七牛云存储

const config = {
  // 七牛
  qiniu: {
    bucket: 'db-dump-server',
    accessKey: 'ssK9nIwjU**********2UOoQcW5eWOosHh7yX09',
    secretKey: 'EQ0IRb**************4QbsJO5sbZcizuM'
  }
}
const qiniu = require('qiniu')

// 七牛票据
const bucket = config.qiniu.bucket
const accessKey = config.qiniu.accessKey
const secretKey = config.qiniu.secretKey
const mac = new qiniu.auth.digest.Mac(accessKey, secretKey)
const cfg = new qiniu.conf.Config()
// const bucketManager = new qiniu.rs.BucketManager(mac, cfg)

const options = {
  scope: bucket
}
const putPolicy = new qiniu.rs.PutPolicy(options)
const uploadToken = putPolicy.uploadToken(mac)

// 空间对应的机房
// config.zone = qiniu.zone.Zone_z2
const formUploader = new qiniu.form_up.FormUploader(cfg)
const putExtra = new qiniu.form_up.PutExtra()

// 本地-上传到七牛
const uploadToQiniu = (localFile, key) => {
  const msg = '本地资源,上传七牛出错'
  // 文件上传
  return new Promise((resolve, reject) => {
    formUploader.putFile(
      uploadToken,
      key,
      localFile,
      putExtra,
      (respErr, respBody, respInfo) => {
        if (respErr) {
          reject({ msg, respErr })
        } else {
          if (respInfo.statusCode === 200) {
            resolve(respBody)
          } else {
            reject({ msg, respInfo })
          }
        }
      }
    )
  })
}

// !(async () => {
//   console.log('start')
//   const res = await uploadToQiniu(path.resolve(__dirname, './README.md'))
//   console.log('end', res)
// })()

module.exports = uploadToQiniu

shelljs 脚本编写

const path = require('path')
const shell = require('shelljs')
// shell.echo('mongodump start')

// 封装异步 exec
const exec = async exec => {
  return new Promise((resolve, reject) => {
    shell.exec(exec, { async: true }, data => {
      resolve(data)
    })
  })
}

const target = './dump'
const targetPath = path.join(target)

/**
 * 数据导出并压缩打包
 * @param {string} db 数据库实例名称
 */
const mongodump = async db => {
  await exec(`mongodump -h localhost:27017 -d ${db} -o ${targetPath}`)
  const key = `${db}-dump.${new Date().getTime()}.tar.gz`
  const newFile = path.join(target, db, key)
  const sourceFilePath = path.join(target, db)
  await exec(`tar -zcvf ${newFile} ${sourceFilePath}`)
  // console.log(res)
  return {
    newFile,
    key
  }
}
// mongodump()

// 删除压缩文件
const removeFile = async (path = targetPath) => {
  await exec(`rm -rf ${path}}`)
}

module.exports = {
  mongodump,
  removeFile
}

nodejs 定时执行

const schedule = require('node-schedule')
const upToQiniu = require('./upToQiniu')
const { mongodump, removeFile } = require('./shell/')

const run = async () => {
  console.log('start upload and remove....')
  const st = new Date().getTime()
  // 执行shell打包压缩mongodb-data
  const { newFile, key } = await mongodump('card')
  // 打包后上传
  await upToQiniu(newFile, key)
  // 上传后删除本地打包文件
  await removeFile()
  const et = new Date().getTime()
  console.log(`打包压缩上传耗时：${(et - st) / 1000}s`)
}

run()
// // 每周一凌晨3点30分执行备份
// schedule.scheduleJob('30 3 * * */1', async () => {
//   run()
// })

参考资料

数据库导出