试试S3+minio进行大文件上传

一、什么是Amazon S3

Amazon S3 是一种对象存储服务，提供行业领先的可扩展性、数据可用性、安全性和性能，他会将数据以对象形式存储在存储桶中，对象指的是一个文件和描述该文件的任何元数据，存储桶是对象的容器。

二、什么是Minio

Minio 是一个高性能对象存储库，他是与 Amazon S3 云存储服务兼容的 API，同时 minio 号称也是世界上速度最快的对象存储服务器，在标准硬件上对象存储的读写速度最高可以达到183GB/s和171GB/s，具体详细介绍可以查看官网。

三、安装minio镜像

1. 使用docker搜索minio

docker search minio

这个就是我们要安装的镜像。

2. 安装镜像

docker pull minio/minio:latest

3. 查看镜像是否安装成功

docker images

4. 创建容器

// 创建保存文件的目录
mkdir -p ～/docker/data

// 创建容器
docker run \
  -p 9000:9000 \
  -p 9001:9001 \
  --name minio \
  -v ~/docker/data:/data \
  // 账号密码
  -e "MINIO_ROOT_USER=xxxx" \  
  -e "MINIO_ROOT_PASSWORD=xxxx" \
  minio/minio server /data --console-address ":9001"

创建成功后浏览器打开 http://127.0.0.1:9000 就能看到 minio 的登陆页。

5. 创建桶

将桶的权限设置为可读写。

四、使用S3上传文件

1、Upload类

// Upload.js
const events = require('events');
const controller = require('@aws-sdk/abort-controller')
const clientS3 = require('@aws-sdk/client-s3')

const {EventEmitter} = events;
const {AbortController} = controller;
const {CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand} = clientS3;

// 获取文件切片
function* getChunk(data, partSize){
    let partNumber = 1;
    let startByte = 0;
    let endByte = partSize;
    while(endByte < data.byteLength) {
        yield {
            partNumber,
            data: data.slice(startByte, endByte)
        }
        partNumber++;
        startByte = endByte;
        endByte = endByte + partSize;
    }
    yield {
        partNumber,
        data: data.slice(startByte),
        lastPart: true
    }
}

// 切片最小值
const MIN_PART_SIZE = 1024 * 1024 * 5;
class Upload extends EventEmitter {
    // 发送请求的通道数量
    queueSize = 4;
    // 允许上传的的最大分片数量
    MAX_PARTS = 10000;
    partSize = MIN_PART_SIZE;
    // S3Client
    client;
    params;
    // 数据总大小
    totalBytes;
    // 中断请求
    abortController;
    // 切片数组
    uploadedParts = [];
    // 每一项都是一个promise，用来描述当前通道的上传过程
    concurrentUploaders = [];
    createMultiPartPromise;
    // 迄今为止已近上传的文件大小
    bytesUploadedSoFar = 0;
    // 上传文件时注册的事件
    uploadEvent = 'httpUploadProgress';
    // 上传单个文件的结果
    singleUploadResult;
    // 是否是分片上传
    isMultiPart = true;
    uploadId;
    constructor(options) {
        super();
        this.queueSize = options.queueSize || this.queueSize;
        this.partSize = options.partSize || this.partSize;
        this.client = options.client;
        this.params = options.params;
        // Body为Buffer数据
        this.totalBytes = this.params.Body.byteLength;
        this.abortController = new AbortController();
    }
    // 中断请求
    abort() {
        this.abortController.abort();
    }
    // 发送请求，如果请求成功或者被中断则退出
    async done() {
        return await Promise.race([
            this.doMultipartUpload(),
            this.abortTimeout(this.abortController.signal)
        ])
    }
    // 注册进度事件的回调函数
    on(event = 'httpUploadProgress', callback){
        this.uploadEvent = event
        // 往EventEmitter上挂载事件
        return super.on(event, callback);
    }
    async abortTimeout(abortAignal) {
        return new Promise((resolve, reject) => {
            abortAignal.onabort = () => {
                const error = new Error(' Upload is aborted')
                reject(error);
            }
        })
    }
    async doMultipartUpload() {
        const dataFeeder = getChunk(this.params.Body, this.partSize);
        // 四个通道并行上传文件
        for(let index = 0; index < this.queueSize; index++) {
            const currentUpload = this.doConcurrentUpload(dataFeeder);
            this.concurrentUploaders.push(currentUpload);
        }
        // console.log(this.concurrentUploaders)
        await Promise.all(this.concurrentUploaders);
        let result;
        if (this.isMultiPart) {
            // 通过PartNumber对已经上传的切片进行排序
            this.uploadedParts.sort((a, b) => a.PartNumber - b.PartNumber);

            const uploadCompleteParams = {
                ...this.params,
                Body: undefined,
                UploadId: this.uploadId,
                MultipartUpload: {
                    Parts: this.uploadedParts,
                },
            };
            // console.log(uploadCompleteParams.Parts)
            // 在将所有的数据Part上传完成之后，必须调用CompleteMultipartUploadCommand API来完成整个文件的上传
            result = await this.client.send(new CompleteMultipartUploadCommand(uploadCompleteParams));
        } else {
            result = this.singleUploadResult;
        }
        return result;
    }

    async doConcurrentUpload(dataFeeder) {
        // dataPart有可能是一个Promise，所以需要使用await获取resolve的值
        for await (let dataPart of dataFeeder) {
            if (this.abortController.signal.aborted) {
                return;
            }
            if (this.uploadedParts.length > this.MAX_PARTS) {
                throw new Error(`每个通道最多只能上传${this.MAX_PARTS}个分片`);
            }
            // 如果只有一个分片
            if (dataPart.partNumber === 1 && dataPart.lastPart) {
                return await this.uploadUsingPut(dataPart);
            }
            // 如果没有uploadId
            if(!this.uploadId) {
                await this.createMultipartUpload();
                if (this.abortController.signal.aborted) {
                    return;
                }
            }
            const partResult = await this.client.send(new UploadPartCommand({
                ...this.params,
                UploadId: this.uploadId,
                Body: dataPart.data,
                PartNumber: dataPart.partNumber
            }))
            if (this.abortController.signal.aborted) {
                return;
            }
            this.uploadedParts.push({
                PartNumber: dataPart.partNumber,
                ETag: partResult.ETag
            });
            this.bytesUploadedSoFar += dataPart.data.byteLength;
            this.notifyProgress({
              loaded: this.bytesUploadedSoFar,
              total: this.totalBytes,
              part: dataPart.partNumber,
              Key: this.params.Key,
              Bucket: this.params.Bucket,
            });
        }
    }
    async uploadUsingPut(dataPart) {
        this.isMultiPart = false;
        const params = { ...this.params, Body: dataPart.data };
        await Promise.all([
          this.client.send(new PutObjectCommand(params)),
          this.client.config.endpoint(),
        ]);
        const [putResult] = await Promise.all([
            this.client.send(new PutObjectCommand(params)),
            this.client.config.endpoint(),
        ]);
      
        // 单个文件上传的结果信息
        this.singleUploadResult = {
            ...putResult,
            Bucket: this.params.Bucket,
            Key: this.params.Key
        };
        const totalSize = dataPart.data.byteLength;
        // 触发progress的回调函数
        this.notifyProgress({
            loaded: totalSize,
            total: totalSize,
            part: 1,
            Key: this.params.Key,
            Bucket: this.params.Bucket,
        });
    }

    // 创建UploadId
    async createMultipartUpload() {
        if (!this.createMultiPartPromise) {
            const createCommandParams = { ...this.params, Body: undefined };
            this.createMultiPartPromise = this.client.send(new CreateMultipartUploadCommand(createCommandParams));
        }
        const createMultipartUploadResult = await this.createMultiPartPromise;
        this.uploadId = createMultipartUploadResult.UploadId;
    }
    // 上传进度事件
    notifyProgress(progress) {
        // 如果注册了回调函数则通知执行
        if (this.uploadEvent) {
            this.emit(this.uploadEvent, progress);
        }
    }
}

module.exports = Upload;

2、建立minio连接

// index.js
var AWS = require('aws-sdk')
const fs = require('fs');
const path = require('path')
const Upload = require('./components/Upload/Upload.js')
const clinet = require('@aws-sdk/client-s3');

const {S3Client} = clinet;

// 创建S3Client实例并于minio建立连接
const ss3 = new S3Client({
    region: 'us-east-1',
    endpoint: {
      protocol: 'http',
      hostname: '127.0.0.1',
      port: 9000,
      path: '/',
    },
    forcePathStyle: true,
    credentials: { accessKeyId: 'xxxx', secretAccessKey: 'xxx' },
})

// 读取文件
const fileObject = fs.readFileSync(path.resolve(__dirname, './file.zip'))

const upload = async (file) => {
    const upload = new Upload({
        params: {
          Bucket: 'minio-test', // 创建的桶名
          Key: 'filename', // 上传到对应的桶后的名称
          Body: fileObject, // 要上传的文件
        },
        client: ss3,
        queueSize: 3,
        partSize: 10*1024*1024
    });
    
    upload.on("httpUploadProgress", (progress) => {
        // console.log(progress);
    });
    // 中断上传
    // setTimeout(() => {
    //     console.log(" Aborting ....");
    //     let res = upload.abort();
    //     console.log('res: ', upload.uploadedParts)
    // }, 2000)
    let uploadResult;
    try {
        uploadResult = await upload.done();
        console.log("done uploading", uploadResult);
    } catch (error) {
        console.log("error", error);
    }
}

upload(fileObject)

这样我们就能在本地顺利的进行大文件上传，作者也是刚接触 S3 和 minio，大家有什么问题欢迎在评论留言。