CV即用:大文件上传的前端设计——批量上传、切片上传、断点续传、秒传、暂停上传
整体思路
处理文件/文件夹选择事件
处理文件/文件夹拖拽事件
上传文件入队列
文件切片
全部暂停/取消 暂停/取消 功能的实现
文章内所有代码已整理 见文章最底部
文章比较干 请自备饮用水
切入正题
思路:实现一个UploadManager以及UploadItem和UploadChunk用于上传
1. 实现一个UploadManager 用于管理上传任务
const MAX_QUEUE_NUM = 5; // 全局锁定最大上传线程
class UploadManager {
queue = null; //任务队列
config = {
maxQueueNum: MAX_QUEUE_NUM,
chunkSize: 1024 * 1024 * 10, //10MB
};
// 任务列表 包含所有状态的任务
list = [];
events = {};
constructor(config) {
this.config = { ...this.config, ...config };
// 初始化一个队列
this.queue = new Queue(this.config.maxQueueNum);
this.queue
.on("complete", () => {
console.log("queue complete");
})
.on("next", (item) => {
console.log("start upload item", item);
});
}
}
const globalUploadManager = new UploadManager();
export default globalUploadManager;
以上是一个代码完成了一个最简单的UploadManger 现在我们为其添加上传功能
uploadList(fileList) {
for (const file of fileList) {
this.uploadFile(file);
}
}
uploadFile(file) {
const uploadItem = new uploadItem(file, this.config);
this.list.push(uploadItem);
// 当传入这个文件的时候 想要直接开始上传这个文件 如果队列满了会从unk状态变为await、stop
this.start(uploadItem.key);
return uploadItem;
}
async remove(key) {
const itemIndex = this.list.findIndex((item) => item.key === key);
if (itemIndex != -1) {
this.list.splice(itemIndex, 1);
}
}
通过增加uploadFile方法 目前我们成功的把需要上传的文件列表推入了this.list这个数组中 并且调用的this.start方法准备开始上传这个文件列表 下面我们实现一下start方法
async start(key) {
const uploadItem = this.list.find((item) => item.key === key);
if (!uploadItem) throw new Error("uploadItem not found");
uploadItem.updateUploadStatus(uploadFileStatus.await);
this.queue.push(uploadItem, this.processItem.bind(this));
this.queue.process(true);
}
async processItem(uploadItem) {
// 不要问这里为什么这么写 这里是删减代码
await uploadItem.upload();
}
这里我们实现了一个start方法 并成功的把这个文件压入到了队列当中 接下来我们来看一下 uploadItem的实现
2.1 实现UploadItem
export default class UploadItem {
//文件对象
file = null;
taskId = "";
errorMsg = ""; // 上传错误原因
// 上传配置
config = {};
constructor(file, config) {
this.file = file;
this.config = config;
}
}
2.2 实现一些UploadItem的上传状态
/**
* 上传文件的状态常量对象
* @typedef {Object} UploadFileStatus
* @property {string} stop - 文件状态:暂停上传
* @property {string} await - 文件状态:等待
* @property {string} uploading - 文件状态:上传中
* @property {string} successQuick - 文件状态:秒传上传成功
* @property {string} chunking - 文件状态:解析中...
* @property {string} success - 文件状态:成功
* @property {string} error - 文件状态:失败
* @property {string} cancel - 文件状态:取消
*/
/**
* 上传文件的状态常量
* @type {UploadFileStatus}
*/
export const uploadFileStatus = {
unknown: "", //刚推入队列的时候的状态
stop: "暂停上传",
await: "等待上传",
uploading: "上传中",
successQuick: "秒传上传成功",
chunking: "解析中...",
success: "上传成功",
error: "上传失败",
cancel: "取消上传",
};
export default class UploadItem {
...
status = uploadFileStatus.unknown; // 上传状态
...
}
#### 2.3 实现UploadItem的文件信息相关的一些东西
```javascript
export default class UploadItem {
...
fileName = ""; // 文件名称
fileExtName = ""; //文件后缀
key = ""; // 唯一key
chunks = []; // 切片数组
...
constructor(file, config) {
...
this.key = this.generateUniqueKey(); //生成短时唯一key
// 获取文件名称以及文件后缀方法可以二合一简写 但我就不写 凑行数
this.fileName = this.getFileName(); // 拿到文件名称
this.fileExtName = this.getFileExtName(); //拿到后缀名称
this.chunks = this.getFileChunks(); //获取切片数组
// 别急 我知道你在想hash什么时候算 我知道你很急 但你先别急
}
/**
*
* @returns 生成唯一key
*/
generateUniqueKey() {
return `${new Date()
.getTime()
.toString(16)}-xxxx-4xxx-yxxx-xxxxxxxxxxxx`.replace(
/[xy]/g,
function (c) {
var r = (Math.random() * 16) | 0,
v = c == "x" ? r : (r & 0x3) | 0x8;
return v.toString(16);
}
);
}
getFileName() {
try {
const index = this.file?.name?.lastIndexOf(".") || -1;
if (index != -1) {
return this.file.name.slice(0, index);
}
return "";
} catch (e) {
console.log("getFileName error", e);
return "";
}
}
getFileExtName() {
try {
const index = this.file?.name?.lastIndexOf(".") || -1;
if (index != -1) {
return this.file?.name?.slice(index + 1).toLocaleLowerCase();
}
return "";
} catch (e) {
console.log("getFileExtName error", e);
return "";
}
}
/**
* 获取文件的切片对象列表
* @returns {FileChunk[]}
*/
getFileChunks() {
let chunkList = [];
let count = 0;
let index = 0;
while (count < this.file.size) {
const content = this.file.slice(count, count + this.config.chunkSize);
const chunk = new FileChunk({ index: ++index, content: content });
count += this.config.chunkSize;
chunkList.push(chunk);
}
return chunkList;
}
}
2.3 实现一些UploadItem的上传方法
在UploadManager中 我们调用了UploadItem.upload方法用于上传 现在实现一下这个方法
export default class UploadItem {
...
async upload() {
if (this.taskId) {
// 已有id 应该是暂停后的继续上传 不再处理taskId等等 直接启动队列上传
await this.startUpload();
} else {
// 切片上传 获取任务Id
await this.createUploadTaskId();
}
return this;
}
async createUploadTaskId() {
let data = {
fileName: this.fileName,
fileType: this.fileExtName,
fileSize: this.file.size,
totalChunkNum: this.chunks.length,
};
try {
const responds = await createTask(data);
if (code != 200) throw responds;
await this.handleChunkTastId(data);
} catch (error) {
this.updateUploadStatus(uploadFileStatus.error);
this.errorMsg = error.message || "服务器错误,请稍后重试";
}
}
async handleChunkTastId(data) {
// 设置任务id
this.taskId = data.taskId;
// 开始上传
await this.startUpload();
}
async startUpload() {
const nextIndex = this.chunks.findIndex(
(item) => item.status != chunStatus.uploaded
);
await this.updateUploadStatus(uploadFileStatus.uploading);
await this.uploadChunk(nextIndex);
return this;
}
async uploadChunk(index) {
const uploadItem = this.chunks[index];
try {
await this.buildChuckUploadRequest(uploadItem);
if (++index < this.chunks.length) {
return this.uploadChunk(index);
}
} catch (error) {
if (error?.message?.indexOf("Abort.") != -1) {
consolelog("用户暂停/取消上传")
} else {
this.updateUploadStatus(uploadFileStatus.error);
this.errorMsg = error.message || "服务器错误,请稍后重试";
}
}
}
async buildChuckUploadRequest(chunk) {
return new Promise(async (resovle, reject) => {
const fd = new FormData();
fd.append("taskId", this.taskId);
fd.append("multipartFile", chunk.content);
fd.append("fileExtName", this.fileExtName);
fd.append("chunkNum", parseInt(chunk.index));
// 大文件在上传切片的时候再进行md5计算
await chunk.buildHash();
fd.append("chunkMd5", chunk.md5);
chunk.status = chunStatus.uploading;
chunk.request = uploadRequset("uploadChunk", fd);
chunk.request.promise
.then((responds) => {
chunk.status = chunStatus.uploaded;
resovle(responds);
})
.catch((err) => reject(err));
});
}
...
}
至此 关于UploadItem最基础的上传流程已经走完了 下面我们来实现以下最简单的UploadChunk
3.1 实现UploadChunk
export const chunStatus = {
await: "等待",
hash: "计算MD5中",
hashEnd: "MD5计算完成,等待中",
uploading: "上传中",
uploaded: "上传完毕",
};
export default class FileChunk {
md5 = "";
index = "";
content = null;
woker = null; // 此处每一个切片单独启用一个woker 效率高
status = chunStatus.await;
uploadedSize = 0; //用于计算已经上传的大小
request = null;
constructor({ index, content }) {
this.index = index;
this.content = content;
}
buildHash() {
return new Promise((resolve) => {
this.status = chunStatus.hash;
this.woker = new Worker("/hash_one.js");
this.woker.postMessage(this.content);
this.woker.onmessage = (e) => {
const { hash } = e.data;
this.md5 = hash;
this.status = chunStatus.hashEnd;
this.woker.terminate();
this.woker.onmessage = null;
this.woker = null;
resolve();
};
});
}
}
UploadChunk的代码就非常的简洁了 对外抛出了一个buildHash方法来通过woker实现计算文件切片的hash值,下面是woker的代码 通过引入spark-md5这个库来计算hash
self.importScripts("/spark-md5.min.js"); // 导入脚本
// 生成文件 hash
self.onmessage = async (e) => {
const fileChunk = e.data;
const spark = new self.SparkMD5.ArrayBuffer();
const reader = new FileReader();
reader.readAsArrayBuffer(fileChunk);
reader.onload = ({ target }) => {
spark.append(target.result);
self.postMessage({
hash: spark.end(),
});
};
};
通过以上代码 我们简单实现了一个带队列的上传管理器 下面我们来继续完善一下 给它增加 全部暂停/暂停、全部开始/开始以及全部取消/取消功能
首先从UploadItem开始完善
4.1 完善上传
为了方便UploadItem和UploadManager通信 以及UploadManger的文件上传完毕的事件获取 下面我们为其增加一个EventManager
不要问为什么不封装Event类 问就是想要JSDoc的提示
export default class UploadItem {
events = {};
...
/**
* 添加事件监听器。
* @param {"process"|"complete"|"uploadStatusChange"} event - 事件名称。
* @param {Function} callback - 回调函数。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
on(event, callback) {
if (!this.events[event]) {
this.events[event] = [];
}
this.events[event].push(callback);
return this;
}
/**
* 触发事件监听器。
* @param {"process"|"complete"|"uploadStatusChange"} event - 事件名称。
* @param {any} data - 回调数据。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
trigger(event, data) {
if (this.events[event]) {
this.events[event].forEach((callback) => {
callback(data);
});
}
return this;
}
off(event, callback) {
if (this.events[event]) {
this.events[event] = this.events[event].filter((cb) => cb !== callback);
}
return this;
}
...
}
同样的 给UploadManger也增加一个EventManager
class UploadManager {
events = {};
...
/**
* 添加事件监听器。
* @param {"itemUploaded"} event - 事件名称。
* @param {Function} callback - 回调函数。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
on(event, callback) {
if (!this.events[event]) {
this.events[event] = [];
}
this.events[event].push(callback);
return this;
}
/**
* 触发事件监听器。
* @param {"itemUploaded"} event - 事件名称。
* @param {any} data - 回调数据。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
trigger(event, data) {
if (this.events[event]) {
this.events[event].forEach((callback) => {
callback(data);
});
}
return this;
}
off(event, callback) {
if (this.events[event]) {
this.events[event] = this.events[event].filter((cb) => cb !== callback);
}
return this;
}
...
}
有了EventManager之后我们就可以简单的改造一下之前的方法
首先是UploadManger
class UploadManager {
...
constructor(config){
...
this.queue
.on("complete", () => {
console.log("queue complete");
})
.on("next", (item) => {
console.log("start upload item", item);
});
}
...
uploadFile(file) {
const uploadItem = new uploadItem(file, this.config);
uploadItem
.on("complete", (item) => {
this.remove(item.key);
this.trigger("itemUploaded", item);
})
.on("uploadStatusChange", ({ newStatus, oldStatus, item }) => {
// 如果是【正在上传】以及【等待上传】的状态 并且 【切换到暂停状态】 则需要从队列中删除这个任务
const inQueueStatus = [
uploadFileStatus.uploading,
uploadFileStatus.await,
].includes(oldStatus);
if (inQueueStatus && newStatus == uploadFileStatus.stop) {
this.queue.checkRemove();
}
// 如果任务失败 就暂停这个任务
// TODO:这里可以优化一下 因为每次status变动都会触发这个事件 导致性能问题
// 优化方案为修改下面的processItem 加入catch 然后resolve这个任务
if (newStatus == uploadFileStatus.error) {
this.stop(item.key);
}
});
this.list.push(uploadItem);
// 直接默认开始上传这个文件 如果队列满了会从unk状态变为await、stop
this.start(uploadItem.key);
return uploadItem;
}
// 增加一些控制方法
// 全部开始
async startAll() {
// 开启所有 停止 状态的任务
for (const uploadItem of this.list.filter((item) => {
return [uploadFileStatus.stop].includes(item.status);
})) {
await this.start(uploadItem.key);
}
this.queue.start();
}
// 全部暂停
async stopAll() {
// 暂停 正在上传以及等待上传的任务
for (const uploadItem of this.list.filter((item) => {
return [
uploadFileStatus.uploading,
uploadFileStatus.await,
uploadFileStatus.stop,
].includes(item.status);
})) {
uploadItem.stop();
uploadItem.deleted = true;
this.queue.checkRemove(); // 从队列中删除这个任务
}
}
// 全部取消
async cancelAll() {
// 取消 【正在上传】、【等待上传】、【暂停上传】的任务
for (const uploadItem of this.list.filter((item) => {
return [
uploadFileStatus.uploading,
uploadFileStatus.await,
uploadFileStatus.stop,
].includes(item.status);
})) {
uploadItem.cancel();
uploadItem.deleted = true;
this.queue.checkRemove(); // 从队列中删除这个任务
setTimeout(() => {
this.remove(uploadItem.key);
}, 1000);
}
}
// 取消某个任务
async cancel(key) {
const uploadItem = this.list.find((item) => item.key === key);
if (!uploadItem) throw new Error("uploadItem not found");
uploadItem.cancel();
uploadItem.deleted = true;
this.queue.checkRemove(); // 从队列中删除这个任务
this.queue.process();
setTimeout(() => {
this.remove(uploadItem.key);
}, 1000);
}
}
其次是首先是UploadItem
export default class UploadItem {
...
async createUploadTaskId() {
let data = {
fileName: this.fileName,
fileType: this.fileExtName,
fileSize: this.file.size,
totalChunkNum: this.chunks.length,
};
try {
const responds = await createTask(data);
if (code != 200) throw responds;
await this.handleChunkTastId(data);
} catch (error) {
this.updateUploadStatus(uploadFileStatus.error);
this.errorMsg = error.message || "服务器错误,请稍后重试";
this.trigger("error", { message: "createTaskError", error });
}
}
async handleChunkTastId(data) {
// 秒传
if (data?.quickUpload) {
this.updateUploadStatus(uploadFileStatus.successQuick);
// 触发上传完成回调
this.trigger("complete", this);
return;
}
// 设置任务id
this.taskId = data.taskId;
// 开始上传
await this.startUpload();
}
async uploadChunk(index) {
const uploadItem = this.chunks[index];
try {
await this.buildChuckUploadRequest(uploadItem);
if (++index < this.chunks.length) {
return this.uploadChunk(index);
}
} catch (error) {
if (error?.message?.indexOf("Abort.") != -1) {
this.trigger("cancel", {
message: `用户暂停/取消上传 ${this.key} ${this.fileName}`,
error,
});
} else {
this.updateUploadStatus(uploadFileStatus.error);
this.errorMsg = error.message || "服务器错误,请稍后重试";
this.trigger("error", { message: "uploadChunkError", error });
}
}
}
async buildChuckUploadRequest(chunk) {
...
chunk.request.promise
.then((responds) => {
chunk.status = chunStatus.uploaded;
// 判断一下chunks有没有上传完成
if (this.isChunksDone) {
this.updateUploadStatus(uploadFileStatus.success);
this.trigger("complete", this);
}
resovle(responds);
})
.catch((err) => reject(err));
}
// 判断切片任务是否上传完成
get isChunksDone() {
return (
this.chunks.filter((item) => {
return (
item.status == chunStatus.uploading || item.status == chunStatus.await
);
}).length <= 0
);
}
// 暂停上传 切片文件可恢复
async stop() {
if (this.status == uploadFileStatus.uploading) {
const uploadingChunks = this.chunks.filter(
(item) => item.status === chunStatus.uploading
);
for (const chunk of uploadingChunks) {
chunk.status = chunStatus.await;
chunk.request.cancel();
}
this.progressData.uploadChunkSize = this.getUploadedChunkSize();
}
this.updateUploadStatus(uploadFileStatus.stop);
}
// 取消上传
async cancel() {
// 先暂停任务
await this.stop();
// 修改状态到取消
this.updateUploadStatus(uploadFileStatus.cancel);
}
/**
* 获取切片列表已经上传的字节大小
* @returns {number} 已上传的字节数
*/
getUploadedChunkSize() {
return this.chunks
.filter((item) => item.status == chunStatus.uploaded)
.reduce((accumulator, currentItem) => {
return accumulator + currentItem.uploadedSize;
}, 0);
}
}
这样基本上一个文件上传的逻辑就写好了 下面是一些文件上传是的辅助代码
// 计算上传速度
export default class UploadItem {
...
//用于计算上传速度
startTime = 0; // 本次上传开始时间
lastTime = 0; // 预估上传剩余时间
startLoaded = 0; // 最后一次上传总大小
uploadSpeed = 0; //上传速度 kb
// 进度条数据
progressData = {
size: 0,
uploadChunkSize: 0,
percentage: 0,
};
finishedTime = ""; //上传完成时间
constructor(file, config){
...
this.progressData.size = file.size; // 设置进度条总大小’
}
// 暂停之后还有可能因为节流 导致上传进度会更新一次
throttleProgress = throttle((progressEvent) => {
const { loaded } = progressEvent;
chunk.uploadedSize = loaded;
this.handleOnUploadChunkProgressChange(progressEvent);
}, 1000);
async buildChuckUploadRequest(chunk) {
...
chunk.status = chunStatus.uploading;
chunk.request = uploadRequset("uploadChunk", fd, {
onUploadProgress: this.throttleProgress,
});
...
}
async handleOnUploadChunkProgressChange(progressEvent) {
const { loaded, total } = progressEvent;
this.progressData.uploadChunkSize = this.getUploadedChunkSize() + loaded;
const newPercentage = parseInt(
(this.progressData.uploadChunkSize / this.progressData.size) * 100
);
this.progressData.percentage = Math.max(
this.progressData.percentage,
newPercentage
);
this.progressData.percentage = Math.min(100, this.progressData.percentage);
const _lastTime = this.getLastUploadTime({
loaded: this.progressData.uploadChunkSize,
total: this.progressData.size,
});
this.lastTime = _lastTime || this.lastTime;
}
getLastUploadTime({ loaded, total }) {
if (!this.startTime) this.startTime = new Date().getTime();
if (!this.startLoaded) this.startLoaded = 0;
const currentTime = new Date().getTime();
const uploadedBytes = loaded - this.startLoaded;
const totalBytes = total;
const loadedBytes = loaded;
const elapsedTime = nearestMultipleOf1000(currentTime - this.startTime);
if (elapsedTime < 500) {
// 如果距离上次计算时间过短,则不更新上传速度 (触发情况大概是一片刚上传完成 下一片的进度马上就更新了)
this.startLoaded = loadedBytes;
return;
}
const uploadSpeed = parseInt(uploadedBytes / elapsedTime);
this.uploadSpeed = uploadSpeed * 1000 || 0;
this.uploadSpeed = this.uploadSpeed < 0 ? 0 : this.uploadSpeed;
const remainingBytes = totalBytes - loadedBytes;
const remainingTime = remainingBytes / uploadSpeed;
let remainingSeconds = remainingTime / 1000;
this.startLoaded = loaded;
this.startTime = currentTime;
remainingSeconds = remainingSeconds < 0 ? 0 : remainingSeconds;
remainingSeconds =
remainingSeconds == Infinity ? 0 : remainingSeconds.toFixed(2);
return remainingSeconds;
}
}
【食用方法】处理文件/文件夹的选择事件
<!-- 多文件的选择上传 -->
<input type="file" multiple @change="onUploadFileChange" />
<!-- 文件夹的选择上传 -->
<input type="file" multiple webkitdirectory mozdirectory odirectory @change="onUploadDirChange" />
<!-- 文件/文件夹的拖拽上传 -->
<div class="drop-file-area" @drop="handleDrop"></div>
export default {
data() {
return {
files: [],
};
},
mounted() {
this.debounceOnFileChange = debounce(this.onUploadFileChangeThen, 100);
},
activated() {},
beforeDestroy() {},
deactivated() {},
methods: {
async onUploadFileChange(even) {
if (even instanceof DataTransferItem) {
this.files.push(even.getAsFile());
} else {
this.files = this.getFiles(even);
}
return this.debounceOnFileChange(); //给拖拽调用做个保底
},
async onUploadFileChangeThen() {
const files = this.files;
this.files = [];
const fileList = this.traverseFileTree(files);
fileList.sort((a, b) => a.size - b.size);
this.$refs.uploadFile.value = ""; // 清空input的值 防止其他异常错误
},
async onUploadDirChange(even) {
let files = [];
if (even instanceof DataTransferItem) {
// 拖拽对象
files = await this.traverseDataTransfer(even);
} else {
// 选择对象
files = this.getFiles(even);
}
const fileList = this.traverseFileTree(files);
fileList.sort((a, b) => a.size - b.size);
this.$refs.uploadFileMultiple.value = ""; // 清空input的值 防止其他异常错误
},
async onFileChange(even) {
for (const item of even.dataTransfer.items) {
if (item.webkitGetAsEntry && item.webkitGetAsEntry().isDirectory) {
this.onUploadDirChange(item)
} else {
this.onUploadFileChange(item)
}
}
},
async getFiles(even) {
if (even?.target?.files) {
return even.target.files;
} else if (even?.dataTransfer?.items) {
return even.dataTransfer.items;
} else if (even?.dataTransfer?.files) {
return even.dataTransfer.files;
} else {
return [];
}
},
traverseFileTree(filesList, path = "") {
let fileList = [];
let tempFilesList = filesList || [];
for (let i = 0; i < tempFilesList.length; i++) {
let file = tempFilesList[i];
if (file instanceof File) {
fileList.push(file);
} else if (file.isDirectory) {
// 如果是文件夹,则递归处理其中的文件和子文件夹
let dirReader = file.createReader();
dirReader.readEntries((entries) => {
let subdirFiles = this.traverseFileTree(
entries,
path + file.name + "/"
);
fileList = fileList.concat(subdirFiles);
});
}
}
return fileList;
},
async traverseDataTransfer(item) {
const ret = [];
const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : item;
if (entry?.isDirectory) {
const directoryReader = entry.createReader();
const entries = await new Promise((resolve, reject) => {
directoryReader.readEntries(resolve, reject);
});
for (const _entry of entries) {
ret.push(await this.traverseDataTransfer(_entry));
}
} else {
const path = item?.fullPath?.substring(1) || "";
const file = await new Promise((resolve, reject) => {
item.file(resolve, reject);
});
const newFile = Object.defineProperty(file, "webkitRelativePath", {
value: path,
});
ret.push(newFile);
}
return ret.flat();
},
},
};
完整代码
UploadeManager.js
import { Queue } from "./queue";
import { uploadFileStatus } from "./item";
import { debounce } from "./utils";
const MAX_QUEUE_NUM = 5; // 全局锁定最大上传线程
const MAX_CHUNK_PROCESS_NUM = 1; //切片只支持单任务上传
class UploadManager {
queue = null; //任务队列
debounceSaveUploadedList = null;
config = {
maxQueueNum: MAX_QUEUE_NUM,
maxChunkProcessNum: MAX_CHUNK_PROCESS_NUM,
chunkSize: 1024 * 1024 * 10, //10MB
};
// 任务列表 包含所有状态的任务
list = [];
events = {};
uploadedList = []; //已经上传的任务列表
get uploadQueueNum() {
//返回上传任务数 不包含取消以及错误的任务
return this.list.filter((item) => {
return ![uploadFileStatus.cancel, uploadFileStatus.error].includes(
item.status
);
}).length;
}
constructor(config) {
this.debounceSaveUploadedList = debounce(this.saveUploadedList, 1000);
this.config = { ...this.config, ...config };
this.queue = new Queue(this.config.maxQueueNum);
this.queue
.on("complete", () => {
console.log("queue complete");
})
.on("next", (item) => {
console.log("start upload item", item);
});
}
updateQueueMaxNum(num) {
if (!num) return;
num = Math.min(num, MAX_QUEUE_NUM);
this.maxQueueNum = num;
this.queue.maxConcurrency = num;
console.log("update maxQueueNum", num);
}
/**
* @param {Array[File]} fileList
*
*/
uploadList(fileList) {
for (const file of fileList) {
this.uploadFile(file);
}
}
/**
* @param {File} file
*
* @returns UploadItem
*/
uploadFile(file) {
const uploadItem = new uploadItem(file, this.config);
uploadItem
.on("complete", (item) => {
item.finishedTime = new Date().toLocaleString();
this.uploadedList = [item, ...this.uploadedList];
this.debounceSaveUploadedList(this.uploadedList);
this.remove(item.key);
this.trigger("itemUploaded", item);
})
.on("uploadStatusChange", ({ newStatus, oldStatus, item }) => {
// 如果是【正在上传】以及【等待上传】的状态 并且 【切换到暂停状态】 则需要从队列中删除这个任务
const inQueueStatus = [
uploadFileStatus.uploading,
uploadFileStatus.await,
].includes(oldStatus);
if (inQueueStatus && newStatus == uploadFileStatus.stop) {
this.queue.checkRemove();
}
// 如果任务失败 就暂停这个任务
// TODO:这里可以优化一下 因为每次status变动都会触发这个事件 导致性能问题
// 优化方案为修改下面的processItem 加入catch 然后resolve这个任务
if (newStatus == uploadFileStatus.error) {
this.stop(item.key);
}
});
this.list.push(uploadItem);
// 直接默认开始上传这个文件 如果队列满了会从unk状态变为await、stop
this.start(uploadItem.key);
return uploadItem;
}
// 任务的Promise builder
async processItem(uploadItem) {
return new Promise((resolve) => {
uploadItem
.on("complete", resolve)
.upload()
.catch((err) => {
console.log("err", err.message);
resolve();
});
});
}
// 全部开始
async startAll() {
// 开启所有 停止 状态的任务
for (const uploadItem of this.list.filter((item) => {
return [uploadFileStatus.stop].includes(item.status);
})) {
await this.start(uploadItem.key);
}
this.queue.start();
}
// 全部暂停
async stopAll() {
// 暂停 正在上传以及等待上传的任务
for (const uploadItem of this.list.filter((item) => {
return [
uploadFileStatus.uploading,
uploadFileStatus.await,
uploadFileStatus.stop,
].includes(item.status);
})) {
uploadItem.stop();
uploadItem.deleted = true;
this.queue.checkRemove(); // 从队列中删除这个任务
}
}
// 全部取消
async cancelAll() {
// 取消 【正在上传】、【等待上传】、【暂停上传】的任务
for (const uploadItem of this.list.filter((item) => {
return [
uploadFileStatus.uploading,
uploadFileStatus.await,
uploadFileStatus.stop,
].includes(item.status);
})) {
uploadItem.cancel();
uploadItem.deleted = true;
this.queue.checkRemove(); // 从队列中删除这个任务
setTimeout(() => {
this.remove(uploadItem.key);
}, 1000);
}
}
// 开始某个任务
async start(key) {
const uploadItem = this.list.find((item) => item.key === key);
if (!uploadItem) throw new Error("uploadItem not found");
uploadItem.updateUploadStatus(uploadFileStatus.await);
await uploadItem.chunks[0].buildHash(); //先切一片hash来判断是否是同一个文件
const result = this.queue.processing
.filter((item) => {
return item.data.key != uploadItem.key;
})
.every((item) => {
return item.data.chunks[0].md5 != uploadItem.chunks[0].md5;
});
if (result) {
this.queue.push(uploadItem, this.processItem.bind(this));
this.queue.process(true);
} else {
uploadItem.updateUploadStatus(uploadFileStatus.stop);
uploadItem.errorMsg = `该文件正在上传,请勿重复上传`;
}
}
// 取消某个任务
async cancel(key) {
const uploadItem = this.list.find((item) => item.key === key);
if (!uploadItem) throw new Error("uploadItem not found");
uploadItem.cancel();
uploadItem.deleted = true;
this.queue.checkRemove(); // 从队列中删除这个任务
this.queue.process();
setTimeout(() => {
this.remove(uploadItem.key);
}, 1000);
}
async remove(key) {
const itemIndex = this.list.findIndex((item) => item.key === key);
if (itemIndex != -1) {
this.list.splice(itemIndex, 1);
}
}
get LocalListKey() {
const userId = "";
return `uploadManager_uploadedList_${userId}`;
}
getUploadedList = () => {
try {
const list = JSON.parse(localStorage.getItem(this.LocalListKey)) || [];
this.uploadedList = list.filter((item) => {
// 过滤大于30天的记录
return (
new Date(item.finishedTime).getTime() + 1000 * 60 * 60 * 24 * 30 >
new Date().getTime()
);
});
return this.uploadedList;
} catch (e) {
console.log(e);
return [];
}
};
saveUploadedList = (uploadedList) => {
this.uploadedList = uploadedList.filter((item) => {
// 过滤大于30天的记录
return (
new Date(item.finishedTime).getTime() + 1000 * 60 * 60 * 24 * 30 >
new Date().getTime()
);
});
localStorage.setItem(this.LocalListKey, JSON.stringify(this.uploadedList));
};
clearu = (UploadedList = () => {
this.uploadedList = [];
localStorage.setItem(this.LocalListKey, JSON.stringify([]));
});
deleteUploadedListItem(item) {
const index = this.uploadedList.findIndex((file) => file.key === item.key);
if (index != -1) {
this.uploadedList.splice(index, 1);
}
this.saveUploadedList(this.uploadedList);
}
/**
* 添加事件监听器。
* @param {"itemUploaded"} event - 事件名称。
* @param {Function} callback - 回调函数。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
on(event, callback) {
if (!this.events[event]) {
this.events[event] = [];
}
this.events[event].push(callback);
return this;
}
/**
* 触发事件监听器。
* @param {"itemUploaded"} event - 事件名称。
* @param {any} data - 回调数据。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
trigger(event, data) {
if (this.events[event]) {
this.events[event].forEach((callback) => {
callback(data);
});
}
return this;
}
off(event, callback) {
if (this.events[event]) {
this.events[event] = this.events[event].filter((cb) => cb !== callback);
}
return this;
}
}
const globalUploadManager = new UploadManager();
export default globalUploadManager;
UploadItem.js
import FileChunk, { chunStatus } from "./chunk";
import uploadRequset from "./request";
/**
* 上传文件的状态常量对象
* @typedef {Object} UploadFileStatus
* @property {string} stop - 文件状态:暂停上传
* @property {string} await - 文件状态:等待
* @property {string} uploading - 文件状态:上传中
* @property {string} successQuick - 文件状态:秒传上传成功
* @property {string} chunking - 文件状态:解析中...
* @property {string} success - 文件状态:成功
* @property {string} error - 文件状态:失败
* @property {string} cancel - 文件状态:取消
*/
/**
* 上传文件的状态常量
* @type {UploadFileStatus}
*/
export const uploadFileStatus = {
unknown: "", //刚推入队列的时候的状态
stop: "暂停上传",
await: "等待上传",
uploading: "上传中",
successQuick: "秒传上传成功",
chunking: "解析中...",
success: "上传成功",
error: "上传失败",
cancel: "取消上传",
};
export default class UploadItem {
file = null; //文件对象
deleted = false;
//用于计算上传速度
startTime = 0; // 本次上传开始时间
lastTime = 0; // 预估上传剩余时间
startLoaded = 0; // 最后一次上传总大小
uploadSpeed = 0; //上传速度 kb
fileName = ""; // 文件名称
fileExtName = ""; //文件后缀
errorMsg = ""; // 新增errorMsg 当上传发生错误时记录 开始上传时清空
key = ""; // 短时唯一key
taskId = ""; //上传任务ID
// 进度条数据
progressData = {
size: 0,
uploadChunkSize: 0,
percentage: 0,
};
finishedTime = ""; //上传完成时间
status = uploadFileStatus.unknown; // 上传状态
chunks = []; // 切片数组
events = {}; // 订阅事件
constructor(file, config) {
this.key = this.generateUniqueKey(); //生成短时唯一key
this.file = file;
this.fileName = this.getFileName(); // 拿到文件名称
this.fileExtName = this.getFileExtName(); //拿到后缀名称
this.progressData.size = file.size; // 设置进度条总大小’
this.chunks = this.getFileChunks();
}
async upload() {
// 开始上传时清空msg
this.errorMsg = "";
if (this.taskId) {
// 已有id 应该是暂停后的继续上传 不再处理taskId等等 直接启动队列上传
await this.startUpload();
} else {
// 切片上传 获取任务Id
await this.createUploadTaskId();
}
return this;
}
async createUploadTaskId() {
let data = {
fileName: this.fileName,
fileType: this.fileExtName,
fileSize: this.file.size,
totalChunkNum: this.chunks.length,
};
try {
const responds = await createTask(data);
if (code != 200) throw responds;
await this.handleChunkTastId(data);
} catch (error) {
this.updateUploadStatus(uploadFileStatus.error);
this.errorMsg = error.message || "服务器错误,请稍后重试";
this.trigger("error", { message: "createTaskError", error });
}
}
async handleChunkTastId(data) {
// 秒传
if (data?.quickUpload) {
// 设置上传进度
this.progressData.percentage = 100;
this.progressData.uploadChunkSize = this.progressData.size;
// 设置上传状态
this.updateUploadStatus(uploadFileStatus.successQuick);
// 触发上传完成回调
this.trigger("complete", this);
return;
}
// 设置任务id
this.taskId = data.taskId;
// 开始上传
await this.startUpload();
}
/**
* 分片任务
* @returns {uploadItem} 当前实例
*/
async startUpload() {
const nextIndex = this.chunks.findIndex(
(item) => item.status != chunStatus.uploaded
);
await this.updateUploadStatus(uploadFileStatus.uploading);
await this.uploadChunk(nextIndex);
return this;
}
async uploadChunk(index) {
const uploadItem = this.chunks[index];
try {
await this.buildChuckUploadRequest(uploadItem);
if (++index < this.chunks.length) {
return this.uploadChunk(index);
}
} catch (error) {
if (error?.message?.indexOf("Abort.") != -1) {
this.trigger("cancel", {
message: `用户暂停/取消上传 ${this.key} ${this.fileName}`,
error,
});
} else {
this.updateUploadStatus(uploadFileStatus.error);
this.errorMsg = error.message || "服务器错误,请稍后重试";
this.trigger("error", { message: "uploadChunkError", error });
}
}
}
// 暂停之后还有可能因为节流 导致上传进度会更新一次
throttleProgress = throttle((progressEvent) => {
const { loaded } = progressEvent;
chunk.uploadedSize = loaded;
this.handleOnUploadChunkProgressChange(progressEvent);
}, 1000); // 注意此处的1000 下面计算上传速度会取1000的整倍数
// 切片任务上传
async buildChuckUploadRequest(chunk) {
return new Promise(async (resovle, reject) => {
const fd = new FormData();
fd.append("taskId", this.taskId);
fd.append("multipartFile", chunk.content);
fd.append("fileExtName", this.fileExtName);
fd.append("chunkNum", parseInt(chunk.index));
// 大文件在上传切片的时候再进行md5计算
await chunk.buildHash();
fd.append("chunkMd5", chunk.md5);
chunk.status = chunStatus.uploading;
chunk.request = uploadRequset("uploadChunk", fd, {
onUploadProgress: this.throttleProgress,
});
chunk.request.promise
.then((responds) => {
chunk.status = chunStatus.uploaded;
// 判断一下chunks有没有上传完成
if (this.isChunksDone) {
this.updateUploadStatus(uploadFileStatus.success);
this.trigger("complete", this);
}
resovle(responds);
})
.catch((err) => reject(err));
});
}
// 暂停上传 切片文件可恢复
async stop() {
if (this.status == uploadFileStatus.uploading) {
const uploadingChunks = this.chunks.filter(
(item) => item.status === chunStatus.uploading
);
for (const chunk of uploadingChunks) {
chunk.status = chunStatus.await;
chunk.request.cancel();
}
this.progressData.uploadChunkSize = this.getUploadedChunkSize();
}
this.updateUploadStatus(uploadFileStatus.stop);
}
// 取消上传
async cancel() {
// 先暂停任务
await this.stop();
// 修改状态到取消
this.updateUploadStatus(uploadFileStatus.cancel);
}
async handleOnUploadChunkProgressChange(progressEvent) {
const { loaded, total } = progressEvent;
this.progressData.uploadChunkSize = this.getUploadedChunkSize() + loaded;
const newPercentage = parseInt(
(this.progressData.uploadChunkSize / this.progressData.size) * 100
);
this.progressData.percentage = Math.max(
this.progressData.percentage,
newPercentage
);
this.progressData.percentage = Math.min(100, this.progressData.percentage);
const _lastTime = this.getLastUploadTime({
loaded: this.progressData.uploadChunkSize,
total: this.progressData.size,
});
this.lastTime = _lastTime || this.lastTime;
}
// 判断切片任务是否上传完成
get isChunksDone() {
return (
this.chunks.filter((item) => {
return (
item.status == chunStatus.uploading || item.status == chunStatus.await
);
}).length <= 0
);
}
/**
* 获取切片列表已经上传的字节大小
* @returns {number} 已上传的字节数
*/
getUploadedChunkSize() {
return this.chunks
.filter((item) => item.status == chunStatus.uploaded)
.reduce((accumulator, currentItem) => {
return accumulator + currentItem.uploadedSize;
}, 0);
}
/**
* 获取文件的切片对象列表
* @returns {FileChunk[]}
*/
getFileChunks() {
let chunkList = [];
let count = 0;
let index = 0;
while (count < this.file.size) {
const content = this.file.slice(count, count + this.config.chunkSize);
const chunk = new FileChunk({ index: ++index, content: content });
count += this.config.chunkSize;
chunkList.push(chunk);
}
return chunkList;
}
getFileName() {
try {
const index = this.file?.name?.lastIndexOf(".") || -1;
if (index != -1) {
return this.file.name.slice(0, index);
}
return "";
} catch (e) {
console.log("getFileName error", e);
return "";
}
}
getFileExtName() {
try {
const index = this.file?.name?.lastIndexOf(".") || -1;
if (index != -1) {
return this.file?.name?.slice(index + 1).toLocaleLowerCase();
}
return "";
} catch (e) {
console.log("getFileExtName error", e);
return "";
}
}
/**
*
* @returns 生成短时唯一key
*/
generateUniqueKey() {
return `${new Date()
.getTime()
.toString(16)}-xxxx-4xxx-yxxx-xxxxxxxxxxxx`.replace(
/[xy]/g,
function (c) {
var r = (Math.random() * 16) | 0,
v = c == "x" ? r : (r & 0x3) | 0x8;
return v.toString(16);
}
);
}
getLastUploadTime({ loaded, total }) {
if (!this.startTime) this.startTime = new Date().getTime();
if (!this.startLoaded) this.startLoaded = 0;
// 当前时间
const currentTime = new Date().getTime();
// 已经上传的数据量
const uploadedBytes = loaded - this.startLoaded;
// 总数据量
const totalBytes = total;
// 已完成数据量
const loadedBytes = loaded;
// 计算上传速度(字节/毫秒)
const elapsedTime = nearestMultipleOf1000(currentTime - this.startTime);
if (elapsedTime < 500) {
// 如果距离上次计算时间过短,则不更新上传速度 (触发情况大概是一片刚上传完成 下一片的进度马上就更新了)
this.startLoaded = loadedBytes;
return;
}
const uploadSpeed = parseInt(uploadedBytes / elapsedTime);
this.uploadSpeed = uploadSpeed * 1000 || 0;
this.uploadSpeed = this.uploadSpeed < 0 ? 0 : this.uploadSpeed;
// 计算剩余数据量
const remainingBytes = totalBytes - loadedBytes;
// 计算剩余时间
const remainingTime = remainingBytes / uploadSpeed;
// 将毫秒转换为秒
let remainingSeconds = remainingTime / 1000;
this.startLoaded = loaded;
this.startTime = currentTime;
remainingSeconds = remainingSeconds < 0 ? 0 : remainingSeconds;
remainingSeconds =
remainingSeconds == Infinity ? 0 : remainingSeconds.toFixed(2);
return remainingSeconds;
}
/**
* 更新当前任务的上传状态
* @param {UploadFileStatus} status
*/
async updateUploadStatus(status) {
this.trigger("uploadStatusChange", {
newStatus: status,
oldStatus: this.status,
item: this,
});
this.status = status;
}
/**
* 添加事件监听器。
* @param {"process"|"complete"|"uploadStatusChange"} event - 事件名称。
* @param {Function} callback - 回调函数。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
on(event, callback) {
if (!this.events[event]) {
this.events[event] = [];
}
this.events[event].push(callback);
return this;
}
/**
* 触发事件监听器。
* @param {"process"|"complete"|"uploadStatusChange"} event - 事件名称。
* @param {any} data - 回调数据。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
trigger(event, data) {
if (this.events[event]) {
this.events[event].forEach((callback) => {
callback(data);
});
}
return this;
}
off(event, callback) {
if (this.events[event]) {
this.events[event] = this.events[event].filter((cb) => cb !== callback);
}
return this;
}
}
export const throttle = (func, limit) => {
let lastFunc;
let lastRan;
return function () {
const context = this;
const args = arguments;
if (!lastRan) {
func.apply(context, args);
lastRan = Date.now();
} else {
if (lastFunc) clearTimeout(lastFunc);
lastFunc = setTimeout(function () {
if (Date.now() - lastRan >= limit) {
func.apply(context, args);
lastRan = Date.now();
}
}, limit - (Date.now() - lastRan));
}
};
};
export const nearestMultipleOf1000 = (num) => {
return Math.floor(num / 1000) * 1000;
};
UploadChunk.js
export const chunStatus = {
await: "等待",
hash: "计算MD5中",
hashEnd: "MD5计算完成,等待中",
uploading: "上传中",
uploaded: "上传完毕",
};
export default class FileChunk {
md5 = "";
index = "";
content = null;
woker = null; // 此处每一个切片单独启用一个woker 效率高
status = chunStatus.await;
uploadedSize = 0; //用于计算已经上传的大小
request = null;
constructor({ index, content }) {
this.index = index;
this.content = content;
}
buildHash() {
return new Promise((resolve) => {
this.status = chunStatus.hash;
this.woker = new Worker("/hash_one.js");
this.woker.postMessage(this.content);
this.woker.onmessage = (e) => {
const { hash } = e.data;
this.md5 = hash;
this.status = chunStatus.hashEnd;
this.woker.terminate();
this.woker.onmessage = null;
this.woker = null;
resolve();
};
});
}
}
Queue.js
export class Queue {
queue = [];
processing = [];
maxConcurrency = 5;
events = {};
constructor(maxConcurrency) {
this.maxConcurrency = maxConcurrency;
}
get concurrency() {
return this.processing.length;
}
get size() {
return this.queue.length;
}
push(data, callback) {
if (this.queue.find((item) => item.data.key == data.key)) {
throw "该任任务已存在";
}
data.deleted = false; // 增加删除标记 用于友好的删除队列
const item = { data, resolve: null, callback: callback || null };
const promise = new Promise((resolve) => (item.resolve = resolve));
this.queue.push(item);
return promise;
}
async process() {
if (this.concurrency >= this.maxConcurrency) return; // 如果队列满了 则不再启动新任务
if (this.queue.length === 0) return; //如果队列没有任务了
const item = this.queue.shift();
this.processing.push(item);
this.trigger("next", item.data);
try {
await item.callback(item.data);
} catch (err) {
console.log(err);
}
item.data.deleted = true;
this.checkRemove();
if (this.queue.length === 0) {
this.trigger("complete");
}
this.process();
item.resolve();
}
/**
* 启动队列
*/
async start() {
for (let i = 0; i < this.maxConcurrency; i++) {
this.process();
}
}
checkRemove() {
// 从两个队列中移除标记为删除的任务
this.queue = this.queue.filter((item) => !item.data.deleted);
this.processing = this.processing.filter((item) => !item.data.deleted);
}
/**
* 添加事件监听器。
* @param {"complete"|"next"} event - 事件名称。
* @param {Function} callback - 回调函数。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
on(event, callback) {
if (!this.events[event]) {
this.events[event] = [];
}
this.events[event].push(callback);
return this;
}
/**
* 触发事件监听器。
* @param {"complete"|"next"} event - 事件名称。
* @param {any} data - 回调数据。
* @returns {this} 当前 EventManager 实例,支持链式调用。
*/
trigger(event, data) {
if (this.events[event]) {
this.events[event].forEach((callback) => {
callback(data);
});
}
return this;
}
off(event, callback) {
if (this.events[event]) {
this.events[event] = this.events[event].filter((cb) => cb !== callback);
}
return this;
}
}
Request.js
import axios from "axios";
// 创建axios实例
const service = axios.create({
headers: {
"Content-Type": "multipart/form-data",
},
});
var serverbaseUrl = window.global_url.baseURL;
service.interceptors.request.use(
function (config) {
let token = "Bearer "; // TODO:此处放入Token
if (token) {
config.headers["Authorization"] = token;
return config;
}
},
function (error) {
return Promise.reject(error);
}
);
service.interceptors.response.use(
(res) => {
if (res.data.code === 200) {
return res.data;
}
const msg = res.data.message || res.data.msg || "系统错误,请稍后再试";
return Promise.reject(new Error(msg));
},
(error) => {
return Promise.reject(error);
}
);
const upload = (url, data = null, config = {}) => {
const source = axios.CancelToken.source();
config.cancelToken = source.token;
const promise = service({
url: serverbaseUrl + url,
method: "post",
data,
...config,
})
.then((response) => {
if (response.code == 200) {
return response.data;
} else {
throw response;
}
})
.catch((error) => {
if (axios.isCancel(error)) {
console.log("Request canceled", error.message);
} else {
console.log("Error", error.message);
}
throw error; // 继续抛出错误,让调用方处理
});
const cancel = () => {
source.cancel("Abort. Request canceled by the user.");
};
return { promise, cancel };
};
export default upload;
Utils.js
/**
* @param {Function} func
* @param {number} wait
* @param {boolean} immediate
* @return {any}
*/
export function debounce(func, wait, immediate) {
let timeout, args, context, timestamp, result;
const later = function () {
const last = +new Date() - timestamp;
if (last < wait && last > 0) {
timeout = setTimeout(later, wait - last);
} else {
timeout = null;
if (!immediate) {
result = func.apply(context, args);
if (!timeout) context = args = null;
}
}
};
return function (...args) {
context = this;
timestamp = +new Date();
const callNow = immediate && !timeout;
if (!timeout) timeout = setTimeout(later, wait);
if (callNow) {
result = func.apply(context, args);
context = args = null;
}
return result;
};
}
hash_one.js
self.importScripts("/spark-md5.min.js"); // 导入脚本
// 生成文件 hash
self.onmessage = async (e) => {
const fileChunk = e.data;
const spark = new self.SparkMD5.ArrayBuffer();
const reader = new FileReader();
reader.readAsArrayBuffer(fileChunk);
reader.onload = ({ target }) => {
spark.append(target.result);
self.postMessage({
hash: spark.end(),
});
};
};