大文件分片上传 ( 切片上传; 断点续传)
由于浏览器和网络环境的限制，直接上传大文件可能导致失败或性能问题。通过分片上传可以将大文件分成小块逐步上传，并在服务器端合并
这其中的解决方案主要包括分片上传、断点续传和秒传等技术，这些技术可以有效解决大文件上传过程中遇到的问题。
大文件上传的常见问题及解决方案
- 分片上传：将大文件分成多个小文件进行上传，可以缩短每个请求的时间，并且如果某个请求失败，不需要重新上传整个文件。前端可以通过获取文件的专属MD5或其他唯一编码来确保文件的完整性。1
- 断点续传：在上传过程中，如果遇到中断，可以从上次中断的地方继续传输，避免重新上传整个文件。这可以通过记录已经上传的分片来实现。
- 秒传：对于已经上传过的文件，无需再次上传，直接使用已经存储在服务器上的版本。这可以通过比较文件的MD5码来实现。
大文件上传的实现步骤和技术细节
- 前端实现：在前端，可以使用JavaScript或相关库（如Web Uploaders）来分割文件并上传每个分片。通过获取文件的二进制内容，将其拆分成指定大小的切片，然后逐个上传。2
- 后端实现：在后端，接收到所有分片后，将这些分片重新组合成一个完整的文件。这通常通过记录每个分片的上传状态和顺序来实现。
本文给出的是Vue+ElementUI的示例, 其他框架可参照本文的代码逻辑适配即可
代码流程图:
Template部分
<el-form
ref="form"
:rules="rules"
:model="form"
label-width="100px"
label-position="right"
style="width: 380px"
class="components-upload"
>
<el-form-item label="选择文件" required>
<el-upload
ref="upload"
class="upload-demo"
action
accept=".zip"
multiple
:file-list="fileList"
:show-file-list="true"
:auto-upload="false"
:on-error="uploadError"
:on-change="uploadFileChange"
:on-remove="handleRemoveFile"
:http-request="httpRequest"
>
<el-button>
<svg-icon icon-class="upload" />
</el-button>
</el-upload>
<!--进度条-->
<el-row v-if="uploadProgress > 0 && !!uploadLoading" :gutter="10">
<el-col :span="24">
<el-progress
:text-inside="true"
:stroke-width="14"
:percentage="uploadProgress"
:color="customColors"
/>
<span v-if="errMsg" style="color: red">*{{ errMsg }}</span>
</el-col>
</el-row>
</el-form-item>
</el-form>
Script部分
import { transitionSha256 } from '@/utils/sha256';
import { bindFileToBusiness } from '@/api/business';
import { getFileIsExists, getMultiPartV2, mergeFile, uploadPartFile } from '@/api/file';
import { getFileType } from '@/utils';
export default {
name: 'UploadComponent',
props: {
isShow: {
type: Boolean,
default: false
},
softwareVersionId: {
type: Number,
default: null
}
},
data() {
return {
fileList: [],
fileSha256: '',
uploadLoading: false,
uploadProgress: 0,
customColors: '',
errMsg: '',
successCount: 0,
unUploadFileNum: 0,
fileInfo: null,
form: {}
};
},
computed: {
dialogVisible: {
get() {
return this.isShow;
},
set() {
return this.isShow;
}
}
},
watch: {
uploadProgress: {
handler(val) {
if (val < 50) {
this.customColors = '#f1d702';
} else if (val < 80) {
this.customColors = '#f68104';
} else {
this.customColors = '#0cf804';
}
}
}
},
methods: {
/**
* 提交上传
*/
submitUpload() {
if (this.fileList.length === 0) {
this.$message.error(this.$t('errMsg.mustUploadFile'));
return;
}
this.$refs.form.validate((valid) => {
if (valid) {
this.$refs.upload.submit();
}
});
},
/**
* 上传控件变更
* @param file
* @param fileList
*/
uploadFileChange(file, fileList) {
this.fileList = [file];
},
/**
* 移除文件
* @param file
* @param fileList
*/
handleRemoveFile(file, fileList) {
this.fileList = fileList;
this.$refs.upload.clearFiles();
},
/**
* 手动触发上传后调用
* @param param 文件信息
*/
httpRequest(param) {
this.uploadLoading = true;
this.fileInfo = param.file; // 相当于input里取得的files
this.uploadProgress = 5;
if (!this.validateFile(this.fileInfo)) {
this.uploadProgress = 0;
this.uploadLoading = false;
return;
}
this.checkFileSha256(this.fileInfo)
.then((res) => {
if (!res) {
this.bindFile()
.then(() => {
this.uploadProgress = 100;
this.uploadLoading = false;
})
.catch((err) => {
console.log(err);
this.uploadLoading = false;
});
} else {
const unUploadPartList = res.list.map((sha) => {
return {
id: res.id,
partNum: sha.partNum,
sha256: sha.sha256,
blob: sha.blob
};
});
this.unUploadFileNum = unUploadPartList.length;
this.successCount = 0;
Promise.all(
this._.range(10).map((n) => this.uploadPartFile(unUploadPartList))
)
.then(() => {
mergeFile(res.id).then(() => {
this.bindFile()
.then(() => {
this.uploadLoading = false;
})
.catch((err) => {
console.log(err);
this.uploadLoading = false;
});
}).catch(err => {
console.log(err);
this.uploadLoading = false;
this.$message.error(this.$t('common.uploadFailed'));
});
})
.catch((err) => {
console.log(err);
this.uploadLoading = false;
this.$message.error(this.$t('common.uploadFailed'));
});
}
})
.catch((err) => {
console.log(err);
this.uploadLoading = false;
});
},
/**
* 校验文件合法性
* @param file
* @returns {boolean}
*/
validateFile(file) {
if (getFileType(file.name) !== 'zip') {
this.$message.error(this.$t('errMsg.mustUploadZip'));
return false;
}
if (file.size > 4 * 1024 * 1024 * 1024) {
this.$message.error(this.$t('errMsg.fileMustLess', { fileSize: '4G' }));
return false;
}
return true;
},
/**
* 上传分片文件
*/
uploadPartFile(unUploadPartList, retryNum = 0) {
return new Promise((resolve, reject) => {
const part = unUploadPartList.shift();
if (!part) {
return resolve();
}
const file = new FormData();
file.append('file', part.blob);
file.append('id', part.id);
file.append('partNum', part.partNum);
uploadPartFile(file)
.then(() => {
this.successCount++;
this.uploadProgress = Math.floor(
20 + (this.successCount / this.unUploadFileNum) * 75
);
return resolve(this.uploadPartFile(unUploadPartList, retryNum));
})
.catch((err) => {
console.log(err);
if (retryNum <= 4) {
unUploadPartList.unshift(part); // 将元素重新加入数组开头
return resolve(this.uploadPartFile(unUploadPartList, retryNum + 1));
}
return reject(err);
});
});
},
/**
* 上传成功后调用绑定接口将文件绑定到相关业务
* @returns {Promise<unknown>}
*/
bindFile() {
const obj = {
...this.form,
fileKey: this.fileSha256,
name: this.fileInfo.name,
size: this.fileInfo.size,
status: 'SUCCESS'
};
return new Promise((resolve, reject) => {
bindFileToBusiness(obj, this.softwareVersionId)
.then(() => {
this.$message.success(this.$t('common.uploadSuccessful'));
this.uploadProgress = 100;
this.handleClose(true);
resolve();
})
.catch((err) => {
this.errMsg = err.message;
reject(err);
});
});
},
/**
* 判断文件是否存在
*/
checkFileSha256(file) {
return new Promise((resolve, reject) => {
transitionSha256(file)
.then((sha256) => {
this.fileSha256 = sha256;
this.uploadProgress = 10;
getFileIsExists(sha256)
.then((res) => {
const { data } = res;
if (data) {
// 存在即返回false
resolve(false);
} else {
// 如不存在则获取文件未上传的分片文件列表
const obj = {
fileName: file.name,
fileKey: sha256,
fileSize: file.size
};
this.getMultiPart(obj)
.then((part) => {
if (
part.unfinishPartNumList &&
part.unfinishPartNumList.length > 0
) {
// 对文件进行切片并返回分片文件列表
this.sliceFile(file, part.partSize).then((parts) => {
// 未上传成功的文件分片
const unFinishPartList = this.checkPartsIsUpload(
parts,
part.unfinishPartNumList
);
// 获取所有未上传分片的sha256
const promise = unFinishPartList.map(async(item) => {
return await transitionSha256(item.blob).then(
(partSha) => {
item.sha256 = partSha;
return item;
}
);
});
this.uploadProgress = 15;
Promise.all(promise).then(() => {
this.uploadProgress = 15;
const obj = {
id: part.id,
list: unFinishPartList.map((item) => {
return {
partNum: item.partNum,
sha256: item.sha256,
blob: item.blob
};
})
};
this.uploadProgress = 20;
resolve(obj);
});
});
} else {
resolve(false);
}
})
.catch((err) => {
console.log(err);
this.uploadLoading = false;
});
}
})
.catch((err) => {
console.log(err);
this.uploadLoading = false;
});
})
.catch((err) => {
console.log(err);
this.uploadLoading = false;
});
});
},
/**
* 获取未上传切片列表
*/
getMultiPart(obj) {
return new Promise((resolve, reject) => {
getMultiPartV2(obj)
.then((res) => {
const { data } = res;
resolve(data);
})
.catch((err) => {
reject(err);
});
});
},
/**
* 对文件进行切片
* @param file
* @param partSize
* @returns {Promise<unknown>}
*/
sliceFile(file, partSize = 100 * 1024 * 1024) {
return new Promise((resolve) => {
if (file.size <= 0) {
resolve([]);
return;
}
const fileList = [];
const totalParts = Math.ceil(file.size / partSize);
for (let i = 0; i < totalParts; i++) {
const start = i * partSize;
const end = Math.min(start + partSize, file.size);
fileList.push({
start,
end,
partNum: i + 1,
blob: file.slice(start, end),
});
}
resolve(fileList);
});
},
/**
* 上传失败
* @param res
* @param file
*/
uploadError(res, file) {
this.$message.error({
title: this.$t('table.fail'),
message: res.message,
duration: 5000
});
},
/**
* 检查未上传的分片
* @param parts
* @param unUploads
* @returns {*[]}
*/
checkPartsIsUpload(parts, unUploads) {
const arr = [];
unUploads.forEach((item) => {
arr.push(parts[item - 1]);
});
return arr;
}
}
};
计算文件哈希值
/**
* 计算文件SHA256值
* @param file
* @returns {Promise<unknown>}
*/
export function transitionSha256(file) {
const fileSize = file.size;
// 切割的每个文件流的大小
const chunkSize = 100 * 1024 * 1024;
let offset = 0;
// 创建Sha256对象
const sha256 = CryptoJS.algo.SHA256.create();
// 创建FileReader对象
const fileReader = new FileReader();
return new Promise((resolve, reject) => {
fileReader.onprogress = e => {
// console.log(Math.floor(e.loaded / e.total * 100));
};
// fileReader.readAsArrayBuffer调用后触发onload事件
fileReader.onload = (e) => {
if (e.target.error === null) {
offset += e.loaded;
const wordArray = CryptoJS.lib.WordArray.create(fileReader.result);
// 将切割的文件流暂存
sha256.update(wordArray);
} else {
reject(e.target.error);
}
if (offset < fileSize) {
fileReader.readAsArrayBuffer(file.slice(offset, chunkSize + offset));
} else {
// 监测加密完成后返回文件的Sha256值
return resolve(sha256.finalize().toString());
}
};
fileReader.readAsArrayBuffer(file?.slice(offset, chunkSize + offset));
});
}
