前言
本文将从零搭建前端和服务端，实现从单文件上传，到切片上传，文件秒传，断点续传的整个逻辑递进过程。
前端：react + antd
后端：express
单文件上传
前端
基本原理：使用window.URL.createObjectURL方法，生成前端预览文件的url；运用FormData对象，添加文件，通过xhr传输到后端，为了界面美观，运用antd快速进行开发。
//Upload.js
import { Row, Col, Button, message, Progress, Table, Space } from "antd";
import { useState, useEffect, useMemo } from "react";
import request from "./request";
const MAX_SIZE = 1024 * 1024 * 10; //控制文件大小
const VALID_TYPE_LIST = ["image/jpeg", "image/jpg", "image/gif"]; //支持的文件类型
function Upload() {
const [currentFile, setCurrentFile] = useState();
const [previewUrl, setPreviewUrll] = useState();
const handleChange = (e) => {
setCurrentFile(e.target.files[0]);
};
//校验文件格式和大小
const isValid = (file) => {
const { type, size } = file;
const isValidType = VALID_TYPE_LIST.includes(type);
if (!isValidType) {
message.error("文件类型错误");
}
const isValidSize = size <= MAX_SIZE;
if (!isValidSize) {
message.error("文件大小错误");
}
return isValidType && isValidSize;
};
//处理上传逻辑
const handleUpload = async () => {
if (!currentFile) {
return message.error("请选择文件");
}
if (isValid(currentFile)) {
const formData = new FormData();
formData.append("filename", currentFile.name);
formData.append("chunk", currentFile);
const result = await request({
url: "/upload",
method: "POST",
data: formData,
});
message.success("上传成功");
}
useEffect(() => {
if (!currentFile) return;
const urlObject = window.URL.createObjectURL(currentFile);
setPreviewUrll(urlObject);
return () => {
window.URL.revokeObjectURL(currentFile);//更换文件后释放内存
};
}, [currentFile]);
return (
<div>
<Row>
<Col span={12}>
<input type="file" onChange={handleChange}></input>
<Space size={20}>
<Button
style={{ margin: "10px 0" }}
onClick={handleUpload}
type="primary"
>
上传
</Button>
</Space>
</Col>
<Col span={12}>
{previewUrl && (
<img style={{ width: 200 }} alt="预览" src={previewUrl} />
)}
</Col>
</Row>
</div>
);
}
封装request请求
function request(options) {
const defaultOptions = {
method: "GET",
baseURL: "http://localhost:4000",
Headers: {},
data: {},
};
options = {
...defaultOptions,
...options,
headers: { ...defaultOptions.headers, ...options.headers },
};
return new Promise(function (resolve, reject) {
const xhr = new XMLHttpRequest();
xhr.open(options.method, options.baseURL + options.url);
for (const key in options.headers) {
xhr.setRequestHeader(key, options.headers[key]);
}
xhr.responseType = "json";
xhr.onreadystatechange = function () {
if (xhr.readyState === 4 && xhr.status === 200) {
resolve(xhr.response);
}
};
xhr.send(options.data);
});
}
export default request;
后端
使用express快速搭建后端服务，通过cors模块，对跨域进行处理，通过multiparty模块，进行前端传输数据的解析
import express from "express";
import { INTERNAL_SERVER_ERROR } from "http-status-codes";
import createError from "http-errors";
import cors from "cors";
import path from "path";
import { fileURLToPath } from "url";
import fs from "fs-extra";
import multiparty from "multiparty";
import logger from "morgan";
const app = express();
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
export const PUBLIC_DIR = path.resolve(__dirname, "public");
app.use(logger("dev"));
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(cors());
app.use(express.static(path.resolve(__dirname, "public")));
//处理upload请求
app.post("/upload", function (req, res, next) {
const form = new multiparty.Form();
form.parse(req, async function (err, fields, files) {
if (err) return next(err);
const filename = fields.filename[0];
const chunk = files.chunk[0];
//将文件存储到指定目录
await fs.move(chunk.path, path.resolve(PUBLIC_DIR, filename), {
overwrite: true,
});
res.json({ success: true });
});
});
//生成错误信息
app.use(function (req, res, next) {
next(createError(404));
});
//返回错误信息
app.use(function (error, res, req, next) {
res.status(error.status | INTERNAL_SERVER_ERROR);
res.json({ success: false, error });
});
export default app;
分片上传
当单个文件较大，接口对文件大小限制等因素时，需要我们把大的文件分割成多个部分，进行分片上传，其核心原理是利用Blob.prototype.slice方法，把文件分割成多个部分，传输到服务器后，再按照顺序进行拼接，为了对文件进行准确的区分，这里需要计算文件hash值作为文件名称，这里我们通过spark-md5这个库来处理.
如果文件较大，计算hash的过程会比较耗时，为了不对页面交互造成阻塞，我们使用web-worker来计算。同时通过通信机制，可以将计算的进度同步到主进程。
前端将文件切片并上传
//hash.js
//引入spark-md5
self.importScripts(
"https://cdn.bootcdn.net/ajax/libs/spark-md5/3.0.2/spark-md5.min.js"
);
self.onmessage = async (event) => {
let { chunkList } = event.data;
const spark = new self.SparkMD5.ArrayBuffer();
let percent = 0;
let perChunk = 100 / chunkList.length;
let buffers = await Promise.all(
chunkList.map(({ chunk, size }) => {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsArrayBuffer(chunk);
reader.onload = function (event) {
percent += perChunk;
//将进度同步给主进程
self.postMessage({
percent: Number(percent.toFixed(2)),
});
resolve(event.target.result);
};
});
})
);
buffers.forEach((buffer) => spark.append(buffer));
self.postMessage({ percent: 100, hash: spark.end() });
self.close();
};
在上传的页面初始化web-worker,并接收hash生成进度
//Upload.js
import { Row, Col, Button, message, Progress, Table, Space } from "antd";
import { useState, useEffect, useMemo } from "react";
import request from "./request";
const MAX_SIZE = 1024 * 1024 * 10; //控制文件大小
const VALID_TYPE_LIST = ["image/jpeg", "image/jpg", "image/png"]; //支持的文件类型
function Upload() {
const [currentFile, setCurrentFile] = useState();
const [previewUrl, setPreviewUrll] = useState();
+ const [hashPercent, setHashPercent] = useState(0);
+ const [filename, setFilename] = useState("");
+ const [chunkList, setChunkList] = useState([]);
const handleChange = (e) => {
setCurrentFile(e.target.files[0]);
};
//校验文件格式和大小
const isValid = (file) => {
const { type, size } = file;
const isValidType = VALID_TYPE_LIST.includes(type);
if (!isValidType) {
message.error("文件类型错误");
}
const isValidSize = size <= MAX_SIZE;
if (!isValidSize) {
message.error("文件大小错误");
}
return isValidType && isValidSize;
};
//处理上传逻辑
const handleUpload = async () => {
if (!currentFile) {
return message.error("请选择文件");
}
+ const chunkList = createChunks(currentFile); //文件分片信息
+ const hash = await createHash(chunkList); //文件hash值
+ const dotIndex = currentFile.name.lastIndexOf(".");
+ const extname = currentFile.name.slice(dotIndex); //文件扩展名
+ const filename = `${hash}${extname}`;
+ setFilename(filename);
+ chunkList.forEach((chunk, index) => {
+ chunk.filename = filename;
+ chunk.chunk_name = filename + "-" + index; //切片名称
+ });
+ setChunkList(chunkList);
+ uploadChunks(chunkList, filename); //上传切片
- if (isValid(currentFile)) {
- const formData = new FormData();
- formData.append("filename", currentFile.name);
- formData.append("chunk", currentFile);
- const result = await request({
- url: "/upload",
- method: "POST",
- data: formData,
- });
- message.success("上传成功");
- }
+ //上传切片
+ async function uploadChunks(chunkList, filename) {
+ let requests = await createRequests(chunkList, uploadedList);
+ await Promise.all(requests);
+ //请求合并文件
+ await request({
+ url: `/merge/${filename}`,
+ });
+ }
+ // 上传请求
+ // 为了方便后端用流来处理文件数据，这里将data只包含文件部分，文件名等信息放在请求url上
+ async function createRequests(chunkList, uploadedList) {
+ return chunkList.map((data) =>
+ request({
+ method: "POST",
+ url: `/upload/${data.filename}/${data.chunk_name}`,
+ headers: { "Content-Type": "application/octet-stream" },
+ data: data.chunk,
+ })
+ );
+ }
+ // 生成文件hash
+ function createHash(chunkList) {
+ return new Promise((resolve, reject) => {
+ const worker = new Worker("/hash.js");
+ worker.postMessage({ chunkList });
+ worker.onmessage = (event) => {
+ let { percent, hash } = event.data;
+ setHashPercent(percent);
+ if (hash) {
+ resolve(hash);
+ }
+ };
+ });
+ }
+ //进行文件切片
+ const createChunks = (file) => {
+ let current = 0;
+ let chunkList = [];
+ while (current < file.size) {
+ const chunk = file.slice(current, current + DEFAULT_SIZE);
+ chunkList.push({ chunk, size: chunk.size });
+ current += DEFAULT_SIZE;
+ }
+ return chunkList;
+ };
useEffect(() => {
if (!currentFile) return;
const urlObject = window.URL.createObjectURL(currentFile);
setPreviewUrll(urlObject);
return () => {
window.URL.revokeObjectURL(currentFile);//更换文件后释放内存
};
}, [currentFile]);
return (
<div>
<Row>
<Col span={12}>
<input type="file" onChange={handleChange}></input>
<Space size={20}>
<Button
style={{ margin: "10px 0" }}
onClick={handleUpload}
type="primary"
>
上传
</Button>
</Space>
</Col>
<Col span={12}>
{previewUrl && (
<img style={{ width: 200 }} alt="预览" src={previewUrl} />
)}
</Col>
</Row>
+ <Row>
+ <Col span={4}>生成hash进度</Col>
+ <Col span={20}>
+ <Progress percent={hashPercent} />
+ </Col>
+ </Row>
</div>
);
}
添加进度显示
为了让上传的过程更加清晰，我们接下来添加上传的文件进度条。原理是给每个request请求，添加onprogress，并通过每个文件的进度计算出整个文件的总进度。
//Upload.js
import { Row, Col, Button, message, Progress, Table, Space } from "antd";
import { useState, useEffect, useMemo } from "react";
import request from "./request";
const MAX_SIZE = 1024 * 1024 * 10; //控制文件大小
const VALID_TYPE_LIST = ["image/jpeg", "image/jpg", "image/png"]; //支持的文件类型
function Upload() {
const [currentFile, setCurrentFile] = useState();
const [previewUrl, setPreviewUrll] = useState();
const [hashPercent, setHashPercent] = useState(0);
const [filename, setFilename] = useState("");
const [chunkList, setChunkList] = useState([]);
+ const totalPercent = useMemo(() => {
+ const sum = chunkList.reduce((acc, cur) => acc + cur.percent / chunkList.length,0);
+ return sum;
+ }, [chunkList])
const handleChange = (e) => {
setCurrentFile(e.target.files[0]);
};
//校验文件格式和大小
const isValid = (file) => {
const { type, size } = file;
const isValidType = VALID_TYPE_LIST.includes(type);
if (!isValidType) {
message.error("文件类型错误");
}
const isValidSize = size <= MAX_SIZE;
if (!isValidSize) {
message.error("文件大小错误");
}
return isValidType && isValidSize;
};
+ const columns = [
+ {
+ title: "切片名称",
+ dataIndex: "chunk_name",
+ width: "25%",
+ },
+ {
+ title: "进度",
+ dataIndex: "percent",
+ width: "75%",
+ render: (val) => <Progress percent={val} />,
+ },
+ ];
//处理上传逻辑
const handleUpload = async () => {
if (!currentFile) {
return message.error("请选择文件");
}
const chunkList = createChunks(currentFile); //文件分片信息
const hash = await createHash(chunkList); //文件hash值
const dotIndex = currentFile.name.lastIndexOf(".");
const extname = currentFile.name.slice(dotIndex); //文件扩展名
const filename = `${hash}${extname}`;
setFilename(filename);
chunkList.forEach((chunk, index) => {
chunk.filename = filename;
chunk.chunk_name = filename + "-" + index; //切片名称
+ chunk.percent = 0; //初始进度为0
});
setChunkList(chunkList);
uploadChunks(chunkList, filename); //上传切片
//上传切片
async function uploadChunks(chunkList, filename) {
let requests = await createRequests(chunkList, uploadedList);
await Promise.all(requests);
//请求合并文件
await request({
url: `/merge/${filename}`,
});
}
// 上传请求
// 为了方便后端用流来处理文件数据，这里将data只包含文件部分，文件名等信息放在请求url上
async function createRequests(chunkList, uploadedList) {
return chunkList.map((data) =>
request({
method: "POST",
url: `/upload/${data.filename}/${data.chunk_name}`,
headers: { "Content-Type": "application/octet-stream" },
data: data.chunk,
+ onprogress: (event) => {
+ data.percent = Number((((data.loaded + event.loaded) / data.size) * 100).toFixed(2));
+ setChunkList([...chunkList]);
+ },
})
);
}
// 生成文件hash
function createHash(chunkList) {
return new Promise((resolve, reject) => {
const worker = new Worker("/hash.js");
worker.postMessage({ chunkList });
worker.onmessage = (event) => {
let { percent, hash } = event.data;
setHashPercent(percent);
if (hash) {
resolve(hash);
}
};
});
}
//进行文件切片
const createChunks = (file) => {
let current = 0;
let chunkList = [];
while (current < file.size) {
const chunk = file.slice(current, current + DEFAULT_SIZE);
chunkList.push({ chunk, size: chunk.size });
current += DEFAULT_SIZE;
}
return chunkList;
};
useEffect(() => {
if (!currentFile) return;
const urlObject = window.URL.createObjectURL(currentFile);
setPreviewUrll(urlObject);
return () => {
window.URL.revokeObjectURL(currentFile);//更换文件后释放内存
};
}, [currentFile]);
return (
<div>
<Row>
<Col span={12}>
<input type="file" onChange={handleChange}></input>
<Space size={20}>
<Button
style={{ margin: "10px 0" }}
onClick={handleUpload}
type="primary"
>
上传
</Button>
</Space>
</Col>
<Col span={12}>
{previewUrl && (
<img style={{ width: 200 }} alt="预览" src={previewUrl} />
)}
</Col>
</Row>
<Row>
<Col span={4}>生成hash进度</Col>
<Col span={20}>
<Progress percent={hashPercent} />
</Col>
</Row>
+ <Row>
+ <Col span={4}>文件上传总进度</Col>
+ <Col span={20}>
+ <Progress percent={totalPercent} />
+ </Col>
+ </Row>
+ <Table rowKey="chunk_name" columns={columns} dataSource={chunkList} />
</div>
);
}
同时在request工具中，调用onprogress
function request(options) {
const defaultOptions = {
method: "GET",
baseURL: "http://localhost:4000",
Headers: {},
data: {},
};
options = {
...defaultOptions,
...options,
headers: { ...defaultOptions.headers, ...options.headers },
};
return new Promise(function (resolve, reject) {
const xhr = new XMLHttpRequest();
xhr.open(options.method, options.baseURL + options.url);
for (const key in options.headers) {
xhr.setRequestHeader(key, options.headers[key]);
}
xhr.responseType = "json";
xhr.onreadystatechange = function () {
if (xhr.readyState === 4 && xhr.status === 200) {
resolve(xhr.response);
}
};
+ xhr.upload.onprogress = options.onprogress;
xhr.send(options.data);
});
}
export default request;
进度条显示效果如下：
存储文件切片
对应修改文件接收逻辑，用文件名作为临时的目录名称，里面存放所有文件的切片数据，当前端请求合并时，读取所有切信息，然后合并成完整的文件。
import express from "express";
import { INTERNAL_SERVER_ERROR } from "http-status-codes";
import createError from "http-errors";
import cors from "cors";
import path from "path";
import { fileURLToPath } from "url";
import fs from "fs-extra";
- import multiparty from "multiparty"
+ import { mergeChunks, TEMP_DIR } from "./utils.js";//引入合并文件方法
import logger from "morgan";
const app = express();
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
export const PUBLIC_DIR = path.resolve(__dirname, "public");
app.use(logger("dev"));
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(cors());
app.use(express.static(path.resolve(__dirname, "public")));
//处理upload请求
- app.post("/upload", function (req, res, next) {
- const form = new multiparty.Form();
- form.parse(req, async function (err, fields, files) {
- if (err) return next(err);
- const filename = fields.filename[0];
- const chunk = files.chunk[0];
- //将文件存储到指定目录
- await fs.move(chunk.path, path.resolve(PUBLIC_DIR, filename), {
- overwrite: true,
- });
- res.json({ success: true });
- });
- });
+ app.post("/upload/:filename/:chunk_name",async function (req, res, next) {
+ let { filename, chunk_name } = req.params
+ let fileDir = path.resolve(TEMP_DIR, filename);
+ const exist = await fs.pathExists(fileDir);
+ if (!exist) await fs.mkdirp(fileDir);//目录不存在就新建
+ const chunkFilePath = path.resolve(fileDir, chunk_name);
+ const ws = fs.createWriteStream(chunkFilePath, { start:0, flags: "a" });
+ req.on("end", () => {
+ ws.close();
+ res.json({ success: true });
+ });
+ req.on("error", () => {
+ ws.close();
+ });
+ req.on("close", () => {
+ ws.close();
+ });
+ req.pipe(ws);
+ }
+ );
+ //合并文件
+ app.get("/merge/:filename", async (req, res, next) => {
+ const { filename } = req.params;
+ await mergeChunks(filename);
+ res.json({ success: true });
+ });
//生成错误信息
app.use(function (req, res, next) {
next(createError(404));
});
//返回错误信息
app.use(function (error, res, req, next) {
res.status(error.status | INTERNAL_SERVER_ERROR);
res.json({ success: false, error });
});
export default app;
合并文件
合并文件一共分3步
- 读取切片缓存目录
- 把目录文件根据顺序进行合并
- 删除切片数据
//utils.js
import path from "path";
import { fileURLToPath } from "url";
import fs from "fs-extra";
import { PUBLIC_DIR } from "./app.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const defaultSize = 100 * 1024;//默认切片大小
export const TEMP_DIR = path.resolve(__dirname, "temp");//切片缓存目录
//合并文件方法
export async function mergeChunks(filename, size = defaultSize) {
const filePath = path.resolve(PUBLIC_DIR, filename);
const chunksDir = path.resolve(TEMP_DIR, filename);
const chunkFIles = await fs.readdir(chunksDir);//读取文件列表
chunkFIles.sort((a, b) => Number(a.split("-")[1]) - Number(b.split("-")[1]));//对文件进行排序
await Promise.all(
chunkFIles.map((chunkPath, index) =>
pipeStream(
path.resolve(chunksDir, chunkPath),
fs.createWriteStream(filePath, { start: index * size })//写入流
)
)
);
await fs.rmdir(chunksDir);//删除缓存目录
}
//文件读取
function pipeStream(filePath, ws) {
return new Promise(function (resolve, reject) {
const rs = fs.createReadStream(filePath);//创建读取流
rs.on("end", async function () {
rs.close();//读取完毕后关闭流
await fs.unlink(filePath);
resolve();
});
rs.pipe(ws);
});
}
文件秒传
原理上就是在文件上传前，根据文件的hash值，判断服务器上是否已经存在此文件，如果存在，即上传成功，不必重新上传！
后端新增和文件验证接口
import express from "express";
import { INTERNAL_SERVER_ERROR } from "http-status-codes";
import createError from "http-errors";
import cors from "cors";
import path from "path";
import { fileURLToPath } from "url";
import fs from "fs-extra";
import multiparty from "multiparty"
import { mergeChunks, TEMP_DIR } from "./utils.js";//引入合并文件方法
import logger from "morgan";
const app = express();
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
export const PUBLIC_DIR = path.resolve(__dirname, "public");
app.use(logger("dev"));
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(cors());
app.use(express.static(path.resolve(__dirname, "public")));
+ //验证文件是否存在
+ app.get("/verify/:filename", async (req, res, next) => {
+ const { filename } = req.params;
+ // 添加秒传
+ const filePath = path.resolve(PUBLIC_DIR, filename);
+ const fileExist = await fs.pathExists(filePath);
+ if (fileExist) // 文件存在不需要重复上传
+ return res.json({
+ success: true,
+ needUpload: false,
+ });
+ res.json({
+ success: true,
+ needUpload: true,
+ });
+ });
//处理upload请求
app.post("/upload/:filename/:chunk_name",async function (req, res, next) {
let { filename, chunk_name } = req.params
let fileDir = path.resolve(TEMP_DIR, filename);
const exist = await fs.pathExists(fileDir);
if (!exist) await fs.mkdirp(fileDir);//目录不存在就新建
const chunkFilePath = path.resolve(fileDir, chunk_name);
const ws = fs.createWriteStream(chunkFilePath, { start:0, flags: "a" }); req.on("end", () => {
ws.close();
res.json({ success: true });
});
req.on("error", () => {
ws.close();
});
req.on("close", () => {
ws.close();
});
req.pipe(ws);
}
);
//合并文件
app.get("/merge/:filename", async (req, res, next) => {
const { filename } = req.params;
await mergeChunks(filename);
res.json({ success: true });
});
//生成错误信息
app.use(function (req, res, next) {
next(createError(404));
});
//返回错误信息
app.use(function (error, res, req, next) {
res.status(error.status | INTERNAL_SERVER_ERROR);
res.json({ success: false, error });
});
export default app;
前端添加验证逻辑
//Upload.js
import { Row, Col, Button, message, Progress, Table, Space } from "antd";
import { useState, useEffect, useMemo } from "react";
import request from "./request";
const MAX_SIZE = 1024 * 1024 * 10; //控制文件大小
const VALID_TYPE_LIST = ["image/jpeg", "image/jpg", "image/png"]; //支持的文件类型
function Upload() {
const [currentFile, setCurrentFile] = useState();
const [previewUrl, setPreviewUrll] = useState();
const [hashPercent, setHashPercent] = useState(0);
const [filename, setFilename] = useState("");
const [chunkList, setChunkList] = useState([]);
const totalPercent = useMemo(() => {
const sum = chunkList.reduce((acc, cur) => acc + cur.percent / chunkList.length,0);
return sum;
}, [chunkList])
const handleChange = (e) => {
setCurrentFile(e.target.files[0]);
};
//校验文件格式和大小
const isValid = (file) => {
const { type, size } = file;
const isValidType = VALID_TYPE_LIST.includes(type);
if (!isValidType) {
message.error("文件类型错误");
}
const isValidSize = size <= MAX_SIZE;
if (!isValidSize) {
message.error("文件大小错误");
}
return isValidType && isValidSize;
};
const columns = [
{
title: "切片名称",
dataIndex: "chunk_name",
width: "25%",
},
{
title: "进度",
dataIndex: "percent",
width: "75%",
render: (val) => <Progress percent={val} />,
},
];
//处理上传逻辑
const handleUpload = async () => {
if (!currentFile) {
return message.error("请选择文件");
}
const chunkList = createChunks(currentFile); //文件分片信息
const hash = await createHash(chunkList); //文件hash值
const dotIndex = currentFile.name.lastIndexOf(".");
const extname = currentFile.name.slice(dotIndex); //文件扩展名
const filename = `${hash}${extname}`;
setFilename(filename);
chunkList.forEach((chunk, index) => {
chunk.filename = filename;
chunk.chunk_name = filename + "-" + index; //切片名称
chunk.percent = 0; //初始进度为0
});
setChunkList(chunkList);
uploadChunks(chunkList, filename); //上传切片
//上传切片
async function uploadChunks(chunkList, filename) {
+ const { needUpload, uploadedList } = await verify(filename);
+ if (!needUpload) return message.success("上传成功");
let requests = await createRequests(chunkList, uploadedList);
await Promise.all(requests);
//请求合并文件
await request({
url: `/merge/${filename}`,
});
}
+ //验证文件是否上传过
+ async function verify(filename) {
+ return await request({
+ url: `/verify/${filename}`,
+ });
+ }
// 上传请求
// 为了方便后端用流来处理文件数据，这里将data只包含文件部分，文件名等信息放在请求url上
async function createRequests(chunkList, uploadedList) {
return chunkList.map((data) =>
request({
method: "POST",
url: `/upload/${data.filename}/${data.chunk_name}`,
headers: { "Content-Type": "application/octet-stream" },
data: data.chunk,
onprogress: (event) => {
data.percent = Number((((data.loaded + event.loaded) / data.size) * 100).toFixed(2));
setChunkList([...chunkList]);
},
})
);
}
// 生成文件hash
function createHash(chunkList) {
return new Promise((resolve, reject) => {
const worker = new Worker("/hash.js");
worker.postMessage({ chunkList });
worker.onmessage = (event) => {
let { percent, hash } = event.data;
setHashPercent(percent);
if (hash) {
resolve(hash);
}
};
});
}
//进行文件切片
const createChunks = (file) => {
let current = 0;
let chunkList = [];
while (current < file.size) {
const chunk = file.slice(current, current + DEFAULT_SIZE);
chunkList.push({ chunk, size: chunk.size });
current += DEFAULT_SIZE;
}
return chunkList;
};
useEffect(() => {
if (!currentFile) return;
const urlObject = window.URL.createObjectURL(currentFile);
setPreviewUrll(urlObject);
return () => {
window.URL.revokeObjectURL(currentFile);//更换文件后释放内存
};
}, [currentFile]);
return (
<div>
<Row>
<Col span={12}>
<input type="file" onChange={handleChange}></input>
<Space size={20}>
<Button
style={{ margin: "10px 0" }}
onClick={handleUpload}
type="primary"
>
上传
</Button>
</Space>
</Col>
<Col span={12}>
{previewUrl && (
<img style={{ width: 200 }} alt="预览" src={previewUrl} />
)}
</Col>
</Row>
<Row>
<Col span={4}>生成hash进度</Col>
<Col span={20}>
<Progress percent={hashPercent} />
</Col>
</Row>
<Row>
<Col span={4}>文件上传总进度</Col>
<Col span={20}>
<Progress percent={totalPercent} />
</Col>
</Row>
<Table rowKey="chunk_name" columns={columns} dataSource={chunkList} />
</div>
);
}
断点续传
如果文件只上传了部分切片，当用户再次上传，只需要上传缺少的切片文件，为了更好的模拟，我们需要添加上传暂停和继续的功能，
后端
后端需要通过接口告知前端目前已经存在的文件切片。同时在合并切片文件时，需要在已有文件的尾部拼接。
import express from "express";
import { INTERNAL_SERVER_ERROR } from "http-status-codes";
import createError from "http-errors";
import cors from "cors";
import path from "path";
import { fileURLToPath } from "url";
import fs from "fs-extra";
import multiparty from "multiparty"
import { mergeChunks, TEMP_DIR } from "./utils.js";//引入合并文件方法
import logger from "morgan";
const app = express();
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
export const PUBLIC_DIR = path.resolve(__dirname, "public");
app.use(logger("dev"));
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(cors());
app.use(express.static(path.resolve(__dirname, "public")));
//验证文件是否存在
app.get("/verify/:filename", async (req, res, next) => {
const { filename } = req.params;
// 添加秒传
const filePath = path.resolve(PUBLIC_DIR, filename);
const fileExist = await fs.pathExists(filePath);
if (fileExist) // 文件存在不需要重复上传
return res.json({
success: true,
needUpload: false,
});
+ let fileDir = path.resolve(TEMP_DIR, filename);
+ const exist = await fs.pathExists(fileDir);
+ let uploadedList = [];//已经存在的文件切片
+ if (exist) {
+ uploadedList = await fs.readdir(fileDir);
+ uploadedList = await Promise.all(
+ uploadedList.map(async (filename) => {
+ const stat = await fs.stat(path.resolve(fileDir, filename));
+ return {
+ filename,
+ size: stat.size,
+ };
+ })
+ );
+ }
res.json({
success: true,
needUpload: true,
+ uploadedList
});
});
//处理upload请求
- app.post("/upload/:filename/:chunk_name",async function (req, res,next) {
+ app.post("/upload/:filename/:chunk_name/:start",async function (req, res,next) {
- let { filename, chunk_name} = req.params
+ let { filename, chunk_name, start} = req.params
+ start = isNaN(start) ? 0 : Number(start);
let fileDir = path.resolve(TEMP_DIR, filename);
const exist = await fs.pathExists(fileDir);
if (!exist) await fs.mkdirp(fileDir);//目录不存在就新建
const chunkFilePath = path.resolve(fileDir, chunk_name);
- const ws = fs.createWriteStream(chunkFilePath, { start:0, flags: "a" });
+ const ws = fs.createWriteStream(chunkFilePath, { start, flags: "a" });
req.on("end", () => {
ws.close();
res.json({ success: true });
});
req.on("error", () => {
ws.close();
});
req.on("close", () => {
ws.close();
});
req.pipe(ws);
}
);
//合并文件
app.get("/merge/:filename", async (req, res, next) => {
const { filename } = req.params;
await mergeChunks(filename);
res.json({ success: true });
});
//生成错误信息
app.use(function (req, res, next) {
next(createError(404));
});
//返回错误信息
app.use(function (error, res, req, next) {
res.status(error.status | INTERNAL_SERVER_ERROR);
res.json({ success: false, error });
});
export default app;
前端
上传文件时需要过滤掉已经上传过的切片，并且要截取那些没有上传完的切片片段。
//Upload.js
import { Row, Col, Button, message, Progress, Table, Space } from "antd";
import { useState, useEffect, useMemo } from "react";
import request from "./request";
const MAX_SIZE = 1024 * 1024 * 10; //控制文件大小
const VALID_TYPE_LIST = ["image/jpeg", "image/jpg", "image/png"]; //支持的文件类型
function Upload() {
const [currentFile, setCurrentFile] = useState();
const [previewUrl, setPreviewUrll] = useState();
const [hashPercent, setHashPercent] = useState(0);
const [filename, setFilename] = useState("");
const [chunkList, setChunkList] = useState([]);
const totalPercent = useMemo(() => {
const sum = chunkList.reduce((acc, cur) => acc + cur.percent / chunkList.length,0);
return sum;
}, [chunkList])
const handleChange = (e) => {
setCurrentFile(e.target.files[0]);
};
//校验文件格式和大小
const isValid = (file) => {
const { type, size } = file;
const isValidType = VALID_TYPE_LIST.includes(type);
if (!isValidType) {
message.error("文件类型错误");
}
const isValidSize = size <= MAX_SIZE;
if (!isValidSize) {
message.error("文件大小错误");
}
return isValidType && isValidSize;
};
const columns = [
{
title: "切片名称",
dataIndex: "chunk_name",
width: "25%",
},
{
title: "进度",
dataIndex: "percent",
width: "75%",
render: (val) => <Progress percent={val} />,
},
];
+ //暂停上传
+ async function handlePause() {
+ chunkList.forEach((chunkData) => chunkData.xhr && chunkData.xhr.abort());
+ }
+ //继续上传
+ async function handleResume() {
+ uploadChunks(chunkList, filename);
+ }
//处理上传逻辑
const handleUpload = async () => {
if (!currentFile) {
return message.error("请选择文件");
}
const chunkList = createChunks(currentFile); //文件分片信息
const hash = await createHash(chunkList); //文件hash值
const dotIndex = currentFile.name.lastIndexOf(".");
const extname = currentFile.name.slice(dotIndex); //文件扩展名
const filename = `${hash}${extname}`;
setFilename(filename);
chunkList.forEach((chunk, index) => {
chunk.filename = filename;
chunk.chunk_name = filename + "-" + index; //切片名称
chunk.percent = 0; //初始进度为0
+ chunk.loaded = 0; // 已经上传过的数据量
});
setChunkList(chunkList);
uploadChunks(chunkList, filename); //上传切片
//上传切片
async function uploadChunks(chunkList, filename) {
const { needUpload, uploadedList } = await verify(filename);
if (!needUpload) return message.success("上传成功");
let requests = await createRequests(chunkList, uploadedList);
await Promise.all(requests);
//请求合并文件
await request({
url: `/merge/${filename}`,
});
}
//验证文件是否上传过
async function verify(filename) {
return await request({
url: `/verify/${filename}`,
});
}
// 上传请求
// 为了方便后端用流来处理文件数据，这里将data只包含文件部分，文件名等信息放在请求url上
async function createRequests(chunkList, uploadedList) {
- return chunkList.map((data) =>
+ return chunkList
+ .filter((chunkData) => {
+ const uploaded = uploadedList.find(
+ (file) => file.filename === chunkData.chunk_name
+ );
+ if (!uploaded) { //没有上传过此切片
+ chunkData.loaded = 0;
+ chunkData.percent = 0;
+ return true;
+ }
+ if (uploaded.size < chunkData.size) {//此切片没有上传完整
+ chunkData.loaded = uploaded.size; //已经上传的大小
+ chunkData.percent = Number(
+ //已经上传的百分比
+ ((uploaded.size / chunkData.size) * 100).toFixed(2)
+ );
+ return true;
+ }
+ return false;
+ })
.map((data) =>
request({
method: "POST",
- url: `/upload/${data.filename}/${data.chunk_name}`,
+ url: `/upload/${data.filename}/${data.chunk_name}/${data.loaded}`,
headers: { "Content-Type": "application/octet-stream" },
- data: data.chunk,
+ data: data.chunk.slice(data.loaded),//截取未上传的片段
onprogress: (event) => {
data.percent = Number((((data.loaded + event.loaded) / data.size) * 100).toFixed(2));
setChunkList([...chunkList]);
},
+ setXHR: (xhr) => (data.xhr = xhr), //给每个切片添加xhr引用
})
);
}
// 生成文件hash
function createHash(chunkList) {
return new Promise((resolve, reject) => {
const worker = new Worker("/hash.js");
worker.postMessage({ chunkList });
worker.onmessage = (event) => {
let { percent, hash } = event.data;
setHashPercent(percent);
if (hash) {
resolve(hash);
}
};
});
}
//进行文件切片
const createChunks = (file) => {
let current = 0;
let chunkList = [];
while (current < file.size) {
const chunk = file.slice(current, current + DEFAULT_SIZE);
chunkList.push({ chunk, size: chunk.size });
current += DEFAULT_SIZE;
}
return chunkList;
};
useEffect(() => {
if (!currentFile) return;
const urlObject = window.URL.createObjectURL(currentFile);
setPreviewUrll(urlObject);
return () => {
window.URL.revokeObjectURL(currentFile);//更换文件后释放内存
};
}, [currentFile]);
return (
<div>
<Row>
<Col span={12}>
<input type="file" onChange={handleChange}></input>
<Space size={20}>
<Button
style={{ margin: "10px 0" }}
onClick={handleUpload}
type="primary"
>
上传
</Button>
+ <Button
+ style={{ margin: "10px 0" }}
+ onClick={handlePause}
+ type="primary"
+ >
+ 暂停
+ </Button>
+ <Button
+ style={{ margin: "10px 0" }}
+ onClick={handleResume}
+ type="primary"
+ >
+ 继续
+ </Button>
</Space>
</Col>
<Col span={12}>
{previewUrl && (
<img style={{ width: 200 }} alt="预览" src={previewUrl} />
)}
</Col>
</Row>
<Row>
<Col span={4}>生成hash进度</Col>
<Col span={20}>
<Progress percent={hashPercent} />
</Col>
</Row>
<Row>
<Col span={4}>文件上传总进度</Col>
<Col span={20}>
<Progress percent={totalPercent} />
</Col>
</Row>
<Table rowKey="chunk_name" columns={columns} dataSource={chunkList} />
</div>
);
}
//request.js
function request(options) {
const defaultOptions = {
method: "GET",
baseURL: "http://localhost:4000",
Headers: {},
data: {},
};
options = {
...defaultOptions,
...options,
headers: { ...defaultOptions.headers, ...options.headers },
};
return new Promise(function (resolve, reject) {
const xhr = new XMLHttpRequest();
+ options.setXHR && options.setXHR(xhr);
xhr.open(options.method, options.baseURL + options.url);
for (const key in options.headers) {
xhr.setRequestHeader(key, options.headers[key]);
}
xhr.responseType = "json";
xhr.onreadystatechange = function () {
if (xhr.readyState === 4 && xhr.status === 200) {
resolve(xhr.response);
}
};
xhr.upload.onprogress = options.onprogress;
xhr.send(options.data);
});
}
export default request;
以上就完成了整个上传的逻辑，当然也有很多可以优化的地方，比如文件切片并发太多，可以适当控制数量，各位看官可以自由补充发挥。感谢观看。
