阿里云OSS能获取进度的简单上传及线程池实现分片上传

分片上传借鉴代码自博客：blog.csdn.net/qq_40274351…

application.yml

#阿里OSS分片上传配置
aliOss:
  multipartUpload:
    partCount: 100     # 分片数量
    threshold: 524288000 # 启用分片上传阈值：500M  500 * 1024 * 1024
    threadCount: 3     # 线程数

controller层

@RestController
@RequestMapping("/api/import")
public class ImportController {
    @Resource
    private ImportService importService;

    @Resource
    private OssService ossService;

    private static final Logger logger = LoggerFactory.getLogger(ImportController.class);

    @PostMapping("/upload")
    @ResponseBody
    public CommonResult<String> upload(MultipartFile file, HttpServletRequest request, HttpSession session) throws IOException {
        return importService.upload(file, session);
    }


    /**
     * 简单列举分片上传事件
     * @param request
     * @return
     */
    @RequestMapping ("/listMultipartUploads")
    @ResponseBody
    public CommonResult<String> listMultipartUploads(HttpServletRequest request, HttpSession session) {
        return importService.listMultipartUploads(session);
    }

    /**
     * 通过fileName获取实时长传进度
     * @param request
     * @return
     */
    @RequestMapping ("/percent")
    @ResponseBody
    public CommonResult<String> getUploadPercent(HttpServletRequest request, String fileName) {
        HttpSession session = request.getSession();
        String name = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
        Double percent = session.getAttribute(name) == null ? 0 : Double.valueOf(session.getAttribute(name).toString());
        logger.debug(name + ":" + session.getAttribute(name));
        return new CommonResult<>(String.valueOf(percent));
    }

    /**
     * 重置上传进度
     * @param request
     * @return
     */
    @PostMapping("/percent/reset")
    public CommonResult resetPercent(HttpServletRequest request, String fileName) {
        HttpSession session = request.getSession();
        String name = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
        session.setAttribute(name, 0);
        return new CommonResult(CommonResultEmnu.OK);
    }

    /**
     * @Description: OSS删除
     */
    @RequestMapping ("/ossDel")
    public CommonResult ossDel(String filePath) {
        ossService.OssDel(filePath);
        return new CommonResult(CommonResultEmnu.OK);
    }
}

service层

@Service
@Primary
@Slf4j
public class ImportServiceOss implements ImportService {
    @Value("${oss.filePrefix}")
    private String filePrefix;

    /**
     * 阈值，达到阈值使用分片上传
     */
    @Value("${aliOss.multipartUpload.threshold}")
    private int threshold;

    @Override
    public CommonResult<String> upload(MultipartFile file, HttpSession session) {
        if (file == null) {
            return new CommonResult<>(CommonResultEmnu.ERROR, "文件为空");
        }

        if (!file.isEmpty()) {
                String fileName = file.getOriginalFilename();
                if (fileName == null) {
                    return new CommonResult<>(CommonResultEmnu.INVALID_PARAMS, "获取文件名失败");
                }
//                InputStream inputStream = file.getInputStream();
                String suffix = fileName.substring(fileName.lastIndexOf(".") + 1);
                String newName = IdGenerator.getUUID() + "." + suffix.toLowerCase(Locale.ENGLISH);
                String filePath = filePrefix + "_upload" + ResourceUtil.createFilePath().replaceAll("/", "_") + newName;

                try {
                    //MultipartFile转File
                    File newFile = new File(fileName);
                    FileOutputStream os = new FileOutputStream(newFile);
                    os.write(file.getBytes());
                    os.close();
                    //将上传文件写入目标文件
                    file.transferTo(newFile);

                    //大于阈值走分片上传，否则直接简单上传
                    long length = newFile.length();
                    if (length < threshold) {
                        return this.simpleUpload(newFile, session, fileName, filePath);
                    } else {
                        return this.multipartUpload(newFile, session, fileName, filePath);
                    }
                } catch (Exception e) {
                    log.error("MultipartFile转File失败： ", e);
                    return new CommonResult<>(CommonResultEmnu.ERROR, "MultipartFile转File失败！");
                }

        } else {
            return new CommonResult<>(CommonResultEmnu.ERROR, "上传失败，因为文件是空的");
        }
    }


    /**
     * 简单上传
     */
    public CommonResult<String> simpleUpload(File f, HttpSession session, String oldFileName, String filePath) {
        // 创建OSSClient实例。
        OSS ossClient = new OSSClientBuilder().build(OSSConfig.ENDPOINT, OSSConfig.ACCESSKEYID, OSSConfig.ACCESSKEYSECRET);

        // 异步处理，可以使用线程池
        Thread t = new Thread(() -> {
            try {
                // 带进度条的上传。
                ossClient.putObject(new PutObjectRequest(OssUtil.getBucketName(filePath), filePath, f).
                        withProgressListener(new PutObjectProgressListener(session, oldFileName)));
            } catch (Exception e) {
                log.error("上传OSS文件出现异常：", e);

            } finally {
                //保证删除服务器上临时文件
                f.delete();
                // 关闭OSSClient。
                ossClient.shutdown();
            }
        }, "简单上传-" + oldFileName);

        t.start();

        String url = OssUtil.getUrl(filePath);
        log.info("upload_url{} ", url);
        return new CommonResult<>(url);
    }

    /**
     * 分片上传
     */
    public CommonResult<String> multipartUpload(File newFile, HttpSession session, String oldFileName, String filePath) {

        // 异步处理，可以使用线程池
        Thread t = new Thread(() -> {
            try {
                AliyunOSSUtil.upLoad(newFile, session, filePath, oldFileName);
            } catch (Throwable e) {
                log.error("上传OSS文件出现异常：", e);
            } finally {
                //保证删除服务器上临时文件
                newFile.delete();
            }
        });

        t.start();

        String url = OssUtil.getUrl(filePath);
        log.info("upload_url:{} ", url);
        return new CommonResult<>(url);
    }

    /**
     * 查询所有未完成的分片上传信息
     */
    @Override
    public CommonResult<String> listMultipartUploads(HttpSession session) {
        // 创建OSSClient实例。
        OSS ossClient = new OSSClientBuilder().build(OSSConfig.ENDPOINT, OSSConfig.ACCESSKEYID, OSSConfig.ACCESSKEYSECRET);

        String bucketName = OssUtil.getBucketName(filePrefix + "_");

        // 列举分片上传事件。默认列举1000个分片。
        ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
        MultipartUploadListing multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);

        for (MultipartUpload multipartUpload : multipartUploadListing.getMultipartUploads()) {
            // 获取uploadId。
            multipartUpload.getUploadId();
            // 获取Key。
            multipartUpload.getKey();
            // 获取分片上传的初始化时间。
            multipartUpload.getInitiated();
        }
        // 关闭OSSClient。
        ossClient.shutdown();
        return null;
    }

简单上传的自定义监听器，用于获取进度

@Slf4j
public class PutObjectProgressListener implements ProgressListener {
    private long bytesWritten = 0;
    private long totalBytes = -1;
    private boolean succeed = false;
    private HttpSession session;
    private String fileName;

    private static final Logger logger = LoggerFactory.getLogger(PutObjectProgressListener.class);

    //构造方法中加入session
    public PutObjectProgressListener() {
    }

    public PutObjectProgressListener(HttpSession mSession, String fileName) {
        this.session = mSession;
        int percent = 0;
        this.fileName = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
        session.setAttribute(this.fileName, percent);
    }

    @Override
    public void progressChanged(ProgressEvent progressEvent) {
        long bytes = progressEvent.getBytes();
        ProgressEventType eventType = progressEvent.getEventType();
        switch (eventType) {
            case TRANSFER_STARTED_EVENT:
                logger.debug("Start to upload......");
                break;
            case REQUEST_CONTENT_LENGTH_EVENT:
                this.totalBytes = bytes;
                logger.debug(this.totalBytes + " bytes in total will be uploaded to OSS");
                break;
            case REQUEST_BYTE_TRANSFER_EVENT:
                this.bytesWritten += bytes;
                if (this.totalBytes != -1) {
                    int percent = (int) (this.bytesWritten * 100.0 / this.totalBytes);
                    //将进度percent放入session中 官网demo中没有放入session这一步
                    session.setAttribute(this.fileName, percent);
                    logger.debug(bytes + " bytes have been written at this time, upload progress: " + percent + "%(" + this.bytesWritten + "/" + this.totalBytes + ")");
                } else {
                    logger.debug(bytes + " bytes have been written at this time, upload ratio: unknown" + "(" + this.bytesWritten + "/...)");
                }
                break;
            case TRANSFER_COMPLETED_EVENT:
                this.succeed = true;
                logger.debug("Succeed to upload, " + this.bytesWritten + " bytes have been transferred in total");
                break;
            case TRANSFER_FAILED_EVENT:
                logger.debug("Failed to upload, " + this.bytesWritten + " bytes have been transferred");
                break;
            default:
                break;
        }
    }

    public boolean isSucceed() {
        return succeed;
    }

可以获取进度的分片上传方法实现

@Component
public class AliyunOSSUtil {
    /**
     * 分片数量，给静态变量用@Value赋值  需要写在set方法上
     */
    private static int defaultPartCount;

    /**
     * 线程数量
     */
    private static int threadCount;

    private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AliyunOSSUtil.class);


    /**
     * 上传文件
     **/
    public static void upLoad(File file, HttpSession session, String filePath, String oldFileName) {

        //创建一个可重用固定线程数的无界队列线程池  最多一百零一个任务，无需担心队列存放任务太多
        ExecutorService executorService = Executors.newFixedThreadPool(threadCount);

        //判断文件
        if (file == null) {
            return;
        }

        ClientBuilderConfiguration conf = new ClientBuilderConfiguration();
        //连接空闲超时时间，超时则关闭
        conf.setIdleConnectionTime(2000);
        //创建OSSClient实例，需要放到这里，不能把OSS作为静态变量，如果多个客户同时上传，客户A关闭ossClient，而客户B正用着ossClient
        OSS ossClient = new OSSClientBuilder().build(OSSConfig.ENDPOINT, OSSConfig.ACCESSKEYID, OSSConfig.ACCESSKEYSECRET, conf);
        //记录上传片数，线程安全，不能静态变量，客户A和客户B同时上传，片数到最后大于partSize
        AtomicInteger uploadedPart = new AtomicInteger(0);
        /*分片上传*/
        /*1.初始化一个分片上传事件*/
        InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(OssUtil.getBucketName(filePath), filePath);
        InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request);
        String uploadId = result.getUploadId();
//        session.setAttribute("uploadId", uploadId);
        try {
            /*2.上传分片**/
            long partSize;
            long fileLength = file.length();

            //按默认片数计算每片大小
            partSize = (int) fileLength / defaultPartCount;

            //如果100片不能分完，就是101片
            int partCount;
            if (fileLength % partSize != 0) {
                partCount = defaultPartCount + 1;
            } else {
                partCount = defaultPartCount;
            }

            //返回uploadId，它是分片上传事件的唯一标识，可以根据这个ID来发起相关操作，如取消分片上传、查询分片上传等
            // 装饰者模式加锁，保证线程安全
            List<PartETag> partETags = Collections.synchronizedList(new ArrayList<>(partCount));
  
            //遍历分片上传
            for (int i = 0; i < partCount; i++) {
                long startPos = i * partSize;
                //是否为最后一块分片
                long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
                //getBucketName(filePath)是Bucket的名字
                executorService.execute(new PartUploader(ossClient, OssUtil.getBucketName(filePath), file, startPos, curPartSize, i + 1, uploadId, partETags, partCount, session, filePath, oldFileName,uploadedPart));
            }
            //等待所有的分片完成    执行后不再接收新任务，如果里面有任务，就执行完
            executorService.shutdown();
            while (!executorService.isTerminated()) {
                try {
                    executorService.awaitTermination(5, TimeUnit.SECONDS);
                } catch (InterruptedException e) {
                    logger.error(e.getMessage());
                }
            }

            //重新设为0片
            uploadedPart.set(0);

            //验证是否所有的分片都完成
            if (partETags.size() != partCount) {
                throw new IllegalStateException("文件的某些部分上传失败！");
            }

            /*3.完成分片上传**/
            //排序。partETags必须按分片号升序排列
            partETags.sort(new Comparator<PartETag>() {
                @Override
                public int compare(PartETag o1, PartETag o2) {
                    return o1.getPartNumber() - o2.getPartNumber();
                }
            });
            //在执行该操作时，需要提供所有有效的partETags。OSS收到提交的partETags后，会逐一验证每个分片的有效性。当所有的数据分片验证通过后，OSS将把这些分片组合成一个完整的文件。
            CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(OssUtil.getBucketName(filePath), filePath, uploadId, partETags);
            ossClient.completeMultipartUpload(completeMultipartUploadRequest);
            /*//设置权限
            ossClient.setBucketAcl(OssUtil.getBucketName(filePath), CannedAccessControlList.PublicRead);*/

        } catch (OSSException oe) {
            logger.error(oe.getMessage());
        } catch (ClientException ce) {
            logger.error(ce.getErrorMessage());
        } finally {
            //关闭OSSClient
            ossClient.shutdown();
        }
    }

    /**
     * 实现并启动线程
     **/
    private static class PartUploader implements Runnable {
        private final String bucketName;
        private final int partCount;
        private final File localFile;
        private final long startPos;
        private final long partSize;
        private final int partNumber;
        private final String uploadId;
        private List<PartETag> partETags;
        private final HttpSession session;
        private final String key;
        private final String oldFileName;
        private final OSS ossClient;
        private AtomicInteger uploadedPart;

        public PartUploader(OSS ossClient, String bucketName, File localFile, long startPos, long partSize, int partNumber, String uploadId, List<PartETag> partETags, int partCount, HttpSession session, String key, String oldFileName, AtomicInteger uploadedPart) {
            this.ossClient = ossClient;
            this.bucketName = bucketName;
            this.localFile = localFile;
            this.startPos = startPos;
            this.partNumber = partNumber;
            this.uploadId = uploadId;
            this.partSize = partSize;
            this.partETags = partETags;
            this.partCount = partCount;
            this.session = session;
            this.key = key;
            this.oldFileName = oldFileName;
            this.uploadedPart = uploadedPart;
        }

        @Override
        public void run() {
            Thread.currentThread().setName("文件分片上传pool-" + oldFileName + "-" + partNumber);
            InputStream inputStream = null;
            try {
                inputStream = new FileInputStream(this.localFile);
                //跳过已经上传的分片
                inputStream.skip(startPos);
                UploadPartRequest uploadPartRequest = new UploadPartRequest();
                uploadPartRequest.setBucketName(bucketName);
                uploadPartRequest.setKey(key);
                uploadPartRequest.setUploadId(this.uploadId);
                uploadPartRequest.setInputStream(inputStream);
                //设置分片大小。除了最后一个分片没有大小限制，其他分片最小为100KB
                uploadPartRequest.setPartSize(this.partSize);
                //设置分片号。每一个上传的分片都有一个分片号，取值范围是1~10000，如果超出这个范围,OSS将返回InvalidArgum的错误码
                uploadPartRequest.setPartNumber(this.partNumber);
                //每个分片不需要按顺序上传，甚至可以在不同客户端上传，OSS会根据分片号排序组成完整的文件。
                UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
//                System.out.println("Part#" + this.partNumber + "done");
//                System.out.println();
                //已经上传的片数加1
                uploadedPart.incrementAndGet();
                //每次上传分片之后，OSS的返回结果会包含一个PartETag。PartETag将被保存到PartETags中。
                this.partETags.add(uploadPartResult.getPartETag());

                //计算进度
                int percent = (int) (uploadedPart.intValue() * 100.0 / partCount);
                session.setAttribute(oldFileName, percent);

            } catch (Exception e) {
                logger.error(e.getMessage());
            } finally {
                if (inputStream != null) {
                    try {
                        inputStream.close();
                    } catch (IOException e) {
                        logger.error(e.getMessage());
                    }
                }
            }
        }
    }


    @Value("${aliOss.multipartUpload.partCount}")
    public void setDefaultPartCount(int defaultPartCount) {
        AliyunOSSUtil.defaultPartCount = defaultPartCount;
    }

    @Value("${aliOss.multipartUpload.threadCount}")
    public void setThreadCount(int threadCount) {
        AliyunOSSUtil.threadCount = threadCount;
    }

获取进度条接口，前端可以循环调用以下接口，传入文件名，获取进度

    @RequestMapping ("/percent")
    @ResponseBody
    public CommonResult<Object> getUploadPercent(HttpServletRequest request, String fileName) {
        HttpSession session = request.getSession();
        String name = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
        return new CommonResult<>(session.getAttribute(name));
    }