s3依次拉取文件下载到客户端
废话不说，直接上代码：
使用zip方式下载和tar压缩方式下载
package com.lenovo.controller;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/**
* desc:交付物接口
*
* @author qinzl2
* @date 2020/11/11
*/
@RestController
@RequestMapping("/test/download")
public class DownloadController {
private static final Logger logger = LoggerFactory.getLogger(DownloadController.class);
@Autowired
private AmazonS3 amazonS3;
/**
* 接收anywhere_delivery job 的回调，更新delivery的stage
*/
@RequestMapping(value = "zip")
public void zip(HttpServletResponse response) throws UnsupportedEncodingException {
// 清空response
response.reset();
String fileName = URLEncoder.encode("安裝包.zip", StandardCharsets.UTF_8.name());
response.setHeader("Content-Disposition", "attachment;filename=" + fileName);
response.setContentType("APPLICATION/zip");
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
// 设置response的Header
try (OutputStream out = response.getOutputStream();
ZipOutputStream zipOut = new ZipOutputStream(out);
WritableByteChannel destChannel = Channels.newChannel(zipOut)) {
String bucketName = "scello";
String fileKey = "ansible-deploy-playbook/53d72cf64d3ef43f60a22461cc0e49bc";
writeFile(destChannel, zipOut, bucketName, fileKey, "11111.sh");
fileKey = "ansible-deploy-playbook/5dfd961a8a121f6cba4093a7b802012c";
writeFile(destChannel, zipOut, bucketName, fileKey, "22222.sh");
fileKey = "sql/137cb351039ce656ba16070e5a201d18";
writeFile(destChannel, zipOut, bucketName, fileKey, "flyway.config");
zipOut.flush();
zipOut.finish();
} catch (Exception e) {
logger.error("download file error.", e);
}
}
private void writeFile(WritableByteChannel destChannel, ZipOutputStream zipOut, String bucketName, String fileKey, String fileName) throws IOException {
S3Object object = amazonS3.getObject(new GetObjectRequest(bucketName, fileKey));
try (InputStream s3ObjectInputStream = object.getObjectContent();
ReadableByteChannel sourceChannel = Channels.newChannel(s3ObjectInputStream)) {
zipOut.putNextEntry(new ZipEntry(fileName));
channelCopy(sourceChannel, destChannel);
zipOut.closeEntry();
} catch (IOException e) {
throw e;
}
}
/**
* 接收anywhere_delivery job 的回调，更新delivery的stage
*/
@RequestMapping(value = "tar")
public void tar(HttpServletResponse response) throws UnsupportedEncodingException {
// 清空response
response.reset();
String fileName = URLEncoder.encode("安裝包.tar.gz", StandardCharsets.UTF_8.name());
response.setHeader("Content-Disposition", "attachment;filename=" + fileName);
response.setContentType("APPLICATION/zip");
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
// 设置response的Header
try (OutputStream out = response.getOutputStream();
BufferedOutputStream buffOut = new BufferedOutputStream(out);
GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(buffOut);
TarArchiveOutputStream tarOut = new TarArchiveOutputStream(gzOut);
WritableByteChannel destChannel = Channels.newChannel(tarOut)) {
tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
tarOut.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
String bucketName = "scello";
String fileKey = "ansible-deploy-playbook/53d72cf64d3ef43f60a22461cc0e49bc";
writeTarFile(destChannel, tarOut, bucketName, fileKey, "11111.sh");
fileKey = "ansible-deploy-playbook/5dfd961a8a121f6cba4093a7b802012c";
writeTarFile(destChannel, tarOut, bucketName, fileKey, "22222.sh");
fileKey = "sql/137cb351039ce656ba16070e5a201d18";
writeTarFile(destChannel, tarOut, bucketName, fileKey, "flyway.config");
// tarOut.closeArchiveEntry();
tarOut.flush();
tarOut.finish();
} catch (Exception e) {
logger.error("download file error.", e);
}
}
private void writeTarFile(WritableByteChannel destChannel, TarArchiveOutputStream tarOut, String bucketName, String fileKey, String fileName) throws IOException {
S3Object object = amazonS3.getObject(new GetObjectRequest(bucketName, fileKey));
try (InputStream s3ObjectInputStream = object.getObjectContent();
ReadableByteChannel sourceChannel = Channels.newChannel(s3ObjectInputStream)) {
long size = object.getObjectMetadata().getInstanceLength();
TarArchiveEntry tarEntry = new TarArchiveEntry("/"+fileName);
tarEntry.setSize(size);
tarOut.putArchiveEntry(tarEntry);
channelCopy(sourceChannel, destChannel);
tarOut.closeArchiveEntry();
} catch (IOException e) {
throw e;
}
}
private void channelCopy(ReadableByteChannel src, WritableByteChannel dest) throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(10 * 1024);//TODO 目前先使用这种方式申请对外内存，以后有时间弄一个资源池,10k
while (src.read(buffer) != -1) {
buffer.flip();
while (buffer.hasRemaining()) {
dest.write(buffer);
}
buffer.clear();
}
}
}
可以加上异常处理代码
package com.knowdee.server.share.controler;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.*;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
/**
* desc:交付物接口
*
* @author qinzl2
* @since 2024/04/10
*/
@RestController
@RequestMapping("/download")
@Slf4j
public class DownloadController {
private static final String splitString = "/";
@Autowired
private AmazonS3 amazonS3;
@GetMapping(value = "/schema/{entity}/{version}/{fileName}")
public void download(HttpServletResponse response, @PathVariable("entity") String entity, @PathVariable("version") String version, @PathVariable("fileName") String fileName) throws IOException {
String bucketName = "cyxinda";
String fileKey = "schema" + splitString + entity + splitString + version + splitString + fileName;
try(
S3Object object = amazonS3.getObject(new GetObjectRequest(bucketName, fileKey));
OutputStream out = response.getOutputStream();
WritableByteChannel destinationChannel = Channels.newChannel(out);
InputStream s3ObjectInputStream = object.getObjectContent();
ReadableByteChannel sourceChannel = Channels.newChannel(s3ObjectInputStream)) {
// 清空response
// 设置response的Header，会影响前端浏览器的行为；如果设置，发生错误的化，仍然会弹出下载页面，不利于错误信息提示。
// 比如，如果把Content-Disposition和setContentType去掉后，不会下载文件；会在浏览器上面看到文件内容；详情见：https://zhuanlan.zhihu.com/p/682763203
response.reset();
String name = URLEncoder.encode(fileName, StandardCharsets.UTF_8);
response.setHeader("Content-Disposition", "attachment;filename=" + name);
response.setContentType("APPLICATION/zip");
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
channelCopy(sourceChannel, destinationChannel);
out.flush();
} catch (Exception e) {
log.error("download file error.", e);
response.sendError(HttpServletResponse.SC_NOT_FOUND, "error......");
}
}
private void channelCopy(ReadableByteChannel src, WritableByteChannel destinationChannel) throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(10 * 1024);
while (src.read(buffer) != -1) {
buffer.flip();
while (buffer.hasRemaining()) {
destinationChannel.write(buffer);
}
buffer.clear();
}
}
}
另一种更好的写法：
package com.knowdee.server.share.controler;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.*;
import java.nio.charset.StandardCharsets;
/**
* desc:交付物接口
*
* @author qinzl2
* @since 2024/04/10
*/
@RestController
@RequestMapping("/download")
@Slf4j
public class DownloadController {
private static final String splitString = "/";
@Autowired
private AmazonS3 amazonS3;
@GetMapping(value = "/schema/{entity}/{version}/{fileName}")
public ResponseEntity<InputStreamResource> download(@PathVariable("entity") String entity, @PathVariable("version") String version, @PathVariable("fileName") String fileName) throws IOException {
// 清空response
// 设置response的Header
String bucketName = "cyxinda";
String fileKey = "schema" + splitString + entity + splitString + version + splitString + fileName;
return writeFile(bucketName, fileKey);
}
private ResponseEntity<InputStreamResource> writeFile(String bucketName, String fileKey) throws IOException {
try {
S3Object object = amazonS3.getObject(new GetObjectRequest(bucketName, fileKey));
InputStream s3ObjectInputStream = object.getObjectContent();
InputStreamResource resource = new InputStreamResource(s3ObjectInputStream);
if (resource.exists() || resource.isReadable()) {
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType("application/octet-stream"))
.body(resource);
} else {
return ResponseEntity.badRequest().body(stringToInputStreamResource("File not found!"));
}
} catch (SdkClientException e) {
log.error("download file error.", e);
Resource page = new ClassPathResource("templates/page-404.html");
InputStreamResource inputStreamResource = new InputStreamResource(page.getInputStream());
return ResponseEntity.status(404).contentType(MediaType.TEXT_HTML).body(inputStreamResource);
// return ResponseEntity.status(404).body(stringToInputStreamResource("An error occurred while processing your request."));
}
}
private InputStreamResource stringToInputStreamResource(String str) {
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
return new InputStreamResource(byteArrayInputStream);
}
}
src/main/resources/templates/page-404.html的内容如下：
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>File Download Error</title>
</head>
<body>
File not found
</body>
</html>
