关于大报文传输问题
背景
报文传输流向分为 请求与响应
- 响应基本上现在所有的框架基本上只需要简单配置即可以自动完成压缩与解压，如gzip .
- 请求的话，需要请求方与服务方进行约定，且编码完成。
方案
请求
引用说明
有时客户端给服务器传输的JSON比较大，几兆甚至几十兆，网络传输占用时间较多，而json都是文本格式，压缩率较高。因此考虑传输压缩后的格式。
HTTP协议在相应部分支持Content-Encoding:gzip，压缩response body，而没有压缩request body的设计，这也很合理，因为在客户端发起请求时并不知道服务器是否支持压缩。因此没法通过http自身解决，只能增加些程序。
压缩和解压都消耗CPU时间，需要根据实际情况决定是否使用压缩方案，一般内网（微服务）调用启用压缩后有可能得不偿失。在网速较慢的情况下才会有意义。
考虑到通用性，仿效response的header Content-Encoding:gzip方式。
客户端把压缩过的json作为post-body传输，然后增加一个request header: Content-Encoding: gzip来告诉服务器端是压缩的格式。
服务端增加一个Filter，对request头进行检查，如果有Content-Encoding则解压缩后继续。这样不影响现有程序。
服务端还有另外一个解决方案，在NGINX反向代理处增加一个插件（需要去找实现这种功能的插件，官方未提供）把解压缩后的request body传给应用服务器(tomcat)。
参照 http://www.pataliebre.net/howto-make-nginx-decompress-a-gzipped-request.html#.XBToGi276i5
服务端自定义解压
基于springboot
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.web.filter.OncePerRequestFilter;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
@Service
public class ContentEncodingFilter extends OncePerRequestFilter {
Logger logger = LoggerFactory.getLogger(ContentEncodingFilter.class);
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response,
FilterChain chain) throws ServletException, IOException {
String conentEncoding = request.getHeader("Content-Encoding");
if(conentEncoding != null && ("gzip".equalsIgnoreCase(conentEncoding) || "deflate".equalsIgnoreCase(conentEncoding))){
logger.trace("Content-Encoding: {}", conentEncoding);
chain.doFilter(new GZIPRequestWrapper(request), response);
return;
}
chain.doFilter(request, response);
}
}
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ReadListener;
import javax.servlet.ServletInputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.zip.DeflaterInputStream;
import java.util.zip.GZIPInputStream;
public class GZIPRequestWrapper extends HttpServletRequestWrapper {
private final static Logger logger = LoggerFactory.getLogger(GZIPRequestWrapper.class);
protected HttpServletRequest request;
public GZIPRequestWrapper(HttpServletRequest request){
super(request);
this.request = request;
}
@Override
public ServletInputStream getInputStream() throws IOException {
ServletInputStream sis = request.getInputStream();
InputStream is = null;
String conentEncoding = request.getHeader("Content-Encoding");
if("gzip".equalsIgnoreCase(conentEncoding)){
is = new GZIPInputStream(sis);
}else if("deflate".equalsIgnoreCase(conentEncoding)){
is = new DeflaterInputStream(sis);
}else{
throw new UnsupportedEncodingException(conentEncoding + " is not supported.");
}
final InputStream compressInputStream = is;
return new ServletInputStream(){
ReadListener readListener;
@Override
public int read() throws IOException {
int b = compressInputStream.read();
if(b == -1 && readListener != null) {
readListener.onAllDataRead();
}
return b;
}
@Override
public boolean isFinished(){
try {
return compressInputStream.available() == 0;
} catch (IOException e) {
logger.error("error", e);
if(readListener != null) {
readListener.onError(e);
}
return false;
}
}
@Override
public boolean isReady() {
try {
return compressInputStream.available() > 0;
} catch (IOException e) {
logger.error("error", e);
if(readListener != null) {
readListener.onError(e);
}
return false;
}
}
@Override
public void setReadListener(final ReadListener readListener) {
this.readListener = readListener;
sis.setReadListener(new ReadListener() {
@Override
public void onDataAvailable() throws IOException {
logger.trace("onDataAvailable");
if(readListener != null){
readListener.onDataAvailable();
}
}
@Override
public void onAllDataRead() throws IOException {
logger.trace("onAllDataRead");
}
@Override
public void onError(Throwable throwable) {
logger.error("onError", throwable);
if(readListener != null){
readListener.onError(throwable);
}
}
});
}
};
}
}
客户端调试
echo '{"type": "json", "length": 2222}' | gzip > body.gz
curl -v -i http://127.0.0.1:8080/ss -H 'Content-Encoding: gzip' -H 'Content-Type:application/json' --data-binary @body.gz
## 此处介绍如何先成gz压缩内容，且进行调用
curl -i --location --request POST 'localhost:8080/api/event/batch/save' \
--header 'Content-Type: application/json' \
--header 'Content-Encoding: gzip' \
--data-binary '@test-200.json.gz' \
-sS \
-w "📤 上传大小：%{size_upload} 字节\n📥 下载大小：%{size_download} 字节\n总耗时：%{time_total} 秒\n连接耗时：%{time_connect} 秒\n响应开始耗时：%{time_starttransfer} 秒\n" \
-o /dev/null
curl -i --location --request POST 'localhost:8080/api/event/batch/save' \
--header 'Content-Type: application/json' \
--data-binary '@test-200.json' \
-sS \
-w "📤 上传大小：%{size_upload} 字节\n📥 下载大小：%{size_download} 字节\n总耗时：%{time_total} 秒\n连接耗时：%{time_connect} 秒\n响应开始耗时：%{time_starttransfer} 秒\n" \
-o /dev/null
使用RestTemplate
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpRequest;
import org.springframework.http.client.ClientHttpRequestExecution;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.util.CollectionUtils;
import org.springframework.web.client.RestTemplate;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPOutputStream;
@Configuration
public class RestClientConfig {
private final static Logger logger = LoggerFactory.getLogger(RestClientConfig.class);
/**
* request body超过1M时，自动开启压缩模式；（需要服务端支持）
* 压缩和解压会增加程序处理时间，一般内网调用不压缩速度会更快；压缩只是在网络环境较差的情况下有使用意义。
* @return
*/
@Bean
public RestTemplate restTemplate() {
RestTemplate restTemplate = new RestTemplate();
List<ClientHttpRequestInterceptor> interceptors = restTemplate.getInterceptors();
if (CollectionUtils.isEmpty(interceptors)) {
interceptors = new ArrayList<>();
}
interceptors.add(new RestTemplateCompressInterceptor());
restTemplate.setInterceptors(interceptors);
return restTemplate;
}
class RestTemplateCompressInterceptor implements ClientHttpRequestInterceptor {
@Override
public ClientHttpResponse intercept(HttpRequest request, byte[] body,
ClientHttpRequestExecution execution) throws IOException {
logger.trace("RestTemplateCompressInterceptor:intercept() body-length: {}", body == null ? 0 : body.length);
byte[] newBody = null;
if(body == null || body.length < 1024 * 1024){
// 小于1M不开启压缩模式
logger.trace("{}不需要压缩", body == null ? 0 : body.length);
newBody = body;
}else{
logger.trace("{}开启压缩模式", body.length);
request.getHeaders().add("Content-Encoding", "gzip");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream gzip;
try {
gzip = new GZIPOutputStream(baos);
gzip.write(body);
gzip.close();
} catch (IOException e) {
logger.error("压缩request body出错.", e);
throw(e);
}
newBody = baos.toByteArray();
logger.trace("压缩后大小 {}", newBody.length);
}
ClientHttpResponse response = execution.execute(request, newBody);
return response;
}
}
}
响应
基于springboot 只需要简单配置即， 客户端基本上，会自动处理响应回来的压缩报文
server:
compression:
enabled: true
mime-types: application/json,application/xml,text/html,text/plain,text/css,application/x-javascript
