业务场景结合分布式链路日志的探索

ASSASSIN
276 阅读5分钟

背景

  在业务系统中，为了排错或者查询，需要根据业务属性去串联和查询链路日志，比如工作流移交，其中涉及表单保存，业务数据校验，工作流事件执行，环节移交等操作，需要根据业务号查询，然后串联成一个时间轴的方式来展示(类似分布式链路监控)。如果每个场景都是由硬编码去埋点和串联，带来的代价是非常大。业务系统本身也有很多模块组合而成，有基于SpringCloud架构的，有基于SpringMVC架构的，一笔业务操作，可能横跨多个不同架构的模块，所以需要一套自动化提取业务链路日志的手段。

  因为有基于SpringCloud的的分布式框架，所以尝试通过接入了Zipkin中间件来实现分布式链路日志，并且结合业务场景，来解决业务链路日志的问题。

解决思路

通过自定义filter过滤器，在请求开始阶段，就产生guid或者获取上一个环节的guid(这个guid后续可以替换成自定义的业务属性，用于前端界面查询)

  1. 如果当前服务是头一个服务，就自己产生一个guid，并写到requestHeader里面(一般操作都是由前端发起，前端在请求后台之前，在requestHeader中写上guid或业务属性字段，以及相关操作内容)
  2. 如果不是，就从requestHeader里面获取guid
  3. 把自定义输入写入到链路trace的span里面
  4. 通过默认接口，或者自定义es查询模型，去按照guid获取数据

SpringBoot场景下

添加过滤器ZipkinFilter

核心就是ZipkinFilter，可以添加到网关里，用于解决RequestHeader继承的问题。

其中xguid是测试的，全局唯一guid写入到链路日志中的

public class ZipkinFilter implements Filter {

    Logger logger = LoggerFactory.getLogger(ZipkinFilter.class);

    @Override
    public void destroy() {
        logger.info("----Filter销毁----");
    }

    @Override
    public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException {
        // 对request、response进行一些预处理

        HttpServletRequest servletRequest = (HttpServletRequest) request;
        ZipkinFilterWrapper zipkinFilterWrapper = new ZipkinFilterWrapper(servletRequest);
        Tracer tracer = SpringBeanUtil.getBean("tracer");
        String uuid =  servletRequest.getHeader("xguid");
        logger.info("当前xguid是 {}", uuid);
        //如果没有xguid传过来，就说明这个是第一个，为了保持完整性，第一个就手动产生xguid
        if(StringUtils.isEmpty(uuid)){
            uuid = UUID.randomUUID().toString();
            zipkinFilterWrapper.putHeader("xguid",uuid);
        }
        //重点是这句代码
        tracer.currentSpan().tag("xguid", uuid);
        // 执行目标资源，放行
        filterChain.doFilter(request, response);
        logger.info("----调用service之后执行一段代码----");
    }

    @Override
    public void init(FilterConfig arg0) throws ServletException {
        logger.info("----Filter初始化----");
    }

}

SpringBeanUtil


/**
 * SpringBean获取, 主要用来获取 trace的Bean，用于自定义添加链路标签
 */
@Component
public class SpringBeanUtil implements ApplicationContextAware {
    private static ApplicationContext applicationContext;

    private static void setStaticApplicationContext(ApplicationContext applicationContext){
        SpringBeanUtil.applicationContext = applicationContext;
    }


    @Override
    public void setApplicationContext(@NonNull ApplicationContext applicationContext) throws BeansException
    {
        setStaticApplicationContext(applicationContext);
    }

    public static ApplicationContext getApplicationContext() {
        return applicationContext;
    }

    @SuppressWarnings("unchecked")
    public static <T> T getBean(String name) throws BeansException {
        return (T)applicationContext.getBean(name);
    }

    public static <T> T getBean(Class<T> name) throws BeansException {
        return applicationContext.getBean(name);
    }
}
****

ZipkinFilterWrapper

这个可以不用，后台写入requestHeader比较麻烦，所以加了这个

/**
 * 自定义方法，添加requestHeader
 */
public class ZipkinFilterWrapper extends HttpServletRequestWrapper {

    private final Map<String, String> customHeaders;

    public ZipkinFilterWrapper(HttpServletRequest request){
        super(request);
        this.customHeaders = new HashMap<>(10);
    }

    public void putHeader(String name, String value){
        this.customHeaders.put(name, value);
    }

    @Override
    public String getHeader(String name) {
        String headerValue = customHeaders.get(name);

        if (headerValue != null){
            return headerValue;
        }
        return ((HttpServletRequest) getRequest()).getHeader(name);
    }

    @Override
    public Enumeration<String> getHeaderNames() {
        Set<String> set = new HashSet<>(customHeaders.keySet());

        Enumeration<String> e = ((HttpServletRequest) getRequest()).getHeaderNames();
        while (e.hasMoreElements()) {
            String n = e.nextElement();
            set.add(n);
        }

        return Collections.enumeration(set);
    }
}

配置文件

spring:
  sleuth:
    web:
      client:
        enabled: true
    sampler:
      probability: 1.0
    propagation:
      tag:
        whitelisted-keys: #链路额外传播字段
          - xguid
    propagation-keys:
      - xguid
  zipkin:
    base-url: http://localhost:9411

引用

<dependency>
    <groupId>org.springframework.cloud</groupId>
    <artifactId>spring-cloud-starter-zipkin</artifactId>
</dependency>

SpringMVC场景下

增加BraveConfig

import brave.Tracing;
import brave.http.HttpTracing;
import brave.propagation.B3Propagation;
import brave.propagation.ExtraFieldPropagation;
import brave.servlet.TracingFilter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import zipkin2.Span;
import zipkin2.reporter.AsyncReporter;
import zipkin2.reporter.Sender;
import zipkin2.reporter.okhttp3.OkHttpSender;

import javax.servlet.Filter;

@Configuration
public class BraveConfig {
    @Bean
    public HttpTracing httpTracing(Tracing tracing){
        return HttpTracing.create(tracing);
    }

    @Bean
    public Tracing tracing(){

        return Tracing.newBuilder().localServiceName("PlatformDesigner").spanReporter(spanReporter())
//配置propagation工厂类
                .propagationFactory(ExtraFieldPropagation.newFactory(B3Propagation.FACTORY, "SIGNATURE")).build();
    }

    @Bean
    Filter tracingFilter(HttpTracing httpTracing) {
        return TracingFilter.create(httpTracing);
    }

    @Bean
    public AsyncReporter<Span> spanReporter() {
        zipkin2.reporter.AsyncReporter.Builder reporterBuilder = AsyncReporter.builder(sender());
        return reporterBuilder.build();
    }

    @Bean
    public Sender sender() {
        String zipkinUrl = Global.getConfig("zipkin.url");
        return OkHttpSender.newBuilder()
                .endpoint(zipkinUrl).build();
    }
}

增加ZipkinFilter

import brave.Tracer;
import brave.Tracing;
import xxxxx.SpringBeanUtil;
import xxxxx.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.servlet.*;
import javax.servlet.annotation.WebFilter;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;

/**
 * Zipkin过滤器
 */
@WebFilter("/*")
public class ZipkinFilter implements Filter {

    Logger logger = LoggerFactory.getLogger(ZipkinFilter.class);

    @Override
    public void destroy() {
        logger.info("----Filter销毁----");
    }

    @Override
    public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException {
        // 对request、response进行一些预处理

        HttpServletRequest servletRequest = (HttpServletRequest) request;
        System.out.println(servletRequest.getRequestURI());
        Tracing tracing = SpringBeanUtil.getBean("tracing");
        String uuid =  servletRequest.getHeader("xguid");
        logger.info("当前xguid是 {}", uuid);
        //如果没有xguid传过来，就说明这个是第一个，为了保持完整性，第一个就手动产生xguid
        if(!StringUtils.isEmpty(uuid)){
            tracing.tracer().currentSpan().tag("xguid",uuid);
            //tracing.currentTraceContext()("xguid", uuid);
        }
        // 执行目标资源，放行
        filterChain.doFilter(request, response);
        logger.info("----调用service之后执行一段代码----");
    }

    @Override
    public void init(FilterConfig arg0) throws ServletException {
        logger.info("----Filter初始化----");
    }
}

添加webxml

<filter>
    <filter-name>tracingFilter</filter-name>
    <filter-class>brave.spring.webmvc.DelegatingTracingFilter</filter-class>
</filter>
<filter-mapping>
    <filter-name>tracingFilter</filter-name>
    <url-pattern>/*</url-pattern>
</filter-mapping>

config.properties

增加配置属性

zipkin.enable=false
zipkin.url=http://localhost:9411/api/v2/spans

涉及的jar包

<dependency>
        <groupId>io.zipkin.brave</groupId>
        <artifactId>brave</artifactId>
        <version>5.0.0</version>
</dependency>

<dependency>
        <groupId>io.zipkin.brave</groupId>
        <artifactId>brave-spring-beans</artifactId>
        <version>5.0.0</version>
</dependency>

<dependency>
        <groupId>io.zipkin.brave</groupId>
        <artifactId>brave-context-log4j12</artifactId>
        <version>5.0.0</version>
</dependency>

<dependency>
        <groupId>io.zipkin.brave</groupId>
        <artifactId>brave-context-slf4j</artifactId>
        <version>5.0.0</version>
</dependency>

<dependency>
        <groupId>io.zipkin.brave</groupId>
        <artifactId>brave-instrumentation-spring-web</artifactId>
        <version>5.0.0</version>
</dependency>

<dependency>
        <groupId>io.zipkin.brave</groupId>
        <artifactId>brave-instrumentation-httpclient</artifactId>
        <version>5.0.0</version>
</dependency>

<dependency>
        <groupId>io.zipkin.brave</groupId>
        <artifactId>brave-instrumentation-spring-webmvc</artifactId>
        <version>5.0.0</version>
</dependency>

<dependency>
        <groupId>io.zipkin.reporter2</groupId>
        <artifactId>zipkin-sender-okhttp3</artifactId>
        <version>2.7.6</version>
</dependency>

<dependency>
        <groupId>io.zipkin.reporter2</groupId>
        <artifactId>zipkin-sender-amqp-client</artifactId>
        <version>2.7.6</version>
</dependency>

HttpClient场景

4.5版本以后的httpclient-core默认支持zipkin链路日志传递， 其他诸如Fegin调用等都是同一个道理

目前代码中会碰到内部用httpclient调用第三方服务，这个可以通过两种方式解决。

一个是采用TracingHttpClientBuilder

public static String executeRequest(HttpUriRequest request,
                                        String username, String password) {
        CloseableHttpClient client = null;
        CloseableHttpResponse response = null;
        try {
            Tracing tracing = null;
            tracing = SpringBeanUtil.getBean("tracing");
            LinkedList<CloseableHttpResponse> httpResponses = new LinkedList<>();
            CredentialsProvider provider = new BasicCredentialsProvider();
            UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(
                    username, password);
            provider.setCredentials(AuthScope.ANY, credentials);
            if (tracing != null) {
                client = TracingHttpClientBuilder.create(tracing).setDefaultCredentialsProvider(provider).build();
            }else {
                client = HttpClientBuilder.create()
                        .setDefaultCredentialsProvider(provider).build();
            }
            response = client.execute(request);
            int statusCode = response.getStatusLine().getStatusCode();
            httpResponses.add(response);

            InputStream inputStream = null;
            String output = "";
            HttpEntity entity = response.getEntity();
            if (entity != null && entity.getContent() != null) {
                inputStream = response.getEntity().getContent();
                output = IOUtils.toString(inputStream, "UTF-8");
                inputStream.close();
            }

            return output;
        } catch (Exception e) {
            e.printStackTrace();
            throw new AssassinException(e.getMessage());
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (client != null) {
                try {
                    client.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

一种是自己埋点

 public static String get(String serviceUrl, String username,
                              String password) {
        HttpURLConnection connect = null;
        try {

//            //获取tracer对象，位于io.zipkin.brave 我用的是5.12.7版本
//            Tracer tracer = SpringBeanUtil.getBean("tracer");
//
//            //最外部链路上增加一些标签，
//            tracer.currentSpan().tag("background-sleep-millis",String.valueOf(RandomUtils.nextDouble()));
//
//            //创建新的链路。名称为了友好，直接用url地址替代好了
//            Span span = tracer.nextSpan().name(serviceUrl).start();
//
//            //增加一些查看的标签
//            span.tag("http.method", "get");
             //默认差不多是这些标签
//            http.method	GET
//            http.path	/testZipkin
//            mvc.controller.class	MainController
//            mvc.controller.method	testZipkin
//            Client Address	192.168.100.100:4101
            // 记录启动状态
            // span.annotate("ServerStart");
            // 实例一个URL资源
            URL url = new URL(serviceUrl);
            // 实例一个HTTP CONNECT
            connect = (HttpURLConnection) url.openConnection();
            connect.setRequestMethod("GET");
            byte[] bytes = Base64.encodeBase64(
                    (username + ":" + password).getBytes(StandardCharsets.UTF_8)
            );
            String encoded = new String(
                    bytes,StandardCharsets.UTF_8
            );
            connect.setRequestProperty("Authorization", ConstValue.TOKEN_START + encoded);
            connect.connect();
            int code = connect.getResponseCode();

            // 将返回的值存入到String中
            StringBuilder sb;
            try (BufferedReader brd = new BufferedReader(new InputStreamReader(
                    connect.getInputStream(), StandardCharsets.UTF_8))) {
                sb = new StringBuilder();
                String line;

                while ((line = brd.readLine()) != null) {
                    sb.append(line);
                }
            }
//            //记录结束状态
//            span.annotate("ServerEnd");
//            //这个链路结束
//            span.finish();
            return code +sb.toString();
        }
        catch (RuntimeException o){
            throw o;
        }
        catch (Exception e) {
           return "999";
        } finally {
            if (connect != null) {
                connect.disconnect();
            }
        }
    }

数据查询

直接用zip的接口处理数据

接口访问地址：

http://xxxx:9411/zipkin/api/v2//traces?annotationQuery=xguid%3D4aaada24-15b6-418c-b44e-c16c2f89949b

其中 annotationQuery节点用于查询自定义的tag属性，比如xguid

语法格式是 xguid=xxx and http.methd=get 中间用and隔开 翻阅源代码以后，要求and前后有且只有一个空格

其他语法参阅 zipkin.io/zipkin-api/…

可以按照自己的需要去查询数据，然后拼接数据

基于es数据查询

一些插件查看es数据，比如Elastic Head，或者Dbeaver等

ES JAVA代码查询器(关键代码)

 public EsTableDataEntity searchAll(EsIndexEntity esIndexEntity, List<EsTableColumnEntity> esTableColumnEntityList, Integer currentIndex,Integer pageSize) throws IOException {
        RestHighLevelClient restHighLevelClient = EsConstValue.REST_CLIENT_MAP.get(esIndexEntity.getConnectUuid());
        SearchRequest searchRequest = new SearchRequest(esIndexEntity.getIndexName());
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.matchAllQuery());
        searchSourceBuilder.from(currentIndex * pageSize);
        searchSourceBuilder.size(pageSize);

        FieldSortBuilder fieldSortBuilder = new FieldSortBuilder("id");
        fieldSortBuilder.order(SortOrder.DESC);


        //创建查询条件
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        for(EsTableColumnEntity esTableColumnEntity: esTableColumnEntityList){
            if(esTableColumnEntity.getValue()!=null && !StringUtils.isEmpty(esTableColumnEntity.getValue().toString())){
                if(StringUtils.equals(
                        esTableColumnEntity.getType(),
                        EsConstValue.DATE_TYPE_NAME)){
                    String[] values = esTableColumnEntity.getValue().toString().split("-");
                    String valueFrom = values[0];
                    String valueTo =values[1];
                    RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(esTableColumnEntity.getDataIndex());
                    rangeQueryBuilder.from(valueFrom);
                    rangeQueryBuilder.to(valueTo);
                    boolQueryBuilder.must(rangeQueryBuilder);
                }
                else if(StringUtils.equals(esTableColumnEntity.getType(),EsConstValue.OBJECT_TYPE_NAME)){
                    //如果是嵌套的Object类型，则需要嵌套方式查询
                    Map<String,String> parseAnnotationQuery = parseAnnotationQuery(esTableColumnEntity.getValue().toString());
                    for (Map.Entry<String, String> kv : parseAnnotationQuery.entrySet()) {
                        if (kv.getValue().isEmpty()) {
                            MatchPhraseQueryBuilder termsQueryBuilder = new MatchPhraseQueryBuilder("_q", kv.getKey());
                            boolQueryBuilder.must(termsQueryBuilder);
                        } else {
                            MatchPhraseQueryBuilder termsQueryBuilder = new MatchPhraseQueryBuilder("_q", kv.getKey() + "=" + kv.getValue());
                            boolQueryBuilder.must(termsQueryBuilder);
                        }
                    }
                }
                else{
                    MatchPhraseQueryBuilder termsQueryBuilder = new MatchPhraseQueryBuilder(esTableColumnEntity.getDataIndex(),
                            esTableColumnEntity.getValue());
                    boolQueryBuilder.must(termsQueryBuilder);
                }

            }
        }
        searchSourceBuilder.query(boolQueryBuilder);

        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse =  restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        return getEsTableDataEntity(esIndexEntity,restHighLevelClient,searchResponse);
    }
avatar
文章
阅读
粉丝