集成opentracing

进程内/http/grpc携带trace信息

进程内通过ctx携带traceId

package main

import (
    "context"
    "fmt"
    "go.etcd.io/etcd/pkg/v3/stringutil"
    "log"
    "runtime"
    "strings"
    "time"
)

func main() {
    ctx := context.Background()
    ctx = context.WithValue(ctx, "trace_id", stringutil.RandString(20))
    ctx = context.WithValue(ctx, "user_id", 2)
    var msg string = visitWeb(ctx)
    fmt.Println(msg)
}

func visitWeb(ctx context.Context) string {
    begin := time.Now()
    userId := ctx.Value("user_id").(int)
    traceId := ctx.Value("trace_id").(string)
    defer func() {
       pc, _, _, _ := runtime.Caller(0)
       funcName := runtime.FuncForPC(pc).Name()
       log.Printf("func: %s, begin: %d, end: %d, userId: %d, traceId: %s", funcName, begin.Nanosecond(), time.Since(begin).Nanoseconds(), userId, traceId)
    }()
    go recordUV(ctx)
    var content string = getRecommond(ctx)
    return content
}

func recordUV(ctx context.Context) {
    begin := time.Now()
    userId := ctx.Value("user_id").(int)
    traceId := ctx.Value("trace_id").(string)
    defer func() {
       pc, _, _, _ := runtime.Caller(0)
       funcName := runtime.FuncForPC(pc).Name()
       log.Printf("func: %s, begin: %d, end: %d, userId: %d, traceId: %s", funcName, begin.Nanosecond(), time.Since(begin).Nanoseconds(), userId, traceId)
    }()

    time.Sleep(time.Millisecond * 3)
}

func getRecommond(ctx context.Context) string {
    begin := time.Now()
    userId := ctx.Value("user_id").(int)
    traceId := ctx.Value("trace_id").(string)
    defer func() {
       pc, _, _, _ := runtime.Caller(0)
       funcName := runtime.FuncForPC(pc).Name()
       log.Printf("func: %s, begin: %d, end: %d, userId: %d, traceId: %s", funcName, begin.Nanosecond(), time.Since(begin).Nanoseconds(), userId, traceId)
    }()

    var role string = getUserRole(ctx)
    content := make([]string, 0)
    if strings.ToLower(role) != "vip" {
       content = append(content, "广告")
    }
    content = append(content, "grpc")
    content = append(content, "gorm")
    return strings.Join(content, "\n")
}

func getUserRole(ctx context.Context) string {
    begin := time.Now()
    userId := ctx.Value("user_id").(int)
    traceId := ctx.Value("trace_id").(string)
    defer func() {
       pc, _, _, _ := runtime.Caller(0)
       funcName := runtime.FuncForPC(pc).Name()
       log.Printf("func: %s, begin: %d, end: %d, userId: %d, traceId: %s", funcName, begin.Nanosecond(), time.Since(begin).Nanoseconds(), userId, traceId)
    }()
    return "VIP"
}

//
//2024/08/15 19:47:25 func: main.getUserRole.func1, begin: 679226000, end: 15890, userId: 2, traceId: qHSPs5kzXY35ScUM3puI
//2024/08/15 19:47:25 func: main.getRecommond.func1, begin: 679226000, end: 183109, userId: 2, traceId: qHSPs5kzXY35ScUM3puI
//2024/08/15 19:47:25 func: main.visitWeb.func1, begin: 679219000, end: 194119, userId: 2, traceId: qHSPs5kzXY35ScUM3puI

http通过header携带traceId

• server/main.go

package main

import (
    "github.com/gin-gonic/gin"
    "log"
    "net/http"
    "time"
)

func main() {
    router := gin.Default()
    router.Use(traceMiddleawre)
    router.GET("/greet", greet)
    router.Run("127.0.0.1:8080")
}

func traceMiddleawre(ctx *gin.Context) {
    begin := time.Now()
    traceId := ctx.GetHeader("trace_id")
    userId := ctx.GetHeader("user_id")
    ctx.Next()
    log.Printf("trace_id %s user_id %s  %s begin time %d use time %d ns\n", traceId, userId, ctx.Request.RequestURI, begin.Nanosecond(), time.Since(begin).Nanoseconds())
}

func greet(ctx *gin.Context) {
    time.Sleep(time.Millisecond * 100)
    ctx.String(http.StatusOK, "Hello World")
}

• client/main.go

package main

import (
    "context"
    "fmt"
    "go.etcd.io/etcd/pkg/v3/stringutil"
    "io"
    "log"
    "net/http"
    "strconv"
    "time"
)

func main() {
    userId := 8
    ctx := context.Background()
    fmt.Println(greet(ctx, userId))
}

func greet(ctx context.Context, userId int) string {
    url := "http://127.0.0.1:8080/greet"
    begin := time.Now()
    traceId := stringutil.RandString(20)
    defer func() {
       log.Printf("trace_id %s user_id %s %s begin time %d use time %d ns\n", traceId, strconv.Itoa(userId), url, begin.Nanosecond(), time.Since(begin).Nanoseconds())
    }()
    client := &http.Client{}
    request, err := http.NewRequest("GET", url, nil)
    request.Header.Set("trace_id", traceId)
    request.Header.Set("user_id", strconv.Itoa(userId))
    if err != nil {
       log.Fatal(err)
    }
    response, err := client.Do(request)
    if err != nil {
       log.Fatal(err)
    }
    defer response.Body.Close()
    bs, err := io.ReadAll(response.Body)
    if err != nil {
       log.Fatal(err)
    }
    return string(bs)
}

grpc通过meta携带traceId

//服务端
grpc.ChainUnaryInterceptor(tracerInterceptor)
metadata.FromIncomingContext

//客户端
grpc.WithChainUnaryInterceptor(traceInterceptor)
metadata.AppendToOutgoingContext

• proto/hello.proto

syntax = "proto3";
package helloworld;

option go_package="./;proto";


service Greeter{
  rpc SayHello(HelloReq) returns (HelloRes);
}


message HelloReq{
  string name=1;
}

message HelloRes{
  string msg=1;
}

//protoc --go_out=./proto --go-grpc_out=./proto ./proto/hello.proto

• server/main.go

package main

import (
    "context"
    "fmt"
    "google.golang.org/grpc"
    "google.golang.org/grpc/metadata"
    pb "grpc-demo/grpc_trace/proto"
    "log"
    "net"
    "time"
)

type server struct {
    pb.UnimplementedGreeterServer
}

func (s *server) SayHello(ctx context.Context, req *pb.HelloRequest) (*pb.HelloReply, error) {
    fmt.Println(req.Name)
    return &pb.HelloReply{Message: "Hello " + req.Name}, nil
}

func main() {
    lis, err := net.Listen("tcp", "localhost:8080")
    if err != nil {
       log.Fatalf("failed to listen: %v", err)
    }
    s := grpc.NewServer(grpc.ChainUnaryInterceptor(tracerInterceptor))
    pb.RegisterGreeterServer(s, &server{})
    err = s.Serve(lis)
    if err != nil {
       log.Fatalf("failed to serve: %v", err)
    }
}

func tracerInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) {
    begin := time.Now()
    var traceId string
    if md, exist := metadata.FromIncomingContext(ctx); exist {
       if v, ok := md["traceid"]; ok {
          traceId = v[0]
       }
    }
    res, err := handler(ctx, req)
    log.Printf("trace id: %v, begin %d, end %d", traceId, begin.Nanosecond(), time.Since(begin).Nanoseconds())
    return res, err
}

• client/main.go

package main

import (
    "context"
    "fmt"
    "go.etcd.io/etcd/pkg/v3/stringutil"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
    "google.golang.org/grpc/metadata"
    pb "grpc-demo/grpc_trace/proto"
    "log"
    "time"
)

func main() {
    conn, err := grpc.NewClient("localhost:8080",
       grpc.WithTransportCredentials(insecure.NewCredentials()),
       grpc.WithChainUnaryInterceptor(traceInterceptor),
    )

    if err != nil {
       log.Fatal(err)
    }
    client := pb.NewGreeterClient(conn)
    res, err := client.SayHello(context.Background(), &pb.HelloRequest{Name: "mmm"})
    if err != nil {
       log.Fatal(err)
    }
    fmt.Println(res.Message)

}

func traceInterceptor(ctx context.Context, method string, req any, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
    begin := time.Now()
    traceId := stringutil.RandString(20)
    ctx = metadata.AppendToOutgoingContext(ctx, "traceid", traceId)
    err := invoker(ctx, method, req, reply, cc, opts...)
    log.Printf("traceid: %s, begin: %d,end %d", traceId, begin.Nanosecond(), time.Since(begin).Nanoseconds())
    return err
}

集成opentracing

opentracing主要的interface

docker run --rm --name jaeger \
  -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
  -p 6831:6831/udp \
  -p 6832:6832/udp \
  -p 5778:5778 \
  -p 16686:16686 \
  -p 4317:4317 \
  -p 4318:4318 \
  -p 14250:14250 \
  -p 14268:14268 \
  -p 14269:14269 \
  -p 9411:9411 \
  jaegertracing/all-in-one:1.60

go get github.com/opentracing/opentracing-go
go get github.com/uber/jaeger-client-go

• 1.Jaeger: open source, end-to-end distributed tracing Monitor and troubleshoot transactions in complex distributed systems

Jaeger 解决的问题
* distributed transaction monitoring分布式事务监控
* performance and latency optimization性能和延迟优化
* root cause analysis根本原因分析
* service dependency analysis服务依赖分析
* distributed context propagation分布式上下文传播

• 2. Tracer 创建span, 传递BaggageItem,
  • pubilc: 其中BaggageItem可以在服务之间传递,是公共的
  • private: 对应的tag和log则是属于该span.
• 3. "Span" 描述一个操作或一次调用的生命周期,如执行一个函数的时间,一次rpc的时间.

主要接口

type Span interface {
    Finish()
    FinishWithOptions(opts FinishOptions)
    Context() SpanContext
    SetOperationName(operationName string) Span
    SetTag(key string, value interface{}) Span
    LogFields(fields ...log.Field)
    LogKV(alternatingKeyValues ...interface{})
    SetBaggageItem(restrictedKey string, value string) Span
    BaggageItem(restrictedKey string) string
    Tracer() Tracer
    LogEvent(event string)
    LogEventWithPayload(event string, payload interface{})
    Log(data LogData)
}


type Tracer interface {
    StartSpan(operationName string, opts ...StartSpanOption) Span
    Inject(sm SpanContext, format interface{}, carrier interface{}) error
    Extract(format interface{}, carrier interface{}) (SpanContext, error)
}

type SpanContext interface {
    ForeachBaggageItem(handler func(k string, v string) bool)
}

span 两种关系

ChildOf      //父子关系, 父依赖子的返回值
FollowsFrom  //父**不依赖**子的返回

进程之间传递依赖Inject和Extract, Extract负责从Carrier里反序列化出SpanContext, Opentracing实现了3种序列化器

Inject(sm SpanContext, format interface{}, carrier interface{}) error
Extract(format interface{}, carrier interface{}) (SpanContext, error)


Binary       不透明的二进制, Inject使用Carrier符合io.Writer,反序列化符合io.Reader接口
TextMap      <k,v>格式,k,v可以是任意字符串, 序列化时遵守TextMapWriter接口, 反序列化支持TextMapReader接口
HTTPHeaders  <k,v>格式, 只能包含英文,不能包含中文.

NOTE: 可以获取traceID,SpanID注入到log中.

进程内追踪,集成opentracing

package main

import (
    "context"
    "fmt"
    "github.com/opentracing/opentracing-go"
    "github.com/opentracing/opentracing-go/log"
    "github.com/uber/jaeger-client-go"
    "github.com/uber/jaeger-client-go/config"
    "github.com/uber/jaeger-lib/metrics"
    "go.etcd.io/etcd/pkg/v3/stringutil"
    "io"
    "strings"
    "time"
)

func main() {
    jaeger, closer, err := NewJaegerTracer("my-svc", "127.0.0.1:6831")
    if err != nil {
       panic(err)
    }
    defer closer.Close()
    opentracing.SetGlobalTracer(jaeger)
    ctx := context.Background()
    var msg string = visitWeb(ctx)
    fmt.Println(msg)
}

func NewJaegerTracer(ServiceName string, jaegerHost string) (opentracing.Tracer, io.Closer, error) {
    configuration := config.Configuration{
       ServiceName: ServiceName,
       Sampler: &config.SamplerConfig{
          Type:  jaeger.SamplerTypeConst,
          Param: 1,
       },
       Reporter: &config.ReporterConfig{
          BufferFlushInterval: 1 * time.Second,
          LogSpans:            true,
          LocalAgentHostPort:  jaegerHost,
       },
    }
    tracer, closer, err := configuration.NewTracer(config.Logger(jaeger.NullLogger), config.Metrics(metrics.NullFactory))
    if err != nil {
       panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
    }
    opentracing.SetGlobalTracer(tracer)
    return tracer, closer, err
}

func visitWeb(ctx context.Context) string {
    span := opentracing.GlobalTracer().StartSpan("visitWeb")
    defer span.Finish()
    time.Sleep(time.Second * 3)
    span.SetTag("tagk1", "tagv1")
    span.LogFields(
       log.Int("user_id", 8),
       log.String("visit_page", "/home"),
    )
    span.SetBaggageItem("trace_id", stringutil.RandString(20))
    span.SetBaggageItem("user_id", "8")
    recordUV(span)
    var content string = getRecommond(span)
    return content
}

func recordUV(parentSpan opentracing.Span) {
    span := opentracing.GlobalTracer().StartSpan("recordUV", opentracing.FollowsFrom(parentSpan.Context()))
    defer span.Finish()
    time.Sleep(time.Second * 3)
}

func getRecommond(parentSpan opentracing.Span) string {
    span := opentracing.GlobalTracer().StartSpan("getRecommond", opentracing.ChildOf(parentSpan.Context()))
    defer span.Finish()
    time.Sleep(time.Second * 2)
    var role string = getUserRole(span)
    content := make([]string, 0)
    if strings.ToLower(role) != "vip" {
       content = append(content, "广告")
    }
    content = append(content, "grpc")
    content = append(content, "gorm")
    return strings.Join(content, "\n")
}

func getUserRole(parentSpan opentracing.Span) string {
    span := opentracing.GlobalTracer().StartSpan("getUserRole", opentracing.ChildOf(parentSpan.Context()))
    defer span.Finish()
    time.Sleep(time.Second * 1)
    fmt.Println("getUserRole 打印baggage")
    span.Context().ForeachBaggageItem(func(k, v string) bool {
       fmt.Println(k, v)
       return true
    })
    return "VIP"
}

http集成opentracing

• server/main.go

package main

import (
    "fmt"
    "github.com/gin-gonic/gin"
    "github.com/opentracing/opentracing-go"
    "github.com/opentracing/opentracing-go/ext"
    "github.com/uber/jaeger-client-go"
    "github.com/uber/jaeger-client-go/config"
    "github.com/uber/jaeger-lib/metrics"
    "io"
    "log"
    "net/http"
    "time"
)

func NewJaegerTracer(ServiceName string, jaegerHost string) (opentracing.Tracer, io.Closer, error) {
    configuration := config.Configuration{
       ServiceName: ServiceName,
       Sampler: &config.SamplerConfig{
          Type:  jaeger.SamplerTypeConst,
          Param: 1,
       },
       Reporter: &config.ReporterConfig{
          BufferFlushInterval: 1 * time.Second,
          LogSpans:            true,
          LocalAgentHostPort:  jaegerHost,
       },
    }
    tracer, closer, err := configuration.NewTracer(config.Logger(jaeger.NullLogger), config.Metrics(metrics.NullFactory))
    if err != nil {
       panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
    }
    opentracing.SetGlobalTracer(tracer)

    return tracer, closer, err
}

func InitHttpJaeger() {
    tracer, _, err := NewJaegerTracer("my_http_server", "127.0.0.1:6831")
    if err != nil {
       log.Fatal(err)
    }
    //defer closer.Close()
    opentracing.SetGlobalTracer(tracer)
}
func main() {
    InitHttpJaeger()
    router := gin.Default()
    router.Use(traceMiddleawre)
    router.GET("/greet", greet)
    router.Run("127.0.0.1:8080")
}

func traceMiddleawre(ctx *gin.Context) {
    clientSpanCtx, err := opentracing.GlobalTracer().Extract(
       opentracing.HTTPHeaders,
       opentracing.HTTPHeadersCarrier(ctx.Request.Header))
    if err != nil {
       log.Printf("extract trace context err: %v", err)
    }
    operatorName := ctx.Request.RequestURI
    serverSpan := opentracing.GlobalTracer().StartSpan(operatorName, ext.RPCServerOption(clientSpanCtx))
    defer serverSpan.Finish()
    for k, v := range ctx.Request.Header {
       if k == "Uber-Trace-Id" {
          continue
       }
       serverSpan.SetTag(k, v)
    }
    ctx.Next()
}

func greet(ctx *gin.Context) {
    time.Sleep(time.Second * 3)
    ctx.String(http.StatusOK, "Hello World")
}

• client/main.go

package main

import (
    "context"
    "fmt"
    "github.com/opentracing/opentracing-go"
    "github.com/uber/jaeger-client-go"
    "github.com/uber/jaeger-client-go/config"
    "github.com/uber/jaeger-lib/metrics"
    "go.etcd.io/etcd/pkg/v3/stringutil"
    "io"
    "log"
    "net/http"
    "strconv"
    "time"
)

func main() {
    //InitHttpJaeger()  //如果这样初始化, 则会出现client span上报失败 `invalid parent span IDs=0f79e6fcbfb0a1fe; skipping clock skew adjustment`
    jaeger, closer, err := NewJaegerTracer("my_http_client", "127.0.0.1:6831")
    if err != nil {
       panic(err)
    }
    defer closer.Close()
    opentracing.SetGlobalTracer(jaeger)
    
    userId := 8
    ctx := context.WithValue(context.Background(), "trace_id", stringutil.RandString(20))
    ctx = context.WithValue(ctx, "user_id", strconv.Itoa(userId))
    fmt.Println(greet(ctx))

}

func NewJaegerTracer(ServiceName string, jaegerHost string) (opentracing.Tracer, io.Closer, error) {
    configuration := config.Configuration{
       ServiceName: ServiceName,
       Sampler: &config.SamplerConfig{
          Type:  jaeger.SamplerTypeConst,
          Param: 1,
       },
       Reporter: &config.ReporterConfig{
          BufferFlushInterval: 1 * time.Second,
          LogSpans:            true,
          LocalAgentHostPort:  jaegerHost,
       },
    }
    tracer, closer, err := configuration.NewTracer(config.Logger(jaeger.NullLogger), config.Metrics(metrics.NullFactory))
    if err != nil {
       panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
    }
    opentracing.SetGlobalTracer(tracer)
    return tracer, closer, err
}

func InitHttpJaeger() {
    jaeger, closer, err := NewJaegerTracer("my_http_server", "127.0.0.1:6831")
    if err != nil {
       log.Fatal(err)
    }
    defer closer.Close()
    opentracing.SetGlobalTracer(jaeger)
}

func greet(ctx context.Context) string {
    span := opentracing.GlobalTracer().StartSpan("greet")
    defer span.Finish()

    url := "http://127.0.0.1:8080/greet"
    client := http.Client{}
    request, err := http.NewRequest("GET", url, nil)
    request.Header.Set("trace_id", ctx.Value("trace_id").(string))
    request.Header.Set("user_id", ctx.Value("user_id").(string))
    //------------------------------------

    if span != nil {
       for k, v := range request.Header {
          span.SetTag(k, v[0])
       }
       if err := opentracing.GlobalTracer().Inject(
          span.Context(),
          opentracing.HTTPHeaders,
          opentracing.HTTPHeadersCarrier(request.Header),
       ); err != nil {
          log.Fatal(err)
       } else {
          for k, v := range request.Header { //header里多出了Uber-Trace-Id
             fmt.Println(k, v[0])
          }
       }
    }

    response, err := client.Do(request)
    if err != nil {
       log.Fatal(err)
    }
    defer response.Body.Close()
    bs, err := io.ReadAll(response.Body)
    if err != nil {
       log.Fatal(err)
    }
    return string(bs)
}

• 测试

• 出现的错误: 客户端span无法上报: invalid parent span IDs=0f79e6fcbfb0a1fe; skipping clock skew adjustment

解决: 不使用InitHttpJaeger(),将它的逻辑直接写到main

func main() {
    //InitHttpJaeger()  //如果这样初始化, 则会出现client span上报失败 `invalid parent span IDs=0f79e6fcbfb0a1fe; skipping clock skew adjustment`
    jaeger, closer, err := NewJaegerTracer("my_http_client", "127.0.0.1:6831")
    if err != nil {
       panic(err)
    }
    defer closer.Close()
    opentracing.SetGlobalTracer(jaeger)

grpc集成opentracing

• server/main.go

package main

import (
    "context"
    "fmt"
    "github.com/opentracing/opentracing-go"
    "github.com/opentracing/opentracing-go/ext"
    "github.com/uber/jaeger-client-go"
    "github.com/uber/jaeger-client-go/config"
    "github.com/uber/jaeger-lib/metrics"
    "google.golang.org/grpc"
    "google.golang.org/grpc/metadata"
    pb "grpc-demo/grpc_trace/proto"
    "io"
    "log"
    "net"
    "time"
)

type server struct {
    pb.UnimplementedGreeterServer
}

// SayHello implements helloworld.GreeterServer
func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
    log.Printf("Received: %v", in.GetName())
    return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil
}

func NewJaegerTracer(ServiceName string, jaegerHost string) (opentracing.Tracer, io.Closer, error) {
    configuration := config.Configuration{
       ServiceName: ServiceName,
       Sampler: &config.SamplerConfig{
          Type:  jaeger.SamplerTypeConst,
          Param: 1,
       },
       Reporter: &config.ReporterConfig{
          BufferFlushInterval: 1 * time.Second,
          LogSpans:            true,
          LocalAgentHostPort:  jaegerHost,
       },
    }
    tracer, closer, err := configuration.NewTracer(config.Logger(jaeger.NullLogger), config.Metrics(metrics.NullFactory))
    if err != nil {
       panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
    }
    opentracing.SetGlobalTracer(tracer)
    return tracer, closer, err
}

func main() {
    jaeger, closer, err := NewJaegerTracer("my_grpc_server", "127.0.0.1:6831")
    if err != nil {
       log.Fatal(err)
    }
    defer closer.Close()
    opentracing.SetGlobalTracer(jaeger)

    lis, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", 50051))
    if err != nil {
       log.Fatalf("failed to listen: %v", err)
    }

    s := grpc.NewServer(grpc.UnaryInterceptor(traceInterceptor))
    pb.RegisterGreeterServer(s, &server{})
    log.Printf("server listening at %v", lis.Addr())
    if err := s.Serve(lis); err != nil {
       log.Fatalf("failed to serve: %v", err)
    }
}

func traceInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) {
    md, ok := metadata.FromIncomingContext(ctx)
    if !ok {
       md = metadata.New(map[string]string{})
    }
    for k, v := range md {
       fmt.Println(k, v[0])
    }
    clientSpanCtx, err := opentracing.GlobalTracer().Extract(opentracing.TextMap, MDCarrier(md))
    if err != nil {
       log.Printf("extract trace from metadata: %v", err)
       return handler(ctx, req)
    }
    span := opentracing.GlobalTracer().StartSpan(
       info.FullMethod,                    //完整的方法名为span名
       ext.RPCServerOption(clientSpanCtx), //把client的metadata带进来, 一方面指明集成关系, 另一方便指明span.kine=server
       opentracing.Tag{Key: string(ext.Component), Value: "gRPC服务端"},
    )
    for k, v := range md {
       if k == "uber-trace-id" {
          continue
       }
       span.SetTag(k, v[0]) //为span加tag
    }
    defer span.Finish()
    return handler(ctx, req)
}

// MDCarrier 类似TextMapCarrier, 但是能承载的mapkey是slice
// type TextMapCarrier map[string]string
// 实现opentracing.TextMapReader和opentracing.TextMapWriter接口
type MDCarrier map[string][]string

func (m MDCarrier) ForeachKey(handler func(key string, val string) error) error {
    for k, strs := range m {
       for _, v := range strs {
          if err := handler(k, v); err != nil {
             return err
          }

       }
    }
    return nil
}

func (m MDCarrier) Set(key, val string) {
    m[key] = append(m[key], val)
}

• client/main.go

package main

import (
    "context"
    "fmt"
    "github.com/opentracing/opentracing-go"
    "github.com/opentracing/opentracing-go/ext"
    "github.com/uber/jaeger-client-go"
    "github.com/uber/jaeger-client-go/config"
    "github.com/uber/jaeger-lib/metrics"
    "go.etcd.io/etcd/pkg/v3/stringutil"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
    "google.golang.org/grpc/metadata"
    pb "grpc-demo/grpc_trace/proto"
    "io"
    "log"
    "strconv"
    "time"
)

func NewJaegerTracer(ServiceName string, jaegerHost string) (opentracing.Tracer, io.Closer, error) {
    configuration := config.Configuration{
       ServiceName: ServiceName,
       Sampler: &config.SamplerConfig{
          Type:  jaeger.SamplerTypeConst,
          Param: 1,
       },
       Reporter: &config.ReporterConfig{
          BufferFlushInterval: 1 * time.Second,
          LogSpans:            true,
          LocalAgentHostPort:  jaegerHost,
       },
    }
    tracer, closer, err := configuration.NewTracer(config.Logger(jaeger.NullLogger), config.Metrics(metrics.NullFactory))
    if err != nil {
       panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
    }
    opentracing.SetGlobalTracer(tracer)
    return tracer, closer, err
}

func main() {
    jaeger, closer, err := NewJaegerTracer("my_grpc_client", "127.0.0.1:6831")
    if err != nil {
       log.Fatal(err)
    }
    defer closer.Close()
    opentracing.SetGlobalTracer(jaeger)

    conn, err := grpc.Dial("127.0.0.1:50051",
       grpc.WithTransportCredentials(insecure.NewCredentials()),
       grpc.WithChainUnaryInterceptor(traceInterCeptor),
    )

    if err != nil {
       log.Fatalf("did not connect: %v", err)
    }
    defer conn.Close()
    c := pb.NewGreeterClient(conn)
    userId := 8
    ctx := context.WithValue(context.Background(), "user_id", strconv.Itoa(userId))
    ctx = metadata.NewOutgoingContext(ctx, metadata.New(map[string]string{"org": "xxx"}))
    SayHello(ctx, c)
}

func SayHello(ctx context.Context, client pb.GreeterClient) string {
    r, err := client.SayHello(ctx, &pb.HelloRequest{Name: fmt.Sprintf("%s", "mmm")})
    if err != nil {
       log.Fatalf("could not greet: %v", err)
    }
    return r.Message
}

func traceInterCeptor(ctx context.Context, method string, req any, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
    traceId := stringutil.RandString(20)
    userId := ctx.Value("user_id").(string)

    //ctx = metadata.AppendToOutgoingContext(ctx, "trace_id", traceId)
    //ctx = metadata.AppendToOutgoingContext(ctx, "user_id", userId)
    md, ok := metadata.FromIncomingContext(ctx)
    if !ok {
       md = metadata.New(map[string]string{})
    }
    md.Append("trace_id", traceId)
    md.Append("user_id", userId)
    span := opentracing.GlobalTracer().StartSpan(
       method, //grpc方法名为Span命名
       opentracing.Tag{Key: string(ext.Component), Value: "gRPC客户端"}, //ext.Component是一个全局变量  string(ext.Component)="component"
       ext.SpanKindRPCClient,
    )
    defer span.Finish()
    for k, v := range md {
       span.SetTag(k, v[0])
       fmt.Println(k, v[0])
    }
    if err := opentracing.GlobalTracer().Inject(
       span.Context(),
       opentracing.TextMap,
       MDCarrier(md)); err != nil {
       log.Printf("SpanContext inject error: %v", err)
    }
    fmt.Println("---------------------------")
    for k, v := range md {
       fmt.Println(k, v[0]) //md里多出一个uber-trace-id
    }
    fmt.Println("---------------------------")
    ctx = metadata.NewOutgoingContext(ctx, md) //执行rpc时,把user-trace-id传给对方,否则对方在执行Extrace时候报错, SpanContext not found inExtrace carrier
    err := invoker(ctx, method, req, reply, cc, opts...)
    return err
}

// MDCarrier 类似TextMapCarrier, 但是能承载的mapkey是slice
// type TextMapCarrier map[string]string
// 实现opentracing.TextMapReader和opentracing.TextMapWriter接口
type MDCarrier map[string][]string

func (m MDCarrier) ForeachKey(handler func(key string, val string) error) error {
    for k, strs := range m {
       for _, v := range strs {
          if err := handler(k, v); err != nil {
             return err
          }

       }
    }
    return nil
}

func (m MDCarrier) Set(key, val string) {
    m[key] = append(m[key], val)
}