server启动多实例,注册到etcd, 客户端从etcd拿地址,均衡的去访问server.
客户端通过前缀/service/grpc/greeter-service/获取etcd子key列表, 挑选一台建立连接.
启动单例etcd,本地开发用
docker run -itd --restart always \
-p 2379:2379 -p 2380:2380 \
--name etcd_3_5_15 \
quay.io/coreos/etcd:v3.5.15 \
/usr/local/bin/etcd --data-dir=/etcd-data --name node1 \
--auto-compaction-mode=periodic --auto-compaction-retention=120h \
--listen-client-urls http://0.0.0.0:2379 \
--advertise-client-urls http://0.0.0.0:2379 \
--listen-peer-urls http://0.0.0.0:2380 \
--initial-advertise-peer-urls http://0.0.0.0:2380 \
--initial-cluster node1=http://0.0.0.0:2380 \
--log-level info --logger zap --log-outputs stderr
etcd key结构
/service/grpc/greeter-service/
127.0.0.1:9001
127.0.0.1:9002
...
客户端有两种方案实现,
- 1.获取etcd服务器节点列表后, 挑一个地址临时生成conn去调用(本文介绍的)
- 2.etcd skd自带的解析器
目录结构
go mod init grpc-etcd-registy-demo
├── client
│ └── main.go
├── go.mod
├── go.sum
├── proto
│ ├── helloworld.pb.go
│ ├── helloworld.proto
│ └── helloworld_grpc.pb.go
├── readme.txt
└── server
└── main.go
代码
- proto/helloworld.proto
syntax = "proto3";
option go_package=".;pb";
package helloworld;
// The greeting service definition.
service Greeter {
// Sends a greeting
rpc SayHello (HelloRequest) returns (HelloReply) {}
}
// The request message containing the user's name.
message HelloRequest {
string name = 1;
}
// The response message containing the greetings
message HelloReply {
string message = 1;
}
//protoc --go_out=./proto --go-grpc_out=./proto ./proto/*.proto
- server/main.go
package main
import (
"context"
"flag"
"fmt"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
etcdv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc"
pb "grpc-etcd-registy-demo/proto"
"log"
"net"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
)
const (
SERVICE_ROOT_PATH = "/service/grpc"
GREETER_SERVICE = "greeter-service"
)
type ServiceHub struct {
client *etcdv3.Client
heartbeatFrequency int64
}
var (
serviceHub *ServiceHub
hubOnce sync.Once
)
var ETCD_CLUSTER = []string{"127.0.0.1:2379"}
func GetServiceHub(etcdServers []string, heartbeatFrequency int64) *ServiceHub {
hubOnce.Do(func() {
if serviceHub == nil {
client, err := etcdv3.New(etcdv3.Config{
Endpoints: etcdServers,
DialTimeout: 3 * time.Second,
})
if err != nil {
log.Fatalf("can not connect etcd server: %v", err)
} else {
serviceHub = &ServiceHub{
client: client,
heartbeatFrequency: heartbeatFrequency,
}
}
}
})
return serviceHub
}
func (hub *ServiceHub) Register(service string, endpoint string, leaseID etcdv3.LeaseID) (etcdv3.LeaseID, error) {
ctx := context.Background()
if leaseID <= 0 {
//create a lease as heartbeatFrequency
if lease, err := hub.client.Grant(ctx, hub.heartbeatFrequency); err != nil {
log.Printf("create lease faild: %v", err)
return 0, err
} else {
key := strings.TrimRight(SERVICE_ROOT_PATH, "/") + "/" + service + "/" + endpoint
if _, err := hub.client.Put(ctx, key, "", etcdv3.WithLease(lease.ID)); err != nil {
log.Printf("put service:%s endpoint:%s lease failed: %v", service, endpoint, err)
return lease.ID, err
} else {
return lease.ID, nil
}
}
} else {
//keepalive
if _, err := hub.client.KeepAliveOnce(ctx, leaseID); err == rpctypes.ErrLeaseNotFound {
return hub.Register(service, endpoint, 0) //找不到租约,走注册流程(把leaseID置为0)
} else if err != nil {
log.Printf("refresh lease faild: %v", err)
return 0, err
} else {
return leaseID, nil
}
}
}
func (hub *ServiceHub) UnRegister(service string, endpoint string) error {
ctx := context.Background()
key := strings.TrimRight(SERVICE_ROOT_PATH, "/") + "/" + service + "/" + endpoint
if _, err := hub.client.Delete(ctx, key); err != nil {
log.Printf("delete service:%s endpoint:%s lease failed: %v", service, endpoint, err)
return err
} else {
log.Printf("unRegister service:%s endpoint:%s lease success", service, endpoint)
return nil
}
}
var port string
func init() {
flag.StringVar(&port, "port", "8004", "启动端口号")
flag.Parse()
}
// server is used to implement helloworld.GreeterServer.
type server struct {
pb.UnimplementedGreeterServer
}
// SayHello implements helloworld.GreeterServer
func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
log.Printf("Received: %v", in.GetName())
return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil
}
func main() {
lis, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%s", port))
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
hub := GetServiceHub(ETCD_CLUSTER, 3)
leaseID, err := hub.Register(GREETER_SERVICE, "127.0.0.1:"+port, 0)
if err != nil {
panic(err)
}
go func() {
for {
hub.Register(GREETER_SERVICE, "127.0.0.1:"+port, leaseID)
time.Sleep(time.Duration(3)*time.Second - 100*time.Millisecond)
}
}()
go func() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
sig := <-c
log.Printf("receive a signal: %s", sig.String())
hub.UnRegister(GREETER_SERVICE, "127.0.0.1:"+port)
os.Exit(0)
}()
s := grpc.NewServer()
pb.RegisterGreeterServer(s, &server{})
log.Printf("server listening at %v", lis.Addr())
if err := s.Serve(lis); err != nil {
hub.UnRegister(GREETER_SERVICE, "127.0.0.1:"+port)
log.Fatalf("failed to serve: %v", err)
}
}
- client/main.go
package main
import (
"context"
"fmt"
etcdv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc"
pb "grpc-etcd-registy-demo/proto"
"log"
"math/rand/v2"
"strings"
"sync"
"time"
)
const (
SERVICE_ROOT_PATH = "/service/grpc"
GREETER_SERVICE = "greeter-service"
)
var ETCD_CLUSTER = []string{"127.0.0.1:2379"}
type ServiceHub struct {
client *etcdv3.Client
endpointCache sync.Map
watched sync.Map
}
var (
serviceHub *ServiceHub
hubOnce sync.Once
)
func GetServiceHub(etcdServers []string) *ServiceHub {
hubOnce.Do(func() {
if serviceHub == nil {
if client, err := etcdv3.New(etcdv3.Config{
Endpoints: etcdServers,
DialTimeout: 5 * time.Second,
}); err != nil {
log.Fatalf("create etcd client failed: %v", err)
} else {
serviceHub = &ServiceHub{
client: client,
endpointCache: sync.Map{},
watched: sync.Map{},
}
}
}
})
return serviceHub
}
func (hub *ServiceHub) getServiceEndpoints(service string) []string {
ctx := context.Background()
prefix := strings.TrimRight(SERVICE_ROOT_PATH, "/") + "/" + service + "/"
if resp, err := hub.client.Get(ctx, prefix, etcdv3.WithPrefix()); err != nil {
log.Printf("get service: %s faild: %v", service, err)
return nil
} else {
endpoints := make([]string, 0, len(resp.Kvs))
for _, kv := range resp.Kvs {
path := strings.Split(string(kv.Key), "/")
endpoints = append(endpoints, path[len(path)-1])
}
log.Printf("refresh service: %s endpoints: %v", service, endpoints)
return endpoints
}
}
func (hub *ServiceHub) watchEndpointsOfService(service string) {
if _, exists := hub.watched.LoadOrStore(service, true); exists {
return
}
ctx := context.Background()
prefix := strings.TrimRight(SERVICE_ROOT_PATH, "/") + "/" + service + "/"
ch := hub.client.Watch(ctx, prefix, etcdv3.WithPrefix())
log.Printf("监听服务%s的节点变化", service)
go func() {
for wresp := range ch {
for _, ev := range wresp.Events {
path := strings.Split(string(ev.Kv.Key), "/")
if len(path) > 2 {
service := path[len(path)-2]
endpoints := hub.getServiceEndpoints(service)
if len(endpoints) > 0 {
hub.endpointCache.Store(service, endpoints)
} else {
hub.endpointCache.Delete(service)
}
}
}
}
}()
}
func (hub *ServiceHub) GetServiceEndpointWithCache(service string) []string {
hub.watchEndpointsOfService(service)
if endpoints, ok := hub.endpointCache.Load(service); ok {
return endpoints.([]string)
} else {
endpoints := hub.getServiceEndpoints(service)
if len(endpoints) > 0 {
hub.endpointCache.Store(service, endpoints)
}
return endpoints
}
}
var serverConn = sync.Map{}
func GetClient() pb.GreeterClient {
hub := GetServiceHub(ETCD_CLUSTER)
servers := hub.GetServiceEndpointWithCache(GREETER_SERVICE)
if len(servers) == 0 {
log.Printf("no cahce for server: %s", GREETER_SERVICE)
return nil
} else {
idx := rand.IntN(len(servers))
server := servers[idx]
fmt.Println(server)
if client, ok := serverConn.Load(server); ok {
return client.(pb.GreeterClient)
} else {
conn, err := grpc.Dial(server, grpc.WithInsecure())
if err != nil {
log.Printf("connect to %s failed: %v", server, err)
return nil
}
client := pb.NewGreeterClient(conn)
serverConn.Store(server, client)
return client
}
}
}
func main() {
for {
rpc()
time.Sleep(time.Second)
}
}
func rpc() {
client := GetClient()
if client == nil {
log.Printf("connect to %s failed", ETCD_CLUSTER)
}
hello, err := client.SayHello(context.Background(), &pb.HelloRequest{Name: "Rust"})
fmt.Println(hello, err)
}
测试
go run server/main.go --port 9001
go run server/main.go --port 9002
go run client/main.go
