简介
kafka 客户端的基本使用
kakfa的本地启动
---
version: '3'
services:
zookeeper:
image: confluentinc/cp-zookeeper:7.3.0
container_name: zookeeper
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
broker:
image: confluentinc/cp-kafka:7.3.0
container_name: broker
ports:
# To learn about configuring Kafka for access across networks see
# https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/
- "9092:9092"
depends_on:
- zookeeper
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
代码实现
安装
github.com/segmentio/kafka-go
1、consomer.gp
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"time"
"github.com/segmentio/kafka-go"
)
func main() {
type User struct {
Id int
Name string
}
// make a new reader that consumes from topic-A, partition 0, at offset 42
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092"},
GroupID: "consumer-group-id",
Topic: "quickstart",
Partition: 0,
MinBytes: 10e3, // 10KB
MaxBytes: 10e6, // 10MB
CommitInterval: time.Second, // flushes commits to Kafka every second 要 加 GroupID 才能自动提交
})
// r.SetOffset(2)
ctx := context.Background()
for {
m, err := r.ReadMessage(ctx)
if err != nil {
break
}
var user User
err2 := json.Unmarshal(m.Value, &user)
if err2 != nil {
fmt.Printf("json unmarshal err: %v \n", err2)
}
fmt.Printf("--user-: %v\n", user)
fmt.Printf("message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value))
}
// if err := r.Close(); err != nil {
// log.Fatal("failed to close reader:", err)
// }
}
2、producer.gp
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"github.com/segmentio/kafka-go"
)
func main() {
type User struct {
Id int
Name string
}
user := &User{
1,
"A",
}
str, err2 := json.Marshal(user)
if err2 != nil {
fmt.Printf("err: %v\n", err2)
}
// make a writer that produces to topic-A, using the least-bytes distribution
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092"),
Topic: "quickstart",
Balancer: &kafka.LeastBytes{},
}
err := w.WriteMessages(context.Background(),
kafka.Message{
Value: str,
},
// kafka.Message{
// Key: []byte("Key-B"),
// Value: []byte("One!"),
// },
// kafka.Message{
// Key: []byte("Key-C"),
// Value: []byte("Two!"),
// },
)
if err != nil {
log.Fatal("failed to write messages:", err)
}
if err := w.Close(); err != nil {
log.Fatal("failed to close writer:", err)
}
}
参考：
