Springboot + Kafka Schema Registry + Avro

Daniel_Qu
920 阅读2分钟
  1. Gradle依赖
plugins {
    id 'com.github.davidmc24.gradle.plugin.avro' version '1.3.0'
}

repositories {
    mavenCentral()
    maven { url "https://packages.confluent.io/maven/" }
}

dependencies {
    implementation "io.confluent:kafka-streams-avro-serde:7.0.1"
    implementation "io.confluent:kafka-avro-serializer:7.0.1"
    implementation 'org.apache.avro:avro:1.11.0'
    // this is to generate avsc file 
    implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-avro:2.12.1'
    
    implementation "org.mapstruct:mapstruct:1.4.2.Final"
    compileOnly "org.mapstruct:mapstruct-processor:1.4.2.Final"
    annotationProcessor "org.mapstruct:mapstruct-processor:1.4.2.Final"
}
  1. 定义model
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class MyTestObject {
    private String field1;
    private long field2;
    private int field3;
    private boolean field4;
    private List<String> field5;
    private List<MyAvroTestInnerList1> field6;
    private Map<String, MyAvroTestInnerMap1> field7;
}
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class MyAvroTestInnerList1 {
    private String innerListField1;
}
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class MyAvroTestInnerMap1 {
    private String innerMapField1;
}
  1. 使用代码自动生成avsc文件内容：
public static void main(String[] args) throws JsonMappingException {
    ObjectMapper mapper = new ObjectMapper(new AvroFactory());
    AvroSchemaGenerator gen = new AvroSchemaGenerator();
    mapper.acceptJsonFormatVisitor(MyAvroTestObject.class, gen);
    AvroSchema schemaWrapper = gen.getGeneratedSchema();

    Schema avroSchema = schemaWrapper.getAvroSchema();
    String asJson = avroSchema.toString(true);
    System.out.println(asJson);
}

将生成的内容拷贝到文件中，防止在main/avro/目录下 适当修改namespace和name. src/main/avro/avro_test.avsc

{
  "type" : "record",
  "name" : "MyAvroTestObject",
  "namespace" : "com.example.springbootstudygradle.model.avro",
  "fields" : [ {
    "name" : "field1",
    "type" : [ "null", "string" ]
  }, {
    "name" : "field2",
    "type" : {
      "type" : "long",
      "java-class" : "java.lang.Long"
    }
  }, {
    "name" : "field3",
    "type" : {
      "type" : "int",
      "java-class" : "java.lang.Integer"
    }
  }, {
    "name" : "field4",
    "type" : "boolean"
  }, {
    "name" : "field5",
    "type" : [ "null", {
      "type" : "array",
      "items" : "string"
    } ]
  }, {
    "name" : "field6",
    "type" : [ "null", {
      "type" : "array",
      "items" : {
        "type" : "record",
        "name" : "MyAvroTestInnerList1",
        "fields" : [ {
          "name" : "innerListField1",
          "type" : [ "null", "string" ]
        } ]
      }
    } ]
  }, {
    "name" : "field7",
    "type" : [ "null", {
      "type" : "map",
      "values" : {
        "type" : "record",
        "name" : "MyAvroTestInnerMap1",
        "fields" : [ {
          "name" : "innerMapField1",
          "type" : [ "null", "string" ]
        } ]
      }
    } ]
  } ]
}
  1. 使用gradle命令生成class文件
gradle assemble
gradle build

image.png
5. 使用MapStruct将object转换成avro object

@Mapper(
        builder = @Builder(disableBuilder = true),
        collectionMappingStrategy = CollectionMappingStrategy.ADDER_PREFERRED,
        nullValueCheckStrategy = NullValueCheckStrategy.ALWAYS,
        nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.SET_TO_NULL,
        unmappedSourcePolicy = ReportingPolicy.ERROR,
        unmappedTargetPolicy = ReportingPolicy.ERROR)
public abstract class MtTestObjectMapper extends BaseMapper {

    public static MtTestObjectMapper INSTANCE = Mappers.getMapper(MtTestObjectMapper.class);

    @BeanMapping(ignoreUnmappedSourceProperties = { "specificData", "schema" })
    public abstract MyTestObject map(MyAvroTestObject myAvroTestObject);
    public abstract MyAvroTestObject map(MyTestObject myTestObject);

}
  1. Kafka Producer code
Properties properties = new Properties();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class);
properties.put(KafkaAvroSerializerConfig.SCHEMA_REGISTRY_URL_CONFIG, "localhost:8081");

KafkaProducer<String, MyAvroTestObject> producer = new KafkaProducer<String, MyAvroTestObject>(properties);

MyAvroTestObject myAvroTestObject = MtTestObjectMapper.INSTANCE.map(build());
ProducerRecord<String, MyAvroTestObject> record = new ProducerRecord<String, MyAvroTestObject>("avro_test", myAvroTestObject);
producer.send(record);
  1. 发送之后可以在registry上看到schema
{
"subject": "avro_test-value",
"version": 1,
"id": 367,
"schema": "{\"type\":\"record\",\"name\":\"MyAvroTestObject\",\"namespace\":\"com.example.springbootstudygradle.model.avro\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]},{\"name\":\"field2\",\"type\":{\"type\":\"long\",\"java-class\":\"java.lang.Long\"}},{\"name\":\"field3\",\"type\":{\"type\":\"int\",\"java-class\":\"java.lang.Integer\"}},{\"name\":\"field4\",\"type\":\"boolean\"},{\"name\":\"field5\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]},{\"name\":\"field6\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"MyAvroTestInnerList1\",\"fields\":[{\"name\":\"innerListField1\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]}]}}]},{\"name\":\"field7\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"record\",\"name\":\"MyAvroTestInnerMap1\",\"fields\":[{\"name\":\"innerMapField1\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}]}]},\"avro.java.string\":\"String\"}]}]}"
}
  1. Kafka Streams 8.1 整合schema registry
public static void main(String[] args) {
    final StreamsBuilder builder = new StreamsBuilder();
    KStream<String, String> source =
            builder.stream("input-topic", Consumed.with(Serdes.String(), Serdes.String()));
    KStream<String, MyTestObject> testObject = source.map((k, v) -> new KeyValue<>(k, build()));
    testObject.to("test_stream_avro_1", Produced.with(Serdes.String(),
            new MyTestObjectSerde(mySpecificAvroSerde())));
    final Topology topology = builder.build();

    Properties properties = new Properties();
    properties.put("application.id","application_id_test_01");
    properties.put("bootstrap.servers","http://localhost:9092");

    final KafkaStreams streams = new KafkaStreams(topology, properties);
    final CountDownLatch latch = new CountDownLatch(1);


    streams.setUncaughtExceptionHandler((exception) -> {
        return StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.REPLACE_THREAD;
    });

    Runtime.getRuntime()
            .addShutdownHook(
                    new Thread("application_id_test_01_shutdown_hook") {
                        @Override
                        public void run() {
                            streams.close();
                            latch.countDown();
                        }
                    });

    try {
        streams.start();
        latch.await();
    } catch (Exception e) {
        System.exit(1);
    }
    System.exit(0);

}

//create SpecificAvroSerde
public static SpecificAvroSerde<MyAvroTestObject> mySpecificAvroSerde() {
    SpecificAvroSerde<MyAvroTestObject> specificAvroSerde = new SpecificAvroSerde<>();
    // When you want to override serdes explicitly/selectively
    final Map<String, String> serdeConfig = Collections.singletonMap("schema.registry.url", "http://localhost:8081");
    specificAvroSerde.configure(serdeConfig, false);
    return specificAvroSerde;
}

private static MyTestObject build(){
    HashMap<String, MyAvroTestInnerMap1> map = new HashMap<>();
    map.put("k", MyAvroTestInnerMap1.builder().innerMapField1("v").build());
    return MyTestObject.builder()
            .field1("s")
            .field2(1)
            .field3(2)
            .field4(true)
            .field5(Arrays.asList("d"))
            .field6(Arrays.asList(MyAvroTestInnerList1.builder().innerListField1("l").build()))
            .field7(map)
            .field8(LocalDateTime.now())
            .build();
}

8.2 自定义Serde

public class MyTestObjectSerde implements Serde<MyTestObject> {

    private final SpecificAvroSerde<MyAvroTestObject> myAvroTestObjectSpecificAvroSerde;

    private final MyTestObjectSerializer myTestObjectSerializer;

    private final MyTestObjectDeserializer myTestObjectDeserializer;

    public MyTestObjectSerde(SpecificAvroSerde<MyAvroTestObject> specificAvroSerde) {
        this.myAvroTestObjectSpecificAvroSerde = specificAvroSerde;
        this.myTestObjectSerializer = new MyTestObjectSerializer(this.myAvroTestObjectSpecificAvroSerde);
        this.myTestObjectDeserializer = new MyTestObjectDeserializer(this.myAvroTestObjectSpecificAvroSerde);

    }

    @Override
    public Serializer<MyTestObject> serializer() {
        return this.myTestObjectSerializer;
    }

    @Override
    public Deserializer<MyTestObject> deserializer() {
        return this.myTestObjectDeserializer;
    }

    public static class MyTestObjectSerializer implements Serializer<MyTestObject> {

        private final Serializer<MyAvroTestObject> myTestObjectSerializer;

        public MyTestObjectSerializer(SpecificAvroSerde<MyAvroTestObject> specificAvroSerde) {
            this.myTestObjectSerializer = specificAvroSerde.serializer();
        }

        @Override
        public byte[] serialize(String topic, MyTestObject myTestObject) {
            if (myTestObject == null) {
                return null;
            }
            MyAvroTestObject myAvroTestObject = MtTestObjectMapper.INSTANCE.map(myTestObject);
            return myTestObjectSerializer.serialize(topic, myAvroTestObject);
        }
    }

    public static class MyTestObjectDeserializer implements Deserializer<MyTestObject> {

        private final Deserializer<MyAvroTestObject> myTestObjectDeserializer;

        public MyTestObjectDeserializer(SpecificAvroSerde<MyAvroTestObject> specificAvroSerde) {
            this.myTestObjectDeserializer = specificAvroSerde.deserializer();
        }

        @Override
        public MyTestObject deserialize(String topic, byte[] data) {
            if (data == null) {
                return null;
            }
            MyAvroTestObject myAvroTestObject = myTestObjectDeserializer.deserialize(topic, data);
            return MtTestObjectMapper.INSTANCE.map(myAvroTestObject);
        }

    }
}

8.3 测试

producer：  
confluent-6.2.0 zhhqu$ kafka-console-producer --bootstrap-server localhost:9092 --topic input-topic
consumer:  
kafka-console-consumer --bootstrap-server localhost:9092 --topic test_stream_avro_1 --from-beginning --formatter io.confluent.kafka.formatter.AvroMessageFormatter --property schema.registry.url=http://localhost:8081
avatar
文章
阅读
粉丝