Debezium Engine Postgres

Debezium Engine 监听 postgrsql 数据库表的行级变更,实现数据同步

Debezium Engine 简介

Debezium connectors 配合 kafka connect 能够捕捉数据库的变更日志，将对应的数据库的行级变化发送大kafka的主题中，通常一张表对应kafka中的一个主题。消费kafka对应的主题，从而将数据库的数据迁移到ElasticSearch 或者其他存储。

Debezium engine 可以内嵌入普通的java应用中，通过Debezium engine 可以获得数据库表行级变更。 不用安装Kafka，适合数据量较少的业务。 Debezium Engine

Postgres 配置

创建用户

安装 postgresql 数据库，postgresql.conf 配置文件中修改 wal_level = logical

wal_level = logical

maven工程

创建一个 java maven工程项目，引入依赖

pom.xml

<properties>
    <maven.compiler.source>11</maven.compiler.source>
    <maven.compiler.target>11</maven.compiler.target>
    <version.debezium>2.0.0.Final</version.debezium>
</properties>

<dependencies>
    <dependency>
        <groupId>io.debezium</groupId>
        <artifactId>debezium-api</artifactId>
        <version>${version.debezium}</version>
    </dependency>

    <dependency>
        <groupId>io.debezium</groupId>
        <artifactId>debezium-embedded</artifactId>
        <version>${version.debezium}</version>
    </dependency>

    <dependency>
        <groupId>io.debezium</groupId>
        <artifactId>debezium-connector-postgres</artifactId>
        <version>${version.debezium}</version>
    </dependency>

    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>2.0.19</version>
    </dependency>
</dependencies>

Debe.java

package com.lagou;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.debezium.embedded.Connect;
import io.debezium.engine.ChangeEvent;
import io.debezium.engine.DebeziumEngine;
import io.debezium.engine.format.Json;
import org.apache.kafka.connect.data.Field;
import org.apache.kafka.connect.data.Struct;
import org.apache.kafka.connect.source.SourceRecord;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * https://github.com/samuelcastle/debezium-artemis-poc/blob/master/src/main/java/be/i8c/dataliberation/poc/MyAMQDebeziumEngine.java
 */
public class Debe {
    public static void main(String[] args) {
    final Properties props = new Properties();
        props.setProperty("name", "engine");
        props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore");
        props.setProperty("offset.storage.file.filename", "offsets.dat");
        props.setProperty("offset.flush.interval.ms", "6000");
        /* begin connector properties */
        props.setProperty("connector.class", "io.debezium.connector.postgresql.PostgresConnector");
        props.setProperty("plugin.name", "pgoutput");
        props.setProperty("database.hostname", "127.0.0.1");
        props.setProperty("database.dbname", "debezium");
        props.setProperty("database.port", "5432");
        props.setProperty("database.user", "root");
        props.setProperty("database.password", "root");
        props.setProperty("schema.include.list", "public");
        props.setProperty("table.include.list", "public.user");
        props.setProperty("topic.prefix", "my_app_connector");
        props.setProperty("schema.history.internal",
                "io.debezium.storage.file.history.FileSchemaHistory");
        props.setProperty("schema.history.internal.file.filename",
                "schemahistory.dat");
        props.setProperty("snapshot.mode", "never");
        props.setProperty("key.converter.schemas.enable", "false");
        props.setProperty("key.converter", "org.apache.kafka.connect.json.JsonConverter");
        props.setProperty("value.converter.schemas.enable", "false");
        props.setProperty("value.converter", "org.apache.kafka.connect.json.JsonConverter");
        props.setProperty("decimal.handling.mode", "string");
        props.setProperty("heartbeat.interval.ms", "90000");

        CountDownLatch latch = new CountDownLatch(1);
        try (DebeziumEngine<ChangeEvent<String, String>> engine = DebeziumEngine.create(Json.class)
                .using(props)
                .notifying((records, committer) -> {
                    for (ChangeEvent<String, String> record : records) {
                        // 跳过心跳
                        if (record.destination().startsWith("__debezium-heartbeat")) {
                            continue;
                        }

                        System.out.println(record.value());
                        committer.markProcessed(record);
                    }
                    committer.markBatchFinished();
                })
                .using(((success, message, error) -> {
                    if (error != null) {
                        log.error("", error);
                    }

                    latch.countDown();
                }))
                .build()
        ) {

            // Run the engine asynchronously ...
            ExecutorService executor = Executors.newSingleThreadExecutor();
            executor.execute(engine);
            Thread.sleep(10000);
            latch.await();
            // 关闭
            executor.shutdown();
            while (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
                log.info("Waiting another 5 seconds for the embedded engine to shut down");
            }
        }
}

参考链接

Convert

debezium-artemis-poc