kafka connect distributed 模式

1. 配置connect-distributed.properties

bootstrap.servers=vm01:9092,vm02:9092,vm03:9092
group.id=connect-cluster
key.converter=org.apache.kafka.connect.json.JsonConverter
value.converter=org.apache.kafka.connect.json.JsonConverter
key.converter.schemas.enable=false
value.converter.schemas.enable=false
offset.storage.topic=connect-offsets
offset.storage.replication.factor=2
config.storage.topic=connect-configs
config.storage.replication.factor=2
status.storage.topic=connect-status
status.storage.replication.factor=2
offset.flush.interval.ms=10000
rest.advertised.host.name=vm01
rest.advertised.port=8083
plugin.path=/home/data_center/kafka_2.12-2.5.1/connect

2. 修改内存connect-distributed.sh

if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
  export KAFKA_HEAP_OPTS="-Xms1G -Xmx1G"
fi

3. 启动distributed模式

bin/connect-distributed.sh config/connect-distributed.properties 1>/dev/null 2>&1 &
or
bin/connect-distributed.sh -daemon config/connect-distributed.properties

kafka connect rest api

1. 查询当前活动的connectors

curl -X GET http://localhost:8083/connectors | jq

2. 查看

curl -X GET http://localhost:8083/connectors/mysql-source | jq

3. 删除

curl -X DELETE http://localhost:8083/connectors/sqlserver-001

4. 暂停

curl -X PUT http://VM02:8083/connectors/sqlserver-mytable-001/pause

5. 恢复

curl -X put http://VM02:8083/connectors/sqlserver-mytable-001/resume

6. 重启

curl -X POST http://VM02:8083/connectors/sqlserver-mytable-001/restart

7. 获取状态

curl -X GET http://VM02:8083/connectors/sqlserver-mytable-001/status

8. 创建connector(sql server)

curl 'http://localhost:8083/connectors' -X POST -i -H "Content-Type:application/json" -d '
{
	"name": "sqlserver-001",
	"config": {
		"connector.class": "io.debezium.connector.sqlserver.SqlServerConnector",
		"database.hostname": "***",
		"database.port": "1433",
		"database.user": "data_center",
		"database.password": "******",
		"database.dbname": "mydb",
		"database.server.name": "debezium_sqlserver",
		"table.whitelist": "dbo.mytable",
		"database.history.kafka.bootstrap.servers": "vm01:9092,vm02:9092,vm03:9092",
		"database.history.kafka.topic": "dbhistory_sqlserver",
		"snapshot.mode": "initial",
		"snapshot.locking.mode": "none",
		"poll.interval.ms": 1000
	}
}
'

9. 创建connector(mysql)

curl 'http://localhost:8083/connectors' -X POST -i -H "Content-Type:application/json" -d '
{
	"name": "mysql_mydb_student",
	"config": {
		"connector.class": "io.debezium.connector.mysql.MySqlConnector",
		"database.hostname": "vm01",
		"database.port": "3306",
		"database.user": "root",
		"database.password": "123456",
		"database.dbname": "mydb",
		"database.server.name": "mydb_server",
		"table.whitelist": "mydb.student",
		"database.history.kafka.bootstrap.servers": "vm01:9092,vm02:9092,vm03:9092",
		"database.history.kafka.topic": "his_mydb",
		"tasks.max": "2"
	}
}
'