序
本文主要研究一下MaxwellKafkaProducer
MaxwellKafkaProducer
maxwell-1.25.1/src/main/java/com/zendesk/maxwell/producer/MaxwellKafkaProducer.java
public class MaxwellKafkaProducer extends AbstractProducer {
private final ArrayBlockingQueue<RowMap> queue;
private final MaxwellKafkaProducerWorker worker;
public MaxwellKafkaProducer(MaxwellContext context, Properties kafkaProperties, String kafkaTopic) {
super(context);
this.queue = new ArrayBlockingQueue<>(100);
this.worker = new MaxwellKafkaProducerWorker(context, kafkaProperties, kafkaTopic, this.queue);
Thread thread = new Thread(this.worker, "maxwell-kafka-worker");
thread.setDaemon(true);
thread.start();
}
@Override
public void push(RowMap r) throws Exception {
this.queue.put(r);
}
@Override
public StoppableTask getStoppableTask() {
return this.worker;
}
@Override
public KafkaProducerDiagnostic getDiagnostic() {
return new KafkaProducerDiagnostic(worker, context.getConfig(), context.getPositionStoreThread());
}
}
- MaxwellKafkaProducer继承了AbstractProducer，其构造器会创建ArrayBlockingQueue、ArrayBlockingQueue；其push方法则往queue中put数据
MaxwellKafkaProducerWorker
maxwell-1.25.1/src/main/java/com/zendesk/maxwell/producer/MaxwellKafkaProducer.java
class MaxwellKafkaProducerWorker extends AbstractAsyncProducer implements Runnable, StoppableTask {
static final Logger LOGGER = LoggerFactory.getLogger(MaxwellKafkaProducer.class);
private final Producer<String, String> kafka;
private final String topic;
private final String ddlTopic;
private final MaxwellKafkaPartitioner partitioner;
private final MaxwellKafkaPartitioner ddlPartitioner;
private final KeyFormat keyFormat;
private final boolean interpolateTopic;
private final ArrayBlockingQueue<RowMap> queue;
private Thread thread;
private StoppableTaskState taskState;
private String deadLetterTopic;
private final ConcurrentLinkedQueue<Pair<ProducerRecord<String,String>, KafkaCallback>> deadLetterQueue;
public static MaxwellKafkaPartitioner makeDDLPartitioner(String partitionHashFunc, String partitionKey) {
if ( partitionKey.equals("table") ) {
return new MaxwellKafkaPartitioner(partitionHashFunc, "table", null, "database");
} else {
return new MaxwellKafkaPartitioner(partitionHashFunc, "database", null, null);
}
}
public MaxwellKafkaProducerWorker(MaxwellContext context, String kafkaTopic, ArrayBlockingQueue<RowMap> queue,
Producer<String,String> producer)
{
super(context);
if ( kafkaTopic == null ) {
this.topic = "maxwell";
} else {
this.topic = kafkaTopic;
}
this.interpolateTopic = this.topic.contains("%{");
this.kafka = producer;
String hash = context.getConfig().kafkaPartitionHash;
String partitionKey = context.getConfig().producerPartitionKey;
String partitionColumns = context.getConfig().producerPartitionColumns;
String partitionFallback = context.getConfig().producerPartitionFallback;
this.partitioner = new MaxwellKafkaPartitioner(hash, partitionKey, partitionColumns, partitionFallback);
this.ddlPartitioner = makeDDLPartitioner(hash, partitionKey);
this.ddlTopic = context.getConfig().ddlKafkaTopic;
this.deadLetterTopic = context.getConfig().deadLetterTopic;
this.deadLetterQueue = new ConcurrentLinkedQueue<>();
if ( context.getConfig().kafkaKeyFormat.equals("hash") )
keyFormat = KeyFormat.HASH;
else
keyFormat = KeyFormat.ARRAY;
this.queue = queue;
this.taskState = new StoppableTaskState("MaxwellKafkaProducerWorker");
}
public MaxwellKafkaProducerWorker(MaxwellContext context, Properties kafkaProperties, String kafkaTopic,
ArrayBlockingQueue<RowMap> queue)
{
this(context, kafkaTopic, queue,
new KafkaProducer<String,String>(kafkaProperties, new StringSerializer(), new StringSerializer()));
}
@Override
public void run() {
this.thread = Thread.currentThread();
while ( true ) {
try {
drainDeadLetterQueue();
RowMap row = queue.take();
if (!taskState.isRunning()) {
taskState.stopped();
return;
}
this.push(row);
} catch ( Exception e ) {
taskState.stopped();
context.terminate(e);
return;
}
}
}
void drainDeadLetterQueue() {
Pair<ProducerRecord<String, String>, KafkaCallback> pair;
while ((pair = deadLetterQueue.poll()) != null) {
sendAsync(pair.getLeft(), pair.getRight());
}
}
//......
}
- MaxwellKafkaProducerWorker继承了AbstractAsyncProducer，实现了Runnable及StoppableTask接口；其run方法使用while循环，不断执行drainDeadLetterQueue、queue.take()、this.push(row)；drainDeadLetterQueue方法从deadLetterQueue拉取数据，然后通过sendAsync再次发送
AbstractAsyncProducer
maxwell-1.25.1/src/main/java/com/zendesk/maxwell/producer/AbstractAsyncProducer.java
public abstract class AbstractAsyncProducer extends AbstractProducer {
public class CallbackCompleter {
private InflightMessageList inflightMessages;
private final MaxwellContext context;
private final MaxwellConfig config;
private final Position position;
private final boolean isTXCommit;
private final long messageID;
public CallbackCompleter(InflightMessageList inflightMessages, Position position, boolean isTXCommit, MaxwellContext context, long messageID) {
this.inflightMessages = inflightMessages;
this.context = context;
this.config = context.getConfig();
this.position = position;
this.isTXCommit = isTXCommit;
this.messageID = messageID;
}
public void markCompleted() {
inflightMessages.freeSlot(messageID);
if(isTXCommit) {
InflightMessageList.InflightMessage message = inflightMessages.completeMessage(position);
if (message != null) {
context.setPosition(message.position);
long currentTime = System.currentTimeMillis();
long age = currentTime - message.sendTimeMS;
messagePublishTimer.update(age, TimeUnit.MILLISECONDS);
messageLatencyTimer.update(Math.max(0L, currentTime - message.eventTimeMS - 500L), TimeUnit.MILLISECONDS);
if (age > config.metricsAgeSlo) {
messageLatencySloViolationCount.inc();
}
}
}
}
}
private InflightMessageList inflightMessages;
public AbstractAsyncProducer(MaxwellContext context) {
super(context);
this.inflightMessages = new InflightMessageList(context);
Metrics metrics = context.getMetrics();
String gaugeName = metrics.metricName("inflightmessages", "count");
metrics.register(gaugeName, (Gauge<Long>) () -> (long) inflightMessages.size());
}
public abstract void sendAsync(RowMap r, CallbackCompleter cc) throws Exception;
@Override
public final void push(RowMap r) throws Exception {
Position position = r.getNextPosition();
// Rows that do not get sent to a target will be automatically marked as complete.
// We will attempt to commit a checkpoint up to the current row.
if(!r.shouldOutput(outputConfig)) {
if ( position != null ) {
inflightMessages.addMessage(position, r.getTimestampMillis(), 0L);
InflightMessageList.InflightMessage completed = inflightMessages.completeMessage(position);
if (completed != null) {
context.setPosition(completed.position);
}
}
return;
}
// back-pressure from slow producers
long messageID = inflightMessages.waitForSlot();
if(r.isTXCommit()) {
inflightMessages.addMessage(position, r.getTimestampMillis(), messageID);
}
CallbackCompleter cc = new CallbackCompleter(inflightMessages, position, r.isTXCommit(), context, messageID);
sendAsync(r, cc);
}
}
- AbstractAsyncProducer继承了AbstractProducer，其push方法主要执行inflightMessages.addMessage及sendAsync
小结
MaxwellKafkaProducer继承了AbstractProducer，其构造器会创建ArrayBlockingQueue、ArrayBlockingQueue；其push方法则往queue中put数据；MaxwellKafkaProducerWorker继承了AbstractAsyncProducer，实现了Runnable及StoppableTask接口；其run方法使用while循环，不断执行drainDeadLetterQueue、queue.take()、this.push(row)；drainDeadLetterQueue方法从deadLetterQueue拉取数据，然后通过sendAsync再次发送
