前言
上次主要讲的是是Maxwell+Kafka的部署，这次我们来处理Kafka发送过来的Binlog
总体上分为两步分：
1、业务端负责es数据的组装，然后通过rpc调用我提供的接口来CRUD数据；
2、服务端处理binlog数据，处理成对象后发给业务端，让业务端做后续操作。
代码实例
pom.xml
<spring-kafka.version>2.6.2</spring-kafka.version>
<spring-boot.version>2.3.1.RELEASE</spring-boot.version>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>${spring-kafka.version}</version>
</dependency>
application.yml
spring:
kafka:
# 参考 kafka的config/server.properties
# listeners=PLAINTEXT://192.168.110.70:9092
# advertised.listeners=PLAINTEXT://192.168.110.70:9092
bootstrap-servers: 192.168.1.220:9092
consumer:
# 参考kafka的config/consumer.properties
# group.id=dev-consumer-group
group-id: dev-consumer-group
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
# 自动提交消息位移数据（Committing Offsets）
# 自动位移能保证不出现消费丢失的情况，但它可能会出现重复消费。
enable-auto-commit: true
producer:
key-serializer: org.apache.kafka.common.serialization.StringSerializer
value-serializer: org.apache.kafka.common.serialization.StringSerializer
kafka mysqlBinlog消费
这里代码还是挺糙的，后面我会在优化后更新一次。
对于数据的同步，我这里还是主要是调用业务端的接口。es数据的组装在业务端。
ConsumerRecordKey
@Getter
@Setter
public class ConsumerRecordKey {
/**
* 数据库
*/
private String database;
/**
* 表名
*/
private String table;
/**
* 主键id
*/
private String pkId;
}
ConsumerRecordValue
@Getter
@Setter
public class ConsumerRecordValue<T> {
public ConsumerRecordValue(T data) {
this.data = data;
}
/**
* 数据库
*/
private String database;
/**
* 表名
*/
private String table;
/**
* 数据操作类型，insert、update、delete
*/
private String type;
/**
* 被修改之后的数据
*/
private T data;
/**
* 被修改之前的数据
*/
private T old;
}
BinLogAspect
@Slf4j
@Aspect
@Component
public class BinLogAspect {
@Pointcut("execution(public * com.tianque.mediation.es.kafka..*.*(..))")
public void binlogCutPoint(){}
@Before("binlogCutPoint()")
public void doBefore(JoinPoint joinPoint) {
// 记录下请求内容
Object[] args = joinPoint.getArgs();
ConsumerRecord<String, String> record = (ConsumerRecord<String, String>) args[0];
String pkId = MySqlBinLogParse.getPkId(record);
log.info("ES数据同步: {}.{} : pkId:{}", joinPoint.getSignature().getDeclaringTypeName(), joinPoint.getSignature().getName(), pkId);
}
}
MySqlBinLogParse
public class MySqlBinLogParse {
private static Gson gson;
private static final Pattern DATE_PATTERN = Pattern.compile("^[-\\+]?[\\d]*$");
static {
GsonBuilder gsonBuilder = new GsonBuilder()
.registerTypeAdapter(LocalDateTime.class, new JsonDeserializer<LocalDateTime>() {
@Override
public LocalDateTime deserialize(JsonElement json, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException {
String datetime = json.getAsJsonPrimitive().getAsString();
String dateFormat = getDateFormat(datetime);
if (StringUtils.isBlank(dateFormat)) {
return null;
}
return LocalDateTime.parse(datetime, DateTimeFormatter.ofPattern(dateFormat));
}
})
.registerTypeAdapter(LocalDate.class, (JsonDeserializer<LocalDate>) (json, type, context) -> {
String datetime = json.getAsJsonPrimitive().getAsString();
String dateFormat = getDateFormat(datetime);
if (StringUtils.isBlank(dateFormat)) {
return null;
}
return LocalDate.parse(datetime, DateTimeFormatter.ofPattern(dateFormat));
})
.registerTypeAdapter(Date.class, (JsonDeserializer<Date>) (json, type, context) -> {
String datetime = json.getAsJsonPrimitive().getAsString();
try {
String dateFormat = getDateFormat(datetime);
if (StringUtils.isBlank(dateFormat)) {
return null;
}
return DateUtils.parseDate(datetime, "yyyy-MM-dd HH:mm:ss");
} catch (ParseException e) {
e.printStackTrace();
}
return null;
})
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES);
gson = gsonBuilder.create();
}
public static <T> KafkaMysqlBaseEvent<T> parse(ConsumerRecord<String, String> record, Type type) {
ConsumerRecordKey consumerRecordKey = parseKey(record);
ConsumerRecordValue<T> consumerRecordValue = parseValue(record, type);
if (consumerRecordKey == null || consumerRecordValue == null) {
return null;
}
return new KafkaMysqlBaseEvent<>(consumerRecordKey.getPkId(), consumerRecordKey, consumerRecordValue);
}
public static ConsumerRecordKey parseKey(ConsumerRecord<String, String> record) {
if (!checkParams(record)) {
return null;
}
String key = parseKey(record.key());
return gson.fromJson(key, ConsumerRecordKey.class);
}
public static <T> ConsumerRecordValue<T> parseValue(ConsumerRecord<String, String> record, Type type) {
if (!checkParams(record)) {
return null;
}
String value = record.value();
return gson.fromJson(value, type);
}
private static String parseKey(String key) {
key = StringUtils.replace(key, "pk.id_", "pk_id");
return StringUtils.replace(key, "pk.id", "pk_id");
}
public static String getPkId(ConsumerRecord<String, String> record) {
ConsumerRecordKey key = parseKey(record);
return key == null ? null : key.getPkId();
}
private static boolean checkParams(ConsumerRecord<String, String> record) {
if (record == null) {
return false;
}
String key = record.key();
String value = record.value();
return !StringUtils.isBlank(key) && !StringUtils.isBlank(value);
}
/**
* 常规自动日期格式识别
*
* @param str 时间字符串
* @return Date
*/
private static String getDateFormat(String str) {
boolean year = false;
if (DATE_PATTERN.matcher(str.substring(0, 4)).matches()) {
year = true;
}
StringBuilder sb = new StringBuilder();
int index = 0;
if (!year) {
if (str.contains("月") || str.contains("-") || str.contains("/")) {
if (Character.isDigit(str.charAt(0))) {
index = 1;
}
} else {
index = 3;
}
}
for (int i = 0; i < str.length(); i++) {
char chr = str.charAt(i);
if (Character.isDigit(chr)) {
if (index == 0) {
sb.append("y");
}
if (index == 1) {
sb.append("M");
}
if (index == 2) {
sb.append("d");
}
if (index == 3) {
sb.append("H");
}
if (index == 4) {
sb.append("m");
}
if (index == 5) {
sb.append("s");
}
if (index == 6) {
sb.append("S");
}
} else {
if (i > 0) {
char lastChar = str.charAt(i - 1);
if (Character.isDigit(lastChar)) {
index++;
}
}
sb.append(chr);
}
}
return sb.toString();
}
}
MysqlBinLogListener
@Slf4j
@Component
public class MysqlBinLogListener {
@Resource
private EsDataService esDataService;
@DubboReference(check = false)
private WorkViewEsFacade workViewEsFacade;
private static Type recordMapJsonType = new TypeToken<ConsumerRecordValue<Map<String, String>>>() {}.getType();
@KafkaListener(id = "listenItemsType",
topics = {KafkaTopicConstant.DEV_DB_CM + KafkaTopicConstant.TABLE_ITEMS_TYPE,
KafkaTopicConstant.TEST_DB_CM + KafkaTopicConstant.TABLE_ITEMS_TYPE,
KafkaTopicConstant.PROD_DB_CM + KafkaTopicConstant.TABLE_ITEMS_TYPE})
public void listenItemsType(ConsumerRecord<String, String> record) {
ConsumerRecordKey key = MySqlBinLogParse.parseKey(record);
ConsumerRecordValue<Map<String, String>> value = MySqlBinLogParse.parseValue(record, recordMapJsonType);
if (key == null || value == null) {
return;
}
if (deleteData(key, value, WorkViewItemsTypeEntity.class)) {
return;
}
String pkId = key.getPkId();
workViewEsFacade.dataSync(WorkViewItemsTypeEntity.class, NumberUtils.toLong(pkId));
}
@KafkaListener(id = "listenActRuTask",
topics = {KafkaTopicConstant.DEV_DB_FLOW + KafkaTopicConstant.TABLE_ACT_RU_TASK,
KafkaTopicConstant.TEST_DB_FLOW + KafkaTopicConstant.TABLE_ACT_RU_TASK,
KafkaTopicConstant.PROD_DB_FLOW + KafkaTopicConstant.TABLE_ACT_RU_TASK})
public void listenActRuTask(ConsumerRecord<String, String> record) {
ConsumerRecordKey key = MySqlBinLogParse.parseKey(record);
ConsumerRecordValue<Map<String, String>> value = MySqlBinLogParse.parseValue(record, recordMapJsonType);
if (key == null || value == null) {
return;
}
if (deleteData(key, value, EsActRuTaskEntity.class)) {
return;
}
Type type = new TypeToken<ConsumerRecordValue<ActRuTask>>() {}.getType();
KafkaMysqlBaseEvent<ActRuTask> kafkaMysqlBaseEvent = MySqlBinLogParse.parse(record, type);
ActRuTaskEvent actRuTaskEvent = new ActRuTaskEvent(kafkaMysqlBaseEvent);
SpringEventUtils.publishEvent(actRuTaskEvent);
}
private boolean deleteData(ConsumerRecordKey key, ConsumerRecordValue<?> value, Class<?> clazz) {
if (StringUtils.isAnyBlank(key.getPkId(), value.getType())) {
return false;
}
if ("delete".equals(value.getType())) {
esDataService.delete(clazz, key.getPkId());
return true;
}
return false;
}
}
异步事件
这一块主要是为了增加消息的消费效率
SpringEventUtils
@Component
public class SpringEventUtils {
private static ApplicationEventPublisher publisher;
public SpringEventUtils(ApplicationEventPublisher publisher) {
if (SpringEventUtils.publisher == null) {
SpringEventUtils.publisher = publisher;
}
}
public static void publishEvent(BaseEvent baseEvent) {
publisher.publishEvent(baseEvent);
}
}
BaseEvent
@Getter
@Setter
public class BaseEvent extends ApplicationEvent {
public BaseEvent(Object source) {
super(source);
}
}
MySqlTableEventHandler
@Slf4j
@Component
public class MySqlTableEventHandler {
@Resource
private EsDataService esDataService;
@Resource
private EsIndexService esIndexService;
@Async
@EventListener(classes = ActRuTaskEvent.class)
public void sent(ActRuTaskEvent event) {
ConsumerRecordValue<ActRuTask> value = event.getValue();
ActRuTask actRuTask = value.getData();
EsActRuTaskEntity esActRuTask = new EsActRuTaskEntity();
esActRuTask.setId(actRuTask.getID_());
esActRuTask.setAssignee(actRuTask.getASSIGNEE_());
esActRuTask.setCandidate(actRuTask.getCandidate_());
esActRuTask.setCreateTime(actRuTask.getCREATE_TIME_() == null ? null : actRuTask.getCREATE_TIME_().getTime());
if (!esIndexService.indexExists(EsActRuTaskEntity.class)) {
esIndexService.createIndex(EsActRuTaskEntity.class);
}
esDataService.save(esActRuTask);
}
}
ActRuTaskEvent
public class ActRuTaskEvent extends KafkaMysqlBaseEvent<ActRuTask> {
public ActRuTaskEvent(String id, ConsumerRecordKey key, ConsumerRecordValue<ActRuTask> value) {
super(id, key, value);
}
public ActRuTaskEvent(KafkaMysqlBaseEvent<ActRuTask> superEvent) {
super(superEvent.getId(),superEvent.getKey(),superEvent.getValue());
}
}
结语
binlog处理逻辑还是挺简单的，但可以优化修改的地方也很多，比如可以将kafka的自动提交改为手动提交，防止消息重复消费等，后续优化的时候会及时更新文章。
