canal.developer
下载canal.developer
开启binlog
/etc/mysql/mysql.conf.d/mysqld.cnf
查看是否打开binlog
show variables like 'log_%';
修改路径文件，加入以下配置：
log-bin=/var/log/mysql/mysql-bin.log # 开启 binlog
binlog-format=ROW # 选择 ROW 模式
server_id=1 # 配置 MySQL replaction 需要定义，不要和 canal 的 slaveId 重复
配置canal mysql
修改canal.deployer-1.1.6\conf\example\instance.properties如下信息
# url
canal.instance.master.address=127.0.0.1:3306
# username/password
canal.instance.dbUsername=root
canal.instance.dbPassword=root
启动canal.developer
切换至bin目录下执行sh startup.sh
rocketmq
下载rocketmq
rocketmq.apache.org/dowloading/…
namesrv启动
sh ./mqnamesrv 或后台启动 nohup sh ./mqnamesrv &
broker启动
nohup sh ./mqbroker -n localhost:9876 &
关闭
sh ./mqshutdown broker
sh ./mqshutdown namesrv
es
下载es
启动
切换至bin目录下执行./elasticsearch
代码
porm
<!-- es-->
<dependency>
<groupId>cn.zxporz</groupId>
<artifactId>esclientrhl</artifactId>
<version>7.0.2</version>
</dependency>
<!--canal-->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>1.1.3</version>
</dependency>
<!-- rocketmq-->
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.9.1</version>
</dependency>
消费者
package org.zxp.esclientrhl.demo.rocketmq;
import com.alibaba.fastjson.JSON;
import lombok.SneakyThrows;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.zxp.esclientrhl.demo.domain.User;
import org.zxp.esclientrhl.demo.repository.UserRepository;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.Map;
/**
* 消费消息
* Created by zzq on 2022/5/13.
*/
@Component
public class Consumer {
@Autowired
private UserRepository userRepository;
@PostConstruct
public void init() throws InterruptedException, MQClientException {
// 实例化消费者
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1");
// 设置NameServer的地址
consumer.setNamesrvAddr("localhost:9876");
// 订阅一个或者多个Topic，以及Tag来过滤需要消费的消息
consumer.subscribe("TopicTest", "*");
// 注册回调实现类来处理从broker拉取回来的消息
consumer.registerMessageListener(new MessageListenerConcurrently() {
@SneakyThrows
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
Map map = JSON.parseObject(msgs.get(0).getBody(), Map.class);
User user = new User();
user.setEvent_type((String) map.get("eventType"));
user.setUser_id((String) ((Map) map.get("afterData")).get("user_id"));
user.setUser_name((String) ((Map) map.get("afterData")).get("username"));
user.setMobile((String) ((Map) map.get("afterData")).get("mobile"));
System.out.println("user:" + user.toString());
//将数据放入es库中
userRepository.save(user);
System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), map);
// 标记该消息已经被成功消费
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
// 启动消费者实例
consumer.start();
System.out.printf("Consumer Started.%n");
}
}
canal客户端
package org.zxp.esclientrhl.demo.canal;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.springframework.util.CollectionUtils;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
import java.util.List;
/**
* canal客户端
* Created by zzq on 2022/5/12.
*/
public class Client {
public static void main(String[] args) throws InterruptedException {
//获取连接
CanalConnector canalConnector = CanalConnectors.newSingleConnector(
new InetSocketAddress("localhost", 11111), "example", "", "");
while (true) {
canalConnector.connect();//连接
canalConnector.subscribe("accountingmanage_zzq.tb_user");//订阅数据库
com.alibaba.otter.canal.protocol.Message message = canalConnector.get(100);//获取数据
List<CanalEntry.Entry> entries = message.getEntries();//获取entry集合
if (CollectionUtils.isEmpty(entries)) {
System.out.println("无数据，等待1s.");
Thread.sleep(1000);
} else {
//单条解析
entries.forEach(e -> {
String tableName = e.getHeader().getTableName();//获取表名
CanalEntry.EntryType entryType = e.getEntryType();//获取类型
ByteString storeValue = e.getStoreValue();//获取序列化后的数据
//判断是否为ROWDATA类型
if (CanalEntry.EntryType.ROWDATA.equals(entryType)) {
try {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(storeValue);//反序列化数据
CanalEntry.EventType eventType = rowChange.getEventType();//获取操作类型
List<CanalEntry.RowData> rowDatasList = rowChange.getRowDatasList();//获取数据集
rowDatasList.forEach(e3 -> {
JSONObject beforeData = new JSONObject();
JSONObject afterData = new JSONObject();
List<CanalEntry.Column> beforeColumnsList = e3.getBeforeColumnsList();
List<CanalEntry.Column> afterColumnsList = e3.getAfterColumnsList();
beforeColumnsList.forEach(e1 -> {
beforeData.put(e1.getName(), e1.getValue());
});
afterColumnsList.forEach(e2 -> {
afterData.put(e2.getName(), e2.getValue());
});
System.out.println("table:" + tableName + ",eventType:" + eventType + ",before:" + beforeData + ",after:" + afterData);
//将数据投入rocketmq中
// 实例化消息生产者Producer
DefaultMQProducer producer = new DefaultMQProducer("group1");
// 设置NameServer的地址
producer.setNamesrvAddr("localhost:9876");
// 启动Producer实例
try {
producer.start();
} catch (MQClientException ex) {
ex.printStackTrace();
}
// 创建消息，并指定Topic，Tag和消息体
Message msg = null;
JSONObject messageMap = new JSONObject();
messageMap.put("eventType", eventType);
messageMap.put("afterData", afterData);
try {
msg = new Message("TopicTest" /* Topic */,
"TagA" /* Tag */,
(JSON.toJSONString(messageMap)).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
);
} catch (UnsupportedEncodingException ex) {
ex.printStackTrace();
}
// 发送消息到一个Broker
try {
SendResult sendResult = producer.send(msg);
System.out.println(111);
} catch (MQClientException ex) {
ex.printStackTrace();
} catch (RemotingException ex) {
ex.printStackTrace();
} catch (MQBrokerException ex) {
ex.printStackTrace();
} catch (InterruptedException ex) {
ex.printStackTrace();
}
});
} catch (InvalidProtocolBufferException ex) {
ex.printStackTrace();
}
} else {
System.out.println(e + "不是待分析类型");
}
});
}
}
}
}
