docker搭建 Cannal + Rocketmq(单机)监听数据库 binlog 实现双写一致性docker搭建 C

创建文件夹

mkdir -p {data/mysql/data,data/mysql/conf,data/canal/logs,data/canal/conf/example,data/mq/broker/conf}

compose 文件 ( 注意版本信息 )

services:
  mysql:
    image: mysql:8.0.35
    container_name: mysql
    environment:
      MYSQL_ROOT_PASSWORD: root123456
      MYSQL_USER: canal
      MYSQL_PASSWORD: canal
      TZ: Asia/Shanghai
    ports:
      - "3306:3306"
    volumes:
      - ./data/mysql/conf/my.cnf:/etc/mysql/my.cnf  
      - ./data/mysql/data:/var/lib/mysql
    networks:
      - canal-network


  # RocketMQ Namesrv
  rmqnamesrv:
    image: apache/rocketmq:5.3.1
    container_name: rmqnamesrv
    ports:
      - 9876:9876
    networks:
      - canal-network
    command: sh mqnamesrv



 # RocketMQ Broker
  rmqbroker:
    image: apache/rocketmq:5.3.1
    container_name: rmqbroker
    ports:
      - 10911:10911
    environment:
      - NAMESRV_ADDR=rmqnamesrv:9876
      - JAVA_OPT_EXT=-Drocketmq.namesrv.addr=rmqnamesrv:9876
    volumes:
      - ./data/mq/broker.conf:/home/rocketmq/rocketmq-5.3.1/conf/broker.conf
    command: sh mqbroker -c /home/rocketmq/rocketmq-5.3.1/conf/broker.conf
    depends_on:
      - rmqnamesrv
    networks:
      - canal-network


 # Canal Server
  canal-server:
    image: canal/canal-server:v1.1.7
    container_name: canal
    depends_on:
      - mysql
      - rmqbroker
    environment:
      - canal.auto.scan=false
      - TZ=Asia/Shanghai  # 修复时区
    volumes:
  #    - ./data/conf/canal.properties:/home/admin/canal-server/conf/canal.properties
  #    - ./data/conf/example/instance.properties:/home/admin/canal-server/conf/example/instance.properties
      - ./data/canal/logs:/home/admin/canal-server/logs
    networks:
      - canal-network


  # RocketMQ Dashboard
  mq-dashboard:
    image: apacherocketmq/rocketmq-dashboard:latest
    container_name: mq-dashboard
    ports:
      - 8080:8080
    environment:
      JAVA_OPTS: -Drocketmq.namesrv.addr=rmqnamesrv:9876 -Dcom.rocketmq.sendMessageWithVIPChannel=false 
    depends_on:
      - rmqnamesrv
    networks:
      - canal-network


  proxy:
    image: apache/rocketmq:5.3.1
    container_name: rmqproxy
    networks:
      - canal-network
    depends_on:
      - rmqbroker
      - rmqnamesrv
    ports:
      - 8081:8081
    restart: on-failure
    environment:
      - NAMESRV_ADDR=rmqnamesrv:9876
    command: sh mqproxy


networks:
  canal-network:
    driver: bridge

启动 docker-compose 拷贝数据

docker cp canal:/home/admin/canal-server/conf/example/instance.properties ./data/canal/conf/example

docker cp canal:/home/admin/canal-server/conf/canal.properties ./data/canal/conf

修改如下配置项

nano ./data/canal/conf/canal.properties

canal.serverMode = rocketMQ

##################################################
#########                   RocketMQ         #############
##################################################
rocketmq.producer.group = canal-producer-group
rocketmq.enable.message.trace = false
rocketmq.customized.trace.topic = canal_test
rocketmq.namespace =
rocketmq.namesrv.addr = 192.168.0.101:9876		# 对应的服务器ip
rocketmq.retry.times.when.send.failed = 0
rocketmq.vip.channel.enabled = false
rocketmq.tag = cannal_tag

nano ./data/canal/conf/example/instance.propertie

# MySQL连接信息

canal.instance.mysql.slaveId=1234 # 这里只要跟主机不冲突就行


# 根据实践情况配
canal.instance.master.address=mysql:3306
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
canal.instance.connectionCharset=UTF-8

canal.mq.topic=canal_test


# 过滤规则（监听所有库）
canal.instance.filter.regex=.*\..*

nano ./data/mysql/conf/my.cnf

[mysqld]
log-bin=mysql-bin
binlog-format=ROW
server-id=1
gtid-mode=on
enforce-gtid-consistency=1
log-slave-updates=1

nano ./data/mq/broker.conf

brokerClusterName = DefaultCluster
#brokerName = broker-a
brokerId = 0
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
brokerIP1 = 192.168.0.101		# 对应服务的ip
# 指定监听端口
listenPort = 10911
autoCreateTopicEnable=true

解开 canal 的数据挂载 , 然后

docker-compose down && docker-compose up-d

检查所有的日志文件是否报错

进入 mysql 检查 binlog 是否启动

docker exec -it mysql bash

SHOW VARIABLES LIKE 'log_bin';          -- 必须返回ON
SHOW VARIABLES LIKE 'binlog_format';    -- 必须返回ROW

设置权限

create user canal@'%' IDENTIFIED by 'canal';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT,SUPER ON *.* TO 'canal'@'%';
FLUSH PRIVILEGES;

校验结果

CREATE DATABASE test;
USE test;
CREATE TABLE user(id INT PRIMARY KEY, name VARCHAR(20));
INSERT INTO user VALUES(1, 'test');

至此 , 收工