Docker安装canal

docker 搭建 canal

环境

本机Ip: 10.60.110.17

MySQL：127.0.0.1

Canal：127.0.0.1

GoLand：    本机

1、创建 MySQL 启动脚本

# 创建数据，配置目录
mkdir -p /docker/mysql/{data,conf.d}
 
# 创建配置文件
vim /docker/mysql/conf.d/my.cnf
 
[mysqld]
log_timestamps=SYSTEM
default-time-zone='+8:00'
log-bin=mysql-bin
server-id=3306
binlog_format=row

启动脚本

vim /docker/mysql/start.sh
 
docker run --name mysql\
-p 3306:3306 \
-v /docker/mysql/conf.d:/etc/mysql/conf.d \
-v /docker/mysql/data:/var/lib/mysql \
-e MYSQL_ROOT_PASSWORD=123456 \
-d mysql:5.7

查看mysql的binlog状态

可参考链接

> show variables like 'log_%';
> show variables like 'binlog_format';

2、创建数据库，表及相关数据[可自行百度]

创建用于 同步的账号

create user canal@'%' IDENTIFIED by 'canal';
grant SELECT, REPLICATION SLAVE, REPLICATION CLIENT on *.* to 'canal'@'%' identified by "canal";
 
flush privileges;

3、创建 canal 启动脚本

vim /docker/canal/start.sh
# 内容
docker run --name canal \
-e canal.instance.master.address=10.60.110.17:3306 \
-e canal.instance.dbUsername=canal \
-e canal.instance.dbPassword=canal \
-p 11111:11111 \
-d canal/canal-server:v1.1.4

4、进入 canal 容器

docker exec -it canal /bin/bash 
 
 # 成功
 2022-06-04 23:55:49.892 [main] INFO c.a.o.c.i.spring.support.PropertyPlaceholderConfigurer - Loading properties file from class path resource [canal.properties]
 2022-06-04 23:55:49.893 [main] INFO c.a.o.c.i.spring.support.PropertyPlaceholderConfigurer - Loading properties file from class path resource [example/instance.properties]
 2022-06-04 23:55:50.324 [main] INFO c.a.otter.canal.instance.spring.CanalInstanceWithSpring - start CannalInstance for 1-example
 2022-06-04 23:55:50.332 [main] WARN c.a.o.canal.parse.inbound.mysql.dbsync.LogEventConvert - --> init table filter : ^.*..*$
 2022-06-04 23:55:50.333 [main] WARN c.a.o.canal.parse.inbound.mysql.dbsync.LogEventConvert - --> init table black filter :
 2022-06-04 23:55:50.340 [main] INFO c.a.otter.canal.instance.core.AbstractCanalInstance - start successful....
 2022-06-04 23:55:51.269 [destination = example , address = /10.60.110.17:3306 , EventParser] WARN c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - --->
 find start position successfully, EntryPosition[included=false,journalName=mysql-bin.000003,position=1355,serverId=3306,gtid=,timestamp=1604192078000] cost : 840ms , the next step is binlog dump

5、golang代码

package main

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	"github.com/withlin/canal-go/client"
	pbe "github.com/withlin/canal-go/protocol/entry"
	"log"
	"os"
	"time"
)

func main() {

	// 10.60.110.17 替换成你的canal server的地址
	// example 替换成-e canal.destinations=example 你自己定义的名字
	connector := client.NewSimpleCanalConnector("10.60.110.17", 11111, "canal", "canal", "example", 60000, 60*60*1000)
	err := connector.Connect()
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	// https://github.com/alibaba/canal/wiki/AdminGuide
	//mysql 数据解析关注的表，Perl正则表达式.
	//
	//多个正则之间以逗号(,)分隔，转义符需要双斜杠(\\)
	//
	//常见例子：
	//
	//  1.  所有表：.*   or  .*\\..*
	//	2.  canal schema下所有表： canal\\..*
	//	3.  canal下的以canal打头的表：canal\\.canal.*
	//	4.  canal schema下的一张表：canal\\.test1
	//  5.  多个规则组合使用：canal\\..*,mysql.test1,mysql.test2 (逗号分隔)

	err = connector.Subscribe(".*\\..*")
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	for {

		message, err := connector.Get(1, nil, nil)
		if err != nil {
			log.Println(err)
			os.Exit(1)
		}
		batchId := message.Id
		if batchId == -1 || len(message.Entries) <= 0 {
			time.Sleep(300 * time.Millisecond)
			//fmt.Println("===没有数据了===")
			continue
		}

		printEntry(message.Entries)

	}
}

func printEntry(entrys []pbe.Entry) {

	for _, entry := range entrys {
		if entry.GetEntryType() == pbe.EntryType_TRANSACTIONBEGIN || entry.GetEntryType() == pbe.EntryType_TRANSACTIONEND {
			continue
		}
		rowChange := new(pbe.RowChange)

		err := proto.Unmarshal(entry.GetStoreValue(), rowChange)
		checkError(err)
		if rowChange != nil {
			eventType := rowChange.GetEventType()
			header := entry.GetHeader()
			fmt.Println(fmt.Sprintf("================> binlog[%s : %d],name[%s,%s], eventType: %s", header.GetLogfileName(), header.GetLogfileOffset(), header.GetSchemaName(), header.GetTableName(), header.GetEventType()))

			for _, rowData := range rowChange.GetRowDatas() {
				if eventType == pbe.EventType_DELETE {
					printColumn(rowData.GetBeforeColumns())
				} else if eventType == pbe.EventType_INSERT {
					printColumn(rowData.GetAfterColumns())
				} else {
					fmt.Println("-------> before")
					printColumn(rowData.GetBeforeColumns())
					fmt.Println("-------> after")
					printColumn(rowData.GetAfterColumns())
				}
			}
		}
	}
}

func printColumn(columns []*pbe.Column) {
	for _, col := range columns {
		fmt.Println(fmt.Sprintf("%s : %s  update= %t", col.GetName(), col.GetValue(), col.GetUpdated()))
	}
}

func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal error: %s", err.Error())
		os.Exit(1)
	}
}