20260106-seatunel同步自动监听ddl变更但是不监听create的ddl语句.只进行当前待同步表的结构变更操作.包含不同的字段类型的测试
source {
MySQL-CDC {
table-names = [
"amzn_data_prd.amzn_api_logs",
"amzn_data_prd.xxx"
]
# 需要字段演进 重要重要重要重要重要重要重要重要重要重要重要重要重要重要1
schema-changes.enabled = true
}
}
sink {
doris {
table = "${table_name}" # 多表写入
database = "amzn_data_prd" # 同库可固定；跨库用 ${database_name} 也行 :contentReference[oaicite:13]{index=13}
# 不允许自动 CREATE 重要重要重要重要重要2
schema_save_mode = ERROR_WHEN_SCHEMA_NOT_EXIST
}
}
schema-changes.enabled 需要特定的引擎支持, 目前不支持yarn平台的flink和spark, 只能用seatunnel本地启动或者 集群启动， 本质要基于 Zeta
schema_save_mode 过滤create的ddl语句，避免binlog监听到create语句去创建表，特别是一些兼容性不怎么好的olap数据平台可能要单独处理数据类型
下面做测试表如下
CREATE TABLE `tiny_id_token` (
`id` int unsigned NOT NULL AUTO_INCREMENT,
`token` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL,
`biz_type` varchar(63) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL,
`remark` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL,
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=37 DEFAULT CHARSET=utf8mb3 ROW_FORMAT=DYNAMIC;
doris的目标表如下
CREATE TABLE `tiny_id_token` (
`id` int NOT NULL,
`token` varchar(255) NOT NULL,
`biz_type` varchar(63) NOT NULL,
`remark` varchar(255) NULL,
`create_time` datetime(3) NULL,
`update_time` datetime(3) NULL,
`version` bigint NOT NULL DEFAULT "0"
) ENGINE=OLAP
UNIQUE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 16
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_medium" = "hdd",
"storage_format" = "V2",
"inverted_index_storage_format" = "V1",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"function_column.sequence_col" = "version",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728",
"enable_mow_light_delete" = "false"
);
在source mysql 端 执行ddl语句监控配置是否生效, 这里的mysql是8.0 ，所以会采用所有的8.0 的数据字段来做一个初步的实现
-- ==================== 数值类型 ====================
-- 1. 增加 TINYINT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_tinyint TINYINT NOT NULL DEFAULT 0 COMMENT '测试tinyint类型';
-- 2. 增加 SMALLINT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_smallint SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '测试smallint类型';
-- 3. 增加 MEDIUMINT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_mediumint MEDIUMINT NOT NULL DEFAULT 0 COMMENT '测试mediumint类型';
-- 4. 增加 BIGINT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_bigint BIGINT NOT NULL DEFAULT 0 COMMENT '测试bigint类型';
-- 5. 增加 DECIMAL 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_decimal DECIMAL(10,2) NOT NULL DEFAULT 0.00 COMMENT '测试decimal类型';
-- 6. 增加 FLOAT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_float FLOAT NOT NULL DEFAULT 0 COMMENT '测试float类型';
-- 7. 增加 DOUBLE 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_double DOUBLE NOT NULL DEFAULT 0 COMMENT '测试double类型';
-- 8. 增加 BIT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_bit BIT(8) NOT NULL DEFAULT b'0' COMMENT '测试bit类型';
-- ==================== 字符串类型 ====================
-- 9. 增加 CHAR 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_char CHAR(10) NOT NULL DEFAULT '' COMMENT '测试char类型';
-- 10. 增加 VARCHAR 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_varchar VARCHAR(100) NOT NULL DEFAULT '' COMMENT '测试varchar类型';
-- 11. 增加 TINYTEXT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_tinytext TINYTEXT COMMENT '测试tinytext类型';
-- 12. 增加 TEXT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_text TEXT COMMENT '测试text类型';
-- 13. 增加 MEDIUMTEXT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_mediumtext MEDIUMTEXT COMMENT '测试mediumtext类型';
-- 14. 增加 LONGTEXT 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_longtext LONGTEXT COMMENT '测试longtext类型';
-- 15. 增加 BINARY 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_binary BINARY(16) COMMENT '测试binary类型';
-- 16. 增加 VARBINARY 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_varbinary VARBINARY(100) COMMENT '测试varbinary类型';
-- 17. 增加 ENUM 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_enum ENUM('ACTIVE', 'INACTIVE', 'PENDING') NOT NULL DEFAULT 'PENDING' COMMENT '测试enum类型';
-- 18. 增加 SET 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_set SET('READ', 'WRITE', 'DELETE') NOT NULL DEFAULT '' COMMENT '测试set类型';
-- ==================== 日期时间类型 ====================
-- 19. 增加 DATE 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_date DATE COMMENT '测试date类型';
-- 20. 增加 TIME 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_time TIME COMMENT '测试time类型';
-- 21. 增加 DATETIME 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_datetime DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '测试datetime类型';
-- 22. 增加 YEAR 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_year YEAR NOT NULL DEFAULT YEAR(CURRENT_DATE) COMMENT '测试year类型';
-- ==================== JSON & 空间类型 ====================
-- 23. 增加 JSON 类型字段 (MySQL 5.7+)
ALTER TABLE tiny_id_token ADD COLUMN test_json JSON COMMENT '测试json类型';
-- 24. 增加 GEOMETRY 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_geometry GEOMETRY COMMENT '测试geometry类型';
-- ==================== BLOB 类型 ====================
-- 25. 增加 TINYBLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_tinyblob TINYBLOB COMMENT '测试tinyblob类型';
-- 26. 增加 BLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_blob BLOB COMMENT '测试blob类型';
-- 27. 增加 MEDIUMBLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_mediumblob MEDIUMBLOB COMMENT '测试mediumblob类型';
-- 28. 增加 LONGBLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_longblob LONGBLOB COMMENT '测试longblob类型';
-- ==================== 字段删除测试 ====================
-- 删除刚才添加的测试字段（按类型顺序逆序删除）
-- 删除 LONGBLOB 类型字段
-- 删除 MEDIUMBLOB 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_mediumblob;
-- 删除 BLOB 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_blob;
-- 删除 TINYBLOB 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_tinyblob;
-- 删除 GEOMETRY 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_geometry;
-- 删除 JSON 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_json;
-- 删除 YEAR 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_year;
-- 删除 DATETIME 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_datetime;
-- 删除 TIME 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_time;
-- 删除 DATE 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_date;
-- 删除 SET 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_set;
-- 删除 ENUM 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_enum;
-- 删除 VARBINARY 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_varbinary;
-- 删除 BINARY 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_binary;
-- 删除 LONGTEXT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_longtext;
-- 删除 MEDIUMTEXT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_mediumtext;
-- 删除 TEXT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_text;
-- 删除 TINYTEXT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_tinytext;
-- 删除 VARCHAR 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_varchar;
-- 删除 CHAR 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_char;
-- 删除 BIT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_bit;
-- 删除 DOUBLE 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_double;
-- 删除 FLOAT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_float;
-- 删除 DECIMAL 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_decimal;
-- 删除 BIGINT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_bigint;
-- 删除 MEDIUMINT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_mediumint;
-- 删除 SMALLINT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_smallint;
-- 删除 TINYINT 类型字段
ALTER TABLE tiny_id_token DROP COLUMN test_tinyint;
每次执行之后，在doris端查看表结构是否出现的变化, ddl是否同步了相关字段上去了
下面是兼容性问题导致的错误的字段类型不能成功同步到doris的异常, 出现此类异常建议重跑
其中对于mysql的类信息中不能好好同步到doris的mysql类型有如下几种
mysql不支持
-- 17. 增加 ENUM 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_enum ENUM('ACTIVE', 'INACTIVE', 'PENDING') NOT NULL DEFAULT 'PENDING' COMMENT '测试enum类型';
-- 18. 增加 SET 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_set SET('READ', 'WRITE', 'DELETE') NOT NULL DEFAULT '' COMMENT '测试set类型';
转换会报错
set
2026-01-06 12:17:40,012 WARN [i.d.c.m.MySqlValueConverters ] [blc-rm-wz9725a798j161q43to.mysql.rds.aliyuncs.com:3306] - Using UTF-8 charset by default for column without charset: test_longtext LONGTEXT CHARSET utf8mb3 DEFAULT VALUE NULL
2026-01-06 12:17:40,598 WARN [elRowDebeziumDeserializeSchema] [BlockingWorker-TaskGroupLocation{jobId=1060768861592223745, pipelineId=1, taskGroupId=2}] - Failed to resolve schemaChangeEvent, just skip.
org.apache.seatunnel.common.exception.SeaTunnelRuntimeException: ErrorCode:[COMMON-17], ErrorDescription:['MySQL' unsupported convert type 'SET' of 'test_set' to SeaTunnel data type.]
at org.apache.seatunnel.common.exception.CommonError.convertToSeaTunnelTypeError(CommonError.java:137) ~[seatunnel-starter.jar:2.3.11]
at org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.mysql.MySqlTypeConverter.convert(MySqlTypeConverter.java:315) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.seatunnel.cdc.mysql.utils.MySqlTypeUtils.convertToSeaTunnelColumn(MySqlTypeUtils.java:139) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.seatunnel.cdc.mysql.source.parser.CustomAlterTableParserListener.toSeatunnelColumn(CustomAlterTableParserListener.java:224) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.base.source.parser.SeatunnelDDLParser.toSeatunnelColumnWithFullTypeInfo(SeatunnelDDLParser.java:35) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.seatunnel.cdc.mysql.source.parser.CustomAlterTableParserListener.lambda$exitAlterByAddColumn$0(CustomAlterTableParserListener.java:100) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser.runIfNotNull(MySqlAntlrDdlParser.java:358) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.seatunnel.cdc.mysql.source.parser.CustomAlterTableParserListener.exitAlterByAddColumn(CustomAlterTableParserListener.java:96) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at io.debezium.ddl.parser.mysql.generated.MySqlParser$AlterByAddColumnContext.exitRule(MySqlParser.java:15459) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at io.debezium.antlr.ProxyParseTreeListenerUtil.delegateExitRule(ProxyParseTreeListenerUtil.java:64) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.seatunnel.cdc.mysql.source.parser.CustomMySqlAntlrDdlParserListener.exitEveryRule(CustomMySqlAntlrDdlParserListener.java:102) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.antlr.v4.runtime.tree.ParseTreeWalker.exitRule(ParseTreeWalker.java:48) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.antlr.v4.runtime.tree.ParseTreeWalker.walk(ParseTreeWalker.java:30) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.antlr.v4.runtime.tree.ParseTreeWalker.walk(ParseTreeWalker.java:28) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.antlr.v4.runtime.tree.ParseTreeWalker.walk(ParseTreeWalker.java:28) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.antlr.v4.runtime.tree.ParseTreeWalker.walk(ParseTreeWalker.java:28) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.antlr.v4.runtime.tree.ParseTreeWalker.walk(ParseTreeWalker.java:28) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.antlr.v4.runtime.tree.ParseTreeWalker.walk(ParseTreeWalker.java:28) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at io.debezium.antlr.AntlrDdlParser.parse(AntlrDdlParser.java:87) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.base.schema.AbstractSchemaChangeResolver.resolve(AbstractSchemaChangeResolver.java:89) ~[connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.debezium.row.SeaTunnelRowDebeziumDeserializeSchema.deserializeSchemaChangeRecord(SeaTunnelRowDebeziumDeserializeSchema.java:129) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.debezium.row.SeaTunnelRowDebeziumDeserializeSchema.deserialize(SeaTunnelRowDebeziumDeserializeSchema.java:112) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.base.source.reader.IncrementalSourceRecordEmitter.emitElement(IncrementalSourceRecordEmitter.java:201) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.base.source.reader.IncrementalSourceRecordEmitter.processElement(IncrementalSourceRecordEmitter.java:147) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.base.source.reader.IncrementalSourceRecordEmitter.emitRecord(IncrementalSourceRecordEmitter.java:102) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.base.source.reader.IncrementalSourceRecordEmitter.emitRecord(IncrementalSourceRecordEmitter.java:62) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.seatunnel.common.source.reader.SourceReaderBase.pollNext(SourceReaderBase.java:110) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.connectors.cdc.base.source.reader.IncrementalSourceReader.pollNext(IncrementalSourceReader.java:119) [connector-cdc-mysql-2.3.11.jar:2.3.11]
at org.apache.seatunnel.engine.server.task.flow.SourceFlowLifeCycle.collect(SourceFlowLifeCycle.java:159) [seatunnel-starter.jar:2.3.11]
at org.apache.seatunnel.engine.server.task.SourceSeaTunnelTask.collect(SourceSeaTunnelTask.java:127) [seatunnel-starter.jar:2.3.11]
at org.apache.seatunnel.engine.server.task.SeaTunnelTask.stateProcess(SeaTunnelTask.java:165) [seatunnel-starter.jar:2.3.11]
at org.apache.seatunnel.engine.server.task.SourceSeaTunnelTask.call(SourceSeaTunnelTask.java:132) [seatunnel-starter.jar:2.3.11]
at org.apache.seatunnel.engine.server.TaskExecutionService$BlockingWorker.run(TaskExecutionService.java:694) [seatunnel-starter.jar:2.3.11]
at org.apache.seatunnel.engine.server.TaskExecutionService$NamedTaskWrapper.run(TaskExecutionService.java:1023) [seatunnel-starter.jar:2.3.11]
at org.apache.seatunnel.api.tracing.MDCRunnable.run(MDCRunnable.java:43) [seatunnel-starter.jar:2.3.11]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_462]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_462]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_462]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_462]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_462]
特殊的转换后类型
-- 20. 增加 TIME 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_time TIME COMMENT '测试time类型';
seatunnel中的规则，对于这种ddl语句，在doris中会转成test_geometry text NULL 来实现兼容
-- 24. 增加 GEOMETRY 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_geometry GEOMETRY COMMENT '测试geometry类型';
seatunnel中的规则，对于这种ddl语句，在doris中会转成 test_time varchar(8) NULL 来实现兼容
-- 25. 增加 TINYBLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_tinyblob TINYBLOB COMMENT '测试tinyblob类型';
seatunnel中的规则，对于这种ddl语句，在doris中会转成 test_tinyblob text NULL, 来实现兼容
-- 26. 增加 BLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_blob BLOB COMMENT '测试blob类型';
seatunnel中的规则，对于这种ddl语句，在doris中会转成 test_blob text NULL 来实现兼容
-- 27. 增加 MEDIUMBLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_mediumblob MEDIUMBLOB COMMENT '测试mediumblob类型';
seatunnel中的规则，对于这种ddl语句，在doris中会转成 test_mediumblob text NULL, 来实现兼容
-- 28. 增加 LONGBLOB 类型字段
ALTER TABLE tiny_id_token ADD COLUMN test_longblob LONGBLOB COMMENT '测试longblob类型';
seatunnel中的规则，对于这种ddl语句，在doris中会转成 test_longblob text NULL 来实现兼容
默认值的问题, doris不支持对应的默认值
会出现异常
2026-01-06 12:21:48,211 WARN [.a.s.c.d.s.SchemaChangeManager] [BlockingWorker-TaskGroupLocation{jobId=1060768861592223745, pipelineId=1, taskGroupId=2}] - Failed to alter table add column, SQL:ALTER TABLE amzn_data_prd.tiny_id_token ADD COLUMN `test_datetime` DATETIME(0) DEFAULT 'CURRENT_TIMESTAMP'
```.preview-wrapper pre::before { position: absolute; top: 0; right: 0; color: #ccc; text-align: center; font-size: 0.8em; padding: 5px 10px 0; line-height: 15px; height: 15px; font-weight: 600; } .hljs.code\_\_pre > .mac-sign { display: flex; } .code\_\_pre { padding: 0 !important; } .hljs.code\_\_pre code { display: -webkit-box; padding: 0.5em 1em 1em; overflow-x: auto; text-indent: 0; } h2 strong { color: inherit !important; }
> 本文使用 [文章同步助手](https://juejin.cn/post/6940875049587097631) 同步
