Flink流水线+Gravitino+Paimon集成
1.数据源管理
1.1 添加Gravitino数据源
添加成功之后，会在Gravitino中创建一个名为配置的中的meatalake
1.2. 添加Paimon数据源
属性gravitinoId可以关联前面创建的Gravitino数据源，关联后，会在gravitino下创建一个该数据源的catalog。
2. 集成演示
2.1 创建任务
-
• 入口：通过顶部菜单栏选择 任务开发，或通过快捷入口 快速创建任务。
-
• 任务类型：选择
FlinkPipeline。
2.2 配置任务
点击任务名称，进入任务详情页。任务节点如下
功能说明：
使用DataGen节点生成100条测试数据，使用JDBCWrite将数据写入mysql的user表，再通过JDBCRead读取mysql的user表，使用SQLExecute写入到Paimon。通过SQLQuery读取Paimon,最后通过ShowData节点输入读取的数据。
{
"flow": {
"engineType": "flink",
"name": "flink_gravition_paimon",
"paths": [
{
"from": "Gravitino_tuky5",
"inport": "",
"outport": "",
"to": "DataGen_r6QRQ"
},
{
"from": "DataGen_r6QRQ",
"inport": "",
"outport": "",
"to": "JDBCWrite__yYGc"
},
{
"from": "JDBCWrite__yYGc",
"inport": "",
"outport": "",
"to": "JDBCRead_WREvh"
},
{
"from": "JDBCRead_WREvh",
"inport": "",
"outport": "",
"to": "SQLExecute_bMeVm"
},
{
"from": "SQLExecute_bMeVm",
"inport": "",
"outport": "",
"to": "SQLQuery_8jwT7"
},
{
"from": "SQLQuery_8jwT7",
"inport": "",
"outport": "",
"to": "ShowData_aMIL3"
}
],
"runMode": "DEBUG",
"stops": [
{
"bundle": "cn.piflow.bundle.flink.catalog.Gravitino",
"customizedProperties": {
},
"name": "Gravitino_tuky5",
"properties": {
"gravitinoUri": "http://127.0.0.1:8092/",
"metalake": "metalake"
},
"uuid": "Gravitino_tuky5"
},
{
"bundle": "cn.piflow.bundle.flink.common.DataGen",
"customizedProperties": {
},
"name": "DataGen_r6QRQ",
"properties": {
"count": "100",
"ratio": "10",
"registerTableName": "datagen_source",
"schema": "[{\"id\":\"323868\",\"filedName\":\"id\",\"filedType\":\"INT\",\"kind\":\"sequence\",\"start\":\"1\",\"end\":\"1000\"},{\"id\":\"479324\",\"filedName\":\"name\",\"filedType\":\"STRING\",\"kind\":\"random\",\"length\":\"5\",\"index\":1}]"
},
"uuid": "DataGen_r6QRQ"
},
{
"bundle": "cn.piflow.bundle.flink.jdbc.JDBCWrite",
"customizedProperties": {
},
"name": "JDBCWrite__yYGc",
"properties": {
"driver": "com.mysql.jdbc.Driver",
"password": "123456",
"properties": "{}",
"tableDefinition": "{\"tableBaseInfo\":{},\"physicalColumnDefinition\":[{\"columnName\":\"id\",\"columnType\":\"INT\",\"length\":null,\"precision\":null,\"scale\":null,\"nullable\":false,\"primaryKey\":false,\"partitionKey\":false,\"comment\":\"\"},{\"columnName\":\"name\",\"columnType\":\"STRING\",\"length\":null,\"precision\":null,\"scale\":null,\"nullable\":false,\"primaryKey\":false,\"partitionKey\":false,\"comment\":\"\"}],\"metadataColumnDefinition\":[],\"computedColumnDefinition\":[],\"watermarkDefinition\":{}}",
"tableName": "user",
"url": "jdbc:mysql://127.0.0.1:3306/test2?characterEncoding=utf8&autoReconnect=true&tinyInt1isBit=false&serverTimezone=Asia/Shanghai",
"username": "root"
},
"uuid": "JDBCWrite__yYGc"
},
{
"bundle": "cn.piflow.bundle.flink.jdbc.JDBCRead",
"customizedProperties": {
},
"name": "JDBCRead_WREvh",
"properties": {
"driver": "com.mysql.jdbc.Driver",
"fetchSize": "10",
"password": "123456",
"properties": "{}",
"tableDefinition": "{\"tableBaseInfo\":{\"catalogName\":\"\",\"dbname\":\"\",\"schema\":\"\",\"registerTableName\":\"t_user_source\",\"registerTableComment\":\"\",\"ifNotExists\":true,\"selectStatement\":\"\",\"likeStatement\":\"\"},\"physicalColumnDefinition\":[{\"columnName\":\"id\",\"columnType\":\"INT\",\"length\":null,\"precision\":null,\"scale\":null,\"nullable\":false,\"primaryKey\":false,\"partitionKey\":false,\"comment\":\"\"},{\"columnName\":\"name\",\"columnType\":\"STRING\",\"length\":null,\"precision\":null,\"scale\":null,\"nullable\":false,\"primaryKey\":false,\"partitionKey\":false,\"comment\":\"\"}],\"metadataColumnDefinition\":[],\"computedColumnDefinition\":[],\"watermarkDefinition\":{}}",
"tableName": "user",
"url": "jdbc:mysql://127.0.0.1:3306/test2?characterEncoding=utf8&autoReconnect=true&tinyInt1isBit=false&serverTimezone=Asia/Shanghai",
"username": "root",
"useTableEnv": "true"
},
"uuid": "JDBCRead_WREvh"
},
{
"bundle": "cn.piflow.bundle.flink.common.SQLQuery",
"customizedProperties": {
},
"name": "SQLQuery_8jwT7",
"properties": {
"registerResultViewName": "",
"registerSourceViewName": "",
"sql": "select id, name from paimon.test.t_user",
"useTableEnv": "true"
},
"uuid": "SQLQuery_8jwT7"
},
{
"bundle": "cn.piflow.bundle.flink.common.SQLExecute",
"customizedProperties": {
},
"name": "SQLExecute_bMeVm",
"properties": {
"sql": "create table if not exists paimon.test.t_user (\r\n id int,\r\n name string\r\n);\r\n\r\ninsert into paimon.test.t_user select * from t_user_source;",
"useTableEnv": "true"
},
"uuid": "SQLExecute_bMeVm"
},
{
"bundle": "cn.piflow.bundle.flink.common.ShowData",
"customizedProperties": {
},
"name": "ShowData_aMIL3",
"properties": {
"changeLog": "false",
"showNumber": "100"
},
"uuid": "ShowData_aMIL3"
}
],
"uuid": "111"
}
}
2.3 运行任务
-
• 点击 运行 按钮启动任务。
🔗 平台体验地址：DataStudio （http://1.94.182.15:8090）
