Flink - DataStream Sink

[DataStream Sink](Overview | Apache Flink) 的不同 API 用于操作数据源写入的方式，可以基于控制台或者文件，也可以基于自定义的方式，实际应用中，基于自定义的方式时比较常用的。

基于控制台和文件

package com.learn.flink.sink;

import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

/**
 * sink - 基于控制台和文件
 */
public class SinkDemo1 {

    public static void main(String[] args) throws Exception {
        //0: env
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
        //1: source
        DataStream<String> ds = env.readTextFile("data/words1");

        //2: transformation

        //3: sink
        ds.print(); // 输出到控制台
        ds.printToErr();// 在控制台以红色输出
        ds.printToErr("输出标识放置前:");
        ds.writeAsText("data/output/result1").setParallelism(1); // 并行度为1 时， result 为文件名
        ds.writeAsText("data/output/result2").setParallelism(2); // 并行度大于1 时， result 为文件夹名

        //4: execute
        env.execute();
    }
}

自定义写入 mysql

package com.learn.flink.sink;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;

/**
 * sink - 自定义
 */
public class SinkDemo2 {

    public static void main(String[] args) throws Exception {
        //0: env
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
        //1: source
        final DataStream<Student> ds = env.fromElements(new Student("004", "wangwu", 20));

        //2: transformation

        //3: sink
        ds.addSink(new MySqlSink());
        //4: execute
        env.execute();
    }

    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    public static class Student {
        private String id;
        private String name;
        private Integer age;
    }

    public static class MySqlSink extends RichSinkFunction<Student> {
        private Connection conn = null;
        private PreparedStatement ps = null;

        /**
         * 适合用于开启资源
         * @param parameters
         * @throws Exception
         */
        @Override
        public void open(Configuration parameters) throws Exception {
            conn = DriverManager.getConnection("jdbc:mysql://node01:3306/bigdata", "root", "root");
            String sql = "INSERT INTO `t_student`(`id`, `name`, `age`) VALUES (?, ?, ?);";
            ps = conn.prepareStatement(sql);
        }

        @Override
        public void invoke(Student student, Context context) throws Exception {
            ps.setString(1, student.getId());
            ps.setString(2, student.getName());
            ps.setInt(3, student.getAge());
            ps.execute();
        }

        /**
         * 用于关闭资源
         * @throws Exception
         */
        @Override
        public void close() throws Exception {
            if (conn != null) conn.close();
            if (ps != null) ps.close();
        }
    }
}