1、启动kafka和flink以及hadoop
为了方便操作，mac可以下载Tabby软件
1.1 启动kafka：进入kafka目录
bin/kafka-server-start.sh config/server.properties
1.2 创建kafka
进入bin目录
./kafka-topics.sh —create —bootstrap-server qf01:9092 —replication-factor 1 —partition 1 —topic test
1.3 启动生产者和消费者
生产者：./kafka-console-producer.sh --broker-list qf01:9092 --topic test
消费者：./kafka-console-consumer.sh --bootstrap-server qf01:9092 --topic test --from-beginning
1.4 启动hadoop集群
start-all.sh
1.5 启动zk集群
zkServer.sh start
1.6 启动flink
start-cluster.sh
2、代码实现
2.1 创建生产者的代码
JProducer.class
@Configuration
@Slf4j
public class JProducer extends Thread {
public static final String broker_list = "ip(换成自己的):9092";
public static final String topic = "test";
public static void main(String[] args) {
JProducer jproducer = new JProducer();
jproducer.start();
}
@Override
public void run() {
try {
producer();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
/**
* 向kafka批量生成记录
*/
@Scheduled(initialDelayString="${kf.flink.init}",fixedDelayString = "${kf.flink.fixRate}")
private void producer() throws InterruptedException {
log.info("启动定时任务");
Properties props = config();//kafka连接
Producer<String, String> producer = new KafkaProducer<>(props);
Date date = new Date();
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String dateString = simpleDateFormat.format(date);
while (true) {
for (int i = 1; i <= Integer.MAX_VALUE; i++) {
String json = "{\"id\":" + i + ",\"ip\":\"192.168.0." + i + "\",\"date\":" + dateString + "}";
String k = "第" + i + "条数据=" + json;
sleep(300);
if (i % 10 == 0) {
sleep(1000);
}
producer.send(new ProducerRecord<String, String>(topic, k, json));
}
producer.close();
}
}
/**
* kafka连接
* @return
*/
private Properties config() {
Properties props = new Properties();
props.put("bootstrap.servers",broker_list);
props.put("acks", "1");
props.put("retries", 0);
props.put("batch.size", 16384);
props.put("linger.ms", 1);
props.put("buffer.memory", 33554432);
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
return props;
}
}
2.2 创建flink程序
Flink.class
public class Flink {
private static final String topic = "test";
public static final String broker_list = "ip(换成自己的):9092";
public static void main(String[] args) {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(1000);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
//读取Kafka数据，主题topic:
DataStream<String> transction = env.addSource(new FlinkKafkaConsumer<String>(topic, new SimpleStringSchema(), props()).setStartFromEarliest());
transction.rebalance().map(new MapFunction<String, Object>() {
private static final long serialVersionUID = 1L;
@Override
public String map(String value) {
System.out.println("ok了");
return value;
}
}).print();
try {
env.execute();
} catch (Exception ex) {
ex.printStackTrace();
}
}
public static Properties props() {
Properties props = new Properties();
props.put("bootstrap.servers", broker_list);
props.put("zookeeper.connect", "192.168.47.130:2182");
props.put("group.id", "kv\_flink");
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
return props;
}
}
**网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。**
**[需要这份系统化资料的朋友，可以戳这里获取](https://gitee.com/vip204888)**
**一个人可以走的很快，但一群人才能走的更远！不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人，都欢迎加入我们的的圈子（技术交流、学习资源、职场吐槽、大厂内推、面试辅导），让我们一起学习成长！**
