Flink+SpringBoot+Vue电商数据实时可视化

Flink实时数据可视化

效果如下

事件13：00分，上海数据：8226 事件13：01分，上海数据：8394

利用Flume采集数据到kafka，再使用Flink消费Kafka的数据至mysql

Flume采集部分

MaxWell数据同步

Flink部分

package com.hito.demo

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.flink.util.Collector
import org.apache.kafka.clients.consumer.ConsumerConfig

import java.sql.{Connection, DriverManager, PreparedStatement}
import java.util.Properties

object statistics_ZJH {
  def main(args: Array[String]): Unit = {
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)

    val kafka_properties = new Properties()
    kafka_properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer")
    kafka_properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer")
    kafka_properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "bigdata1:9092")
    kafka_properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")

    val stream: DataStream[String] = env.addSource(new FlinkKafkaConsumer[String]("order", new SimpleStringSchema(), kafka_properties))

    val main_data: DataStream[Map[String, String]] = stream.filter(_.contains("order_master"))
      .map(
        data => {
          val main_data: String = data.split(","data":")(1).split('{')(1).split('}')(0)
          val arrData: Map[String, String] = main_data.split(",")
            .map(
              data => {
                val keyValue: Array[String] = data.split(":")
                val key: String = keyValue(0).replaceAll(""", "")
                val value: String = keyValue.drop(1).mkString(":").replaceAll(""", "")
                key -> value
              }
            ).toMap
          arrData
        }
      )

    var shanghai = 0
    var zhejiang = 0
    var jiangsu = 0
    val ZJHStream: DataStream[(String, Int)] = main_data.process(new ProcessFunction[Map[String, String], (String, Int)]() {
      override def processElement(i: Map[String, String], context: ProcessFunction[Map[String, String], (String, Int)]#Context, collector: Collector[(String, Int)]) = {
        if (i("city").contains("上海")) {
          shanghai += 1
          collector.collect(("上海", shanghai))
        } else if (i("city").contains("江苏")) {
          jiangsu += 1
          collector.collect(("江苏", jiangsu))
        } else if (i("city").contains("浙江")) {
          zhejiang += 1
          collector.collect(("浙江", zhejiang))
        }
      }
    })

    // 将处理后的数据写入到MySQL中
    ZJHStream.addSink(new MysqlSink)

    env.execute("统计江浙沪的订单占比")
  }

  class MysqlSink extends RichSinkFunction[(String, Int)] {
    var conn: Connection = _
    var inserState: PreparedStatement = _

    override def open(parameters: Configuration): Unit = {
      super.open(parameters)
      Class.forName("com.mysql.jdbc.Driver")
      conn = DriverManager.getConnection("jdbc:mysql://192.168.23.60:3306/ht?characterEncoding=utf-8&useSSL=false", "root", "123456")
      inserState = conn.prepareStatement("INSERT INTO store_JZH(region, result) VALUES (?, ?) ON DUPLICATE KEY UPDATE result = ?")
    }

    override def invoke(value: (String, Int), context: SinkFunction.Context): Unit = {
      inserState.setString(1, value._1)
      inserState.setInt(2, value._2)
      inserState.setInt(3, value._2)
      inserState.executeUpdate()
    }

    override def close(): Unit = {
      inserState.close()
      conn.close()
    }
  }
}

SpringBoot部分(Mybatis-Plus)

实体类

@Data
@AllArgsConstructor
@NoArgsConstructor
@TableName("store_JZH")
public class StoreJzh  {

    @TableId
    private String region;

    private Integer result;

}

mapper层

@Repository
public interface JZHmapper extends BaseMapper<StoreJzh> {

}

service层

public interface StoreJzhService extends IService<StoreJzh> {

}

impl层

@Service("storeJzhService")
public class StoreJzhServiceImpl extends ServiceImpl<JZHmapper, StoreJzh> implements StoreJzhService {
    @Autowired
    JZHmapper jzHmapper;
}

控制层

@Autowired
private StoreJzhService storeJzhService;

@RequestMapping("/jzh_all")
public List<StoreJzh> getJzhCount(){
    return storeJzhService.list();
}

Vue3 部分

<template>
  <div class="container">
      <div id="bar" class="box"></div>
      <div id="pie" class="box"></div>
  </div>
</template>

<script>
import * as echarts from "echarts";
import axios from "axios";
import { ref, onMounted } from 'vue';

export default {
  name: 'jzh',
  setup() {
      const data1 = ref(null);

      const fetchData = async () => {
          try {
              const response = await axios.get("http://localhost:8080/store/jzh_all");
              data1.value = response.data;
              console.log("Fetched data", data1.value);
              // 确保数据获取后再绘制图表
              BarChart();
              PieChart();
          } catch (error) {
              console.log("Has Error", error);
          }
      }

      const PieChart = () => {
          if (!data1.value) {
              return;
          }
          // 判断饼图容器是否存在
          const pieContainer = document.getElementById('pie');
          if (!pieContainer) {
              console.error('Pie chart Container not found');
              return;
          } else {
              console.log("Started to Practice Pie Chart");
          }

          const Pie = echarts.init(pieContainer);

          const opt = {
              title: {
                  text: '江浙沪购买比例',
                  left: 'center'
              },
              tooltip: {
                  trigger: 'item'
              },
              legend: {
                  orient: 'vertical',
                  left: 'left'
              },
              series: [
                  {
                      name: '江浙沪 Data',
                      type: 'pie',
                      radius: '50%',
                      data: data1.value.map(item => ({ value: parseInt(item.result), name: item.region })),
                      emphasis: {
                          itemStyle: {
                              shadowBlur: 10,
                              shadowOffsetX: 0,
                              shadowColor: 'rgba(0,0,0,0.5)'
                          }
                      }
                  }
              ]
          };

          Pie.setOption(opt);
      }

      const BarChart = () => {
          if (!data1.value) {
              return;
          }
          const barContainer = document.getElementById('bar');
          if (!barContainer) {
              console.error("Bar chart Container not found");
              return;
          } else {
              console.log('Started to Practice Bar Chart');
          }

          const Bar = echarts.init(barContainer);

          const opt = {
              title: {
                  text: '江浙沪购买量',
                  left: 'center'
              },
              tooltip: {
                  trigger: 'item'
              },
              xAxis: {
                  type: 'category',
                  data: data1.value.map(data => data.region)
              },
              yAxis: {
                  type: 'value'
              },
              series: [
                  {
                      name: 'JZH Count',
                      type: 'bar',
                      data: data1.value.map(data => parseInt(data.result))
                  }
              ]
          };

          Bar.setOption(opt);
      }

      onMounted(() => {
          fetchData();
      });

      return {
          data1
      }
  }
}
</script>

<style>
.box {
  border: black 5px solid;
  margin: 5px;
  height: 400px; 
  width: 600px; 
}
.container {
  display: flex;
  flex-wrap: wrap;
}
</style>

前端部分应该再写个定时器实时刷新，懒得写了