前端监控系统（pv/uv）可适用于企业级项目（已开源）

一、任务目标

1、统计系统的PV 2、统计统计系统的uv 3、 使用可视化页面展示统计量

二、开发接口

1、开发浏览量上报记录接口 2、开发数据统计接口 3、开发删除浏览量接口

三、技术点介绍：

1、搭建后端egg框架,使用orm框架访问数据库 2、使用vue开发前端展示可视化页面 3、数据存储使用mysql存储

四：数据表介绍

1、数据库表总共使用了3张表，分别是浏览历史记录表，访问用户表，系统表

五、后端架构，文件目录展示

|__ .gitignore
  |__ app
    |__ contract
      |__ browsingData.js
      |__ format.js      
      |__ index.js       
      |__ login.js       
      |__ responst.js    
      |__ user.js        
    |__ controller       
      |__ browsingData.js
      |__ home.js        
      |__ login.js       
      |__ swaggerdoc.md  
      |__ user.js        
    |__ extend
      |__ agent.js       
      |__ application.js 
      |__ context.js     
      |__ helper.js      
      |__ request.js     
      |__ response.js    
    |__ io
      |__ controller     
        |__ home.js      
      |__ middleware     
        |__ connection.js
        |__ packet.js        
    |__ middleware
      |__ jwt.js
    |__ model
      |__ authority.js       
      |__ browsing_history.js
      |__ data_dictionary.js 
      |__ data_type.js       
      |__ role.js
      |__ role_authority.js  
      |__ role_user.js       
      |__ user.js
      |__ visitor.js
    |__ public
    |__ router
      |__ browsingData.js
      |__ home.js
      |__ login.js
      |__ user.js
    |__ router.js
    |__ schedule
      |__ update_cache.js
    |__ service
      |__ browsingData.js
      |__ login.js
      |__ user.js
  |__ config
    |__ config.default.js
    |__ config.prod.js
    |__ plugin.js
  |__ index.js
  |__ package-lock.json
  |__ package.json
  |__ README.md
  |__ yarn.lock
  |__ 开发文档记录.md

解释此项目内置登陆模块，用户模块，鉴权模块

六、路由代码

路由代码存放在app/router/browsingData

module.exports = app => {
  const { router, controller} = app
  router.post('/create', controller.browsingData.create);  //这是上报数据的接口
  router.get('/getData', controller.browsingData.get);  // 这是数据大屏展示的接口
  router.get('/delData', controller.browsingData.delete);  // 这是定时上报数据的接口
};

七、控制器代码

控制器主要是接受路由传递参数，处理参数的地方 控制器存放位置：app/controller/browsingData 核心代码

'use strict';


const Controller = require('egg').Controller;

/**
* @controller 用户管理
*/
class BrowsingData extends Controller {
  

    /** 
    * @summary 数据上报
    * @description 
    * @router post /create
    * @consumes applicatoin/json
    * @Request body browsingDataRequest *create
    * @response 200 baseResponse ok
    */  

     async create () {
        const { ctx } = this;
        // 参数校验
        ctx.validate(ctx.rule.browsingDataRequest);
        // 处理参数
        const payload = ctx.request.body || {};
        const res = await ctx.service.browsingData.create(payload);
        ctx.helper.success({ ctx, res });
    }

    /** 
    * @summary 数据统计展示
    * @description 数据统计展示
    * @router get /getData
    * @response 200 baseResponse ok
    */
    async get () {
        const { ctx } = this;
        // 处理参数
        const systemId = ctx.query.systemId
        const res = await ctx.service.browsingData.get();
        let data = null
        if(systemId) {
            data = res.filter( (res) => {
              return res.data_code === systemId
            })
        }else {
            data = res
        }
        ctx.helper.success({ ctx, res: data });
    }

   /** 
    * @summary 根据时间删除数据
    * @description 根据时间删除数据
    * @router get /delData
    * @response 200 baseResponse ok
    */
  async delete () {
    const { ctx } = this;
        // 处理参数
        const res = await ctx.service.browsingData.delete();
        ctx.helper.success({ ctx, res });
    }

    
}

module.exports = BrowsingData;

八、服务代码

这部分代码比较核心主要是跟数据库打交道的，用来处理返回的数据 控制器存放位置：app/controller/service

'use strict';

const Service = require('egg').Service;
const{ Op,QueryTypes } = require("sequelize");
const sequelize = require("sequelize");
const moment = require('moment');

function formatDate(date) {  
    var y = date.getFullYear();  
    var m = date.getMonth() + 1;  
    m = m < 10 ? '0' + m : m;  
    var d = date.getDate();  
    d = d < 10 ? ('0' + d) : d;  
    return y + '-' + m + '-' + d;  
}

class BrowsingData extends Service {
    async create(params) {
        const ctx = this.ctx;
        // 查看访客表有则不添加没有则添加
    
        const { uuid, systemId } = params
        await ctx.model.Visitor.findOrCreate({
            where: { 
                uuid,
                system_code: systemId
            },
        })
        return await ctx.model.BrowsingHistory.create(params);
    }

    // 获取系统code
    async getSystemCode() {
        const ctx = this.ctx;
        const params = {
            attributes:['data_code','identification'],
            where: {
                data_type:'system'
            }
        }
        return await ctx.model.DataDictionary.findAll(params);
    }

    
    // 删除三天前的数据
    async delete(){
        const ctx = this.ctx;
        const params = {
            where: {  
                create_time: {
                    [Op.lt]: moment().subtract(3, "days").format("YYYY-MM-DD 00:00:00"), //最大时间
                }
            }
        }
        return await ctx.model.BrowsingHistory.destroy(params);
    }
    // 统计每小时产生的数据
     async byHour(date,systemId) {
            const params = String(date)
            return await this.app.model.query(`SELECT HOUR(create_time) as hour,count(*) as Count 
            FROM browsing_history e 
            WHERE DATE_FORMAT(create_time,'%Y-%m-%d') = '${params}' AND systemId = '${systemId}'
            GROUP BY HOUR(create_time)`,{ type: QueryTypes.SELECT });
      }

    //     SELECT HOUR(create_time) as hour,count(*) as Count 
    // FROM browsing_history e 
    // WHERE DATE_FORMAT(create_time,'%Y-%m-%d') = '2021-08-02' 
    // GROUP BY HOUR(create_time)
    async get() {
        const ctx = this.ctx;
        const d =  new Date()
        const Year = d.getFullYear()
        const Month = d.getMonth()
        const date = d.getDate()
        var yeaterday = new Date(date-24*60*60*1000)
        var yeaterdayDate = formatDate(yeaterday) 
        const preDate = d.getDate() - 1 //昨天
        const nowDate = formatDate(d) 
        const dateTime = new Date( Year, Month, date , 0 , 0, 0)
        const preDateTime = new Date( Year, Month, preDate , 0 , 0, 0)
    
        // 查询今日访问次数
        const veryDayParams = {
            where: {
                create_time:{
                    [Op.lt]: new Date(), //最大时间
                    [Op.gt]: dateTime, //最小时间
                },
            }
        }
        // 昨日访问次数
        const preDayParams = {
            where: {
                create_time:{
                    [Op.lt]: dateTime, //最大时间
                    [Op.gt]: preDateTime, //最小时间
                },
            }
        }

        const systemCode =  await this.getSystemCode()
        
        for (let i = 0; i < systemCode.length; i++) {
            let VisitsTodayNumber = 0 // 今日浏览量
            let VisitsYesterdayNumber = 0 // 昨日浏览量
            let VisitorsTodayNumberTotal = 0 // 今日访客
            let VisitorsYesterdayNumberTotal = 0 // 昨日访客

            const item = systemCode[i]
            veryDayParams.where.systemId = item.data_code
            preDayParams.where.systemId = item.data_code
            // 查询今日浏览量
            VisitsTodayNumber = await ctx.model.BrowsingHistory.count(veryDayParams);
            // 查询昨日浏览量
            VisitsYesterdayNumber = await ctx.model.BrowsingHistory.count(preDayParams);
            // 查询今日访客
            const resToday  = await this.app.model.query(`
            SELECT DISTINCT uuid FROM browsing_history WHERE 
            (browsing_history.create_time < '${moment().format("YYYY-MM-DD HH:mm:ss")}' AND browsing_history.create_time > '${moment().startOf('day').format('YYYY-MM-DD HH:mm:ss')}') AND browsing_history.systemId = '${item.data_code}'`)
            VisitorsTodayNumberTotal = resToday[0].length

            // 查询昨日访客
            const resYesterda  = await this.app.model.query(`
            SELECT DISTINCT uuid FROM browsing_history WHERE 
            (browsing_history.create_time < '${moment().startOf('day').format('YYYY-MM-DD HH:mm:ss')}' AND browsing_history.create_time > '${moment().subtract(1, "days").format("YYYY-MM-DD 00:00:00")}') AND browsing_history.systemId = '${item.data_code}'`)
            VisitorsYesterdayNumberTotal = resYesterda[0].length


            // 获取今天每小时产生的数据量
            let nowByHourData =  await this.byHour(nowDate, item.data_code)
            const xAxis = []
            const series = []
            nowByHourData.forEach(res => {
              xAxis.push(res.hour + '时')
              series.push(res.Count)
            })
            nowByHourData = {}
            nowByHourData.xAxis = xAxis
            nowByHourData.series = series
            
            const yeaByHourData =  await this.byHour(yeaterdayDate)
            // yeaterdayDate
           
            // 查询当前系统用户量
           const userTotal = await ctx.model.Visitor.count(
                {
                    where:{
                        system_code: item.data_code
                    }
                }
            )

             // 查询新增用户
           const newUser = await ctx.model.Visitor.count(
                {
                    where:{
                        created_time:{
                            [Op.lt]: new Date(), //最大时间
                            [Op.gt]: dateTime, //最小时间
                        },
                        system_code: item.data_code
                    }
                }
            )
            
            systemCode[i].dataValues.VisitsTodayNumber = VisitsTodayNumber
            systemCode[i].dataValues.VisitsYesterdayNumber = VisitsYesterdayNumber
            systemCode[i].dataValues.VisitorsTodayNumberTotal = VisitorsTodayNumberTotal
            systemCode[i].dataValues.VisitorsYesterdayNumberTotal = VisitorsYesterdayNumberTotal
            systemCode[i].dataValues.nowByHourData = nowByHourData
            systemCode[i].dataValues.yeaByHourData = yeaByHourData
            systemCode[i].dataValues.userTotal = userTotal
            systemCode[i].dataValues.newUser = newUser
        }
            return systemCode
        
    }
}

module.exports = BrowsingData;

九、定时删除无用的数据、历史数据

此部分代码存在于 app/schedule/update_cache ,主要用来删除历史浏览记录

const Subscription = require('egg').Subscription;

module.exports = {
  schedule: {
    cron: '0 0 */8 * * *',
    type: 'all', // 指定所有的 worker 都需要执行
  },
  async task(ctx) {
    const res = await ctx.curl('http://localhost:7001/delData', {
      dataType: 'json',
    });
    console.log("定时任务",res)
  },
};

十、前端核心代码

前端使用的vue框架展示 此处只展示前端核心代码

<template>
  <div class="body">
    <header id="header">
      <h3 class="header-title">大屏数据可视化PV/VU</h3>
    </header>
    <!-- <div class="loader" /> -->
    <div id="container">
      <div id="flexCon">
        <div class="flex-row">
          <div v-for="item in list" :key="item.data_code" class="flex-cell">
            <div class="chart-wrapper">
              <h3 class="chart-title">{{ item.identification }}</h3>
              <div class="chart-div">
                <div class="data">
                  <div>
                    <div class="total"> 昨日访客:  <span>  {{ item.VisitorsYesterdayNumberTotal }}</span> | 昨日访问量:<span>  {{ item.VisitsYesterdayNumber }}</span> </div>
                    <div class="total"> 今日访客: <span> {{ item.VisitorsTodayNumberTotal }}</span> | 今日访问量:<span> {{ item.VisitsTodayNumber }}</span> </div>
                  </div>
                  <div>
                    <div class="total"> 总用户: <span> {{ item.userTotal }}</span> </div>
                    <div class="total"> 新用户: <span> {{ item.newUser }}</span> </div>
                  </div>
                </div>
                <div class="chart-loader">
                  <div class="my-echart" style="width: 100%;height:100%;" />
                </div>
              </div>
            </div>
          </div>
        </div>

      </div>
    </div>
  </div>
</template>

<script>
import './index.css'
import * as echarts from 'echarts'
import axios from 'axios'
export default {
  name: 'DataMonitoring',
  data() {
    return {
      list: null
    }
  },

  mounted() {
    this.getData()
    setInterval(() => {
      this.getData()
    }, 6000)
  },

  methods: {
    getData() {
      axios({
        url: 'http://192.168.2.202:7001/getData',
        methods: 'GET'
      }).then(res => {
        const list = res.data.data
        this.list = list
        this.drawChart(list)
      })
    },
    drawChart(data) {
      // 基于准备好的dom，初始化echarts实例
      this.$nextTick(() => {
        var anyEchart = document.querySelectorAll('.my-echart')
        for (var i = 0; i < anyEchart.length; i++) {
          const { xAxis, series } = data[i].nowByHourData
          var myChart = echarts.init(anyEchart[i])
          // 绘制图表
          myChart.setOption({
            tooltip: {
              trigger: 'axis',
              axisPointer: {
                // 坐标轴指示器，坐标轴触发有效
                type: 'shadow' // 默认为直线，可选为：'line' | 'shadow'
              }
            },
            grid: {
              left: '3%',
              right: '4%',
              bottom: '3%',
              containLabel: true
            },
            axisLabel: {
              formatter: '{value} '
            },
            xAxis: [
              {
                type: 'category',
                data: xAxis,
                axisTick: {
                  alignWithLabel: true
                },
                axisLine: {
                  lineStyle: {
                    color: '#9a9595'
                  }
                }
              }
            ],
            yAxis: [
              {
                type: 'value',
                axisLine: {
                  lineStyle: {
                    color: '#9a9595'
                  }
                },
                // 坐标轴内线的样式
                splitLine: {
                  lineStyle: {
                    color: '#9a9595'
                  }
                }
              }
            ],
            series: [
              {
                name: '当前时间段访问量',
                type: 'bar',
                barWidth: '60%',
                itemStyle: {
                  color: '#1b67b6c7',
                  lineStyle: {
                    color: '#9a9595'
                  }
                },

                data: series
              }
            ]
          })
        }
      })
    }
  }
}
</script>

<style lang="scss" scoped>
  .total {
    color: #c4c4c4;
    font-weight: 700;

  }
  .total span{
    color: #e4e4e4;
    font-weight: 400;
  }
  .data {
    display: flex;
    justify-content: space-between;
  }
  .body {
    position:relative;
    font-family:"Microsoft Yahei", Arial, sans-serif;
    background:#050d3c url("./img/bg.png") 0 0 / 100% 100% no-repeat;
     width:100%;
    height:100%;
    min-width:1200px;
    min-height:600px;
    overflow:hidden;
  }
</style>

以上就是前端监控pv/uv系统的核心代码加逻辑 备注：项目源代码地址：github.com/git-wangyah…