💖💖作者：计算机毕业设计江挽 💙💙个人简介：曾长期从事计算机专业培训教学，本人也热爱上课教学，语言擅长Java、微信小程序、Python、Golang、安卓Android等，开发项目包括大数据、深度学习、网站、小程序、安卓、算法。平常会做一些项目定制化开发、代码讲解、答辩教学、文档编写、也懂一些降重方面的技巧。平常喜欢分享一些自己开发中遇到的问题的解决办法，也喜欢交流技术，大家有技术代码这一块的问题可以问我！ 💛💛想说的话：感谢大家的关注与支持！ 💜💜 网站实战项目 安卓/小程序实战项目 大数据实战项目 深度学习实战项目
基于大数据的水质数据可视化分析系统介绍
水质数据可视化分析系统是一个基于Hadoop+Spark大数据技术栈构建的环境监测类应用系统。本系统采用HDFS作为分布式存储引擎，利用Spark SQL对海量水质监测数据进行快速查询与统计分析，通过Pandas和NumPy完成数据清洗、特征提取等预处理工作。后端采用Spring Boot框架搭建RESTful接口服务，前端使用Vue+ElementUI构建交互界面，借助Echarts图表库将复杂的水质指标数据转化为直观的可视化图表。系统实现了水质数据的批量导入、多维度统计分析、趋势预测以及动态可视化展示等核心功能，支持按时间段、监测点位、污染物类型等多个维度进行数据筛选与对比分析。整个系统充分发挥大数据框架在处理大规模数据集时的性能优势，能够在秒级响应时间内完成百万级水质记录的聚合计算，为环境监测部门提供了一个高效、直观的数据分析工具，让复杂的水质变化趋势通过图表一目了然。
基于大数据的水质数据可视化分析系统演示视频
基于大数据的水质数据可视化分析系统演示图片
基于大数据的水质数据可视化分析系统代码展示
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.functions;
import org.springframework.stereotype.Service;
import java.util.*;
@Service
public class WaterQualityAnalysisService {
private SparkSession spark = SparkSession.builder().appName("WaterQualityAnalysis").master("local[*]").config("spark.sql.warehouse.dir", "/user/hive/warehouse").getOrCreate();
public Map<String, Object> analyzeWaterQualityTrend(String startDate, String endDate, String monitorPoint) {
Dataset<Row> rawData = spark.read().format("jdbc").option("url", "jdbc:mysql://localhost:3306/water_quality").option("dbtable", "water_quality_data").option("user", "root").option("password", "123456").load();
Dataset<Row> filteredData = rawData.filter(rawData.col("monitor_point").equalTo(monitorPoint).and(rawData.col("sample_date").between(startDate, endDate)));
Dataset<Row> trendData = filteredData.groupBy("sample_date").agg(functions.avg("ph_value").alias("avg_ph"), functions.avg("dissolved_oxygen").alias("avg_do"), functions.avg("cod").alias("avg_cod"), functions.avg("ammonia_nitrogen").alias("avg_an"));
Dataset<Row> sortedTrend = trendData.orderBy("sample_date");
List<Row> trendList = sortedTrend.collectAsList();
List<Map<String, Object>> chartData = new ArrayList<>();
for (Row row : trendList) {
Map<String, Object> dataPoint = new HashMap<>();
dataPoint.put("date", row.getAs("sample_date").toString());
dataPoint.put("avgPh", row.getAs("avg_ph"));
dataPoint.put("avgDo", row.getAs("avg_do"));
dataPoint.put("avgCod", row.getAs("avg_cod"));
dataPoint.put("avgAn", row.getAs("avg_an"));
chartData.add(dataPoint);
}
Dataset<Row> qualityLevel = filteredData.withColumn("quality_level", functions.when(filteredData.col("ph_value").between(6.5, 8.5).and(filteredData.col("dissolved_oxygen").gt(6)).and(filteredData.col("cod").lt(15)).and(filteredData.col("ammonia_nitrogen").lt(0.5)), "优").when(filteredData.col("ph_value").between(6.0, 9.0).and(filteredData.col("dissolved_oxygen").gt(5)).and(filteredData.col("cod").lt(20)).and(filteredData.col("ammonia_nitrogen").lt(1.0)), "良").otherwise("差"));
Dataset<Row> levelStats = qualityLevel.groupBy("quality_level").count().orderBy(functions.desc("count"));
List<Row> levelList = levelStats.collectAsList();
Map<String, Long> levelDistribution = new HashMap<>();
for (Row row : levelList) {
levelDistribution.put(row.getAs("quality_level"), row.getAs("count"));
}
Map<String, Object> result = new HashMap<>();
result.put("trendData", chartData);
result.put("levelDistribution", levelDistribution);
result.put("totalRecords", filteredData.count());
return result;
}
public Map<String, Object> compareMultipleMonitorPoints(List<String> monitorPoints, String targetDate) {
Dataset<Row> rawData = spark.read().format("jdbc").option("url", "jdbc:mysql://localhost:3306/water_quality").option("dbtable", "water_quality_data").option("user", "root").option("password", "123456").load();
Dataset<Row> dateFilteredData = rawData.filter(rawData.col("sample_date").equalTo(targetDate));
Dataset<Row> pointFilteredData = dateFilteredData.filter(dateFilteredData.col("monitor_point").isin(monitorPoints.toArray()));
Dataset<Row> comparisonData = pointFilteredData.groupBy("monitor_point").agg(functions.avg("ph_value").alias("avg_ph"), functions.max("ph_value").alias("max_ph"), functions.min("ph_value").alias("min_ph"), functions.avg("dissolved_oxygen").alias("avg_do"), functions.max("dissolved_oxygen").alias("max_do"), functions.min("dissolved_oxygen").alias("min_do"), functions.avg("cod").alias("avg_cod"), functions.max("cod").alias("max_cod"), functions.min("cod").alias("min_cod"), functions.avg("ammonia_nitrogen").alias("avg_an"), functions.max("ammonia_nitrogen").alias("max_an"), functions.min("ammonia_nitrogen").alias("min_an"), functions.count("*").alias("sample_count"));
List<Row> comparisonList = comparisonData.collectAsList();
List<Map<String, Object>> pointComparison = new ArrayList<>();
for (Row row : comparisonList) {
Map<String, Object> pointData = new HashMap<>();
pointData.put("monitorPoint", row.getAs("monitor_point"));
pointData.put("avgPh", row.getAs("avg_ph"));
pointData.put("maxPh", row.getAs("max_ph"));
pointData.put("minPh", row.getAs("min_ph"));
pointData.put("avgDo", row.getAs("avg_do"));
pointData.put("maxDo", row.getAs("max_do"));
pointData.put("minDo", row.getAs("min_do"));
pointData.put("avgCod", row.getAs("avg_cod"));
pointData.put("maxCod", row.getAs("max_cod"));
pointData.put("minCod", row.getAs("min_cod"));
pointData.put("avgAn", row.getAs("avg_an"));
pointData.put("maxAn", row.getAs("max_an"));
pointData.put("minAn", row.getAs("min_an"));
pointData.put("sampleCount", row.getAs("sample_count"));
pointComparison.add(pointData);
}
Dataset<Row> abnormalData = pointFilteredData.filter(pointFilteredData.col("ph_value").lt(6.0).or(pointFilteredData.col("ph_value").gt(9.0)).or(pointFilteredData.col("dissolved_oxygen").lt(3.0)).or(pointFilteredData.col("cod").gt(40)).or(pointFilteredData.col("ammonia_nitrogen").gt(2.0)));
long abnormalCount = abnormalData.count();
List<Row> abnormalList = abnormalData.select("monitor_point", "sample_time", "ph_value", "dissolved_oxygen", "cod", "ammonia_nitrogen").collectAsList();
List<Map<String, Object>> abnormalRecords = new ArrayList<>();
for (Row row : abnormalList) {
Map<String, Object> record = new HashMap<>();
record.put("monitorPoint", row.getAs("monitor_point"));
record.put("sampleTime", row.getAs("sample_time").toString());
record.put("phValue", row.getAs("ph_value"));
record.put("dissolvedOxygen", row.getAs("dissolved_oxygen"));
record.put("cod", row.getAs("cod"));
record.put("ammoniaNitrogen", row.getAs("ammonia_nitrogen"));
abnormalRecords.add(record);
}
Map<String, Object> result = new HashMap<>();
result.put("pointComparison", pointComparison);
result.put("abnormalCount", abnormalCount);
result.put("abnormalRecords", abnormalRecords);
result.put("targetDate", targetDate);
return result;
}
public Map<String, Object> statisticsWaterQualityIndicators(String startDate, String endDate) {
Dataset<Row> rawData = spark.read().format("jdbc").option("url", "jdbc:mysql://localhost:3306/water_quality").option("dbtable", "water_quality_data").option("user", "root").option("password", "123456").load();
Dataset<Row> periodData = rawData.filter(rawData.col("sample_date").between(startDate, endDate));
Dataset<Row> overallStats = periodData.agg(functions.avg("ph_value").alias("avg_ph"), functions.stddev("ph_value").alias("stddev_ph"), functions.avg("dissolved_oxygen").alias("avg_do"), functions.stddev("dissolved_oxygen").alias("stddev_do"), functions.avg("cod").alias("avg_cod"), functions.stddev("cod").alias("stddev_cod"), functions.avg("ammonia_nitrogen").alias("avg_an"), functions.stddev("ammonia_nitrogen").alias("stddev_an"), functions.count("*").alias("total_samples"));
Row statsRow = overallStats.first();
Map<String, Object> overallStatistics = new HashMap<>();
overallStatistics.put("avgPh", statsRow.getAs("avg_ph"));
overallStatistics.put("stddevPh", statsRow.getAs("stddev_ph"));
overallStatistics.put("avgDo", statsRow.getAs("avg_do"));
overallStatistics.put("stddevDo", statsRow.getAs("stddev_do"));
overallStatistics.put("avgCod", statsRow.getAs("avg_cod"));
overallStatistics.put("stddevCod", statsRow.getAs("stddev_cod"));
overallStatistics.put("avgAn", statsRow.getAs("avg_an"));
overallStatistics.put("stddevAn", statsRow.getAs("stddev_an"));
overallStatistics.put("totalSamples", statsRow.getAs("total_samples"));
Dataset<Row> monthlyStats = periodData.withColumn("month", functions.date_format(periodData.col("sample_date"), "yyyy-MM")).groupBy("month").agg(functions.avg("ph_value").alias("monthly_avg_ph"), functions.avg("dissolved_oxygen").alias("monthly_avg_do"), functions.avg("cod").alias("monthly_avg_cod"), functions.avg("ammonia_nitrogen").alias("monthly_avg_an"), functions.count("*").alias("monthly_samples"));
Dataset<Row> sortedMonthlyStats = monthlyStats.orderBy("month");
List<Row> monthlyList = sortedMonthlyStats.collectAsList();
List<Map<String, Object>> monthlyStatistics = new ArrayList<>();
for (Row row : monthlyList) {
Map<String, Object> monthData = new HashMap<>();
monthData.put("month", row.getAs("month"));
monthData.put("avgPh", row.getAs("monthly_avg_ph"));
monthData.put("avgDo", row.getAs("monthly_avg_do"));
monthData.put("avgCod", row.getAs("monthly_avg_cod"));
monthData.put("avgAn", row.getAs("monthly_avg_an"));
monthData.put("sampleCount", row.getAs("monthly_samples"));
monthlyStatistics.add(monthData);
}
Dataset<Row> pointStats = periodData.groupBy("monitor_point").agg(functions.avg("ph_value").alias("point_avg_ph"), functions.avg("dissolved_oxygen").alias("point_avg_do"), functions.avg("cod").alias("point_avg_cod"), functions.avg("ammonia_nitrogen").alias("point_avg_an"), functions.count("*").alias("point_samples")).orderBy(functions.desc("point_samples"));
List<Row> pointList = pointStats.collectAsList();
List<Map<String, Object>> pointStatistics = new ArrayList<>();
for (Row row : pointList) {
Map<String, Object> pointData = new HashMap<>();
pointData.put("monitorPoint", row.getAs("monitor_point"));
pointData.put("avgPh", row.getAs("point_avg_ph"));
pointData.put("avgDo", row.getAs("point_avg_do"));
pointData.put("avgCod", row.getAs("point_avg_cod"));
pointData.put("avgAn", row.getAs("point_avg_an"));
pointData.put("sampleCount", row.getAs("point_samples"));
pointStatistics.add(pointData);
}
Dataset<Row> exceedanceData = periodData.filter(periodData.col("ph_value").lt(6.0).or(periodData.col("ph_value").gt(9.0)).or(periodData.col("cod").gt(30)).or(periodData.col("ammonia_nitrogen").gt(1.5)));
long exceedanceCount = exceedanceData.count();
double exceedanceRate = (double) exceedanceCount / statsRow.getAs("total_samples") * 100;
Map<String, Object> result = new HashMap<>();
result.put("overallStatistics", overallStatistics);
result.put("monthlyStatistics", monthlyStatistics);
result.put("pointStatistics", pointStatistics);
result.put("exceedanceCount", exceedanceCount);
result.put("exceedanceRate", String.format("%.2f", exceedanceRate) + "%");
result.put("periodRange", startDate + " 至 " + endDate);
return result;
}
}
基于大数据的水质数据可视化分析系统文档展示
💖💖作者：计算机毕业设计江挽 💙💙个人简介：曾长期从事计算机专业培训教学，本人也热爱上课教学，语言擅长Java、微信小程序、Python、Golang、安卓Android等，开发项目包括大数据、深度学习、网站、小程序、安卓、算法。平常会做一些项目定制化开发、代码讲解、答辩教学、文档编写、也懂一些降重方面的技巧。平常喜欢分享一些自己开发中遇到的问题的解决办法，也喜欢交流技术，大家有技术代码这一块的问题可以问我！ 💛💛想说的话：感谢大家的关注与支持！ 💜💜 网站实战项目 安卓/小程序实战项目 大数据实战项目 深度学习实战项目
