前言
💖💖作者：计算机程序员小杨 💙💙个人简介：我是一名计算机相关专业的从业者，擅长Java、微信小程序、Python、Golang、安卓Android等多个IT方向。会做一些项目定制化开发、代码讲解、答辩教学、文档编写、也懂一些降重方面的技巧。热爱技术，喜欢钻研新工具和框架，也乐于通过代码解决实际问题，大家有技术代码这一块的问题可以问我！ 💛💛想说的话：感谢大家的关注与支持！ 💕💕文末获取源码联系 计算机程序员小杨 💜💜 网站实战项目 安卓/小程序实战项目 大数据实战项目 深度学习实战项目 计算机毕业设计选题 💜💜
一.开发工具简介
大数据框架：Hadoop+Spark（本次没用Hive，支持定制） 开发语言：Python+Java（两个版本都支持） 后端框架：Django+Spring Boot(Spring+SpringMVC+Mybatis)（两个版本都支持） 前端：Vue+ElementUI+Echarts+HTML+CSS+JavaScript+jQuery 详细技术点：Hadoop、HDFS、Spark、Spark SQL、Pandas、NumPy 数据库：MySQL
二.系统内容简介
《玉米产量数据可视化分析系统》是一个基于Hadoop+Spark大数据技术栈构建的农业数据分析平台，采用Python作为主要开发语言，后端使用Django框架提供稳定的API服务，前端基于Vue+ElementUI+Echarts技术实现交互式数据可视化界面。系统通过HDFS分布式文件系统存储海量玉米产量相关数据，利用Spark及Spark SQL进行高效的数据处理与分析计算，结合Pandas和NumPy进行精细化数据操作，数据持久化采用MySQL数据库管理。系统核心功能涵盖数据质量分析、环境影响分析、生长发育分析、时空分布分析、品种特性分析、产量性能分析以及可视化大屏展示等七大模块，为农业生产决策提供科学的数据支撑和直观的可视化展示，帮助用户深入理解玉米产量的影响因素和变化规律。
三.系统功能演示
四.系统界面展示
五.系统源码展示
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import *
import pandas as pd
import numpy as np
from django.http import JsonResponse
from django.views import View
import json
from datetime import datetime, timedelta
spark = SparkSession.builder.appName("CornYieldAnalysis").config("spark.sql.adaptive.enabled", "true").config("spark.sql.adaptive.coalescePartitions.enabled", "true").getOrCreate()
def data_quality_analysis(data_path):
df = spark.read.option("header", "true").option("inferSchema", "true").csv(data_path)
total_records = df.count()
null_counts = df.select([count(when(col(c).isNull(), c)).alias(c) for c in df.columns])
null_percentages = null_counts.collect()[0].asDict()
duplicate_count = df.count() - df.dropDuplicates().count()
duplicate_percentage = (duplicate_count / total_records) * 100
numeric_cols = [field.name for field in df.schema.fields if field.dataType in [IntegerType(), DoubleType(), FloatType()]]
outlier_results = {}
for col_name in numeric_cols:
stats = df.select(col_name).describe().collect()
mean_val = float([row['summary'] for row in stats if row[col_name] == 'mean'][0])
std_val = float([row['summary'] for row in stats if row[col_name] == 'stddev'][0])
outliers = df.filter((col(col_name) < (mean_val - 3 * std_val)) | (col(col_name) > (mean_val + 3 * std_val))).count()
outlier_results[col_name] = (outliers / total_records) * 100
completeness_score = sum([100 - (null_percentages[col] / total_records * 100) for col in df.columns]) / len(df.columns)
consistency_score = 100 - duplicate_percentage
accuracy_score = 100 - sum(outlier_results.values()) / len(outlier_results) if outlier_results else 100
overall_quality = (completeness_score + consistency_score + accuracy_score) / 3
quality_report = {"total_records": total_records, "null_analysis": null_percentages, "duplicate_analysis": {"count": duplicate_count, "percentage": duplicate_percentage}, "outlier_analysis": outlier_results, "quality_scores": {"completeness": completeness_score, "consistency": consistency_score, "accuracy": accuracy_score, "overall": overall_quality}}
return quality_report
def environment_impact_analysis(data_path):
df = spark.read.option("header", "true").option("inferSchema", "true").csv(data_path)
weather_impact = df.groupBy("region").agg(avg("temperature").alias("avg_temp"), avg("rainfall").alias("avg_rainfall"), avg("humidity").alias("avg_humidity"), avg("yield_per_hectare").alias("avg_yield")).collect()
temp_correlation = df.stat.corr("temperature", "yield_per_hectare")
rainfall_correlation = df.stat.corr("rainfall", "yield_per_hectare")
humidity_correlation = df.stat.corr("humidity", "yield_per_hectare")
soil_analysis = df.groupBy("soil_type").agg(avg("ph_value").alias("avg_ph"), avg("nitrogen_content").alias("avg_nitrogen"), avg("phosphorus_content").alias("avg_phosphorus"), avg("potassium_content").alias("avg_potassium"), avg("yield_per_hectare").alias("avg_yield")).collect()
optimal_conditions = df.filter((col("temperature").between(20, 30)) & (col("rainfall").between(500, 800)) & (col("humidity").between(60, 80))).agg(avg("yield_per_hectare").alias("optimal_yield")).collect()[0]["optimal_yield"]
suboptimal_conditions = df.filter(~((col("temperature").between(20, 30)) & (col("rainfall").between(500, 800)) & (col("humidity").between(60, 80)))).agg(avg("yield_per_hectare").alias("suboptimal_yield")).collect()[0]["suboptimal_yield"]
yield_difference = optimal_conditions - suboptimal_conditions
climate_zones = df.withColumn("climate_zone", when((col("temperature") > 25) & (col("rainfall") > 600), "tropical").when((col("temperature").between(15, 25)) & (col("rainfall").between(400, 800)), "temperate").otherwise("arid")).groupBy("climate_zone").agg(avg("yield_per_hectare").alias("avg_yield"), count("*").alias("sample_count")).collect()
seasonal_analysis = df.withColumn("season", when(col("planting_month").isin([3, 4, 5]), "spring").when(col("planting_month").isin([6, 7, 8]), "summer").when(col("planting_month").isin([9, 10, 11]), "autumn").otherwise("winter")).groupBy("season").agg(avg("yield_per_hectare").alias("avg_yield"), avg("temperature").alias("avg_temp")).collect()
extreme_weather_impact = df.filter((col("temperature") > 35) | (col("temperature") < 10) | (col("rainfall") > 1000) | (col("rainfall") < 200)).agg(avg("yield_per_hectare").alias("extreme_weather_yield"), count("*").alias("extreme_weather_count")).collect()[0]
environment_report = {"weather_correlations": {"temperature": temp_correlation, "rainfall": rainfall_correlation, "humidity": humidity_correlation}, "regional_analysis": [row.asDict() for row in weather_impact], "soil_analysis": [row.asDict() for row in soil_analysis], "optimal_vs_suboptimal": {"optimal_yield": optimal_conditions, "suboptimal_yield": suboptimal_conditions, "yield_difference": yield_difference}, "climate_zones": [row.asDict() for row in climate_zones], "seasonal_patterns": [row.asDict() for row in seasonal_analysis], "extreme_weather_impact": extreme_weather_impact.asDict()}
return environment_report
def yield_performance_analysis(data_path):
df = spark.read.option("header", "true").option("inferSchema", "true").csv(data_path)
variety_performance = df.groupBy("variety").agg(avg("yield_per_hectare").alias("avg_yield"), max("yield_per_hectare").alias("max_yield"), min("yield_per_hectare").alias("min_yield"), stddev("yield_per_hectare").alias("yield_stddev"), count("*").alias("sample_count")).orderBy(desc("avg_yield")).collect()
top_performers = [row.asDict() for row in variety_performance[:5]]
bottom_performers = [row.asDict() for row in variety_performance[-5:]]
yearly_trends = df.groupBy("year").agg(avg("yield_per_hectare").alias("avg_yield"), sum("total_production").alias("total_production"), avg("planted_area").alias("avg_planted_area")).orderBy("year").collect()
yield_growth_rate = []
for i in range(1, len(yearly_trends)):
current_yield = yearly_trends[i]["avg_yield"]
previous_yield = yearly_trends[i-1]["avg_yield"]
growth_rate = ((current_yield - previous_yield) / previous_yield) * 100
yield_growth_rate.append({"year": yearly_trends[i]["year"], "growth_rate": growth_rate})
regional_comparison = df.groupBy("region").agg(avg("yield_per_hectare").alias("avg_yield"), avg("cost_per_hectare").alias("avg_cost")).withColumn("profit_margin", col("avg_yield") * 0.5 - col("avg_cost")).orderBy(desc("profit_margin")).collect()
yield_stability = df.groupBy("variety").agg(stddev("yield_per_hectare").alias("yield_variance"), avg("yield_per_hectare").alias("avg_yield")).withColumn("coefficient_variation", col("yield_variance") / col("avg_yield")).orderBy("coefficient_variation").collect()
production_efficiency = df.groupBy("farming_method").agg(avg("yield_per_hectare").alias("avg_yield"), avg("water_usage").alias("avg_water"), avg("fertilizer_usage").alias("avg_fertilizer")).withColumn("water_efficiency", col("avg_yield") / col("avg_water")).withColumn("fertilizer_efficiency", col("avg_yield") / col("avg_fertilizer")).collect()
benchmark_analysis = df.agg(expr("percentile_approx(yield_per_hectare, 0.9)").alias("top_10_percent"), expr("percentile_approx(yield_per_hectare, 0.75)").alias("top_25_percent"), expr("percentile_approx(yield_per_hectare, 0.5)").alias("median_yield"), expr("percentile_approx(yield_per_hectare, 0.25)").alias("bottom_25_percent")).collect()[0]
performance_categories = df.withColumn("performance_level", when(col("yield_per_hectare") >= benchmark_analysis["top_10_percent"], "excellent").when(col("yield_per_hectare") >= benchmark_analysis["top_25_percent"], "good").when(col("yield_per_hectare") >= benchmark_analysis["median_yield"], "average").otherwise("below_average")).groupBy("performance_level").count().collect()
performance_report = {"variety_rankings": {"top_performers": top_performers, "bottom_performers": bottom_performers}, "temporal_analysis": {"yearly_trends": [row.asDict() for row in yearly_trends], "growth_rates": yield_growth_rate}, "regional_comparison": [row.asDict() for row in regional_comparison], "stability_analysis": [row.asDict() for row in yield_stability], "efficiency_metrics": [row.asDict() for row in production_efficiency], "benchmark_percentiles": benchmark_analysis.asDict(), "performance_distribution": [row.asDict() for row in performance_categories]}
return performance_report
六.系统文档展示
结束
💕💕文末获取源码联系 计算机程序员小杨
