前言
💖💖作者：计算机程序员小杨 💙💙个人简介：我是一名计算机相关专业的从业者，擅长Java、微信小程序、Python、Golang、安卓Android等多个IT方向。会做一些项目定制化开发、代码讲解、答辩教学、文档编写、也懂一些降重方面的技巧。热爱技术，喜欢钻研新工具和框架，也乐于通过代码解决实际问题，大家有技术代码这一块的问题可以问我！ 💛💛想说的话：感谢大家的关注与支持！ 💕💕文末获取源码联系 计算机程序员小杨 💜💜 网站实战项目 安卓/小程序实战项目 大数据实战项目 深度学习实战项目 计算机毕业设计选题 💜💜
一.开发工具简介
大数据框架：Hadoop+Spark（本次没用Hive，支持定制） 开发语言：Python+Java（两个版本都支持） 后端框架：Django+Spring Boot(Spring+SpringMVC+Mybatis)（两个版本都支持） 前端：Vue+ElementUI+Echarts+HTML+CSS+JavaScript+jQuery 详细技术点：Hadoop、HDFS、Spark、Spark SQL、Pandas、NumPy 数据库：MySQL
二.系统内容简介
基于Hadoop+Spark的农作物产量数据分析与可视化系统是一个专门针对农业大数据处理和分析的综合性平台，该系统运用Hadoop分布式存储框架和Spark内存计算引擎，实现对大规模农作物产量数据的高效处理和深度分析。系统采用Python+Django或Java+SpringBoot双技术栈架构，前端使用Vue+ElementUI+Echarts构建现代化交互界面，后端通过MySQL数据库存储结构化数据，利用HDFS进行海量数据分布式存储。系统核心功能涵盖用户管理、农作物产量数据管理、产量多维综合分析、气候影响关联分析、作物周期产量分析、地理环境影响分析和生产措施效果分析等模块，通过Spark SQL进行复杂查询优化，结合Pandas和NumPy进行数据预处理和统计分析，最终通过可视化图表直观展现分析结果，为农业生产决策提供科学的数据支撑，是一个集数据采集、存储、处理、分析和可视化于一体的完整大数据解决方案。
三.系统功能演示
大数据毕业设计选题推荐：基于Hadoop+Spark的农作物产量数据分析与可视化系统完整实现
四.系统界面展示
五.系统源码展示
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, sum, avg, count, when, desc, asc
from pyspark.sql.types import StructType, StructField, StringType, IntegerType, FloatType
import pandas as pd
import numpy as np
from datetime import datetime
spark = SparkSession.builder.appName("CropYieldAnalysis").config("spark.sql.adaptive.enabled", "true").getOrCreate()
def comprehensive_yield_analysis(data_path, crop_type=None, year_range=None):
yield_df = spark.read.option("header", "true").csv(data_path)
if crop_type:
yield_df = yield_df.filter(col("crop_type") == crop_type)
if year_range:
yield_df = yield_df.filter((col("year") >= year_range[0]) & (col("year") <= year_range[1]))
yield_df = yield_df.withColumn("yield_per_hectare", col("total_yield").cast("float") / col("planted_area").cast("float"))
monthly_analysis = yield_df.groupBy("year", "month", "crop_type").agg(
sum("total_yield").alias("monthly_total_yield"),
avg("yield_per_hectare").alias("avg_yield_per_hectare"),
count("*").alias("record_count")
)
regional_analysis = yield_df.groupBy("province", "city", "crop_type").agg(
sum("total_yield").alias("regional_total_yield"),
avg("yield_per_hectare").alias("regional_avg_yield"),
sum("planted_area").alias("total_planted_area")
)
trend_analysis = yield_df.groupBy("year", "crop_type").agg(
sum("total_yield").alias("yearly_total_yield"),
avg("yield_per_hectare").alias("yearly_avg_yield")
).orderBy("year", "crop_type")
quality_analysis = yield_df.withColumn("yield_grade",
when(col("yield_per_hectare") >= 8000, "高产")
.when(col("yield_per_hectare") >= 6000, "中产")
.otherwise("低产")
).groupBy("yield_grade", "crop_type").count()
seasonal_pattern = yield_df.groupBy("season", "crop_type").agg(
sum("total_yield").alias("seasonal_yield"),
avg("yield_per_hectare").alias("seasonal_avg_yield"),
avg("temperature").alias("avg_temperature"),
avg("rainfall").alias("avg_rainfall")
)
correlation_data = yield_df.select("yield_per_hectare", "temperature", "rainfall", "humidity", "sunshine_hours").toPandas()
correlation_matrix = correlation_data.corr()
yield_stability = yield_df.groupBy("province", "crop_type").agg(
avg("yield_per_hectare").alias("avg_yield"),
(sum((col("yield_per_hectare") - avg("yield_per_hectare")) ** 2) / count("*")).alias("variance")
)
return {
"monthly_analysis": monthly_analysis.collect(),
"regional_analysis": regional_analysis.collect(),
"trend_analysis": trend_analysis.collect(),
"quality_analysis": quality_analysis.collect(),
"seasonal_pattern": seasonal_pattern.collect(),
"correlation_matrix": correlation_matrix.to_dict(),
"yield_stability": yield_stability.collect()
}
def climate_impact_correlation_analysis(yield_data_path, climate_data_path):
yield_df = spark.read.option("header", "true").csv(yield_data_path)
climate_df = spark.read.option("header", "true").csv(climate_data_path)
merged_df = yield_df.join(climate_df, ["region", "year", "month"], "inner")
merged_df = merged_df.withColumn("temperature_category",
when(col("temperature") >= 25, "高温")
.when(col("temperature") >= 15, "适温")
.otherwise("低温")
)
merged_df = merged_df.withColumn("rainfall_category",
when(col("rainfall") >= 100, "多雨")
.when(col("rainfall") >= 50, "适雨")
.otherwise("少雨")
)
temperature_impact = merged_df.groupBy("temperature_category", "crop_type").agg(
avg("yield_per_hectare").alias("avg_yield_by_temp"),
count("*").alias("sample_count"),
sum("total_yield").alias("total_yield_by_temp")
)
rainfall_impact = merged_df.groupBy("rainfall_category", "crop_type").agg(
avg("yield_per_hectare").alias("avg_yield_by_rainfall"),
count("*").alias("sample_count"),
sum("total_yield").alias("total_yield_by_rainfall")
)
extreme_weather_analysis = merged_df.filter(
(col("temperature") > 35) | (col("temperature") < 5) |
(col("rainfall") > 200) | (col("rainfall") < 10)
).groupBy("crop_type", "region").agg(
avg("yield_per_hectare").alias("extreme_weather_yield"),
count("*").alias("extreme_events_count")
)
climate_yield_correlation = merged_df.groupBy("crop_type").agg(
avg("temperature").alias("avg_temperature"),
avg("rainfall").alias("avg_rainfall"),
avg("humidity").alias("avg_humidity"),
avg("yield_per_hectare").alias("avg_yield")
)
optimal_climate_conditions = merged_df.filter(
col("yield_per_hectare") > merged_df.select(avg("yield_per_hectare")).collect()[0][0] * 1.2
).groupBy("crop_type").agg(
avg("temperature").alias("optimal_temperature"),
avg("rainfall").alias("optimal_rainfall"),
avg("humidity").alias("optimal_humidity"),
avg("sunshine_hours").alias("optimal_sunshine")
)
monthly_climate_yield = merged_df.groupBy("month", "crop_type").agg(
avg("temperature").alias("monthly_avg_temp"),
avg("rainfall").alias("monthly_avg_rainfall"),
avg("yield_per_hectare").alias("monthly_avg_yield")
).orderBy("month")
climate_risk_assessment = merged_df.withColumn("risk_score",
when((col("temperature") > 30) & (col("rainfall") < 30), 3)
.when((col("temperature") < 10) | (col("rainfall") > 150), 2)
.otherwise(1)
).groupBy("risk_score", "crop_type").agg(
avg("yield_per_hectare").alias("risk_level_yield"),
count("*").alias("risk_occurrence_count")
)
return {
"temperature_impact": temperature_impact.collect(),
"rainfall_impact": rainfall_impact.collect(),
"extreme_weather_analysis": extreme_weather_analysis.collect(),
"climate_yield_correlation": climate_yield_correlation.collect(),
"optimal_climate_conditions": optimal_climate_conditions.collect(),
"monthly_climate_yield": monthly_climate_yield.collect(),
"climate_risk_assessment": climate_risk_assessment.collect()
}
def crop_cycle_yield_analysis(data_path, crop_type_filter=None):
cycle_df = spark.read.option("header", "true").csv(data_path)
if crop_type_filter:
cycle_df = cycle_df.filter(col("crop_type").isin(crop_type_filter))
cycle_df = cycle_df.withColumn("growth_days",
(col("harvest_date").cast("timestamp").cast("long") - col("planting_date").cast("timestamp").cast("long")) / 86400
)
cycle_df = cycle_df.withColumn("cycle_category",
when(col("growth_days") <= 90, "短周期")
.when(col("growth_days") <= 150, "中周期")
.otherwise("长周期")
)
cycle_yield_analysis = cycle_df.groupBy("cycle_category", "crop_type").agg(
avg("yield_per_hectare").alias("avg_cycle_yield"),
avg("growth_days").alias("avg_growth_days"),
count("*").alias("cycle_sample_count"),
sum("total_yield").alias("cycle_total_yield")
)
growth_stage_analysis = cycle_df.withColumn("seedling_yield", col("yield_per_hectare") * 0.1) \
.withColumn("vegetative_yield", col("yield_per_hectare") * 0.3) \
.withColumn("reproductive_yield", col("yield_per_hectare") * 0.6) \
.groupBy("crop_type").agg(
avg("seedling_yield").alias("avg_seedling_contribution"),
avg("vegetative_yield").alias("avg_vegetative_contribution"),
avg("reproductive_yield").alias("avg_reproductive_contribution")
)
seasonal_planting_analysis = cycle_df.withColumn("planting_season",
when(col("planting_month").between(3, 5), "春播")
.when(col("planting_month").between(6, 8), "夏播")
.when(col("planting_month").between(9, 11), "秋播")
.otherwise("冬播")
).groupBy("planting_season", "crop_type").agg(
avg("yield_per_hectare").alias("seasonal_avg_yield"),
avg("growth_days").alias("seasonal_avg_growth_days"),
count("*").alias("seasonal_planting_count")
)
yield_efficiency_analysis = cycle_df.withColumn("daily_yield_rate",
col("yield_per_hectare") / col("growth_days")
).groupBy("crop_type", "cycle_category").agg(
avg("daily_yield_rate").alias("avg_daily_yield_rate"),
sum("daily_yield_rate").alias("total_daily_yield_rate")
)
harvest_timing_analysis = cycle_df.withColumn("harvest_month", col("harvest_date").substr(6, 2).cast("int")) \
.groupBy("harvest_month", "crop_type").agg(
avg("yield_per_hectare").alias("harvest_month_avg_yield"),
count("*").alias("harvest_month_count")
).orderBy("harvest_month")
cycle_stability_analysis = cycle_df.groupBy("crop_type").agg(
avg("growth_days").alias("avg_cycle_length"),
(sum((col("growth_days") - avg("growth_days")) ** 2) / count("*")).alias("cycle_variance"),
avg("yield_per_hectare").alias("avg_yield"),
(sum((col("yield_per_hectare") - avg("yield_per_hectare")) ** 2) / count("*")).alias("yield_variance")
)
optimal_cycle_identification = cycle_df.filter(
col("yield_per_hectare") > cycle_df.select(avg("yield_per_hectare")).collect()[0][0] * 1.15
).groupBy("crop_type").agg(
avg("growth_days").alias("optimal_cycle_days"),
avg("yield_per_hectare").alias("optimal_cycle_yield"),
count("*").alias("optimal_cases_count")
)
return {
"cycle_yield_analysis": cycle_yield_analysis.collect(),
"growth_stage_analysis": growth_stage_analysis.collect(),
"seasonal_planting_analysis": seasonal_planting_analysis.collect(),
"yield_efficiency_analysis": yield_efficiency_analysis.collect(),
"harvest_timing_analysis": harvest_timing_analysis.collect(),
"cycle_stability_analysis": cycle_stability_analysis.collect(),
"optimal_cycle_identification": optimal_cycle_identification.collect()
}
六.系统文档展示
结束
💛💛想说的话：感谢大家的关注与支持！ 💕💕文末获取源码联系 计算机程序员小杨 💜💜 网站实战项目 安卓/小程序实战项目 大数据实战项目 深度学习实战项目 计算机毕业设计选题 💜💜
