【大数据】豆瓣高分电影数据可视化分析系统 计算机毕业设计项目 Anaconda+Hadoop+Spark环境配置 数据科学与大数据技术 附源码+文档+讲解

一、个人简介

💖💖作者：计算机编程果茶熊 💙💙个人简介：曾长期从事计算机专业培训教学，担任过编程老师，同时本人也热爱上课教学，擅长Java、微信小程序、Python、Golang、安卓Android等多个IT方向。会做一些项目定制化开发、代码讲解、答辩教学、文档编写、也懂一些降重方面的技巧。平常喜欢分享一些自己开发中遇到的问题的解决办法，也喜欢交流技术，大家有技术代码这一块的问题可以问我！ 💛💛想说的话：感谢大家的关注与支持！ 💜💜 网站实战项目 安卓/小程序实战项目 大数据实战项目 计算机毕业设计选题 💕💕文末获取源码联系计算机编程果茶熊

二、系统介绍

大数据框架：Hadoop+Spark（Hive需要定制修改） 开发语言：Java+Python（两个版本都支持） 数据库：MySQL 后端框架：SpringBoot(Spring+SpringMVC+Mybatis)+Django（两个版本都支持） 前端：Vue+Echarts+HTML+CSS+JavaScript+jQuery

豆瓣高分电影数据可视化分析系统是一个基于大数据技术栈构建的电影数据分析平台，采用Hadoop+Spark分布式计算框架处理海量电影数据，结合Django后端框架和Vue前端技术实现数据的高效分析与可视化展示。系统利用HDFS存储电影评分、类型、制作、影人等多维度数据，通过Spark SQL进行复杂查询分析，运用Pandas和NumPy进行数据预处理和统计计算，将分析结果通过Echarts图表库以直观的可视化形式呈现。系统核心功能涵盖电影特征可视化分析、电影类型内容分析、电影制作产业分析、影人影响力可视化分析、观众平台覆盖率分析等模块，并提供可视化大屏功能，为电影行业从业者、研究人员和普通观众提供了一个全面了解电影市场趋势、发掘优质内容、分析产业格局的数据分析工具。

三、视频解说

豆瓣高分电影数据可视化分析系统

四、部分功能展示

五、部分代码展示

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, avg, count, desc, when, regexp_replace, split, explode
import pandas as pd
import numpy as np
from django.http import JsonResponse
from collections import Counter

spark = SparkSession.builder.appName("DoubanMovieAnalysis").config("spark.sql.adaptive.enabled", "true").getOrCreate()

def analyze_movie_features(request):
    movie_df = spark.read.option("header", "true").csv("hdfs://localhost:9000/movie_data/movies.csv")
    rating_df = spark.read.option("header", "true").csv("hdfs://localhost:9000/movie_data/ratings.csv")
    joined_df = movie_df.join(rating_df, "movie_id", "inner")
    high_rating_movies = joined_df.filter(col("rating") >= 8.0)
    feature_stats = high_rating_movies.groupBy("genre", "year").agg(
        avg("rating").alias("avg_rating"),
        count("movie_id").alias("movie_count"),
        avg("duration").alias("avg_duration")
    ).orderBy(desc("avg_rating"))
    yearly_trends = high_rating_movies.groupBy("year").agg(
        count("movie_id").alias("total_movies"),
        avg("rating").alias("year_avg_rating")
    ).orderBy("year")
    director_influence = high_rating_movies.groupBy("director").agg(
        count("movie_id").alias("movie_count"),
        avg("rating").alias("director_avg_rating"
    ).filter(col("movie_count") >= 3).orderBy(desc("director_avg_rating"))
    rating_distribution = high_rating_movies.groupBy(
        when(col("rating") >= 9.0, "9.0+")
        .when(col("rating") >= 8.5, "8.5-8.9")
        .otherwise("8.0-8.4").alias("rating_range")
    ).count().orderBy(desc("count"))
    country_performance = high_rating_movies.groupBy("country").agg(
        count("movie_id").alias("movie_count"),
        avg("rating").alias("country_avg_rating")
    ).filter(col("movie_count") >= 5).orderBy(desc("country_avg_rating"))
    feature_results = feature_stats.toPandas().to_dict('records')
    yearly_results = yearly_trends.toPandas().to_dict('records')
    director_results = director_influence.toPandas().to_dict('records')
    rating_dist_results = rating_distribution.toPandas().to_dict('records')
    country_results = country_performance.toPandas().to_dict('records')
    return JsonResponse({
        'feature_stats': feature_results,
        'yearly_trends': yearly_results,
        'director_influence': director_results[:20],
        'rating_distribution': rating_dist_results,
        'country_performance': country_results[:15]
    })

def analyze_movie_genres(request):
    movie_df = spark.read.option("header", "true").csv("hdfs://localhost:9000/movie_data/movies.csv")
    rating_df = spark.read.option("header", "true").csv("hdfs://localhost:9000/movie_data/ratings.csv")
    joined_df = movie_df.join(rating_df, "movie_id", "inner")
    genre_split_df = joined_df.withColumn("genre_array", split(col("genre"), "/"))
    genre_exploded_df = genre_split_df.select("movie_id", "title", "rating", "year", explode("genre_array").alias("single_genre"))
    genre_stats = genre_exploded_df.groupBy("single_genre").agg(
        count("movie_id").alias("total_count"),
        avg("rating").alias("avg_rating"),
        count(when(col("rating") >= 8.0, 1)).alias("high_rating_count")
    ).withColumn("high_rating_ratio", col("high_rating_count") / col("total_count"))
    genre_yearly_trend = genre_exploded_df.groupBy("single_genre", "year").agg(
        count("movie_id").alias("yearly_count"),
        avg("rating").alias("yearly_avg_rating")
    ).orderBy("single_genre", "year")
    top_genres = genre_stats.filter(col("total_count") >= 50).orderBy(desc("avg_rating"))
    genre_combinations = joined_df.groupBy("genre").agg(
        count("movie_id").alias("combo_count"),
        avg("rating").alias("combo_avg_rating")
    ).filter(col("combo_count") >= 10).orderBy(desc("combo_avg_rating"))
    genre_duration_analysis = genre_exploded_df.groupBy("single_genre").agg(
        avg("duration").alias("avg_duration"),
        count("movie_id").alias("movie_count")
    ).filter(col("movie_count") >= 30).orderBy(desc("avg_duration"))
    popular_genre_by_year = genre_exploded_df.groupBy("year").agg(
        count("movie_id").alias("total_yearly_movies")
    ).join(
        genre_exploded_df.groupBy("year", "single_genre").agg(
            count("movie_id").alias("genre_yearly_count")
        ), "year"
    ).withColumn("genre_percentage", col("genre_yearly_count") / col("total_yearly_movies") * 100)
    genre_stats_results = genre_stats.toPandas().to_dict('records')
    yearly_trend_results = genre_yearly_trend.toPandas().to_dict('records')
    top_genres_results = top_genres.toPandas().to_dict('records')
    combo_results = genre_combinations.toPandas().to_dict('records')
    duration_results = genre_duration_analysis.toPandas().to_dict('records')
    popular_by_year_results = popular_genre_by_year.toPandas().to_dict('records')
    return JsonResponse({
        'genre_statistics': genre_stats_results,
        'yearly_trends': yearly_trend_results,
        'top_performing_genres': top_genres_results[:15],
        'genre_combinations': combo_results[:20],
        'duration_analysis': duration_results,
        'popularity_by_year': popular_by_year_results
    })

def analyze_actor_influence(request):
    movie_df = spark.read.option("header", "true").csv("hdfs://localhost:9000/movie_data/movies.csv")
    rating_df = spark.read.option("header", "true").csv("hdfs://localhost:9000/movie_data/ratings.csv")
    actor_df = spark.read.option("header", "true").csv("hdfs://localhost:9000/movie_data/actors.csv")
    movie_rating_df = movie_df.join(rating_df, "movie_id", "inner")
    actor_movie_df = actor_df.join(movie_rating_df, "movie_id", "inner")
    actor_performance = actor_movie_df.groupBy("actor_name").agg(
        count("movie_id").alias("total_movies"),
        avg("rating").alias("avg_movie_rating"),
        count(when(col("rating") >= 8.0, 1)).alias("high_rating_movies"),
        avg("box_office").alias("avg_box_office")
    ).withColumn("high_rating_ratio", col("high_rating_movies") / col("total_movies"))
    prolific_actors = actor_performance.filter(col("total_movies") >= 5).orderBy(desc("avg_movie_rating"))
    actor_career_span = actor_movie_df.groupBy("actor_name").agg(
        min("year").alias("career_start"),
        max("year").alias("career_end"),
        count("movie_id").alias("movie_count")
    ).withColumn("career_span", col("career_end") - col("career_start") + 1)
    actor_genre_preference = actor_movie_df.withColumn("genre_array", split(col("genre"), "/")).select(
        "actor_name", "movie_id", "rating", explode("genre_array").alias("single_genre")
    ).groupBy("actor_name", "single_genre").agg(
        count("movie_id").alias("genre_count"),
        avg("rating").alias("genre_avg_rating")
    ).orderBy("actor_name", desc("genre_count"))
    actor_collaboration = actor_movie_df.groupBy("actor_name", "director").agg(
        count("movie_id").alias("collaboration_count"),
        avg("rating").alias("collaboration_avg_rating")
    ).filter(col("collaboration_count") >= 2).orderBy(desc("collaboration_avg_rating"))
    top_box_office_actors = actor_performance.filter(col("total_movies") >= 3).orderBy(desc("avg_box_office"))
    actor_rating_consistency = actor_movie_df.groupBy("actor_name").agg(
        count("movie_id").alias("movie_count"),
        avg("rating").alias("avg_rating"),
        stddev("rating").alias("rating_stddev")
    ).filter(col("movie_count") >= 5).withColumn(
        "consistency_score", 1 / (col("rating_stddev") + 0.1)
    ).orderBy(desc("consistency_score"))
    performance_results = prolific_actors.toPandas().to_dict('records')
    career_results = actor_career_span.toPandas().to_dict('records')
    genre_pref_results = actor_genre_preference.toPandas().to_dict('records')
    collaboration_results = actor_collaboration.toPandas().to_dict('records')
    box_office_results = top_box_office_actors.toPandas().to_dict('records')
    consistency_results = actor_rating_consistency.toPandas().to_dict('records')
    return JsonResponse({
        'actor_performance': performance_results[:30],
        'career_analysis': career_results,
        'genre_preferences': genre_pref_results[:100],
        'director_collaborations': collaboration_results[:50],
        'box_office_leaders': box_office_results[:20],
        'rating_consistency': consistency_results[:25]
    })

六、部分文档展示

七、END

💕💕文末获取源码联系计算机编程果茶熊