kaggle 房价预测项目学习

学习了一下优秀的选手kaggle入门之 房价预测 - 简书 (jianshu.com)怎么把 House Prices - Advanced Regression Techniques | Kaggle 做到比较高的准确率的，用这个代码达到了6%的名次。
有一些知识点记录一下，方便回顾，代码在前，解释在后。

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import warnings
# % matplotlib inline
plt.style.use('ggplot')

plt.style.use('ggplot')用来设置代码绘图风格

pd.set_option('max_columns', 1000)
pd.set_option('max_row', 300)
pd.set_option('display.float_format', lambda x: '%.5f' % x)

pd.set_option()用来给 pd 设置一些默认参数
pd.get_option()用来获取当前的参数
max_rows和 min_rows的显示逻辑稍复杂，但是设计的挺合理。
当 pd 的行数小于max_rows时，按照实际显示，稍小于max_rows时按照 max_rows来显示。比较大于max_rows时按照min_rows显示。

df_train = pd.read_csv('train.csv')
df_train.columns

df_train['SalePrice'].describe()

sns.distplot(df_train['SalePrice'])

dist:distribution的缩写，也就是直方图的意思

var = 'GrLivArea'
data = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)
data.plot.scatter(x=var, y='SalePrice', ylim=(0, 800000))

pd.caontact:按轴来合并，默认axis=0
scatter：散点图

var = 'TotalBsmtSF'
data = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)
data.plot.scatter(x=var, y='SalePrice', ylim=(0, 800000))

var = 'OverallQual'
data = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)
fig = sns.boxplot(x=var, y='SalePrice', data=data)
fig.axis(ymin=0, ymax=800000)

sns.boxplot：画箱型图 从箱型图可以看出一组数据的上边缘、下边缘、中位数和两个四分位数 有时有异常值

var = 'YearBuilt'
data = pd.concat([df_train['SalePrice'], df_train[var]], axis=1)
f, ax = plt.subplots(figsize=(20, 8))
fig = sns.boxplot(x=var, y='SalePrice', data=data)
fig.axis(ymin=0, ymax=800000)

ax.set(xlim=(xmin, xmax), ylim=(ymin, ymax))设置轴的显示范围
f, ax = plt.subplots(figsize=(20, 8))用来画子图，figsize 定义每个子图的大小，返回的 ax 表示子图的元祖

corrmat = df_train.corr()
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(corrmat, square=True, cmap='YlGnBu')
plt.show()

corrmat = df_train.corr()计算 f的各个feature的相关性,默认是pearson系数，返回的是DataFrame。 sns.heatmap()用来画热点图

cols = corrmat.nlargest(10, 'SalePrice').index
cm = np.corrcoef(df_train[cols].values.T)
sns.set(font_scale=1.25)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10},
                 yticklabels=cols.values, xticklabels=cols.values, cmap='YlGnBu')
plt.show()

DataFrame.nlargest():返回最大的几行
np.corrcoef(x)计算相关性系数,x的一行表示一个变量，每一列表示一个样本

train = pd.read_csv('./train.csv')
test = pd.read_csv('./test.csv')
print(f"The train data size is: {train.shape}")
print(f"The test data size is: {test.shape}")

train = train.drop(train.query('GrLivArea > 4000 and SalePrice < 300000').index)
fig, ax = plt.subplots()
ax.scatter(train['GrLivArea'], train['SalePrice'])
plt.ylabel('SalePrice', fontsize=13)
plt.xlabel('GrLivArea', fontsize=13)

train.query('GrLivArea > 4000 and SalePrice < 300000')：query语句选择数据，这个后面再具体学习

from scipy import stats
from scipy.stats import norm
sns.distplot(train['SalePrice'], fit=stats.norm)
(mu, sigma) = norm.fit(train['SalePrice'])
print(f"mu = {mu} and sigma = {sigma}")
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma= ${:.2f}'.format(mu, sigma)], loc='best')
plt.ylabel('Frequency')
plt.title('SalePrice distribution')
fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
plt.show()

(mu, sigma) = norm.fit(train['SalePrice'])：用正态分布进行参数估计
res = stats.probplot(train['SalePrice'], plot=plt):绘制 qq 图，默认是正态分布 qq 图

train['SalePrice'] = np.log1p(train['SalePrice'])
sns.distplot(train.SalePrice, fit=norm)
mu, sigma = norm.fit(train.SalePrice)
print(f"mu={mu} sigma={sigma} ")
plt.legend([f'Normal dist. (\mu={mu} sigma={sigma}'])
plt.ylabel('Frequency')
plt.title('SalePrice distribution')
fig = plt.figure()
res = stats.probplot(train.SalePrice, plot=plt)
plt.show()
y_train = train.SalePrice.values
n_train = train.shape[0]
n_test = test.shape[0]
all_data = pd.concat([train, test], ignore_index=True)
all_data.drop(['Id', 'SalePrice'], axis=1, inplace=True)
print(all_data.shape)

np.log1p(x)就是log(1+x)

all_data_na = (all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na[all_data_na > 0].sort_values(ascending=False)
all_data_na.head(100)

f, ax = plt.subplots(figsize=(10, 8))
sns.barplot(x=all_data_na.index, y=all_data_na)
plt.xticks(rotation=90)
plt.xlabel('Feature', fontsize=15)
plt.ylabel('Percent of missing values', fontsize=15)
plt.title('Percent missing data by feature', fontsize=15)
plt.show()

sns.barplot(x=all_data_na.index, y=all_data_na):画柱状图

all_data.LotFrontage = all_data.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))

num_cols = ['MasVnrArea', 'BsmtUnfSF', 'TotalBsmtSF', 'GarageCars', 'BsmtFinSF2', 'BsmtFinSF1', 'GarageArea']
for col in num_cols:
    all_data[col].fillna(0, inplace=True)

ob_cols = ["PoolQC", "MiscFeature", "Alley", "Fence", "FireplaceQu", "GarageQual", "GarageCond", "GarageFinish",
           "GarageYrBlt", "GarageType", "BsmtExposure", "BsmtCond", "BsmtQual", "BsmtFinType2", "BsmtFinType1",
           "MasVnrType"]
for col in ob_cols:
    all_data[col].fillna('None', inplace=True)

cat_cols = ["MSZoning", "BsmtFullBath", "BsmtHalfBath", "Utilities", "Functional", "Electrical", "KitchenQual",
            "SaleType", "Exterior1st", "Exterior2nd"]
for col in cat_cols:
    all_data[col].fillna(all_data[col].mode()[0], inplace=True)

mode():出现第一多的数据，有可能存在并列第一的。

all_data.isnull().sum()[all_data.isnull().sum() > 0]

NumStr = ["MSSubClass", "BsmtFullBath", "BsmtHalfBath", "HalfBath", "BedroomAbvGr", "KitchenAbvGr", "MoSold", "YrSold",
          "YearBuilt", "YearRemodAdd", "LowQualFinSF", "GarageYrBlt"]
for col in NumStr:
    all_data[col] = all_data[col].astype(str)

astype()转换类别

all_data['Utilities'].value_counts()

all_data = all_data.drop(['Utilities'], axis=1)

all_data.shape

from sklearn.preprocessing import LabelEncoder
cols = ['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', 'ExterQual', 'ExterCond', 'HeatingQC',
        'PoolQC', 'KitchenQual', 'BsmtFinType1', 'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish',
        'LandSlope', 'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond', 'YrSold',
        'MoSold', 'YearBuilt', 'YearRemodAdd', 'LowQualFinSF', 'GarageYrBlt']
lbl = LabelEncoder()
for c in cols:
    lbl.fit(list(all_data[c].values))
    all_data[c] = lbl.transform(list(all_data[c].values))

这里给把分类型数据转化为数值

all_data.shape

from scipy.stats import norm, skew
numeric_feats = all_data.select_dtypes(exclude=['object']).columns
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
print(' Skew in numerical features: \n')
skewness = pd.DataFrame({'Skew': skewed_feats})
skewness.head(10)

检查特征的偏度

skewness = skewness[abs(skewness) > 0.75]
print(f'{skewness.shape[0]} skewed features')
from scipy.special import boxcox1p
skewed_feats = skewness.index
lam = 0.15
for feat in skewed_feats:
    all_data[feat] = boxcox1p(all_data[feat], lam)

用box-cox变换来处理偏度大的数据

all_data = pd.get_dummies(all_data)
print(all_data.shape)

from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
train = all_data[:n_train]
test = all_data[n_train:]

def rmsle_cv(model):
    rmse = np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv=5))
    return rmse

lasso = make_pipeline(RobustScaler(), Lasso(alpha=0.0005))
ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)

Lasso()：回归，就是带L1惩罚函数的线性回归
Ridge()：岭回归，就是带L2惩罚函数的线性回归
ElasticNet：Lasso 和 Ridge 的结合
KernelRidge：核函数线性回归

GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
                                   max_depth=4, max_features='sqrt',
                                   min_samples_leaf=15, min_samples_split=10,
                                   loss='huber')

GradientBoostingRegressor：新的 model 不断拟合残差

from sklearn.model_selection import GridSearchCV
class grid():
    def __init__(self, model):
        self.model = model
    def grid_set(self, X, y, param_grid):
        grid_search = GridSearchCV(self.model, param_grid, cv=5, scoring='neg_mean_squared_error')
        grid_search.fit(X, y)
        print(grid_search.best_params_, np.sqrt(-grid_search.best_score_))
        grid_search.cv_results_['mean_test_score'] = np.sqrt(-grid_search.cv_results_['mean_test_score'])
        print(pd.DataFrame(grid_search.cv_results_)[['params', 'mean_test_score', 'std_test_score']])

GridSearchCV用来做模型超参数的选择

grid(Lasso()).grid_set(train, y_train, {'alpha': [0.0004, 0.0005, 0.0007, 0.0003, 0.0009, 0.0008], 'max_iter': [10000]})

class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
    def __init__(self, base_models, meta_model, n_folds=5):
        print("init")
        self.base_models = base_models
        self.meta_model = meta_model
        self.n_folds = n_folds
        self.base_models_ = None
        self.meta_model_ = None
    def fit(self, X, y):
        print("fit")
        self.base_models_ = [list() for x in self.base_models]
        self.meta_model_ = clone(self.meta_model)
        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))
        for i, model in enumerate(self.base_models):
            print(f"progress: {i} /{len(self.base_models)}")
            for train_index, holdout_index in kfold.split(X, y):
                instance = clone(model)
                self.base_models_[i].append(instance)
                instance.fit(X[train_index], y[train_index])
                y_pred = instance.predict(X[holdout_index])
                out_of_fold_predictions[holdout_index, i] = y_pred
                print("_", end="")
            print()
        self.meta_model_.fit(out_of_fold_predictions, y)
        return self
    def predict(self, X):
        print("predict")
        print("")
        meta_features = np.column_stack([
            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)
            for base_models in self.base_models_])
        return self.meta_model_.predict(meta_features)

# stacked_averaged_models = StackingAveragedModels(base_models=(ENet, GBoost, KRR), meta_model=lasso)
# score = rmsle_cv(stacked_averaged_models)
# print("Stacking Averaged models score: {:.4f} ({:.4f})".format(score.mean(), score.std()))

stacked_averaged_models = StackingAveragedModels(base_models=(ENet, GBoost, KRR), meta_model=lasso)
stacked_averaged_models.fit(train.values,y_train)

最终选择好模型以及模型参数后 需要在所有的训练数据时上进行一次训练。

test1 = pd.read_csv('test.csv')
pred = np.exp(stacked_averaged_models.predict(test))
result = pd.DataFrame({'Id': test1.Id, 'SalePrice': pred})
result.to_csv("submission.csv", index=False)
result