Kernel Author:
Bhishan Poudel, Ph.D Astrophysics .
Bhishan Poudel, Ph.D Astrophysics .
Data Description¶
In this project, we will predict the probability that an auto insurance policy holder files a claim. This a binary classification problem.
We have more than half a million records and 59 features (including already calculated features).
binary features: _bin
categorical features: _cat
continuous or ordinal feafures: ind, reg, car, calc
missing values: -1
Fullforms
ind = individual
reg = registration
car = car
calc = calculatedThe target columns signifies whether or not a claim was filed for that policy holder.
Evaluation Metric¶
- https://www.kaggle.com/c/porto-seguro-safe-driver-prediction/overview/evaluation
- https://en.wikipedia.org/wiki/Gini_coefficient
From this graph of wikipedia G = A / (A+B). Gini index varies between 0 and 1. Here we have only binary options: rich and poor.
x-axis= number of people (cumulative sum)
y-axis = total income (cumulative sum)
0 = complete equality of richness
1 = complete inequality of richness
This competition
0 = random guessing
1 = maximum score (also remember 2*1-1 = 1 when maximum auc is 1).If we calculate gini from gini = 2*auc -1 it has range (-1,1).
For AUC:
worst binary classifier AUC = 0.5
perfect binary classifier AUC = 1
If AUC is less than below, simply simply invert 0 <==> 1 then we will get roc auc score between 0.5 and 1.0Imports¶
In [ ]:
import os
import time
import gc
import numpy as np
import pandas as pd
import scipy
from scipy import stats
import seaborn as sns
sns.set(color_codes=True)
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
time_start_notebook = time.time()
SEED=100
print([(x.__name__,x.__version__) for x in [np, pd,sns,matplotlib]])
from scipy import sparse as ssp
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
In [ ]:
# Google colab
In [ ]:
%%capture
# capture will not print in notebook
import os
import sys
ENV_COLAB = 'google.colab' in sys.modules
if ENV_COLAB:
#### print
print('Environment: Google Colaboratory.')
# NOTE: If we update modules in gcolab, we need to restart runtime.
Useful Functions¶
In [ ]:
df_eval = pd.DataFrame({'Model': [],
'Description':[],
'Accuracy':[],
'Precision':[],
'Recall':[],
'F1':[],
'AUC':[],
'NormalizedGini': []
})
Load the data¶
In [ ]:
df = pd.read_csv('https://github.com/bhishanpdl/Datasets/blob/master/'
'Porto_seguro_safe_driver_prediction/train.csv.zip?raw=true',compression='zip')
print(df.shape)
df.head()
Out[ ]:
In [ ]:
"""
Comment about file size:
The data is large, it has 595k records and 59 features.
ps = porto seguro
_bin = binary feature
_cat = categorical feature
continuous or ordinal: ind, reg, car, calc
""";
In [ ]:
target = 'target'
Data Processing¶
In [ ]:
# all features except target
cols_all= df.columns.drop(target).to_list()
# categorical features except later created count
cols_cat = [c for c in cols_all if ('cat' in c and 'count' not in c)]
# we exclude calc features in numeric features
cols_num = [c for c in cols_all if ('cat' not in c and 'calc' not in c)]
print(cols_num)
In [ ]:
# missing count
df['missing'] = df.eq(-1).sum(axis=1).astype(float)
cols_num.append('missing')
# individual features
cols_ind = [c for c in cols_all if 'ind' in c]
df['ind_concat'] = df[cols_ind].astype(str).agg('_'.join,axis=1)
# cat count features from whole data
cols_cat_count = []
for col in cols_cat + ['ind_concat']:
d = df[col].value_counts().to_dict()
df[f'{col}_count'] = df[col].apply(lambda x:d.get(x,0))
cols_cat_count.append(f'{col}_count')
# after creating count of ind concat, drop it
df = df.drop('ind_concat',axis=1)
# one hot encoding
df = pd.get_dummies(df, columns=cols_cat, drop_first=True)
Train-test Split with Stratify¶
In [ ]:
from sklearn.model_selection import train_test_split
df_Xtrain, df_Xtest, ser_ytrain, ser_ytest = train_test_split(
df.drop(target,axis=1),df[target],
test_size=0.2,random_state=SEED, stratify=df[target])
# backup and delete id
cols_drop = ['id']
train_id = df_Xtrain[cols_drop]
test_id = df_Xtest[cols_drop]
df_Xtrain = df_Xtrain.drop(cols_drop,axis=1)
df_Xtest = df_Xtest.drop(cols_drop,axis=1)
Xtrain = df_Xtrain.to_numpy()
ytrain = ser_ytrain.to_numpy().ravel()
Xtest = df_Xtest.to_numpy()
ytest = ser_ytest.to_numpy().ravel()
# make sure no nans and no strings
print(Xtrain.sum().sum())
Training Data¶
In [ ]:
pd.set_option('display.max_columns',250)
df_Xtrain.head()
Out[ ]:
In [ ]:
# df_Xtrain.columns # make sure there are no id and index
In [ ]:
Xtr = Xtrain
Xtx = Xtest
ytr = ytrain
ytx = ytest
print(Xtr.shape, Xtx.shape)
In [ ]:
ser_ytest.value_counts(normalize=True)
Out[ ]:
Evaluation Metric¶
In [ ]:
# @numba.jit fails and falls back to object mode.
from sklearn import metrics
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini / (ntrue * (n - ntrue))
return gini
def gini(y, pred):
fpr, tpr, thr = metrics.roc_curve(y, pred, pos_label=1)
g = 2 * metrics.auc(fpr, tpr) -1
return g
def gini_lgb(preds, dtrain):
y = list(dtrain.get_label())
score = gini(y, preds) / gini(y, y)
return 'gini', score, True
Modelling: LightGBM¶
In [ ]:
import joblib
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from lightgbm import LGBMClassifier
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.metrics import accuracy_score,precision_score, recall_score
from sklearn.metrics import f1_score, roc_auc_score
In [ ]:
clf_lgb = lgb.LGBMClassifier(random_state=SEED)
clf_lgb
Out[ ]:
In [ ]:
# fit and predict
clf_lgb.fit(Xtr, ytr)
ypreds = clf_lgb.predict(Xtx)
# model evaluation
average = 'binary'
model_name = 'lgb'
desc = 'default'
row_eval = [model_name,desc,
accuracy_score(ytx, ypreds),
precision_score(ytx, ypreds, average=average),
recall_score(ytx, ypreds, average=average),
f1_score(ytx, ypreds, average=average),
roc_auc_score(ytx, ypreds),
2 * roc_auc_score(ytx, ypreds) - 1,
]
df_eval.loc[len(df_eval)] = row_eval
df_eval = df_eval.drop_duplicates()
display(df_eval)
In [ ]:
# changing threshold
yprobs = clf_lgb.predict_proba(Xtx)[:,1]
thresholds = np.arange(0, 1, 0.001)
auc_scores = [roc_auc_score(ytx, [0 if i <= thr else 1 for i in yprobs])
for thr in thresholds]
idx = np.argmax(auc_scores)
best_thr = thresholds[idx]
ypreds = [0 if i <= best_thr else 1 for i in yprobs]
# model evaluation
average = 'binary'
model_name = 'lgb'
desc = 'default, threshold change'
row_eval = [model_name,desc,
accuracy_score(ytx, ypreds),
precision_score(ytx, ypreds, average=average),
recall_score(ytx, ypreds, average=average),
f1_score(ytx, ypreds, average=average),
roc_auc_score(ytx, ypreds),
2 * roc_auc_score(ytx, ypreds) - 1,
]
df_eval.loc[len(df_eval)] = row_eval
df_eval = df_eval.drop_duplicates()
display(df_eval)
Lightgbm with tuned parameters¶
In [ ]:
learning_rate = 0.1
num_leaves = 15
min_data_in_leaf = 2000
feature_fraction = 0.6
num_boost_round = 10000
params = {"objective": "binary",
"boosting_type": "gbdt",
"learning_rate": learning_rate,
"num_leaves": num_leaves,
"max_bin": 256,
"feature_fraction": feature_fraction,
"verbosity": 0,
"drop_rate": 0.1,
"is_unbalance": False,
"max_drop": 50,
"min_child_samples": 10,
"min_child_weight": 150,
"min_split_gain": 0,
"subsample": 0.9
}
In [ ]:
time_start = time.time()
K = 5
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)
ser_vdprobs = ser_ytrain * 0
ser_vdpreds = ser_ytrain * 0 # all index of validation makes same as train
txprobs = ser_ytest.to_numpy().ravel() * 0.0 # make it zero, to add values from CV
increase = True
best_thr_lst = []
for i, (idx_tr, idx_vd) in enumerate(skf.split(Xtrain, ytrain)):
# print
print( "\nFold ", i)
# data for this fold
df_Xtr = df_Xtrain.iloc[idx_tr,:].copy()
ser_ytr = ser_ytrain.iloc[idx_tr].copy()
df_Xvd = df_Xtrain.iloc[idx_vd,:].copy()
ser_yvd = ser_ytrain.iloc[idx_vd].copy()
df_Xtx = df_Xtest.copy() # we add target encoding features to test
clf = lgb.LGBMClassifier(random_state=SEED,**params)
# Upsample during cross validation to avoid having the same samples
# in both train and validation sets
# Validation set is not up-sampled to monitor overfitting
if increase:
# Get positive examples
pos = pd.Series(ser_ytr == 1)
# Add positive examples
df_Xtr = pd.concat([df_Xtr, df_Xtr.loc[pos]], axis=0)
ser_ytr = pd.concat([ser_ytr, ser_ytr.loc[pos]], axis=0)
# Shuffle data
idx = np.arange(len(df_Xtr))
np.random.seed(SEED)
np.random.shuffle(idx)
df_Xtr = df_Xtr.iloc[idx]
ser_ytr = ser_ytr.iloc[idx]
fit_model = clf.fit(df_Xtr, ser_ytr,
eval_set=[(df_Xtr, ser_ytr), (df_Xvd, ser_yvd)],
eval_metric='auc', # gini_lgb gives error.
early_stopping_rounds=None,
verbose=False)
# valid probs for this fold
vdprobs = fit_model.predict_proba(df_Xvd)[:,1]
print( " Gini (from probs) : ", eval_gini(ser_yvd, vdprobs) )
# find the best threshold using validation data
thresholds = np.arange(0, 1, 0.001)
# using auc instead of gini to find best threshold
auc_scores = [roc_auc_score(ser_yvd, [0 if i <= thr else 1 for i in vdprobs])
for thr in thresholds]
idx = np.argmax(auc_scores)
best_thr = thresholds[idx]
best_auc = auc_scores[idx]
best_thr_lst.append(best_thr)
vdpreds = [0 if i <= best_thr else 1 for i in vdprobs]
print(f' Best threshold : {best_thr:.3f}')
print(f' Best AUC : {best_auc:.5f}')
print( " Gini (from preds) : ", eval_gini(ser_yvd, vdpreds) )
ser_vdprobs.iloc[idx_vd] = vdprobs
ser_vdpreds.iloc[idx_vd] = vdpreds
# accumulate probs
txprobs += fit_model.predict_proba(df_Xtx)[:,1] # test probs
# clean memory
del df_Xtr, ser_ytr, df_Xvd, ser_yvd, df_Xtx
# time taken
time_taken = time.time() - time_start
h,m = divmod(time_taken,60*60)
print(' Time taken : {:.0f} hr '\
'{:.0f} min {:.0f} secs'.format(h, *divmod(m,60)))
txprobs /= K # avg test probs
best_thr = np.mean(best_thr_lst)
txpreds = [0 if i <= best_thr else 1 for i in txprobs]
print()
print(f'Best thresholds for fold : {best_thr_lst}')
print("Gini for full training set (from probs) : ",
eval_gini(ser_ytrain, ser_vdprobs))
print("Gini for full training set (from preds): ",
eval_gini(ser_ytrain, ser_vdpreds))
# predictions
ypreds = txpreds
# model evaluation
average = 'binary'
row_eval = ['lgb','skf cv, upsample, threshold change',
accuracy_score(ytest, ypreds),
precision_score(ytest, ypreds, average=average),
recall_score(ytest, ypreds, average=average),
f1_score(ytest, ypreds, average=average),
roc_auc_score(ytest, ypreds),
2 * roc_auc_score(ytest, ypreds) -1,
]
df_eval.loc[len(df_eval)] = row_eval
df_eval = df_eval.drop_duplicates()
display(df_eval)
In [ ]:
time_start = time.time()
K = 5
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)
ser_vdprobs = ser_ytrain * 0
ser_vdpreds = ser_ytrain * 0 # all index of validation makes same as train
txprobs = ser_ytest.to_numpy().ravel() * 0.0 # make it zero, to add values from CV
increase = False
best_thr_lst = []
for i, (idx_tr, idx_vd) in enumerate(skf.split(Xtrain, ytrain)):
# print
print( "\nFold ", i)
# data for this fold
df_Xtr = df_Xtrain.iloc[idx_tr,:].copy()
ser_ytr = ser_ytrain.iloc[idx_tr].copy()
df_Xvd = df_Xtrain.iloc[idx_vd,:].copy()
ser_yvd = ser_ytrain.iloc[idx_vd].copy()
df_Xtx = df_Xtest.copy() # we add target encoding features to test
clf = lgb.LGBMClassifier(random_state=SEED,**params)
# Upsample during cross validation to avoid having the same samples
# in both train and validation sets
# Validation set is not up-sampled to monitor overfitting
if increase:
# Get positive examples
pos = pd.Series(ser_ytr == 1)
# Add positive examples
df_Xtr = pd.concat([df_Xtr, df_Xtr.loc[pos]], axis=0)
ser_ytr = pd.concat([ser_ytr, ser_ytr.loc[pos]], axis=0)
# Shuffle data
idx = np.arange(len(df_Xtr))
np.random.seed(SEED)
np.random.shuffle(idx)
df_Xtr = df_Xtr.iloc[idx]
ser_ytr = ser_ytr.iloc[idx]
fit_model = clf.fit(df_Xtr, ser_ytr,
eval_set=[(df_Xtr, ser_ytr), (df_Xvd, ser_yvd)],
eval_metric='auc', # gini_lgb gives error.
early_stopping_rounds=None,
verbose=False)
# valid probs for this fold
vdprobs = fit_model.predict_proba(df_Xvd)[:,1]
print( " Gini (from probs) : ", eval_gini(ser_yvd, vdprobs) )
# find the best threshold using validation data
thresholds = np.arange(0, 1, 0.001)
# using auc instead of gini to find best threshold
auc_scores = [roc_auc_score(ser_yvd, [0 if i <= thr else 1 for i in vdprobs])
for thr in thresholds]
idx = np.argmax(auc_scores)
best_thr = thresholds[idx]
best_auc = auc_scores[idx]
best_thr_lst.append(best_thr)
vdpreds = [0 if i <= best_thr else 1 for i in vdprobs]
print(f' Best threshold : {best_thr:.3f}')
print(f' Best AUC : {best_auc:.5f}')
print( " Gini (from preds) : ", eval_gini(ser_yvd, vdpreds) )
ser_vdprobs.iloc[idx_vd] = vdprobs
ser_vdpreds.iloc[idx_vd] = vdpreds
# accumulate probs
txprobs += fit_model.predict_proba(df_Xtx)[:,1] # test probs
# clean memory
del df_Xtr, ser_ytr, df_Xvd, ser_yvd, df_Xtx
# time taken
time_taken = time.time() - time_start
h,m = divmod(time_taken,60*60)
print(' Time taken : {:.0f} hr '\
'{:.0f} min {:.0f} secs'.format(h, *divmod(m,60)))
txprobs /= K # avg test probs
best_thr = np.mean(best_thr_lst)
txpreds = [0 if i <= best_thr else 1 for i in txprobs]
print()
print(f'Best thresholds for fold : {best_thr_lst}')
print("Gini for full training set (from probs) : ",
eval_gini(ser_ytrain, ser_vdprobs))
print("Gini for full training set (from preds): ",
eval_gini(ser_ytrain, ser_vdpreds))
# predictions
ypreds = txpreds
# model evaluation
average = 'binary'
row_eval = ['lgb','skf cv, NO upsample, threshold change',
accuracy_score(ytest, ypreds),
precision_score(ytest, ypreds, average=average),
recall_score(ytest, ypreds, average=average),
f1_score(ytest, ypreds, average=average),
roc_auc_score(ytest, ypreds),
2 * roc_auc_score(ytest, ypreds) -1,
]
df_eval.loc[len(df_eval)] = row_eval
df_eval = df_eval.drop_duplicates()
display(df_eval)
In [ ]: