def xgb_cval(learning_rate,
n_estimators,max_depth,
subsample,colsample_bytree,
gamma,
reg_alpha,reg_lambda,
scale_pos_weight
):
# params
global df_train, ser_ytrain
np.random.seed(SEED)
n_splits = 5
# make integer values as integer
n_estimators = int(n_estimators)
max_depth = int(max_depth)
# fix the cap
subsample = max(min(subsample, 1), 0)
colsample_bytree = max(min(colsample_bytree, 1), 0)
learning_rate = max(learning_rate, 0)
gamma = max(gamma, 0)
reg_alpha = max(reg_alpha, 0)
reg_lambda = max(reg_lambda, 0)
scale_pos_weight = max(scale_pos_weight, 0)
# eval metric
METRIC_NAME = 'Profit'
EVAL_METRIC = metricProfit # auc, aucpr
disable_default_eval_metric = 1 if EVAL_METRIC == metricProfit else 0
# define the model
model = xgb.XGBClassifier(
learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth=max_depth,
subsample=subsample,
colsample_bytree=colsample_bytree,
gamma=gamma,
reg_alpha=reg_alpha,
reg_lambda=reg_lambda,
scale_pos_weight=scale_pos_weight,
disable_default_eval_metric=disable_default_eval_metric
)
# initialize KFold and progress bar
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=SEED)
arr_eval = np.zeros((n_estimators, n_splits))
oof_predictions = np.empty(len(df_train))
progress_bar = tqdm(
enumerate(skf.split(ytrain, ytrain)),
total=n_splits,
leave=False
)
# cross-validation
for fold, (idx_tr, idx_vd) in progress_bar:
# data to fit
Xtr, Xvd = df_train.iloc[idx_tr], df_train.iloc[idx_vd]
ytr, yvd = ser_ytrain.iloc[idx_tr], ser_ytrain.iloc[idx_vd]
# model
eval_set = [(Xtr, ytr), (Xvd, yvd)]
model.fit(Xtr, ytr,
eval_set=eval_set,
eval_metric=EVAL_METRIC,
verbose=False
)
# best fold
arr_eval[:, fold] = model.evals_result_["validation_1"][METRIC_NAME]
best_round = np.argsort(arr_eval[:, fold])[0]
# progress bar update
txt = 'Fold #{}: {:.5f}'.format(fold, arr_eval[:, fold][best_round])
progress_bar.set_description(txt, refresh=True)
# find best round from best mean
mean_eval, std_eval = np.mean(arr_eval, axis=1), np.std(arr_eval, axis=1)
best_round = np.argsort(mean_eval)[0]
val_maximize = mean_eval[best_round]
return val_maximize
#=============================================
pbounds = {
# note: bayes_opt makes all integers float, we need to convert later.
'n_estimators' : (100, 1000),
'max_depth' : (3,10),
# other params
'learning_rate' : (0.01, 1.0),
'subsample' : (0.6, 1.0),
'colsample_bytree': (0.7, 1.0),
'gamma' : (0, 5),
# regularization
'reg_alpha' : (0.001,1),
'reg_lambda' : (0,10),
# class imbalance
'scale_pos_weight': (2,5),
}
optimizer = BayesianOptimization(
f=xgb_cval,
pbounds=pbounds,
random_state=SEED,
)
path_log = "../logs/bo_logs_xgb.json"
# optimizer.maximize(n_iter=20, acq= 'ucb')
# print("Final result:", optimizer.max)
# logger = JSONLogger(path='xgb_bayes.log')
# optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
out = """
params= {'colsample_bytree': 0.8680868060574589,
'gamma': 2.212263351683875, 'learning_rate': 0.6781363901873738,
'max_depth': 6.237263193088246, 'n_estimators': 218.88182747451702,
'reg_alpha': 0.008319041339960368, 'reg_lambda': 6.773808680400034,
'scale_pos_weight': 4.286709496818984, 'subsample': 0.8881322862290755}
""";
