Performance Benchmarking

This tutorial compares Perpetual with a heavily tuned LightGBM model across different datasets (California Housing and Cover Type). Perpetual achieves state-of-the-art results without any hyperparameter tuning.

import sys
from importlib.metadata import version

import numpy as np
import optuna
from lightgbm import LGBMClassifier, LGBMRegressor
from perpetual import PerpetualBooster
from sklearn.datasets import fetch_california_housing, fetch_covtype
from sklearn.metrics import log_loss, mean_squared_error
from sklearn.model_selection import cross_validate, train_test_split

print(sys.version)

print(f"numpy: {version('numpy')}")
print(f"optuna: {version('optuna')}")
print(f"lightgbm: {version('lightgbm')}")
print(f"scikit-learn: {version('scikit-learn')}")
print(f"perpetual: {version('perpetual')}")

task_is_cal_housing = False  # change to False for Cover Types task.

seed = 0  # average results are reported for 5 seeds -> [0, 1, 2, 3, 4]
n_estimators = 1  # results are reported for 100, 300, 1000 n_estimators.
n_trials = 1

if task_is_cal_housing:
    data, target = fetch_california_housing(return_X_y=True, as_frame=True)
    scoring = "neg_mean_squared_error"
    metric_function = mean_squared_error
    metric_name = "mse"
    LGBMBooster = LGBMRegressor
    objective_type = "SquaredLoss"
else:
    data, target = fetch_covtype(return_X_y=True, as_frame=True)
    scoring = "neg_log_loss"
    metric_function = log_loss
    metric_name = "log_loss"
    LGBMBooster = LGBMClassifier
    objective_type = "LogLoss"

X_train, X_test, y_train, y_test = train_test_split(
    data, target, test_size=0.2248, random_state=seed
)

print(f"len(X_train): {len(X_train)}")
print(f"len(X_test): {len(X_test)}")

best_cv_results = None
cv_results = None


def save_best_cv_results(study, trial):
    global best_cv_results
    if study.best_trial.number == trial.number:
        best_cv_results = cv_results

def objective_function(trial):
    global cv_results
    params = {
        "seed": seed,
        "verbosity": -1,
        "n_estimators": n_estimators,
        "learning_rate": trial.suggest_float("learning_rate", 0.001, 0.5, log=True),
        "min_split_gain": trial.suggest_float("min_split_gain", 1e-6, 1.0, log=True),
        "reg_alpha": trial.suggest_float("reg_alpha", 1e-6, 1.0, log=True),
        "reg_lambda": trial.suggest_float("reg_lambda", 1e-6, 1.0, log=True),
        "colsample_bytree": trial.suggest_float("colsample_bytree", 0.2, 1.0),
        "subsample": trial.suggest_float("subsample", 0.2, 1.0),
        "subsample_freq": trial.suggest_int("subsample_freq", 1, 10),
        "max_depth": trial.suggest_int("max_depth", 3, 33),
        "num_leaves": trial.suggest_int("num_leaves", 2, 1024),
        "min_child_samples": trial.suggest_int("min_child_samples", 1, 100),
    }
    model = LGBMBooster(**params)
    cv_results = cross_validate(
        model,
        X_train,
        y_train,
        cv=5,
        scoring=scoring,
        return_train_score=True,
        return_estimator=True,
    )
    return -1 * np.mean(cv_results["test_score"])

sampler = optuna.samplers.TPESampler(seed=seed)
study = optuna.create_study(direction="minimize", sampler=sampler)

study.optimize(objective_function, n_trials=n_trials, callbacks=[save_best_cv_results])

print(f"Number of finished trials: {len(study.trials)}")
print("Best trial:")
print(f"  Number: {study.best_trial.number}")
print(f"  Value: {study.best_trial.value}")
print("  Params: ")
for key, value in study.best_trial.params.items():
    print(f"    {key}: {value}")

print(f"CV train scores: {-1 * best_cv_results['train_score']}")
print(
    f"CV train scores average : {round(np.mean(-1 * best_cv_results['train_score']), 6)}"
)
print(f"CV valid scores: {-1 * best_cv_results['test_score']}")
print(
    f"CV valid scores average : {round(np.mean(-1 * best_cv_results['test_score']), 6)}"
)

models = best_cv_results["estimator"]

for i, model in enumerate(models):
    y_pred = (
        model.predict_proba(X_train)
        if metric_name == "log_loss"
        else model.predict(X_train)
    )
    print(
        f"Model {i}, train {metric_name}: {round(metric_function(y_train, y_pred), 6)}"
    )

for i, model in enumerate(models):
    y_pred = (
        model.predict_proba(X_test)
        if metric_name == "log_loss"
        else model.predict(X_test)
    )
    print(f"Model {i}, test {metric_name}: {round(metric_function(y_test, y_pred), 6)}")

if metric_name == "log_loss":
    y_pred = np.mean([model.predict_proba(X_train) for model in models], axis=0)
else:
    y_pred = np.mean([model.predict(X_train) for model in models], axis=0)
print(f"Train {metric_name}: {round(metric_function(y_train, y_pred), 6)}")

if metric_name == "log_loss":
    y_pred = np.mean([model.predict_proba(X_test) for model in models], axis=0)
else:
    y_pred = np.mean([model.predict(X_test) for model in models], axis=0)
print(f"Test {metric_name}: {round(metric_function(y_test, y_pred), 6)}")

LightGBM n_estimators	Seed	LightGBM mse	LightGBM cpu time
100	0	0.186588	729
100	1	0.194348	1294
100	2	0.197862	990
100	3	0.188629	1143
100	4	0.194338	860
100	avg	0.192196	978
300	0	0.185100	2282
300	1	0.192767	3650
300	2	0.190481	2746
300	3	0.182359	2782
300	4	0.191614	3871
300	avg	0.188464	3066
1000	0	0.179158	9615
1000	1	0.190866	7258
1000	2	0.188030	10997
1000	3	0.179903	7636
1000	4	0.190033	8095
1000	avg	0.185598	8720

model = PerpetualBooster(objective=objective_type)

model.budget = 1.0
model.fit(X_train, y_train)

if metric_name == "log_loss":
    y_pred = model.predict_proba(X_test)
else:
    y_pred = model.predict(X_test)
print(f"Test {metric_name}: {round(metric_function(y_test, y_pred), 6)}")

model.number_of_trees