Gradient Boosting

Gradient Boosting is an ensemble method that builds models sequentially, where each new model corrects the errors (residuals) of the previous model.

Basic Concept

1. Create an initial model (bias)
2. Compute the residuals (errors) of the current model
3. Train a new model to predict the residuals
4. Update the prediction: prediction = prediction + learning_rate × residual
5. Repeat until n_estimators is reached

from sklearn.ensemble import GradientBoostingRegressor
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

X, y = make_regression(n_samples=500, n_features=10, noise=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

gb = GradientBoostingRegressor(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    random_state=42
)
gb.fit(X_train, y_train)

y_pred = gb.predict(X_test)
print(f"MSE: {mean_squared_error(y_test, y_pred):.4f}")
print(f"R²: {gb.score(X_test, y_test):.4f}")

Sequential Learning

Each new tree focuses on the errors of the previous tree:

import numpy as np

# Simulation of simple gradient boosting
def simple_gb(X, y, n_trees=5, lr=0.1):
    predictions = np.zeros(len(y))

    for i in range(n_trees):
        residuals = y - predictions
        # In practice: train a tree to predict the residuals
        # Here it is just a simulation
        tree_pred = residuals * 0.5  # simplified
        predictions += lr * tree_pred
        mse = np.mean((y - predictions) ** 2)
        print(f"Tree {i+1}: MSE = {mse:.4f}")

np.random.seed(42)
y = np.random.randn(100)
simple_gb(None, y, n_trees=10)

Learning Rate

The learning rate determines the contribution of each tree. Smaller → more trees required.

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.datasets import load_iris
from sklearn.model_selection import cross_val_score
import numpy as np

iris = load_iris()
X, y = iris.data, iris.target

learning_rates = [0.01, 0.05, 0.1, 0.2, 0.5]
for lr in learning_rates:
    gb = GradientBoostingClassifier(
        n_estimators=100,
        learning_rate=lr,
        random_state=42
    )
    scores = cross_val_score(gb, X, y, cv=5)
    print(f"lr={lr:5.2f}: {scores.mean():.4f}")

Loss Function

from sklearn.ensemble import GradientBoostingRegressor

# Regression with different losses
for loss in ['squared_error', 'absolute_error', 'huber']:
    gb = GradientBoostingRegressor(loss=loss, random_state=42)
    gb.fit(X_train, y_train)
    print(f"Loss {loss:15s}: R² = {gb.score(X_test, y_test):.4f}")

Feature Importance

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.datasets import load_wine
import pandas as pd

wine = load_wine()
X, y = wine.data, wine.target

gb = GradientBoostingClassifier(n_estimators=100, random_state=42)
gb.fit(X, y)

importance = pd.Series(gb.feature_importances_, index=wine.feature_names)
importance = importance.sort_values(ascending=False)

print("Feature Importance:")
print(importance)

Staged Predictions

Viewing predictions at each iteration:

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.datasets import load_iris
from sklearn.metrics import accuracy_score
import numpy as np

iris = load_iris()
X, y = iris.data, iris.target

gb = GradientBoostingClassifier(n_estimators=100, random_state=42)
gb.fit(X, y)

# Staged predictions
train_scores = []
for y_pred in gb.staged_predict(X):
    train_scores.append(accuracy_score(y, y_pred))

optimal_trees = np.argmax(train_scores) + 1
print(f"Optimal trees: {optimal_trees}")
print(f"Best accuracy: {max(train_scores):.4f}")

Example: Price Prediction

from sklearn.ensemble import GradientBoostingRegressor
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score

housing = fetch_california_housing()
X_train, X_test, y_train, y_test = train_test_split(
    housing.data, housing.target, test_size=0.2, random_state=42
)

gb = GradientBoostingRegressor(
    n_estimators=200,
    learning_rate=0.1,
    max_depth=5,
    random_state=42
)
gb.fit(X_train, y_train)

y_pred = gb.predict(X_test)
print(f"MSE: {mean_squared_error(y_test, y_pred):.4f}")
print(f"R²: {r2_score(y_test, y_pred):.4f}")

Important Hyperparameters

Next chapter: XGBoost, a faster and more efficient implementation of gradient boosting.

Next: XGBoost →