XGBoost

XGBoost (Extreme Gradient Boosting) is an implementation of gradient boosting optimized for performance and speed. It is very popular in data science competitions.

Installation

pip install xgboost

Basic Usage

import xgboost as xgb
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.2, random_state=42
)

model = xgb.XGBClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    random_state=42,
    eval_metric='mlogloss'
)
model.fit(X_train, y_train)

y_pred = model.predict(X_test)
print(f"Accuracy: {accuracy_score(y_test, y_pred):.4f}")

XGBoost Key Features

1. Built-in Regularization

# L1 regularization
model_l1 = xgb.XGBClassifier(reg_alpha=0.1)

# L2 regularization
model_l2 = xgb.XGBClassifier(reg_lambda=1.0)

2. Handling Missing Values

import xgboost as xgb
import numpy as np

X_train = np.array([[1, 2], [3, np.nan], [5, 6], [7, 8]])
y_train = np.array([0, 1, 0, 1])

model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss')
model.fit(X_train, y_train)
print("XGBoost handles missing values natively!")

3. Tree Pruning

model_pruned = xgb.XGBClassifier(
    max_depth=3,
    min_child_weight=1,
    gamma=0.1,  # pruning parameter
    grow_policy='lossguide'  # best-first search
)

Example: Regression

import xgboost as xgb
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score

housing = fetch_california_housing()
X_train, X_test, y_train, y_test = train_test_split(
    housing.data, housing.target, test_size=0.2, random_state=42
)

model = xgb.XGBRegressor(
    n_estimators=200,
    learning_rate=0.1,
    max_depth=6,
    random_state=42
)
model.fit(X_train, y_train)

y_pred = model.predict(X_test)
print(f"MSE: {mean_squared_error(y_test, y_pred):.4f}")
print(f"R²: {r2_score(y_test, y_pred):.4f}")

Cross Validation with XGBoost

import xgboost as xgb
from sklearn.datasets import load_iris
from sklearn.model_selection import cross_val_score

iris = load_iris()
X, y = iris.data, iris.target

model = xgb.XGBClassifier(
    n_estimators=100,
    learning_rate=0.1,
    use_label_encoder=False,
    eval_metric='mlogloss',
    random_state=42
)

scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
print(f"CV Accuracy: {scores.mean():.4f} (+/- {scores.std():.4f})")

Feature Importance

import xgboost as xgb
from sklearn.datasets import load_wine
import pandas as pd

wine = load_wine()
X, y = wine.data, wine.target

model = xgb.XGBClassifier(n_estimators=100, random_state=42,
                           use_label_encoder=False, eval_metric='mlogloss')
model.fit(X, y)

importance = pd.Series(model.feature_importances_, index=wine.feature_names)
print(importance.sort_values(ascending=False))

Important Hyperparameters

Early Stopping

import xgboost as xgb
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.2, random_state=42
)

model = xgb.XGBClassifier(
    n_estimators=500,
    learning_rate=0.1,
    early_stopping_rounds=10,
    eval_metric='mlogloss',
    random_state=42
)

model.fit(
    X_train, y_train,
    eval_set=[(X_test, y_test)],
    verbose=False
)

print(f"Best iteration: {model.best_iteration}")

XGBoost Advantages

• Very fast and efficient
• Built-in regularization
• Handles missing values
• Parallel processing
• Built-in feature importance
• Built-in early stopping
• Winner of many Kaggle competitions

Next chapter: Neural Networks, the foundation of deep learning.

Next: Neural Network →