Data Preprocessing

Data Preprocessing is a crucial step before modeling. Real-world data is rarely perfect — there are missing values, outliers, features with different scales, and other problems that need to be handled.

Missing Values

import pandas as pd
import numpy as np

df = pd.DataFrame({
    'age': [25, 30, np.nan, 35, 40],
    'income': [50000, np.nan, 75000, np.nan, 90000],
    'score': [1, 2, 3, np.nan, 5]
})

print("Missing values:")
print(df.isnull().sum())

Handling Strategies

from sklearn.impute import SimpleImputer, KImputer

# Mean imputation
imputer_mean = SimpleImputer(strategy='mean')
df_imputed = pd.DataFrame(imputer_mean.fit_transform(df), columns=df.columns)

# Median imputation
imputer_median = SimpleImputer(strategy='median')
df_median = pd.DataFrame(imputer_median.fit_transform(df), columns=df.columns)

# KNN Imputation
from sklearn.impute import KNNImputer
imputer_knn = KNNImputer(n_neighbors=2)
df_knn = pd.DataFrame(imputer_knn.fit_transform(df), columns=df.columns)

Scaling

StandardScaler (Z-score normalization)

from sklearn.preprocessing import StandardScaler
import numpy as np

X = np.array([[100, 0.001], [200, 0.002], [300, 0.003]])
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

print(f"Mean: {X_scaled.mean(axis=0)}")
print(f"Std: {X_scaled.std(axis=0)}")

MinMaxScaler

from sklearn.preprocessing import MinMaxScaler

scaler = MinMaxScaler()
X_scaled = scaler.fit_transform(X)

print(f"Min: {X_scaled.min(axis=0)}")
print(f"Max: {X_scaled.max(axis=0)}")

RobustScaler (Outlier-robust)

from sklearn.preprocessing import RobustScaler

scaler = RobustScaler()
X_scaled = scaler.fit_transform(X)

Outlier Detection

IQR Method

import pandas as pd
import numpy as np

np.random.seed(42)
data = np.random.randn(100) * 10 + 50
data = np.append(data, [100, 105, -20])  # add outliers

Q1 = np.percentile(data, 25)
Q3 = np.percentile(data, 75)
IQR = Q3 - Q1

lower = Q1 - 1.5 * IQR
upper = Q3 + 1.5 * IQR

outliers = data[(data < lower) | (data > upper)]
print(f"Outliers found: {len(outliers)}")

Z-Score Method

from scipy import stats
import numpy as np

np.random.seed(42)
data = np.random.randn(100) * 10 + 50

z_scores = np.abs(stats.zscore(data))
outliers = data[z_scores > 3]
print(f"Outliers (z>3): {len(outliers)}")

Train-Test Split

from sklearn.model_selection import train_test_split
from sklearn.datasets import load_iris

iris = load_iris()
X, y = iris.data, iris.target

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

print(f"Train: {X_train.shape[0]} samples")
print(f"Test: {X_test.shape[0]} samples")

Full Preprocessing Pipeline

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np

np.random.seed(42)
df = pd.DataFrame({
    'age': [25, 30, np.nan, 35, 40, 28, np.nan, 33],
    'income': [50000, 60000, np.nan, 80000, 90000, 55000, 70000, np.nan],
    'city': ['A', 'B', 'A', 'C', 'B', 'A', 'C', 'B'],
    'purchased': [0, 1, 0, 1, 1, 0, 1, 0]
})

X = df.drop('purchased', axis=1)
y = df['purchased']

numeric_features = ['age', 'income']
categorical_features = ['city']

numeric_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler())
])

categorical_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('encoder', OneHotEncoder(drop='first', sparse_output=False))
])

preprocessor = ColumnTransformer([
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features)
])

full_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(random_state=42))
])

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)
full_pipeline.fit(X_train, y_train)
print(f"Accuracy: {full_pipeline.score(X_test, y_test):.4f}")

Next chapter: Evaluation Metrics, how to measure model performance correctly.

Next: Evaluation Metrics →