RNN (Recurrent Neural Network)

An RNN is a neural network designed for sequential data (text, time series, audio). It has "memory" that allows information from the previous step to influence the current prediction.

Basic Concept

import numpy as np

# Simple RNN forward pass
def simple_rnn(x, Wxh, Whh, bias):
    h = np.zeros((x.shape[0], Whh.shape[0]))
    outputs = []
    for t in range(x.shape[1]):
        h = np.tanh(x[:, t, :].dot(Wxh) + h.dot(Whh) + bias)
        outputs.append(h)
    return outputs

RNN with Keras

from tensorflow import keras
from tensorflow.keras import layers

model = keras.Sequential([
    layers.SimpleRNN(64, return_sequences=True, input_shape=(10, 1)),
    layers.SimpleRNN(32),
    layers.Dense(1)
])

model.compile(optimizer='adam', loss='mse')
model.summary()

LSTM (Long Short-Term Memory)

LSTMs address the vanishing gradient problem with a gating mechanism.

from tensorflow import keras
from tensorflow.keras import layers

model = keras.Sequential([
    layers.LSTM(64, return_sequences=True, input_shape=(10, 1)),
    layers.LSTM(32),
    layers.Dense(1)
])

model.compile(optimizer='adam', loss='mse')
model.summary()

GRU (Gated Recurrent Unit)

The GRU is a simpler version of the LSTM.

from tensorflow import keras
from tensorflow.keras import layers

model = keras.Sequential([
    layers.GRU(64, return_sequences=True, input_shape=(10, 1)),
    layers.GRU(32),
    layers.Dense(1)
])

model.compile(optimizer='adam', loss='mse')

The Vanishing Gradient Problem

The gradient shrinks exponentially through the time steps
→ LSTMs/GRUs address this with a gating mechanism

Sequence-to-Sequence

from tensorflow import keras
from tensorflow.keras import layers

# Encoder
encoder = keras.Sequential([
    layers.LSTM(128, input_shape=(None, 1))
])

# Decoder
decoder = keras.Sequential([
    layers.RepeatVector(10),
    layers.LSTM(128, return_sequences=True),
    layers.TimeDistributed(layers.Dense(1))
])

model = keras.Sequential([encoder, decoder])
model.compile(optimizer='adam', loss='mse')

Example: Time Series Prediction

import numpy as np
from tensorflow import keras
from tensorflow.keras import layers

# Generate data
np.random.seed(42)
t = np.linspace(0, 100, 1000)
data = np.sin(t) + np.random.randn(1000) * 0.1

# Create sequences
def create_sequences(data, seq_length):
    X, y = [], []
    for i in range(len(data) - seq_length):
        X.append(data[i:i+seq_length])
        y.append(data[i+seq_length])
    return np.array(X), np.array(y)

seq_length = 50
X, y = create_sequences(data, seq_length)
X = X.reshape(-1, seq_length, 1)

split = int(0.8 * len(X))
X_train, X_test = X[:split], X[split:]
y_train, y_test = y[:split], y[split:]

model = keras.Sequential([
    layers.LSTM(32, input_shape=(seq_length, 1)),
    layers.Dense(16, activation='relu'),
    layers.Dense(1)
])

model.compile(optimizer='adam', loss='mse')
model.fit(X_train, y_train, epochs=20, batch_size=32,
          validation_split=0.1, verbose=0)

loss = model.evaluate(X_test, y_test)
print(f"Test MSE: {loss:.4f}")

Bidirectional LSTM

from tensorflow import keras
from tensorflow.keras import layers

model = keras.Sequential([
    layers.Bidirectional(layers.LSTM(64, return_sequences=True), input_shape=(10, 1)),
    layers.Bidirectional(layers.LSTM(32)),
    layers.Dense(1)
])

Comparing RNN Variants

Next chapter: NLP, applying ML to natural language processing.

Next: NLP →