Deep Learning
From Neuron to Transformer

import torch

# Simple example: compute gradient of L = (wx - y)²
w = torch.tensor([2.0], requires_grad=True)   # track gradients
x = torch.tensor([3.0])
y = torch.tensor([9.0])

# Forward pass — builds computation graph
pred = w * x
loss = (pred - y) ** 2

# Backward pass — computes all gradients via chain rule
loss.backward()

print(w.grad)    # → tensor([-18.]) = 2*(wx-y)*x = 2*(6-9)*3 = -18
# Manual check: dL/dw = 2(wx-y)·x = 2(2·3-9)·3 = 2(-3)(3) = -18 ✓

# ─── In a real network ────────────────────────────────────
optimizer.zero_grad()   # clear gradients from last step
output = model(X_batch)  # forward pass — builds graph
loss = criterion(output, y_batch)
loss.backward()          # backward — computes all .grad attributes
optimizer.step()         # use .grad to update weights

import torch
import torch.nn as nn
import numpy as np

# ─── Tensors — identical to NumPy ndarray ─────────────────
x = torch.tensor([1.0, 2.0, 3.0])
X = torch.randn(64, 20)           # batch=64, features=20
W = torch.zeros(20, 10)            # weight matrix
I = torch.eye(5)                   # identity matrix

# Same NumPy operations work on PyTorch tensors:
X @ W                               # matrix multiply (forward pass!)
X.mean(dim=0)                     # mean per feature
X.sum(); X.max(); X.abs()
X.reshape(-1); X.T; X[:10]

# ─── GPU — one line to move everything ────────────────────
device = "cuda" if torch.cuda.is_available() else "cpu"
X = X.to(device)     # move tensor to GPU (or stay on CPU)
model = model.to(device)

# ─── NumPy interop ────────────────────────────────────────
arr = np.array([1, 2, 3], dtype=np.float32)
t = torch.from_numpy(arr)          # shared memory!
t.detach().cpu().numpy()           # safe Tensor → NumPy

# ─── DataLoader — automatic batching ──────────────────────
from torch.utils.data import TensorDataset, DataLoader

dataset = TensorDataset(
    torch.FloatTensor(X_train),
    torch.LongTensor(y_train)        # LongTensor for class labels
)
loader = DataLoader(dataset, batch_size=64, shuffle=True, num_workers=2)

for X_batch, y_batch in loader:    # X_batch: (64, 20), y_batch: (64,)
    pass                             # your training code here

import torch.nn as nn

class MLP(nn.Module):
    def __init__(self, in_dim, hidden, out_dim, dropout=0.3):
        super().__init__()                # ALWAYS call this first
        layers = []
        prev = in_dim
        for h in hidden:
            layers += [
                nn.Linear(prev, h),        # y = xW^T + b
                nn.BatchNorm1d(h),          # normalize activations
                nn.ReLU(),                  # non-linearity
                nn.Dropout(dropout),        # regularization
            ]
            prev = h
        layers.append(nn.Linear(prev, out_dim))  # output (no activation!)
        self.net = nn.Sequential(*layers)

    def forward(self, x):
        return self.net(x)               # Sequential handles the rest

# Instantiate
model = MLP(in_dim=20, hidden=[128, 64, 32], out_dim=2, dropout=0.3).to(device)

# Count parameters
total = sum(p.numel() for p in model.parameters())
print(f"Parameters: {total:,}")

# Weight initialization — He (Kaiming) for ReLU networks
def init_weights(m):
    if isinstance(m, nn.Linear):
        nn.init.kaiming_normal_(m.weight, nonlinearity='relu')
        nn.init.zeros_(m.bias)
model.apply(init_weights)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=50)

best_val, history = float('inf'), {'train':[], 'val':[], 'acc':[]}

for epoch in range(100):

    # ═══ TRAIN ══════════════════════════════════════════════
    model.train()                      # enables dropout + BN train mode
    train_loss = 0.0
    for Xb, yb in train_loader:
        Xb, yb = Xb.to(device), yb.to(device)

        optimizer.zero_grad()          # 1. CLEAR old gradients ← never forget
        logits = model(Xb)              # 2. Forward pass
        loss = criterion(logits, yb)    # 3. Compute loss
        loss.backward()               # 4. Backprop — computes all grads
        nn.utils.clip_grad_norm_(      # 5. Clip gradients (optional)
            model.parameters(), 1.0)
        optimizer.step()              # 6. Update weights with grads
        train_loss += loss.item()

    # ═══ VALIDATE ═══════════════════════════════════════════
    model.eval()                       # disables dropout; BN uses running stats
    val_loss, correct, total = 0.0, 0, 0
    with torch.no_grad():              # no graph built — 4x less memory
        for Xv, yv in val_loader:
            Xv, yv = Xv.to(device), yv.to(device)
            logits = model(Xv)
            val_loss += criterion(logits, yv).item()
            correct += (logits.argmax(1) == yv).sum().item()
            total += yv.size(0)

    scheduler.step()
    if val_loss < best_val:
        best_val = val_loss
        torch.save(model.state_dict(), 'best.pt')   # save best checkpoint

    if epoch % 10 == 0:
        print(f"[{epoch:3d}] train={train_loss/len(train_loader):.4f} val={val_loss/len(val_loader):.4f} acc={correct/total:.3f}")

class CNN(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        self.features = nn.Sequential(
            # Block 1: (3,224,224) → (32,112,112)
            nn.Conv2d(3, 32, kernel_size=3, padding=1),  # same padding
            nn.BatchNorm2d(32), nn.ReLU(),
            nn.MaxPool2d(2),                            # halve spatial dims
            # Block 2: → (64, 56, 56)
            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64), nn.ReLU(),
            nn.MaxPool2d(2),
            # Block 3: → (128, 4, 4) via adaptive pooling
            nn.Conv2d(64, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128), nn.ReLU(),
            nn.AdaptiveAvgPool2d((4, 4)),              # always (128,4,4)
        )
        self.classifier = nn.Sequential(
            nn.Flatten(),                              # (128,4,4) → (2048,)
            nn.Linear(128*4*4, 256), nn.ReLU(), nn.Dropout(0.5),
            nn.Linear(256, num_classes),
        )
    def forward(self, x): return self.classifier(self.features(x))

# Output size formula: out = (in + 2*pad - kernel) / stride + 1
# 224 + 2*1 - 3 / 1 + 1 = 224  ← "same" padding keeps spatial size

class LSTMClassifier(nn.Module):
    def __init__(self, vocab_size, emb_dim, hidden, n_classes, n_layers=2):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, emb_dim, padding_idx=0)
        self.lstm = nn.LSTM(emb_dim, hidden, n_layers,
                            batch_first=True,          # (batch, seq, feat)
                            dropout=0.3,
                            bidirectional=True)
        self.fc = nn.Linear(hidden * 2, n_classes)   # *2 for bidirectional

    def forward(self, x):               # x: (batch, seq_len)
        emb = self.embed(x)             # (batch, seq_len, emb_dim)
        out, (h_n, c_n) = self.lstm(emb)
        # h_n: (n_layers*2, batch, hidden) — final hidden states
        h = torch.cat([h_n[-2], h_n[-1]], dim=1)   # concat fwd + bwd
        return self.fc(h)               # (batch, n_classes)

import math
import torch.nn.functional as F

class MultiHeadAttention(nn.Module):
    def __init__(self, d_model, n_heads):
        super().__init__()
        self.d_k = d_model // n_heads
        self.n_heads = n_heads
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)

    def forward(self, x, mask=None):
        B, T, C = x.shape
        def split_heads(t):
            return t.view(B, T, self.n_heads, self.d_k).transpose(1, 2)
        Q, K, V = split_heads(self.W_q(x)), split_heads(self.W_k(x)), split_heads(self.W_v(x))
        scores = Q @ K.transpose(-2, -1) / math.sqrt(self.d_k)
        if mask is not None: scores = scores.masked_fill(mask == 0, -1e9)
        attn = F.softmax(scores, dim=-1)
        out = (attn @ V).transpose(1, 2).reshape(B, T, C)
        return self.W_o(out)

class TransformerBlock(nn.Module):
    def __init__(self, d_model, n_heads, ff_dim, dropout=0.1):
        super().__init__()
        self.attn = MultiHeadAttention(d_model, n_heads)
        self.ff   = nn.Sequential(nn.Linear(d_model, ff_dim), nn.GELU(), nn.Linear(ff_dim, d_model))
        self.ln1  = nn.LayerNorm(d_model)
        self.ln2  = nn.LayerNorm(d_model)
        self.drop = nn.Dropout(dropout)

    def forward(self, x):
        x = x + self.drop(self.attn(self.ln1(x)))   # residual + attention
        x = x + self.drop(self.ff(self.ln2(x)))     # residual + feedforward
        return x

from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

# ─── Load pretrained BERT ──────────────────────────────────
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased", num_labels=2    # replace classification head
).to(device)

# ─── Tokenize ─────────────────────────────────────────────
texts = ["Great product!", "Terrible experience"]
inputs = tokenizer(texts, padding=True, truncation=True,
                   max_length=128, return_tensors="pt")

# ─── Discriminative learning rates ────────────────────────
optimizer = torch.optim.AdamW([
    {"params": model.bert.embeddings.parameters(), "lr": 1e-5},   # early
    {"params": model.bert.encoder.parameters(),    "lr": 2e-5},   # middle
    {"params": model.classifier.parameters(),      "lr": 5e-5},   # head
])

# ─── Training step ────────────────────────────────────────
labels = torch.tensor([1, 0])
outputs = model(**inputs.to(device), labels=labels.to(device))
loss = outputs.loss       # HuggingFace computes loss for you
preds = outputs.logits.argmax(dim=1)