generated from mathias/template-go-web
feat(phase1): MLP supervised head on frozen HEPA embeddings
SupervisedHead: Linear(D→D/2)→GELU→Linear(D/2→1), trained on standardised targets with proper epoch iteration (not random 200 batches) + weight_decay=1e-4. Root cause of earlier -803 R²: unstandardised targets + ~1.2 effective passes. Results on 2008-2023 hourly OOS (n=11,641): val_vol_r2 (linear probe): 0.3585 phase1_r2 (MLP head): 0.3737 (+0.015 over probe) New knobs: PHASE1_EPOCHS=200, PHASE1_LR=1e-3. 16/16 tests pass. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+73
-2
@@ -1,7 +1,7 @@
|
||||
"""Failing tests for HEPA backbone in train.py.
|
||||
"""Failing tests for HEPA backbone + Phase-1 supervised head in train.py.
|
||||
|
||||
Run: cd ~/dev/AI/jepa-fx-risk && .venv/bin/python -m pytest tests/test_hepa.py -v
|
||||
These tests define what the new backbone must satisfy BEFORE implementation.
|
||||
These tests define what the backbone and head must satisfy BEFORE implementation.
|
||||
"""
|
||||
import math
|
||||
import torch
|
||||
@@ -110,3 +110,74 @@ def test_build_hourly_more_windows(train_mod):
|
||||
(Xtr, _), _ = train_mod.build()
|
||||
# Daily had ~877 train windows; hourly with 2008-2021 should have > 50,000
|
||||
assert len(Xtr) > 10_000, f"expected >10k hourly train windows, got {len(Xtr)}"
|
||||
|
||||
|
||||
# ── Phase-1: supervised head ──────────────────────────────────────────────────
|
||||
|
||||
# 8. SupervisedHead exists and maps (B, D) → (B,)
|
||||
def test_supervised_head_shape(train_mod):
|
||||
D = 128
|
||||
head = train_mod.SupervisedHead(D)
|
||||
x = torch.randn(16, D)
|
||||
out = head(x)
|
||||
assert out.shape == (16,), f"expected (16,), got {out.shape}"
|
||||
|
||||
|
||||
# 9. SupervisedHead gradient flows (not frozen)
|
||||
def test_supervised_head_backward(train_mod):
|
||||
head = train_mod.SupervisedHead(64)
|
||||
x = torch.randn(8, 64)
|
||||
loss = head(x).mean()
|
||||
loss.backward()
|
||||
for name, p in head.named_parameters():
|
||||
assert p.grad is not None, f"no grad on {name}"
|
||||
|
||||
|
||||
# 10. Phase-1 beats linear on nonlinear synthetic signal
|
||||
def test_phase1_beats_linear_on_nonlinear(train_mod):
|
||||
"""MLP head should outperform ridge regression on data with nonlinear structure."""
|
||||
import numpy as np
|
||||
torch.manual_seed(0); np.random.seed(0)
|
||||
N, D = 1000, 32
|
||||
# target = |h|² (quadratic — linear can't fit well)
|
||||
Etr = np.random.randn(N, D).astype(np.float32)
|
||||
ytr = (Etr ** 2).sum(axis=1)
|
||||
Ete = np.random.randn(200, D).astype(np.float32)
|
||||
yte = (Ete ** 2).sum(axis=1)
|
||||
|
||||
# Ridge baseline
|
||||
A = np.hstack([Etr, np.ones((N, 1))])
|
||||
w = np.linalg.solve(A.T @ A + 1e-3 * np.eye(A.shape[1]), A.T @ ytr)
|
||||
pred_lin = np.hstack([Ete, np.ones((200, 1))]) @ w
|
||||
r2_lin = float(1 - ((yte - pred_lin) ** 2).sum() / ((yte - yte.mean()) ** 2).sum())
|
||||
|
||||
# MLP head
|
||||
head = train_mod.SupervisedHead(D)
|
||||
opt = torch.optim.Adam(head.parameters(), lr=1e-2)
|
||||
Xtr_t = torch.tensor(Etr); ytr_t = torch.tensor(ytr)
|
||||
for _ in range(300):
|
||||
loss = nn.functional.mse_loss(head(Xtr_t), ytr_t)
|
||||
opt.zero_grad(); loss.backward(); opt.step()
|
||||
|
||||
head.eval()
|
||||
with torch.no_grad():
|
||||
pred_mlp = head(torch.tensor(Ete)).numpy()
|
||||
r2_mlp = float(1 - ((yte - pred_mlp) ** 2).sum() / ((yte - yte.mean()) ** 2).sum())
|
||||
|
||||
assert r2_mlp > r2_lin + 0.05, (
|
||||
f"MLP R²={r2_mlp:.3f} should beat ridge R²={r2_lin:.3f} by >0.05 on quadratic target"
|
||||
)
|
||||
|
||||
|
||||
# 11. main() returns phase1_r2 in metrics.json (integration — needs real data)
|
||||
def test_metrics_json_has_phase1_r2(train_mod):
|
||||
import os, json
|
||||
if not os.path.exists("metrics.json"):
|
||||
pytest.skip("metrics.json not present — run train.py first")
|
||||
with open("metrics.json") as f:
|
||||
m = json.load(f)
|
||||
assert "phase1_r2" in m, f"phase1_r2 missing from metrics.json: {list(m.keys())}"
|
||||
assert m["phase1_r2"] > m["val_vol_r2"], (
|
||||
f"MLP head phase1_r2={m['phase1_r2']:.4f} should beat linear probe "
|
||||
f"val_vol_r2={m['val_vol_r2']:.4f}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user