fix(eval): correct probe metric to use true year-based OOS split

- train.py build(): year-based split (train≤2021, OOS≥2022) replaces misleading 70/30 mixed-period split; true OOS val_vol_r2 now ~-0.36 vs previously reported +0.18 (artefact of cross-period data leakage) - train.py: EXPORT_EMBEDDINGS block now exports both train+OOS embeddings with dates and HV labels for Go eval harness - cmd/eval: LinearProbeTrainTest uses train stats for standardisation of both sets (no leakage); standardiseCompute/applyStandardise helpers - internal/eval: add LinearProbeTrainTest (fit-on-train, eval-on-OOS) alongside LinearProbe (same-set); 8/8 tests still green Phase-0 gate result: val_vol_r2=-0.36, silhouette=0.043, erank=58.9/64. Backbone produces high-rank embeddings (SIGReg working) but does NOT generalize across 2021→2022 regime boundary. Gate: INCONCLUSIVE/FAIL. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-24 22:57:14 +02:00
parent e11e7d2524
commit 20aeecb971
3 changed files with 186 additions and 44 deletions
@@ -1,11 +1,19 @@
 // cmd/eval — CLI driver for the jepa-fx-risk evaluation harness.
-// Reads embeddings from a parquet/npy-style JSON export (embeddings.json)
-// and targets from eurusd_daily.parquet, then runs the requested metric.
 //
 //	./bin/eval -metric probe|silhouette|erank [-emb embeddings.json]
 //
-// embeddings.json format: {"embeddings": [[...], ...], "dates": ["2022-01-03", ...]}
-// Generated by train.py when run with EXPORT_EMBEDDINGS=1.
+// embeddings.json format (from train.py EXPORT_EMBEDDINGS=1):
+//
+//	{
+//	  "embeddings":         [[...], ...],  // OOS frozen embeddings
+//	  "realized_vol":       [...],         // OOS target (next-day RV)
+//	  "hv_label":           [...],         // binary HV label (top-33%)
+//	  "train_embeddings":   [[...], ...],  // train-set frozen embeddings
+//	  "train_realized_vol": [...]          // train-set RV targets
+//	}
+//
+// eval:probe standardises both sets using train statistics (no leakage).
+// Falls back to internal 70/30 split of OOS if train_embeddings absent.
 package main

 import (
@@ -26,34 +34,55 @@ func main() {

 	log := slog.New(slog.NewJSONHandler(os.Stdout, nil))

-	emb, labels, y, err := loadEmbeddings(*embFile)
+	d, err := readJSON(*embFile)
 	if err != nil {
 		log.Error("load embeddings", "err", err)
 		os.Exit(1)
 	}
-	log.Info("loaded", "n", len(emb), "dim", len(emb[0]), "metric", *metric)
+	log.Info("loaded", "oos", len(d.Embeddings), "dim", len(d.Embeddings[0]),
+		"train", len(d.TrainEmbeddings), "metric", *metric)

 	switch *metric {
 	case "probe":
-		r2 := eval.LinearProbe(emb, y, 1e-3)
+		var r2 float64
+		if len(d.TrainEmbeddings) > 0 {
+			// standardise both sets using train statistics to prevent leakage
+			trEmb, mu, sd := standardiseCompute(d.TrainEmbeddings)
+			oosEmb := applyStandardise(d.Embeddings, mu, sd)
+			r2 = eval.LinearProbeTrainTest(trEmb, d.TrainRealizedVol, oosEmb, d.RealizedVol, 1e-3)
+			log.Info("probe mode", "fit_on", "train_embeddings", "eval_on", "oos")
+		} else {
+			// fallback: internal 70/30 split of OOS embeddings
+			oosEmb, mu, sd := standardiseCompute(d.Embeddings)
+			n70 := int(float64(len(oosEmb)) * 0.7)
+			oos70 := applyStandardise(d.Embeddings[n70:], mu, sd)
+			r2 = eval.LinearProbeTrainTest(oosEmb[:n70], d.RealizedVol[:n70],
+				oos70, d.RealizedVol[n70:], 1e-3)
+			log.Info("probe mode", "fit_on", "oos[0:70%]", "eval_on", "oos[70%:]")
+		}
 		fmt.Printf(`{"metric":"val_vol_r2","value":%.6f}`+"\n", r2)
 		log.Info("linear probe", "val_vol_r2", fmt.Sprintf("%.4f", r2))
+
 	case "silhouette":
-		if labels == nil {
-			log.Error("silhouette requires HV labels in embeddings.json")
+		if len(d.HVLabel) == 0 {
+			log.Error("silhouette requires hv_label in embeddings.json")
 			os.Exit(1)
 		}
-		sil, err := eval.Silhouette(emb, labels)
+		oosEmb := standardise(d.Embeddings)
+		sil, err := eval.Silhouette(oosEmb, d.HVLabel)
 		if err != nil {
 			log.Error("silhouette", "err", err)
 			os.Exit(1)
 		}
 		fmt.Printf(`{"metric":"silhouette","value":%.6f}`+"\n", sil)
 		log.Info("silhouette", "score", fmt.Sprintf("%.4f", sil))
+
 	case "erank":
-		er := eval.EffectiveRank(emb)
+		oosEmb := standardise(d.Embeddings)
+		er := eval.EffectiveRank(oosEmb)
 		fmt.Printf(`{"metric":"effective_rank","value":%.6f}`+"\n", er)
 		log.Info("effective rank", "erank", fmt.Sprintf("%.2f", er))
+
 	default:
 		log.Error("unknown metric", "metric", *metric)
 		os.Exit(1)
@@ -61,32 +90,45 @@ func main() {
 }

 type embJSON struct {
-	Embeddings [][]float64 `json:"embeddings"`
-	Dates      []string    `json:"dates"`
-	RealizedVol []float64  `json:"realized_vol"`
-	HVLabel     []int      `json:"hv_label"`
+	Embeddings       [][]float64 `json:"embeddings"`
+	Dates            []string    `json:"dates"`
+	RealizedVol      []float64   `json:"realized_vol"`
+	HVLabel          []int       `json:"hv_label"`
+	TrainEmbeddings  [][]float64 `json:"train_embeddings"`
+	TrainRealizedVol []float64   `json:"train_realized_vol"`
 }

-func loadEmbeddings(path string) (emb [][]float64, labels []int, y []float64, err error) {
+func readJSON(path string) (*embJSON, error) {
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("open %s: %w", path, err)
+		return nil, fmt.Errorf("open %s: %w", path, err)
 	}
 	defer func() { _ = f.Close() }()
-
 	var d embJSON
 	if err := json.NewDecoder(f).Decode(&d); err != nil {
-		return nil, nil, nil, fmt.Errorf("decode: %w", err)
+		return nil, fmt.Errorf("decode: %w", err)
 	}
 	if len(d.Embeddings) == 0 {
-		return nil, nil, nil, fmt.Errorf("empty embeddings in %s", path)
+		return nil, fmt.Errorf("empty embeddings in %s", path)
 	}
+	return &d, nil
+}

-	// standardise embeddings (zero mean, unit std) per dimension
-	n, dim := len(d.Embeddings), len(d.Embeddings[0])
+// standardise centres + scales to zero mean / unit std; returns normalised rows.
+func standardise(rows [][]float64) [][]float64 {
+	out, _, _ := standardiseCompute(rows)
+	return out
+}
+
+// standardiseCompute centres + scales and returns (normalised, mu, sd) for reuse.
+func standardiseCompute(rows [][]float64) ([][]float64, []float64, []float64) {
+	if len(rows) == 0 {
+		return rows, nil, nil
+	}
+	n, dim := len(rows), len(rows[0])
 	mu := make([]float64, dim)
-	for _, row := range d.Embeddings {
-		for j, v := range row {
+	for _, r := range rows {
+		for j, v := range r {
 			mu[j] += v
 		}
 	}
@@ -94,8 +136,8 @@ func loadEmbeddings(path string) (emb [][]float64, labels []int, y []float64, er
 		mu[j] /= float64(n)
 	}
 	sd := make([]float64, dim)
-	for _, row := range d.Embeddings {
-		for j, v := range row {
+	for _, r := range rows {
+		for j, v := range r {
 			diff := v - mu[j]
 			sd[j] += diff * diff
 		}
@@ -103,16 +145,24 @@ func loadEmbeddings(path string) (emb [][]float64, labels []int, y []float64, er
 	for j := range sd {
 		sd[j] = math.Sqrt(sd[j]/float64(n)) + 1e-8
 	}
-	norm := make([][]float64, n)
-	for i, row := range d.Embeddings {
-		norm[i] = make([]float64, dim)
-		for j, v := range row {
-			norm[i][j] = (v - mu[j]) / sd[j]
+	out := make([][]float64, n)
+	for i, r := range rows {
+		out[i] = make([]float64, dim)
+		for j, v := range r {
+			out[i][j] = (v - mu[j]) / sd[j]
 		}
 	}
+	return out, mu, sd
+}

-	if len(d.HVLabel) > 0 {
-		labels = d.HVLabel
+// applyStandardise normalises rows using pre-computed mu and sd.
+func applyStandardise(rows [][]float64, mu, sd []float64) [][]float64 {
+	out := make([][]float64, len(rows))
+	for i, r := range rows {
+		out[i] = make([]float64, len(r))
+		for j, v := range r {
+			out[i][j] = (v - mu[j]) / sd[j]
+		}
 	}
-	return norm, labels, d.RealizedVol, nil
+	return out
 }
@@ -10,6 +10,58 @@ import (
 	"math"
 )

+// LinearProbeTrainTest fits ridge regression on (trainEmb, trainY) and evaluates
+// on (testEmb, testY). Returns OOS R². Use this for proper held-out evaluation.
+func LinearProbeTrainTest(trainEmb [][]float64, trainY []float64,
+	testEmb [][]float64, testY []float64, lambda float64) float64 {
+	n := len(trainEmb)
+	if n == 0 || len(testEmb) == 0 {
+		return 0
+	}
+	d := len(trainEmb[0])
+	p := d + 1
+
+	A := make([][]float64, n)
+	for i, e := range trainEmb {
+		row := make([]float64, p)
+		copy(row, e)
+		row[d] = 1.0
+		A[i] = row
+	}
+	AtA := make([][]float64, p)
+	for i := range AtA {
+		AtA[i] = make([]float64, p)
+	}
+	Aty := make([]float64, p)
+	for i := 0; i < n; i++ {
+		for j := 0; j < p; j++ {
+			Aty[j] += A[i][j] * trainY[i]
+			for k := 0; k < p; k++ {
+				AtA[j][k] += A[i][j] * A[i][k]
+			}
+		}
+	}
+	for j := 0; j < p; j++ {
+		AtA[j][j] += lambda
+	}
+	w := solveCholesky(AtA, Aty)
+
+	yMean := mean(testY)
+	var ssRes, ssTot float64
+	for i, e := range testEmb {
+		row := make([]float64, p)
+		copy(row, e)
+		row[d] = 1.0
+		pred := dot(row, w)
+		ssRes += (testY[i] - pred) * (testY[i] - pred)
+		ssTot += (testY[i] - yMean) * (testY[i] - yMean)
+	}
+	if ssTot == 0 {
+		return 0
+	}
+	return 1 - ssRes/ssTot
+}
+
 // LinearProbe fits a ridge regression (closed-form) on (emb, y) with regularisation λ
 // and returns R² on the same data. Call with train embeddings; probe on held-out by
 // splitting before calling.
@@ -102,19 +102,23 @@ class Predictor(nn.Module):
 # ── Data ────────────────────────────────────────────────────────────────────

 def build():
+    """Year-based split: encoder trains on 2019-2021; probe evaluates on 2022-2023 OOS."""
    df = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
+    df["date"] = pd.to_datetime(df["date"])
    feats  = df[["ret", "realized_vol"]].to_numpy(np.float32)
    target = df["realized_vol"].to_numpy(np.float32)
-    X, y = [], []
-    for t in range(WINDOW, len(df) - 1):
-        X.append(feats[t - WINDOW:t])
-        y.append(target[t + 1])
-    X = np.stack(X); y = np.array(y, np.float32)
-    n_tr = int(0.7 * len(X))
-    mu  = X[:n_tr].mean((0, 1))
-    sd  = X[:n_tr].std((0, 1)) + 1e-8
-    X   = (X - mu) / sd
-    return (X[:n_tr], y[:n_tr]), (X[n_tr:], y[n_tr:])
+    tr_idx = df.index[df["date"].dt.year <= 2021].tolist()
+    te_idx = df.index[df["date"].dt.year >= 2022].tolist()
+    mu = feats[:tr_idx[-1]+1].mean(0)
+    sd = feats[:tr_idx[-1]+1].std(0) + 1e-8
+    fn = (feats - mu) / sd
+    def windows(idx):
+        X, y = [], []
+        for t in idx:
+            if t - WINDOW >= 0 and t + 1 < len(df):
+                X.append(fn[t - WINDOW:t]); y.append(target[t + 1])
+        return np.stack(X).astype(np.float32), np.array(y, np.float32)
+    return windows(tr_idx), windows(te_idx)


 # ── Training ─────────────────────────────────────────────────────────────────
@@ -169,6 +173,42 @@ def main():
    }, open("metrics.json", "w"), indent=2)
    print("val_vol_r2 = %.4f  (n_test=%d, dev=%s)" % (val_vol_r2, len(yte), dev))

+    # ── EXPORT BLOCK — do NOT edit (agent boundary) ──────────────────────────
+    # Set EXPORT_EMBEDDINGS=1 to write embeddings.json for the Go eval harness.
+    # Uses year-based split (train≤2021, OOS≥2022) regardless of probe split.
+    import os
+    if os.environ.get("EXPORT_EMBEDDINGS") == "1":
+        df2 = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
+        df2["date"] = pd.to_datetime(df2["date"])
+        tr_mask = df2["date"].dt.year <= 2021
+        feats2 = df2[["ret", "realized_vol"]].to_numpy(np.float32)
+        mu2 = feats2[tr_mask].mean(0); sd2 = feats2[tr_mask].std(0) + 1e-8
+        fn2 = (feats2 - mu2) / sd2
+        def _export_windows(year_mask):
+            idx = df2.index[year_mask].tolist()
+            Xs, dates, rvs = [], [], []
+            for t in idx:
+                if t - WINDOW >= 0:
+                    Xs.append(fn2[t - WINDOW:t])
+                    dates.append(str(df2["date"].iloc[t].date()))
+                    rvs.append(float(df2["realized_vol"].iloc[t]))
+            if not Xs:
+                return [], [], []
+            with torch.no_grad():
+                E = enc(torch.tensor(np.stack(Xs), device=dev)).mean(1).cpu().numpy().tolist()
+            return E, dates, rvs
+        Etr, dates_tr, rv_tr = _export_windows(tr_mask)
+        Eoos, dates_oos, rv_oos = _export_windows(df2["date"].dt.year >= 2022)
+        hv_thr = float(np.percentile(rv_oos, 67))
+        hv_label = [1 if v >= hv_thr else 0 for v in rv_oos]
+        json.dump({"embeddings": Eoos, "dates": dates_oos,
+                   "realized_vol": rv_oos, "hv_label": hv_label,
+                   "train_embeddings": Etr, "train_realized_vol": rv_tr},
+                  open("embeddings.json", "w"))
+        print("exported embeddings.json  train=%d  oos=%d  HV=%d/%d" % (
+            len(Etr), len(Eoos), sum(hv_label), len(hv_label)))
+    # ── END EXPORT BLOCK ─────────────────────────────────────────────────────
+

 if __name__ == "__main__":
    main()