diff --git a/cmd/eval/main.go b/cmd/eval/main.go index c1d43a2..4b9cf9d 100644 --- a/cmd/eval/main.go +++ b/cmd/eval/main.go @@ -1,11 +1,19 @@ // cmd/eval — CLI driver for the jepa-fx-risk evaluation harness. -// Reads embeddings from a parquet/npy-style JSON export (embeddings.json) -// and targets from eurusd_daily.parquet, then runs the requested metric. // // ./bin/eval -metric probe|silhouette|erank [-emb embeddings.json] // -// embeddings.json format: {"embeddings": [[...], ...], "dates": ["2022-01-03", ...]} -// Generated by train.py when run with EXPORT_EMBEDDINGS=1. +// embeddings.json format (from train.py EXPORT_EMBEDDINGS=1): +// +// { +// "embeddings": [[...], ...], // OOS frozen embeddings +// "realized_vol": [...], // OOS target (next-day RV) +// "hv_label": [...], // binary HV label (top-33%) +// "train_embeddings": [[...], ...], // train-set frozen embeddings +// "train_realized_vol": [...] // train-set RV targets +// } +// +// eval:probe standardises both sets using train statistics (no leakage). +// Falls back to internal 70/30 split of OOS if train_embeddings absent. package main import ( @@ -26,34 +34,55 @@ func main() { log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) - emb, labels, y, err := loadEmbeddings(*embFile) + d, err := readJSON(*embFile) if err != nil { log.Error("load embeddings", "err", err) os.Exit(1) } - log.Info("loaded", "n", len(emb), "dim", len(emb[0]), "metric", *metric) + log.Info("loaded", "oos", len(d.Embeddings), "dim", len(d.Embeddings[0]), + "train", len(d.TrainEmbeddings), "metric", *metric) switch *metric { case "probe": - r2 := eval.LinearProbe(emb, y, 1e-3) + var r2 float64 + if len(d.TrainEmbeddings) > 0 { + // standardise both sets using train statistics to prevent leakage + trEmb, mu, sd := standardiseCompute(d.TrainEmbeddings) + oosEmb := applyStandardise(d.Embeddings, mu, sd) + r2 = eval.LinearProbeTrainTest(trEmb, d.TrainRealizedVol, oosEmb, d.RealizedVol, 1e-3) + log.Info("probe mode", "fit_on", "train_embeddings", "eval_on", "oos") + } else { + // fallback: internal 70/30 split of OOS embeddings + oosEmb, mu, sd := standardiseCompute(d.Embeddings) + n70 := int(float64(len(oosEmb)) * 0.7) + oos70 := applyStandardise(d.Embeddings[n70:], mu, sd) + r2 = eval.LinearProbeTrainTest(oosEmb[:n70], d.RealizedVol[:n70], + oos70, d.RealizedVol[n70:], 1e-3) + log.Info("probe mode", "fit_on", "oos[0:70%]", "eval_on", "oos[70%:]") + } fmt.Printf(`{"metric":"val_vol_r2","value":%.6f}`+"\n", r2) log.Info("linear probe", "val_vol_r2", fmt.Sprintf("%.4f", r2)) + case "silhouette": - if labels == nil { - log.Error("silhouette requires HV labels in embeddings.json") + if len(d.HVLabel) == 0 { + log.Error("silhouette requires hv_label in embeddings.json") os.Exit(1) } - sil, err := eval.Silhouette(emb, labels) + oosEmb := standardise(d.Embeddings) + sil, err := eval.Silhouette(oosEmb, d.HVLabel) if err != nil { log.Error("silhouette", "err", err) os.Exit(1) } fmt.Printf(`{"metric":"silhouette","value":%.6f}`+"\n", sil) log.Info("silhouette", "score", fmt.Sprintf("%.4f", sil)) + case "erank": - er := eval.EffectiveRank(emb) + oosEmb := standardise(d.Embeddings) + er := eval.EffectiveRank(oosEmb) fmt.Printf(`{"metric":"effective_rank","value":%.6f}`+"\n", er) log.Info("effective rank", "erank", fmt.Sprintf("%.2f", er)) + default: log.Error("unknown metric", "metric", *metric) os.Exit(1) @@ -61,32 +90,45 @@ func main() { } type embJSON struct { - Embeddings [][]float64 `json:"embeddings"` - Dates []string `json:"dates"` - RealizedVol []float64 `json:"realized_vol"` - HVLabel []int `json:"hv_label"` + Embeddings [][]float64 `json:"embeddings"` + Dates []string `json:"dates"` + RealizedVol []float64 `json:"realized_vol"` + HVLabel []int `json:"hv_label"` + TrainEmbeddings [][]float64 `json:"train_embeddings"` + TrainRealizedVol []float64 `json:"train_realized_vol"` } -func loadEmbeddings(path string) (emb [][]float64, labels []int, y []float64, err error) { +func readJSON(path string) (*embJSON, error) { f, err := os.Open(path) if err != nil { - return nil, nil, nil, fmt.Errorf("open %s: %w", path, err) + return nil, fmt.Errorf("open %s: %w", path, err) } defer func() { _ = f.Close() }() - var d embJSON if err := json.NewDecoder(f).Decode(&d); err != nil { - return nil, nil, nil, fmt.Errorf("decode: %w", err) + return nil, fmt.Errorf("decode: %w", err) } if len(d.Embeddings) == 0 { - return nil, nil, nil, fmt.Errorf("empty embeddings in %s", path) + return nil, fmt.Errorf("empty embeddings in %s", path) } + return &d, nil +} - // standardise embeddings (zero mean, unit std) per dimension - n, dim := len(d.Embeddings), len(d.Embeddings[0]) +// standardise centres + scales to zero mean / unit std; returns normalised rows. +func standardise(rows [][]float64) [][]float64 { + out, _, _ := standardiseCompute(rows) + return out +} + +// standardiseCompute centres + scales and returns (normalised, mu, sd) for reuse. +func standardiseCompute(rows [][]float64) ([][]float64, []float64, []float64) { + if len(rows) == 0 { + return rows, nil, nil + } + n, dim := len(rows), len(rows[0]) mu := make([]float64, dim) - for _, row := range d.Embeddings { - for j, v := range row { + for _, r := range rows { + for j, v := range r { mu[j] += v } } @@ -94,8 +136,8 @@ func loadEmbeddings(path string) (emb [][]float64, labels []int, y []float64, er mu[j] /= float64(n) } sd := make([]float64, dim) - for _, row := range d.Embeddings { - for j, v := range row { + for _, r := range rows { + for j, v := range r { diff := v - mu[j] sd[j] += diff * diff } @@ -103,16 +145,24 @@ func loadEmbeddings(path string) (emb [][]float64, labels []int, y []float64, er for j := range sd { sd[j] = math.Sqrt(sd[j]/float64(n)) + 1e-8 } - norm := make([][]float64, n) - for i, row := range d.Embeddings { - norm[i] = make([]float64, dim) - for j, v := range row { - norm[i][j] = (v - mu[j]) / sd[j] + out := make([][]float64, n) + for i, r := range rows { + out[i] = make([]float64, dim) + for j, v := range r { + out[i][j] = (v - mu[j]) / sd[j] } } + return out, mu, sd +} - if len(d.HVLabel) > 0 { - labels = d.HVLabel +// applyStandardise normalises rows using pre-computed mu and sd. +func applyStandardise(rows [][]float64, mu, sd []float64) [][]float64 { + out := make([][]float64, len(rows)) + for i, r := range rows { + out[i] = make([]float64, len(r)) + for j, v := range r { + out[i][j] = (v - mu[j]) / sd[j] + } } - return norm, labels, d.RealizedVol, nil + return out } diff --git a/internal/eval/eval.go b/internal/eval/eval.go index c446ba3..a4c046b 100644 --- a/internal/eval/eval.go +++ b/internal/eval/eval.go @@ -10,6 +10,58 @@ import ( "math" ) +// LinearProbeTrainTest fits ridge regression on (trainEmb, trainY) and evaluates +// on (testEmb, testY). Returns OOS R². Use this for proper held-out evaluation. +func LinearProbeTrainTest(trainEmb [][]float64, trainY []float64, + testEmb [][]float64, testY []float64, lambda float64) float64 { + n := len(trainEmb) + if n == 0 || len(testEmb) == 0 { + return 0 + } + d := len(trainEmb[0]) + p := d + 1 + + A := make([][]float64, n) + for i, e := range trainEmb { + row := make([]float64, p) + copy(row, e) + row[d] = 1.0 + A[i] = row + } + AtA := make([][]float64, p) + for i := range AtA { + AtA[i] = make([]float64, p) + } + Aty := make([]float64, p) + for i := 0; i < n; i++ { + for j := 0; j < p; j++ { + Aty[j] += A[i][j] * trainY[i] + for k := 0; k < p; k++ { + AtA[j][k] += A[i][j] * A[i][k] + } + } + } + for j := 0; j < p; j++ { + AtA[j][j] += lambda + } + w := solveCholesky(AtA, Aty) + + yMean := mean(testY) + var ssRes, ssTot float64 + for i, e := range testEmb { + row := make([]float64, p) + copy(row, e) + row[d] = 1.0 + pred := dot(row, w) + ssRes += (testY[i] - pred) * (testY[i] - pred) + ssTot += (testY[i] - yMean) * (testY[i] - yMean) + } + if ssTot == 0 { + return 0 + } + return 1 - ssRes/ssTot +} + // LinearProbe fits a ridge regression (closed-form) on (emb, y) with regularisation λ // and returns R² on the same data. Call with train embeddings; probe on held-out by // splitting before calling. diff --git a/train.py b/train.py index d6482b1..bf262c7 100644 --- a/train.py +++ b/train.py @@ -102,19 +102,23 @@ class Predictor(nn.Module): # ── Data ──────────────────────────────────────────────────────────────────── def build(): + """Year-based split: encoder trains on 2019-2021; probe evaluates on 2022-2023 OOS.""" df = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True) + df["date"] = pd.to_datetime(df["date"]) feats = df[["ret", "realized_vol"]].to_numpy(np.float32) target = df["realized_vol"].to_numpy(np.float32) - X, y = [], [] - for t in range(WINDOW, len(df) - 1): - X.append(feats[t - WINDOW:t]) - y.append(target[t + 1]) - X = np.stack(X); y = np.array(y, np.float32) - n_tr = int(0.7 * len(X)) - mu = X[:n_tr].mean((0, 1)) - sd = X[:n_tr].std((0, 1)) + 1e-8 - X = (X - mu) / sd - return (X[:n_tr], y[:n_tr]), (X[n_tr:], y[n_tr:]) + tr_idx = df.index[df["date"].dt.year <= 2021].tolist() + te_idx = df.index[df["date"].dt.year >= 2022].tolist() + mu = feats[:tr_idx[-1]+1].mean(0) + sd = feats[:tr_idx[-1]+1].std(0) + 1e-8 + fn = (feats - mu) / sd + def windows(idx): + X, y = [], [] + for t in idx: + if t - WINDOW >= 0 and t + 1 < len(df): + X.append(fn[t - WINDOW:t]); y.append(target[t + 1]) + return np.stack(X).astype(np.float32), np.array(y, np.float32) + return windows(tr_idx), windows(te_idx) # ── Training ───────────────────────────────────────────────────────────────── @@ -169,6 +173,42 @@ def main(): }, open("metrics.json", "w"), indent=2) print("val_vol_r2 = %.4f (n_test=%d, dev=%s)" % (val_vol_r2, len(yte), dev)) + # ── EXPORT BLOCK — do NOT edit (agent boundary) ────────────────────────── + # Set EXPORT_EMBEDDINGS=1 to write embeddings.json for the Go eval harness. + # Uses year-based split (train≤2021, OOS≥2022) regardless of probe split. + import os + if os.environ.get("EXPORT_EMBEDDINGS") == "1": + df2 = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True) + df2["date"] = pd.to_datetime(df2["date"]) + tr_mask = df2["date"].dt.year <= 2021 + feats2 = df2[["ret", "realized_vol"]].to_numpy(np.float32) + mu2 = feats2[tr_mask].mean(0); sd2 = feats2[tr_mask].std(0) + 1e-8 + fn2 = (feats2 - mu2) / sd2 + def _export_windows(year_mask): + idx = df2.index[year_mask].tolist() + Xs, dates, rvs = [], [], [] + for t in idx: + if t - WINDOW >= 0: + Xs.append(fn2[t - WINDOW:t]) + dates.append(str(df2["date"].iloc[t].date())) + rvs.append(float(df2["realized_vol"].iloc[t])) + if not Xs: + return [], [], [] + with torch.no_grad(): + E = enc(torch.tensor(np.stack(Xs), device=dev)).mean(1).cpu().numpy().tolist() + return E, dates, rvs + Etr, dates_tr, rv_tr = _export_windows(tr_mask) + Eoos, dates_oos, rv_oos = _export_windows(df2["date"].dt.year >= 2022) + hv_thr = float(np.percentile(rv_oos, 67)) + hv_label = [1 if v >= hv_thr else 0 for v in rv_oos] + json.dump({"embeddings": Eoos, "dates": dates_oos, + "realized_vol": rv_oos, "hv_label": hv_label, + "train_embeddings": Etr, "train_realized_vol": rv_tr}, + open("embeddings.json", "w")) + print("exported embeddings.json train=%d oos=%d HV=%d/%d" % ( + len(Etr), len(Eoos), sum(hv_label), len(hv_label))) + # ── END EXPORT BLOCK ───────────────────────────────────────────────────── + if __name__ == "__main__": main() \ No newline at end of file