fix(eval): correct probe metric to use true year-based OOS split
CD / Lint / Test / Vet (push) Successful in 4s
CD / Build & Import (push) Failing after 7s
CD / Deploy via GitOps (push) Has been skipped

- train.py build(): year-based split (train≤2021, OOS≥2022) replaces
  misleading 70/30 mixed-period split; true OOS val_vol_r2 now ~-0.36
  vs previously reported +0.18 (artefact of cross-period data leakage)
- train.py: EXPORT_EMBEDDINGS block now exports both train+OOS embeddings
  with dates and HV labels for Go eval harness
- cmd/eval: LinearProbeTrainTest uses train stats for standardisation of
  both sets (no leakage); standardiseCompute/applyStandardise helpers
- internal/eval: add LinearProbeTrainTest (fit-on-train, eval-on-OOS)
  alongside LinearProbe (same-set); 8/8 tests still green

Phase-0 gate result: val_vol_r2=-0.36, silhouette=0.043, erank=58.9/64.
Backbone produces high-rank embeddings (SIGReg working) but does NOT
generalize across 2021→2022 regime boundary. Gate: INCONCLUSIVE/FAIL.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-24 22:57:14 +02:00
co-authored by Claude Sonnet 4.6
parent e11e7d2524
commit 20aeecb971
3 changed files with 186 additions and 44 deletions
+50 -10
View File
@@ -102,19 +102,23 @@ class Predictor(nn.Module):
# ── Data ────────────────────────────────────────────────────────────────────
def build():
"""Year-based split: encoder trains on 2019-2021; probe evaluates on 2022-2023 OOS."""
df = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
df["date"] = pd.to_datetime(df["date"])
feats = df[["ret", "realized_vol"]].to_numpy(np.float32)
target = df["realized_vol"].to_numpy(np.float32)
X, y = [], []
for t in range(WINDOW, len(df) - 1):
X.append(feats[t - WINDOW:t])
y.append(target[t + 1])
X = np.stack(X); y = np.array(y, np.float32)
n_tr = int(0.7 * len(X))
mu = X[:n_tr].mean((0, 1))
sd = X[:n_tr].std((0, 1)) + 1e-8
X = (X - mu) / sd
return (X[:n_tr], y[:n_tr]), (X[n_tr:], y[n_tr:])
tr_idx = df.index[df["date"].dt.year <= 2021].tolist()
te_idx = df.index[df["date"].dt.year >= 2022].tolist()
mu = feats[:tr_idx[-1]+1].mean(0)
sd = feats[:tr_idx[-1]+1].std(0) + 1e-8
fn = (feats - mu) / sd
def windows(idx):
X, y = [], []
for t in idx:
if t - WINDOW >= 0 and t + 1 < len(df):
X.append(fn[t - WINDOW:t]); y.append(target[t + 1])
return np.stack(X).astype(np.float32), np.array(y, np.float32)
return windows(tr_idx), windows(te_idx)
# ── Training ─────────────────────────────────────────────────────────────────
@@ -169,6 +173,42 @@ def main():
}, open("metrics.json", "w"), indent=2)
print("val_vol_r2 = %.4f (n_test=%d, dev=%s)" % (val_vol_r2, len(yte), dev))
# ── EXPORT BLOCK — do NOT edit (agent boundary) ──────────────────────────
# Set EXPORT_EMBEDDINGS=1 to write embeddings.json for the Go eval harness.
# Uses year-based split (train≤2021, OOS≥2022) regardless of probe split.
import os
if os.environ.get("EXPORT_EMBEDDINGS") == "1":
df2 = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
df2["date"] = pd.to_datetime(df2["date"])
tr_mask = df2["date"].dt.year <= 2021
feats2 = df2[["ret", "realized_vol"]].to_numpy(np.float32)
mu2 = feats2[tr_mask].mean(0); sd2 = feats2[tr_mask].std(0) + 1e-8
fn2 = (feats2 - mu2) / sd2
def _export_windows(year_mask):
idx = df2.index[year_mask].tolist()
Xs, dates, rvs = [], [], []
for t in idx:
if t - WINDOW >= 0:
Xs.append(fn2[t - WINDOW:t])
dates.append(str(df2["date"].iloc[t].date()))
rvs.append(float(df2["realized_vol"].iloc[t]))
if not Xs:
return [], [], []
with torch.no_grad():
E = enc(torch.tensor(np.stack(Xs), device=dev)).mean(1).cpu().numpy().tolist()
return E, dates, rvs
Etr, dates_tr, rv_tr = _export_windows(tr_mask)
Eoos, dates_oos, rv_oos = _export_windows(df2["date"].dt.year >= 2022)
hv_thr = float(np.percentile(rv_oos, 67))
hv_label = [1 if v >= hv_thr else 0 for v in rv_oos]
json.dump({"embeddings": Eoos, "dates": dates_oos,
"realized_vol": rv_oos, "hv_label": hv_label,
"train_embeddings": Etr, "train_realized_vol": rv_tr},
open("embeddings.json", "w"))
print("exported embeddings.json train=%d oos=%d HV=%d/%d" % (
len(Etr), len(Eoos), sum(hv_label), len(hv_label)))
# ── END EXPORT BLOCK ─────────────────────────────────────────────────────
if __name__ == "__main__":
main()