generated from mathias/template-go-web
fix(eval): correct probe metric to use true year-based OOS split
- train.py build(): year-based split (train≤2021, OOS≥2022) replaces misleading 70/30 mixed-period split; true OOS val_vol_r2 now ~-0.36 vs previously reported +0.18 (artefact of cross-period data leakage) - train.py: EXPORT_EMBEDDINGS block now exports both train+OOS embeddings with dates and HV labels for Go eval harness - cmd/eval: LinearProbeTrainTest uses train stats for standardisation of both sets (no leakage); standardiseCompute/applyStandardise helpers - internal/eval: add LinearProbeTrainTest (fit-on-train, eval-on-OOS) alongside LinearProbe (same-set); 8/8 tests still green Phase-0 gate result: val_vol_r2=-0.36, silhouette=0.043, erank=58.9/64. Backbone produces high-rank embeddings (SIGReg working) but does NOT generalize across 2021→2022 regime boundary. Gate: INCONCLUSIVE/FAIL. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -102,19 +102,23 @@ class Predictor(nn.Module):
|
||||
# ── Data ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def build():
|
||||
"""Year-based split: encoder trains on 2019-2021; probe evaluates on 2022-2023 OOS."""
|
||||
df = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
|
||||
df["date"] = pd.to_datetime(df["date"])
|
||||
feats = df[["ret", "realized_vol"]].to_numpy(np.float32)
|
||||
target = df["realized_vol"].to_numpy(np.float32)
|
||||
X, y = [], []
|
||||
for t in range(WINDOW, len(df) - 1):
|
||||
X.append(feats[t - WINDOW:t])
|
||||
y.append(target[t + 1])
|
||||
X = np.stack(X); y = np.array(y, np.float32)
|
||||
n_tr = int(0.7 * len(X))
|
||||
mu = X[:n_tr].mean((0, 1))
|
||||
sd = X[:n_tr].std((0, 1)) + 1e-8
|
||||
X = (X - mu) / sd
|
||||
return (X[:n_tr], y[:n_tr]), (X[n_tr:], y[n_tr:])
|
||||
tr_idx = df.index[df["date"].dt.year <= 2021].tolist()
|
||||
te_idx = df.index[df["date"].dt.year >= 2022].tolist()
|
||||
mu = feats[:tr_idx[-1]+1].mean(0)
|
||||
sd = feats[:tr_idx[-1]+1].std(0) + 1e-8
|
||||
fn = (feats - mu) / sd
|
||||
def windows(idx):
|
||||
X, y = [], []
|
||||
for t in idx:
|
||||
if t - WINDOW >= 0 and t + 1 < len(df):
|
||||
X.append(fn[t - WINDOW:t]); y.append(target[t + 1])
|
||||
return np.stack(X).astype(np.float32), np.array(y, np.float32)
|
||||
return windows(tr_idx), windows(te_idx)
|
||||
|
||||
|
||||
# ── Training ─────────────────────────────────────────────────────────────────
|
||||
@@ -169,6 +173,42 @@ def main():
|
||||
}, open("metrics.json", "w"), indent=2)
|
||||
print("val_vol_r2 = %.4f (n_test=%d, dev=%s)" % (val_vol_r2, len(yte), dev))
|
||||
|
||||
# ── EXPORT BLOCK — do NOT edit (agent boundary) ──────────────────────────
|
||||
# Set EXPORT_EMBEDDINGS=1 to write embeddings.json for the Go eval harness.
|
||||
# Uses year-based split (train≤2021, OOS≥2022) regardless of probe split.
|
||||
import os
|
||||
if os.environ.get("EXPORT_EMBEDDINGS") == "1":
|
||||
df2 = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
|
||||
df2["date"] = pd.to_datetime(df2["date"])
|
||||
tr_mask = df2["date"].dt.year <= 2021
|
||||
feats2 = df2[["ret", "realized_vol"]].to_numpy(np.float32)
|
||||
mu2 = feats2[tr_mask].mean(0); sd2 = feats2[tr_mask].std(0) + 1e-8
|
||||
fn2 = (feats2 - mu2) / sd2
|
||||
def _export_windows(year_mask):
|
||||
idx = df2.index[year_mask].tolist()
|
||||
Xs, dates, rvs = [], [], []
|
||||
for t in idx:
|
||||
if t - WINDOW >= 0:
|
||||
Xs.append(fn2[t - WINDOW:t])
|
||||
dates.append(str(df2["date"].iloc[t].date()))
|
||||
rvs.append(float(df2["realized_vol"].iloc[t]))
|
||||
if not Xs:
|
||||
return [], [], []
|
||||
with torch.no_grad():
|
||||
E = enc(torch.tensor(np.stack(Xs), device=dev)).mean(1).cpu().numpy().tolist()
|
||||
return E, dates, rvs
|
||||
Etr, dates_tr, rv_tr = _export_windows(tr_mask)
|
||||
Eoos, dates_oos, rv_oos = _export_windows(df2["date"].dt.year >= 2022)
|
||||
hv_thr = float(np.percentile(rv_oos, 67))
|
||||
hv_label = [1 if v >= hv_thr else 0 for v in rv_oos]
|
||||
json.dump({"embeddings": Eoos, "dates": dates_oos,
|
||||
"realized_vol": rv_oos, "hv_label": hv_label,
|
||||
"train_embeddings": Etr, "train_realized_vol": rv_tr},
|
||||
open("embeddings.json", "w"))
|
||||
print("exported embeddings.json train=%d oos=%d HV=%d/%d" % (
|
||||
len(Etr), len(Eoos), sum(hv_label), len(hv_label)))
|
||||
# ── END EXPORT BLOCK ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user