fix(eval): correct probe metric to use true year-based OOS split
CD / Lint / Test / Vet (push) Successful in 4s
CD / Build & Import (push) Failing after 7s
CD / Deploy via GitOps (push) Has been skipped

- train.py build(): year-based split (train≤2021, OOS≥2022) replaces
  misleading 70/30 mixed-period split; true OOS val_vol_r2 now ~-0.36
  vs previously reported +0.18 (artefact of cross-period data leakage)
- train.py: EXPORT_EMBEDDINGS block now exports both train+OOS embeddings
  with dates and HV labels for Go eval harness
- cmd/eval: LinearProbeTrainTest uses train stats for standardisation of
  both sets (no leakage); standardiseCompute/applyStandardise helpers
- internal/eval: add LinearProbeTrainTest (fit-on-train, eval-on-OOS)
  alongside LinearProbe (same-set); 8/8 tests still green

Phase-0 gate result: val_vol_r2=-0.36, silhouette=0.043, erank=58.9/64.
Backbone produces high-rank embeddings (SIGReg working) but does NOT
generalize across 2021→2022 regime boundary. Gate: INCONCLUSIVE/FAIL.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-24 22:57:14 +02:00
co-authored by Claude Sonnet 4.6
parent e11e7d2524
commit 20aeecb971
3 changed files with 186 additions and 44 deletions
+52
View File
@@ -10,6 +10,58 @@ import (
"math"
)
// LinearProbeTrainTest fits ridge regression on (trainEmb, trainY) and evaluates
// on (testEmb, testY). Returns OOS R². Use this for proper held-out evaluation.
func LinearProbeTrainTest(trainEmb [][]float64, trainY []float64,
testEmb [][]float64, testY []float64, lambda float64) float64 {
n := len(trainEmb)
if n == 0 || len(testEmb) == 0 {
return 0
}
d := len(trainEmb[0])
p := d + 1
A := make([][]float64, n)
for i, e := range trainEmb {
row := make([]float64, p)
copy(row, e)
row[d] = 1.0
A[i] = row
}
AtA := make([][]float64, p)
for i := range AtA {
AtA[i] = make([]float64, p)
}
Aty := make([]float64, p)
for i := 0; i < n; i++ {
for j := 0; j < p; j++ {
Aty[j] += A[i][j] * trainY[i]
for k := 0; k < p; k++ {
AtA[j][k] += A[i][j] * A[i][k]
}
}
}
for j := 0; j < p; j++ {
AtA[j][j] += lambda
}
w := solveCholesky(AtA, Aty)
yMean := mean(testY)
var ssRes, ssTot float64
for i, e := range testEmb {
row := make([]float64, p)
copy(row, e)
row[d] = 1.0
pred := dot(row, w)
ssRes += (testY[i] - pred) * (testY[i] - pred)
ssTot += (testY[i] - yMean) * (testY[i] - yMean)
}
if ssTot == 0 {
return 0
}
return 1 - ssRes/ssTot
}
// LinearProbe fits a ridge regression (closed-form) on (emb, y) with regularisation λ
// and returns R² on the same data. Call with train embeddings; probe on held-out by
// splitting before calling.