generated from mathias/template-go-web
fix(eval): correct probe metric to use true year-based OOS split
- train.py build(): year-based split (train≤2021, OOS≥2022) replaces misleading 70/30 mixed-period split; true OOS val_vol_r2 now ~-0.36 vs previously reported +0.18 (artefact of cross-period data leakage) - train.py: EXPORT_EMBEDDINGS block now exports both train+OOS embeddings with dates and HV labels for Go eval harness - cmd/eval: LinearProbeTrainTest uses train stats for standardisation of both sets (no leakage); standardiseCompute/applyStandardise helpers - internal/eval: add LinearProbeTrainTest (fit-on-train, eval-on-OOS) alongside LinearProbe (same-set); 8/8 tests still green Phase-0 gate result: val_vol_r2=-0.36, silhouette=0.043, erank=58.9/64. Backbone produces high-rank embeddings (SIGReg working) but does NOT generalize across 2021→2022 regime boundary. Gate: INCONCLUSIVE/FAIL. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,58 @@ import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// LinearProbeTrainTest fits ridge regression on (trainEmb, trainY) and evaluates
|
||||
// on (testEmb, testY). Returns OOS R². Use this for proper held-out evaluation.
|
||||
func LinearProbeTrainTest(trainEmb [][]float64, trainY []float64,
|
||||
testEmb [][]float64, testY []float64, lambda float64) float64 {
|
||||
n := len(trainEmb)
|
||||
if n == 0 || len(testEmb) == 0 {
|
||||
return 0
|
||||
}
|
||||
d := len(trainEmb[0])
|
||||
p := d + 1
|
||||
|
||||
A := make([][]float64, n)
|
||||
for i, e := range trainEmb {
|
||||
row := make([]float64, p)
|
||||
copy(row, e)
|
||||
row[d] = 1.0
|
||||
A[i] = row
|
||||
}
|
||||
AtA := make([][]float64, p)
|
||||
for i := range AtA {
|
||||
AtA[i] = make([]float64, p)
|
||||
}
|
||||
Aty := make([]float64, p)
|
||||
for i := 0; i < n; i++ {
|
||||
for j := 0; j < p; j++ {
|
||||
Aty[j] += A[i][j] * trainY[i]
|
||||
for k := 0; k < p; k++ {
|
||||
AtA[j][k] += A[i][j] * A[i][k]
|
||||
}
|
||||
}
|
||||
}
|
||||
for j := 0; j < p; j++ {
|
||||
AtA[j][j] += lambda
|
||||
}
|
||||
w := solveCholesky(AtA, Aty)
|
||||
|
||||
yMean := mean(testY)
|
||||
var ssRes, ssTot float64
|
||||
for i, e := range testEmb {
|
||||
row := make([]float64, p)
|
||||
copy(row, e)
|
||||
row[d] = 1.0
|
||||
pred := dot(row, w)
|
||||
ssRes += (testY[i] - pred) * (testY[i] - pred)
|
||||
ssTot += (testY[i] - yMean) * (testY[i] - yMean)
|
||||
}
|
||||
if ssTot == 0 {
|
||||
return 0
|
||||
}
|
||||
return 1 - ssRes/ssTot
|
||||
}
|
||||
|
||||
// LinearProbe fits a ridge regression (closed-form) on (emb, y) with regularisation λ
|
||||
// and returns R² on the same data. Call with train embeddings; probe on held-out by
|
||||
// splitting before calling.
|
||||
|
||||
Reference in New Issue
Block a user