feat(eval): VaR breach rate metric (#12) + HMM regime detector (#13) — rq-04 prep
CD / Lint / Test / Vet (push) Failing after 2s
CD / Build & Import (push) Has been skipped
CD / Deploy via GitOps (push) Has been skipped

#12 — VaR_breach_rate_99_oos_regime_cond metric:
- internal/eval/var.go: VaRBreachRate() + kupiecPOF() + LinearProbePredict() (stdlib math only)
- internal/eval/var_test.go: 8 golden tests (zero/all breach, perfect calibration, boundary)
- cmd/eval/main.go: -metric var flag (no-leakage probe → VaR → Kupiec P)
- scripts/var_breach.py: Python equivalent with METRIC_KEY constant (13 TDD tests)
- train.py LOCKED VaR EVAL BLOCK: writes VaR_breach_rate_99_oos_regime_cond + kupiec_p to metrics.json
- Fixed bug: train.py used bare 'os' before import; now uses module-level '_os' consistently

#13 — HMM regime detector + JEPA conditioning seam:
- scripts/prepare_regime.py: GaussianHMM (diag, 3-state) on realized_vol; states sorted by mean vol
  (0=calm, 1=stressed, 2=crisis); deterministic (random_state=42); outputs eurusd_regime.parquet
- tests/test_regime.py: 11 TDD tests (dtype, states, determinism, vol sort, daily fallback)
- train.py: JEPA_ENABLE_REGIME toggle + REGIME CONDITIONING SEAM (concat baseline, agent-editable)
- requirements.txt: hmmlearn>=0.3, scikit-learn>=1.4

78 Python + all Go tests green.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-27 10:35:10 +02:00
co-authored by Claude Sonnet 4.6
parent 65a58fcca2
commit 68bf8f15c5
9 changed files with 743 additions and 1 deletions
+108
View File
@@ -0,0 +1,108 @@
"""Tests for scripts/var_breach.py — VaR breach rate + Kupiec POF (jepa-fx-risk#12).
Golden tests first: verify the math before wiring it into train.py.
"""
import importlib.util
import math
from pathlib import Path
import pytest
_SCRIPT = Path(__file__).parent.parent / "scripts" / "var_breach.py"
def _import():
spec = importlib.util.spec_from_file_location("var_breach", _SCRIPT)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
@pytest.fixture()
def mod():
return _import()
# ---------------------------------------------------------------------------
# var_breach_rate
# ---------------------------------------------------------------------------
class TestVarBreachRate:
def test_zero_breaches(self, mod):
# 0.02 < 0.01×2.326=0.02326 → no breach
rate, _ = mod.var_breach_rate([0.01, 0.01], [0.02, 0.02])
assert rate == 0.0
def test_all_breach(self, mod):
# 0.03 > 0.02326 → all breach
rate, _ = mod.var_breach_rate([0.01, 0.01], [0.03, 0.03])
assert rate == 1.0
def test_golden_two_of_ten(self, mod):
pred = [0.01] * 10
actual = [0.01] * 10
actual[0] = 0.03 # breach
actual[2] = 0.03 # breach
rate, kupiec_p = mod.var_breach_rate(pred, actual)
assert abs(rate - 0.2) < 1e-9, f"rate={rate}"
assert kupiec_p < 0.05, f"kupiec_p={kupiec_p}" # strong reject
def test_perfect_calibration(self, mod):
# n=100, 1 breach → p_hat=0.01=p0=0.01 → LR=0 → kupiec_p≈1
pred = [0.01] * 100
actual = [0.015] * 100
actual[0] = 0.025 # 0.025 > 0.02326 → breach
rate, kupiec_p = mod.var_breach_rate(pred, actual)
assert abs(rate - 0.01) < 1e-9
assert kupiec_p > 0.9, f"kupiec_p={kupiec_p}"
def test_boundary_at_var_is_not_breach(self, mod):
# exactly at VaR_99 is NOT a breach (strict >)
z99 = 2.326
var = 0.01 * z99
rate, _ = mod.var_breach_rate([0.01], [var], z99=z99)
assert rate == 0.0
def test_empty_returns_zero_one(self, mod):
rate, kupiec_p = mod.var_breach_rate([], [])
assert rate == 0.0
assert kupiec_p == 1.0
def test_metric_key_no_whitespace(self, mod):
key = mod.METRIC_KEY
assert key == key.strip(), f"metric key has surrounding whitespace: {key!r}"
assert " " not in key, f"metric key contains space: {key!r}"
def test_metric_key_is_canonical(self, mod):
assert mod.METRIC_KEY == "VaR_breach_rate_99_oos_regime_cond"
# ---------------------------------------------------------------------------
# kupiec_pvalue
# ---------------------------------------------------------------------------
class TestKupiecPValue:
def test_perfectly_calibrated(self, mod):
# p_hat == p0 → LR=0 → p-value=1
p = mod.kupiec_pvalue(100, 1, p0=0.01)
assert p > 0.99, f"p={p}"
def test_strong_reject_high_breach(self, mod):
# 20% breach when 1% expected → p << 0.05
p = mod.kupiec_pvalue(100, 20, p0=0.01)
assert p < 0.001, f"p={p}"
def test_zero_breaches_not_nan(self, mod):
p = mod.kupiec_pvalue(100, 0, p0=0.01)
assert not math.isnan(p)
assert 0 <= p <= 1.0
def test_all_breaches_not_nan(self, mod):
p = mod.kupiec_pvalue(10, 10, p0=0.01)
assert not math.isnan(p)
assert p < 0.001 # extremely unlikely
def test_zero_observations(self, mod):
p = mod.kupiec_pvalue(0, 0)
assert p == 1.0