generated from mathias/template-go-web
mathiasandClaude Sonnet 4.6
68bf8f15c5
#12 — VaR_breach_rate_99_oos_regime_cond metric: - internal/eval/var.go: VaRBreachRate() + kupiecPOF() + LinearProbePredict() (stdlib math only) - internal/eval/var_test.go: 8 golden tests (zero/all breach, perfect calibration, boundary) - cmd/eval/main.go: -metric var flag (no-leakage probe → VaR → Kupiec P) - scripts/var_breach.py: Python equivalent with METRIC_KEY constant (13 TDD tests) - train.py LOCKED VaR EVAL BLOCK: writes VaR_breach_rate_99_oos_regime_cond + kupiec_p to metrics.json - Fixed bug: train.py used bare 'os' before import; now uses module-level '_os' consistently #13 — HMM regime detector + JEPA conditioning seam: - scripts/prepare_regime.py: GaussianHMM (diag, 3-state) on realized_vol; states sorted by mean vol (0=calm, 1=stressed, 2=crisis); deterministic (random_state=42); outputs eurusd_regime.parquet - tests/test_regime.py: 11 TDD tests (dtype, states, determinism, vol sort, daily fallback) - train.py: JEPA_ENABLE_REGIME toggle + REGIME CONDITIONING SEAM (concat baseline, agent-editable) - requirements.txt: hmmlearn>=0.3, scikit-learn>=1.4 78 Python + all Go tests green. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
109 lines
3.5 KiB
Python
109 lines
3.5 KiB
Python
"""Tests for scripts/var_breach.py — VaR breach rate + Kupiec POF (jepa-fx-risk#12).
|
||
|
||
Golden tests first: verify the math before wiring it into train.py.
|
||
"""
|
||
|
||
import importlib.util
|
||
import math
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
_SCRIPT = Path(__file__).parent.parent / "scripts" / "var_breach.py"
|
||
|
||
|
||
def _import():
|
||
spec = importlib.util.spec_from_file_location("var_breach", _SCRIPT)
|
||
mod = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(mod)
|
||
return mod
|
||
|
||
|
||
@pytest.fixture()
|
||
def mod():
|
||
return _import()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# var_breach_rate
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestVarBreachRate:
|
||
def test_zero_breaches(self, mod):
|
||
# 0.02 < 0.01×2.326=0.02326 → no breach
|
||
rate, _ = mod.var_breach_rate([0.01, 0.01], [0.02, 0.02])
|
||
assert rate == 0.0
|
||
|
||
def test_all_breach(self, mod):
|
||
# 0.03 > 0.02326 → all breach
|
||
rate, _ = mod.var_breach_rate([0.01, 0.01], [0.03, 0.03])
|
||
assert rate == 1.0
|
||
|
||
def test_golden_two_of_ten(self, mod):
|
||
pred = [0.01] * 10
|
||
actual = [0.01] * 10
|
||
actual[0] = 0.03 # breach
|
||
actual[2] = 0.03 # breach
|
||
rate, kupiec_p = mod.var_breach_rate(pred, actual)
|
||
assert abs(rate - 0.2) < 1e-9, f"rate={rate}"
|
||
assert kupiec_p < 0.05, f"kupiec_p={kupiec_p}" # strong reject
|
||
|
||
def test_perfect_calibration(self, mod):
|
||
# n=100, 1 breach → p_hat=0.01=p0=0.01 → LR=0 → kupiec_p≈1
|
||
pred = [0.01] * 100
|
||
actual = [0.015] * 100
|
||
actual[0] = 0.025 # 0.025 > 0.02326 → breach
|
||
rate, kupiec_p = mod.var_breach_rate(pred, actual)
|
||
assert abs(rate - 0.01) < 1e-9
|
||
assert kupiec_p > 0.9, f"kupiec_p={kupiec_p}"
|
||
|
||
def test_boundary_at_var_is_not_breach(self, mod):
|
||
# exactly at VaR_99 is NOT a breach (strict >)
|
||
z99 = 2.326
|
||
var = 0.01 * z99
|
||
rate, _ = mod.var_breach_rate([0.01], [var], z99=z99)
|
||
assert rate == 0.0
|
||
|
||
def test_empty_returns_zero_one(self, mod):
|
||
rate, kupiec_p = mod.var_breach_rate([], [])
|
||
assert rate == 0.0
|
||
assert kupiec_p == 1.0
|
||
|
||
def test_metric_key_no_whitespace(self, mod):
|
||
key = mod.METRIC_KEY
|
||
assert key == key.strip(), f"metric key has surrounding whitespace: {key!r}"
|
||
assert " " not in key, f"metric key contains space: {key!r}"
|
||
|
||
def test_metric_key_is_canonical(self, mod):
|
||
assert mod.METRIC_KEY == "VaR_breach_rate_99_oos_regime_cond"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# kupiec_pvalue
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestKupiecPValue:
|
||
def test_perfectly_calibrated(self, mod):
|
||
# p_hat == p0 → LR=0 → p-value=1
|
||
p = mod.kupiec_pvalue(100, 1, p0=0.01)
|
||
assert p > 0.99, f"p={p}"
|
||
|
||
def test_strong_reject_high_breach(self, mod):
|
||
# 20% breach when 1% expected → p << 0.05
|
||
p = mod.kupiec_pvalue(100, 20, p0=0.01)
|
||
assert p < 0.001, f"p={p}"
|
||
|
||
def test_zero_breaches_not_nan(self, mod):
|
||
p = mod.kupiec_pvalue(100, 0, p0=0.01)
|
||
assert not math.isnan(p)
|
||
assert 0 <= p <= 1.0
|
||
|
||
def test_all_breaches_not_nan(self, mod):
|
||
p = mod.kupiec_pvalue(10, 10, p0=0.01)
|
||
assert not math.isnan(p)
|
||
assert p < 0.001 # extremely unlikely
|
||
|
||
def test_zero_observations(self, mod):
|
||
p = mod.kupiec_pvalue(0, 0)
|
||
assert p == 1.0
|