feat(wing): backtest metric (coverage/precision at a threshold) and shared TopWing (#205 S1)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
func TestEvaluate_PolicyAtAThreshold(t *testing.T) {
|
||||
samples := []wing.Sample{
|
||||
{Truth: "a", Suggested: "a", Confidence: 0.9, ScorerTop: "a"}, // auto, correct
|
||||
{Truth: "a", Suggested: "b", Confidence: 0.9, ScorerTop: "b"}, // auto, WRONG (both models agree on a wrong wing)
|
||||
{Truth: "a", Suggested: "a", Confidence: 0.5, ScorerTop: "a"}, // below threshold -> exception
|
||||
{Truth: "a", Suggested: "a", Confidence: 0.95, ScorerTop: "b"}, // scorer disagrees -> exception
|
||||
{Truth: "a", Suggested: "", Confidence: 0.99, ScorerTop: "a"}, // abstain -> exception
|
||||
}
|
||||
r := wing.Evaluate(samples, 0.8)
|
||||
assert.Equal(t, 5, r.N)
|
||||
assert.Equal(t, 2, r.Auto)
|
||||
assert.Equal(t, 1, r.Correct)
|
||||
assert.InDelta(t, 0.4, r.Coverage, 1e-9)
|
||||
assert.InDelta(t, 0.5, r.Precision, 1e-9)
|
||||
}
|
||||
|
||||
func TestEvaluate_NothingAutoHasZeroPrecisionNotNaN(t *testing.T) {
|
||||
r := wing.Evaluate([]wing.Sample{{Truth: "a", Suggested: "a", Confidence: 0.1, ScorerTop: "a"}}, 0.8)
|
||||
assert.Zero(t, r.Auto)
|
||||
assert.Zero(t, r.Precision)
|
||||
assert.Zero(t, r.Coverage)
|
||||
}
|
||||
|
||||
func TestEvaluate_EmptyInput(t *testing.T) {
|
||||
r := wing.Evaluate(nil, 0.8)
|
||||
assert.Zero(t, r.N)
|
||||
assert.Zero(t, r.Coverage)
|
||||
}
|
||||
Reference in New Issue
Block a user