diff --git a/ingestion/internal/wing/backtest.go b/ingestion/internal/wing/backtest.go new file mode 100644 index 0000000..f334251 --- /dev/null +++ b/ingestion/internal/wing/backtest.go @@ -0,0 +1,43 @@ +package wing + +// Sample is one backtest observation: the human-decided wing of an existing +// page, and what the two independent opinions said about it with the wing +// label removed. +type Sample struct { + Truth string + Suggested string // classifier's wing, "" if it abstained or failed + Confidence float64 + ScorerTop string // scorer's single best wing, "" if no clear opinion +} + +// Result is the policy outcome at one threshold. +type Result struct { + N int + Auto int // placed without a human + Correct int // of Auto, matching Truth + Coverage float64 // Auto / N + Precision float64 // Correct / Auto; 0 when Auto == 0 +} + +// Evaluate applies the Resolver's classified-path rule (confident, and the +// scorer agrees) to recorded samples. Eligibility is not modelled: callers +// backtest only on wings they would allow. +func Evaluate(samples []Sample, threshold float64) Result { + r := Result{N: len(samples)} + for _, s := range samples { + if s.Suggested == "" || s.Confidence < threshold || s.ScorerTop != s.Suggested { + continue + } + r.Auto++ + if s.Suggested == s.Truth { + r.Correct++ + } + } + if r.N > 0 { + r.Coverage = float64(r.Auto) / float64(r.N) + } + if r.Auto > 0 { + r.Precision = float64(r.Correct) / float64(r.Auto) + } + return r +} diff --git a/ingestion/internal/wing/backtest_test.go b/ingestion/internal/wing/backtest_test.go new file mode 100644 index 0000000..88ebc90 --- /dev/null +++ b/ingestion/internal/wing/backtest_test.go @@ -0,0 +1,38 @@ +package wing_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" +) + +func TestEvaluate_PolicyAtAThreshold(t *testing.T) { + samples := []wing.Sample{ + {Truth: "a", Suggested: "a", Confidence: 0.9, ScorerTop: "a"}, // auto, correct + {Truth: "a", Suggested: "b", Confidence: 0.9, ScorerTop: "b"}, // auto, WRONG (both models agree on a wrong wing) + {Truth: "a", Suggested: "a", Confidence: 0.5, ScorerTop: "a"}, // below threshold -> exception + {Truth: "a", Suggested: "a", Confidence: 0.95, ScorerTop: "b"}, // scorer disagrees -> exception + {Truth: "a", Suggested: "", Confidence: 0.99, ScorerTop: "a"}, // abstain -> exception + } + r := wing.Evaluate(samples, 0.8) + assert.Equal(t, 5, r.N) + assert.Equal(t, 2, r.Auto) + assert.Equal(t, 1, r.Correct) + assert.InDelta(t, 0.4, r.Coverage, 1e-9) + assert.InDelta(t, 0.5, r.Precision, 1e-9) +} + +func TestEvaluate_NothingAutoHasZeroPrecisionNotNaN(t *testing.T) { + r := wing.Evaluate([]wing.Sample{{Truth: "a", Suggested: "a", Confidence: 0.1, ScorerTop: "a"}}, 0.8) + assert.Zero(t, r.Auto) + assert.Zero(t, r.Precision) + assert.Zero(t, r.Coverage) +} + +func TestEvaluate_EmptyInput(t *testing.T) { + r := wing.Evaluate(nil, 0.8) + assert.Zero(t, r.N) + assert.Zero(t, r.Coverage) +} diff --git a/ingestion/internal/wing/resolve.go b/ingestion/internal/wing/resolve.go index 1dd35a2..646ed19 100644 --- a/ingestion/internal/wing/resolve.go +++ b/ingestion/internal/wing/resolve.go @@ -147,8 +147,15 @@ func (r Resolver) scorerTop(text string) (string, bool) { if r.Scorer == nil { return "", false } + return TopWing(r.Scorer.Score(text)) +} + +// TopWing returns the unique highest-scoring wing in scores, or false when +// there is none or the top is tied. Shared by the Resolver and the backtest so +// both apply the same notion of "the scorer has an opinion". +func TopWing(scores map[string]float64) (string, bool) { best, bestScore, tied := "", 0.0, false - for k, v := range r.Scorer.Score(text) { + for k, v := range scores { switch { case v > bestScore: best, bestScore, tied = norm(k), v, false