feat(wing): backtest metric (coverage/precision at a threshold) and shared TopWing (#205 S1)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
package wing
|
||||
|
||||
// Sample is one backtest observation: the human-decided wing of an existing
|
||||
// page, and what the two independent opinions said about it with the wing
|
||||
// label removed.
|
||||
type Sample struct {
|
||||
Truth string
|
||||
Suggested string // classifier's wing, "" if it abstained or failed
|
||||
Confidence float64
|
||||
ScorerTop string // scorer's single best wing, "" if no clear opinion
|
||||
}
|
||||
|
||||
// Result is the policy outcome at one threshold.
|
||||
type Result struct {
|
||||
N int
|
||||
Auto int // placed without a human
|
||||
Correct int // of Auto, matching Truth
|
||||
Coverage float64 // Auto / N
|
||||
Precision float64 // Correct / Auto; 0 when Auto == 0
|
||||
}
|
||||
|
||||
// Evaluate applies the Resolver's classified-path rule (confident, and the
|
||||
// scorer agrees) to recorded samples. Eligibility is not modelled: callers
|
||||
// backtest only on wings they would allow.
|
||||
func Evaluate(samples []Sample, threshold float64) Result {
|
||||
r := Result{N: len(samples)}
|
||||
for _, s := range samples {
|
||||
if s.Suggested == "" || s.Confidence < threshold || s.ScorerTop != s.Suggested {
|
||||
continue
|
||||
}
|
||||
r.Auto++
|
||||
if s.Suggested == s.Truth {
|
||||
r.Correct++
|
||||
}
|
||||
}
|
||||
if r.N > 0 {
|
||||
r.Coverage = float64(r.Auto) / float64(r.N)
|
||||
}
|
||||
if r.Auto > 0 {
|
||||
r.Precision = float64(r.Correct) / float64(r.Auto)
|
||||
}
|
||||
return r
|
||||
}
|
||||
Reference in New Issue
Block a user