44 lines
1.3 KiB
Go
44 lines
1.3 KiB
Go
package wing
|
|
|
|
// Sample is one backtest observation: the human-decided wing of an existing
|
|
// page, and what the two independent opinions said about it with the wing
|
|
// label removed.
|
|
type Sample struct {
|
|
Truth string
|
|
Suggested string // classifier's wing, "" if it abstained or failed
|
|
Confidence float64
|
|
ScorerTop string // scorer's single best wing, "" if no clear opinion
|
|
}
|
|
|
|
// Result is the policy outcome at one threshold.
|
|
type Result struct {
|
|
N int
|
|
Auto int // placed without a human
|
|
Correct int // of Auto, matching Truth
|
|
Coverage float64 // Auto / N
|
|
Precision float64 // Correct / Auto; 0 when Auto == 0
|
|
}
|
|
|
|
// Evaluate applies the Resolver's classified-path rule (confident, and the
|
|
// scorer agrees) to recorded samples. Eligibility is not modelled: callers
|
|
// backtest only on wings they would allow.
|
|
func Evaluate(samples []Sample, threshold float64) Result {
|
|
r := Result{N: len(samples)}
|
|
for _, s := range samples {
|
|
if s.Suggested == "" || s.Confidence < threshold || s.ScorerTop != s.Suggested {
|
|
continue
|
|
}
|
|
r.Auto++
|
|
if s.Suggested == s.Truth {
|
|
r.Correct++
|
|
}
|
|
}
|
|
if r.N > 0 {
|
|
r.Coverage = float64(r.Auto) / float64(r.N)
|
|
}
|
|
if r.Auto > 0 {
|
|
r.Precision = float64(r.Correct) / float64(r.Auto)
|
|
}
|
|
return r
|
|
}
|