feat(wing): score a wing by its best page, not summed hits; add wingbacktest command (#205 S1)
A first backtest showed BM25 summed over a wing's top hits favours big wings (0/4 agreement). Best-page scoring is volume-robust; pinned by a test that failed before the change. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,7 +11,6 @@ import (
|
||||
const (
|
||||
maxQueryTerms = 12
|
||||
minTermLen = 5
|
||||
hitsPerWing = 3
|
||||
)
|
||||
|
||||
// BM25Scorer scores a note against each wing by running the brain's own BM25
|
||||
@@ -38,24 +37,18 @@ func (b *BM25Scorer) Score(text string) map[string]float64 {
|
||||
return out
|
||||
}
|
||||
for _, w := range b.wings {
|
||||
hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: hitsPerWing + 1})
|
||||
hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: 2})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var sum float64
|
||||
n := 0
|
||||
for _, h := range hits {
|
||||
if b.exclude != "" && h.Path == b.exclude {
|
||||
continue
|
||||
}
|
||||
if n == hitsPerWing {
|
||||
break
|
||||
}
|
||||
sum += float64(h.Score)
|
||||
n++
|
||||
}
|
||||
if sum > 0 {
|
||||
out[w] = sum
|
||||
// Hits come best-first: a wing is scored by its single best
|
||||
// page, so a big wing cannot win on volume alone.
|
||||
out[w] = float64(h.Score)
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -43,3 +43,19 @@ func TestBM25Scorer_NoUsableTermsScoresNothing(t *testing.T) {
|
||||
s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "")
|
||||
assert.Empty(t, s.Score("a an to of"))
|
||||
}
|
||||
|
||||
// A big wing must not win just by having more pages that brush the query: a
|
||||
// wing is scored by its single best page, not by the sum of its top hits.
|
||||
func TestBM25Scorer_ABigWingDoesNotWinOnVolume(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePage(t, dir, "wiki/small/facts/target.md", "---\ntitle: Target\n---\nkubernetes cluster scheduling pods deployment nodes\n")
|
||||
for i := 0; i < 8; i++ {
|
||||
writePage(t, dir, filepath.Join("wiki/big/facts", "p"+string(rune('a'+i))+".md"),
|
||||
"---\ntitle: Filler\n---\nkubernetes cluster scheduling mentioned in passing among bookkeeping ledger entries\n")
|
||||
}
|
||||
s := wing.NewBM25Scorer(dir, []string{"small", "big"}, "")
|
||||
got := s.Score("kubernetes cluster scheduling pods deployment nodes")
|
||||
top, ok := wing.TopWing(got)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "small", top)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user