ingestion/cmd/wingbacktest/main.go

@@ -0,0 +1,173 @@
+// Command wingbacktest measures how well the wing resolver would have placed
+// pages whose wing a human already decided (hyperguild#205).
+//
+// It reads pages from the wings given in -wings, hides the wing, asks the
+// local classifier and the BM25 scorer, and reports coverage/precision per
+// confidence threshold. Page text is sent to -llm-url, which MUST be a local
+// model (it is not checked here); sample only wings you would allow.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
+)
+
+var legacyDirs = map[string]bool{"concepts": true, "entities": true, "sources": true}
+
+func main() {
+	brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory (read only)")
+	wingsFlag := flag.String("wings", "", "comma-separated wings to sample pages FROM (must be wings you would allow)")
+	perWing := flag.Int("per-wing", 8, "pages sampled per wing")
+	llmURL := flag.String("llm-url", "http://10.0.1.20:31234/v1", "OpenAI-compatible LOCAL endpoint")
+	model := flag.String("model", "qwen35-9b-fast", "model name")
+	seed := flag.Int64("seed", 1, "sampling seed")
+	flag.Parse()
+
+	if err := run(*brain, strings.Split(*wingsFlag, ","), *perWing, *llmURL, *model, *seed); err != nil {
+		slog.Error("wingbacktest", "err", err)
+		os.Exit(1)
+	}
+}
+
+func run(brain string, sampleWings []string, perWing int, llmURL, model string, seed int64) error {
+	known, err := knownWings(brain)
+	if err != nil {
+		return err
+	}
+	client := llm.New(llmURL, "", model, 3*time.Minute)
+	classifier := wing.NewLLMClassifier(client.Complete)
+	rng := rand.New(rand.NewSource(seed))
+
+	var samples []wing.Sample
+	perWingSamples := map[string][]wing.Sample{}
+	for _, w := range sampleWings {
+		w = strings.TrimSpace(w)
+		if w == "" {
+			continue
+		}
+		pages, err := listPages(brain, w)
+		if err != nil {
+			return err
+		}
+		rng.Shuffle(len(pages), func(i, j int) { pages[i], pages[j] = pages[j], pages[i] })
+		if len(pages) > perWing {
+			pages = pages[:perWing]
+		}
+		for _, rel := range pages {
+			text, err := readBody(brain, rel)
+			if err != nil || len(text) < 200 {
+				continue
+			}
+			sugg, cerr := classifier.Classify(context.Background(), text, known)
+			if cerr != nil {
+				slog.Warn("classify failed", "page", rel, "err", cerr)
+			}
+			top, _ := wing.TopWing(wing.NewBM25Scorer(brain, known, rel).Score(text))
+			s := wing.Sample{Truth: w, Suggested: strings.ToLower(sugg.Wing), Confidence: sugg.Confidence, ScorerTop: top}
+			slog.Info("sample", "page", rel, "truth", w, "classifier", s.Suggested, "conf", s.Confidence, "scorer", top, "reason", sugg.Reason)
+			samples = append(samples, s)
+			perWingSamples[w] = append(perWingSamples[w], s)
+		}
+	}
+	report(samples, perWingSamples)
+	return nil
+}
+
+func report(samples []wing.Sample, per map[string][]wing.Sample) {
+	fmt.Printf("# wing backtest\n\nn=%d pages. Classifier+scorer must agree; threshold applies to classifier confidence.\n\n", len(samples))
+	fmt.Println("| threshold | auto | coverage | precision |")
+	fmt.Println("|---|---|---|---|")
+	for _, t := range []float64{0.5, 0.6, 0.7, 0.8, 0.9, 0.95} {
+		r := wing.Evaluate(samples, t)
+		fmt.Printf("| %.2f | %d/%d | %.0f%% | %.0f%% |\n", t, r.Auto, r.N, 100*r.Coverage, 100*r.Precision)
+	}
+	fmt.Println("\n## per wing (threshold 0.80)\n\n| wing | n | auto | precision |\n|---|---|---|---|")
+	names := make([]string, 0, len(per))
+	for w := range per {
+		names = append(names, w)
+	}
+	sort.Strings(names)
+	for _, w := range names {
+		r := wing.Evaluate(per[w], 0.8)
+		fmt.Printf("| %s | %d | %d | %.0f%% |\n", w, r.N, r.Auto, 100*r.Precision)
+	}
+	fmt.Println("\n## wrong auto-placements at 0.80")
+	fmt.Println()
+	for _, s := range samples {
+		if s.Suggested != "" && s.Confidence >= 0.8 && s.ScorerTop == s.Suggested && s.Suggested != s.Truth {
+			fmt.Printf("- truth=%s -> both said %s (conf %.2f)\n", s.Truth, s.Suggested, s.Confidence)
+		}
+	}
+}
+
+// knownWings lists wiki/<wing> dirs that hold hall subdirectories, skipping
+// the retired legacy layout.
+func knownWings(brain string) ([]string, error) {
+	entries, err := os.ReadDir(filepath.Join(brain, "wiki"))
+	if err != nil {
+		return nil, fmt.Errorf("list wings: %w", err)
+	}
+	var out []string
+	for _, e := range entries {
+		if !e.IsDir() || legacyDirs[e.Name()] {
+			continue
+		}
+		subs, _ := os.ReadDir(filepath.Join(brain, "wiki", e.Name()))
+		for _, s := range subs {
+			if s.IsDir() {
+				out = append(out, e.Name())
+				break
+			}
+		}
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+func listPages(brain, w string) ([]string, error) {
+	var out []string
+	root := filepath.Join(brain, "wiki", w)
+	err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || !d.Type().IsRegular() || !strings.HasSuffix(p, ".md") || d.Name() == "_index.md" {
+			return err
+		}
+		rel, rerr := filepath.Rel(brain, p)
+		if rerr != nil {
+			return rerr
+		}
+		out = append(out, filepath.ToSlash(rel))
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("walk %s: %w", w, err)
+	}
+	return out, nil
+}
+
+// readBody returns the page text without YAML frontmatter, so the wing label
+// (and anything else in frontmatter) cannot leak into the classification.
+func readBody(brain, rel string) (string, error) {
+	b, err := os.ReadFile(filepath.Join(brain, filepath.FromSlash(rel)))
+	if err != nil {
+		return "", fmt.Errorf("read %s: %w", rel, err)
+	}
+	s := string(b)
+	if strings.HasPrefix(s, "---\n") {
+		if end := strings.Index(s[4:], "\n---"); end >= 0 {
+			s = s[4+end+4:]
+		}
+	}
+	return strings.TrimSpace(s), nil
+}

ingestion/internal/wing/bm25scorer.go

@@ -11,7 +11,6 @@ import (
 const (
 	maxQueryTerms = 12
 	minTermLen    = 5
-	hitsPerWing   = 3
 )
 
 // BM25Scorer scores a note against each wing by running the brain's own BM25
@@ -38,24 +37,18 @@ func (b *BM25Scorer) Score(text string) map[string]float64 {
 		return out
 	}
 	for _, w := range b.wings {
-		hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: hitsPerWing + 1})
+		hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: 2})
 		if err != nil {
 			continue
 		}
-		var sum float64
-		n := 0
 		for _, h := range hits {
 			if b.exclude != "" && h.Path == b.exclude {
 				continue
 			}
-			if n == hitsPerWing {
+			// Hits come best-first: a wing is scored by its single best
-				break
+			// page, so a big wing cannot win on volume alone.
-			}
+			out[w] = float64(h.Score)
-			sum += float64(h.Score)
+			break
-			n++
-		}
-		if sum > 0 {
-			out[w] = sum
 		}
 	}
 	return out

ingestion/internal/wing/bm25scorer_test.go

@@ -43,3 +43,19 @@ func TestBM25Scorer_NoUsableTermsScoresNothing(t *testing.T) {
 	s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "")
 	assert.Empty(t, s.Score("a an to of"))
 }
+
+// A big wing must not win just by having more pages that brush the query: a
+// wing is scored by its single best page, not by the sum of its top hits.
+func TestBM25Scorer_ABigWingDoesNotWinOnVolume(t *testing.T) {
+	dir := t.TempDir()
+	writePage(t, dir, "wiki/small/facts/target.md", "---\ntitle: Target\n---\nkubernetes cluster scheduling pods deployment nodes\n")
+	for i := 0; i < 8; i++ {
+		writePage(t, dir, filepath.Join("wiki/big/facts", "p"+string(rune('a'+i))+".md"),
+			"---\ntitle: Filler\n---\nkubernetes cluster scheduling mentioned in passing among bookkeeping ledger entries\n")
+	}
+	s := wing.NewBM25Scorer(dir, []string{"small", "big"}, "")
+	got := s.Score("kubernetes cluster scheduling pods deployment nodes")
+	top, ok := wing.TopWing(got)
+	assert.True(t, ok)
+	assert.Equal(t, "small", top)
+}