diff --git a/ingestion/cmd/wingbacktest/main.go b/ingestion/cmd/wingbacktest/main.go new file mode 100644 index 0000000..08a73e7 --- /dev/null +++ b/ingestion/cmd/wingbacktest/main.go @@ -0,0 +1,173 @@ +// Command wingbacktest measures how well the wing resolver would have placed +// pages whose wing a human already decided (hyperguild#205). +// +// It reads pages from the wings given in -wings, hides the wing, asks the +// local classifier and the BM25 scorer, and reports coverage/precision per +// confidence threshold. Page text is sent to -llm-url, which MUST be a local +// model (it is not checked here); sample only wings you would allow. +package main + +import ( + "context" + "flag" + "fmt" + "io/fs" + "log/slog" + "math/rand" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/mathiasbq/hyperguild/ingestion/internal/llm" + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" +) + +var legacyDirs = map[string]bool{"concepts": true, "entities": true, "sources": true} + +func main() { + brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory (read only)") + wingsFlag := flag.String("wings", "", "comma-separated wings to sample pages FROM (must be wings you would allow)") + perWing := flag.Int("per-wing", 8, "pages sampled per wing") + llmURL := flag.String("llm-url", "http://10.0.1.20:31234/v1", "OpenAI-compatible LOCAL endpoint") + model := flag.String("model", "qwen35-9b-fast", "model name") + seed := flag.Int64("seed", 1, "sampling seed") + flag.Parse() + + if err := run(*brain, strings.Split(*wingsFlag, ","), *perWing, *llmURL, *model, *seed); err != nil { + slog.Error("wingbacktest", "err", err) + os.Exit(1) + } +} + +func run(brain string, sampleWings []string, perWing int, llmURL, model string, seed int64) error { + known, err := knownWings(brain) + if err != nil { + return err + } + client := llm.New(llmURL, "", model, 3*time.Minute) + classifier := wing.NewLLMClassifier(client.Complete) + rng := rand.New(rand.NewSource(seed)) + + var samples []wing.Sample + perWingSamples := map[string][]wing.Sample{} + for _, w := range sampleWings { + w = strings.TrimSpace(w) + if w == "" { + continue + } + pages, err := listPages(brain, w) + if err != nil { + return err + } + rng.Shuffle(len(pages), func(i, j int) { pages[i], pages[j] = pages[j], pages[i] }) + if len(pages) > perWing { + pages = pages[:perWing] + } + for _, rel := range pages { + text, err := readBody(brain, rel) + if err != nil || len(text) < 200 { + continue + } + sugg, cerr := classifier.Classify(context.Background(), text, known) + if cerr != nil { + slog.Warn("classify failed", "page", rel, "err", cerr) + } + top, _ := wing.TopWing(wing.NewBM25Scorer(brain, known, rel).Score(text)) + s := wing.Sample{Truth: w, Suggested: strings.ToLower(sugg.Wing), Confidence: sugg.Confidence, ScorerTop: top} + slog.Info("sample", "page", rel, "truth", w, "classifier", s.Suggested, "conf", s.Confidence, "scorer", top, "reason", sugg.Reason) + samples = append(samples, s) + perWingSamples[w] = append(perWingSamples[w], s) + } + } + report(samples, perWingSamples) + return nil +} + +func report(samples []wing.Sample, per map[string][]wing.Sample) { + fmt.Printf("# wing backtest\n\nn=%d pages. Classifier+scorer must agree; threshold applies to classifier confidence.\n\n", len(samples)) + fmt.Println("| threshold | auto | coverage | precision |") + fmt.Println("|---|---|---|---|") + for _, t := range []float64{0.5, 0.6, 0.7, 0.8, 0.9, 0.95} { + r := wing.Evaluate(samples, t) + fmt.Printf("| %.2f | %d/%d | %.0f%% | %.0f%% |\n", t, r.Auto, r.N, 100*r.Coverage, 100*r.Precision) + } + fmt.Println("\n## per wing (threshold 0.80)\n\n| wing | n | auto | precision |\n|---|---|---|---|") + names := make([]string, 0, len(per)) + for w := range per { + names = append(names, w) + } + sort.Strings(names) + for _, w := range names { + r := wing.Evaluate(per[w], 0.8) + fmt.Printf("| %s | %d | %d | %.0f%% |\n", w, r.N, r.Auto, 100*r.Precision) + } + fmt.Println("\n## wrong auto-placements at 0.80") + fmt.Println() + for _, s := range samples { + if s.Suggested != "" && s.Confidence >= 0.8 && s.ScorerTop == s.Suggested && s.Suggested != s.Truth { + fmt.Printf("- truth=%s -> both said %s (conf %.2f)\n", s.Truth, s.Suggested, s.Confidence) + } + } +} + +// knownWings lists wiki/ dirs that hold hall subdirectories, skipping +// the retired legacy layout. +func knownWings(brain string) ([]string, error) { + entries, err := os.ReadDir(filepath.Join(brain, "wiki")) + if err != nil { + return nil, fmt.Errorf("list wings: %w", err) + } + var out []string + for _, e := range entries { + if !e.IsDir() || legacyDirs[e.Name()] { + continue + } + subs, _ := os.ReadDir(filepath.Join(brain, "wiki", e.Name())) + for _, s := range subs { + if s.IsDir() { + out = append(out, e.Name()) + break + } + } + } + sort.Strings(out) + return out, nil +} + +func listPages(brain, w string) ([]string, error) { + var out []string + root := filepath.Join(brain, "wiki", w) + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !d.Type().IsRegular() || !strings.HasSuffix(p, ".md") || d.Name() == "_index.md" { + return err + } + rel, rerr := filepath.Rel(brain, p) + if rerr != nil { + return rerr + } + out = append(out, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk %s: %w", w, err) + } + return out, nil +} + +// readBody returns the page text without YAML frontmatter, so the wing label +// (and anything else in frontmatter) cannot leak into the classification. +func readBody(brain, rel string) (string, error) { + b, err := os.ReadFile(filepath.Join(brain, filepath.FromSlash(rel))) + if err != nil { + return "", fmt.Errorf("read %s: %w", rel, err) + } + s := string(b) + if strings.HasPrefix(s, "---\n") { + if end := strings.Index(s[4:], "\n---"); end >= 0 { + s = s[4+end+4:] + } + } + return strings.TrimSpace(s), nil +} diff --git a/ingestion/internal/wing/bm25scorer.go b/ingestion/internal/wing/bm25scorer.go index 74cd037..fedd93b 100644 --- a/ingestion/internal/wing/bm25scorer.go +++ b/ingestion/internal/wing/bm25scorer.go @@ -11,7 +11,6 @@ import ( const ( maxQueryTerms = 12 minTermLen = 5 - hitsPerWing = 3 ) // BM25Scorer scores a note against each wing by running the brain's own BM25 @@ -38,24 +37,18 @@ func (b *BM25Scorer) Score(text string) map[string]float64 { return out } for _, w := range b.wings { - hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: hitsPerWing + 1}) + hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: 2}) if err != nil { continue } - var sum float64 - n := 0 for _, h := range hits { if b.exclude != "" && h.Path == b.exclude { continue } - if n == hitsPerWing { - break - } - sum += float64(h.Score) - n++ - } - if sum > 0 { - out[w] = sum + // Hits come best-first: a wing is scored by its single best + // page, so a big wing cannot win on volume alone. + out[w] = float64(h.Score) + break } } return out diff --git a/ingestion/internal/wing/bm25scorer_test.go b/ingestion/internal/wing/bm25scorer_test.go index b648ca3..b6c8901 100644 --- a/ingestion/internal/wing/bm25scorer_test.go +++ b/ingestion/internal/wing/bm25scorer_test.go @@ -43,3 +43,19 @@ func TestBM25Scorer_NoUsableTermsScoresNothing(t *testing.T) { s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "") assert.Empty(t, s.Score("a an to of")) } + +// A big wing must not win just by having more pages that brush the query: a +// wing is scored by its single best page, not by the sum of its top hits. +func TestBM25Scorer_ABigWingDoesNotWinOnVolume(t *testing.T) { + dir := t.TempDir() + writePage(t, dir, "wiki/small/facts/target.md", "---\ntitle: Target\n---\nkubernetes cluster scheduling pods deployment nodes\n") + for i := 0; i < 8; i++ { + writePage(t, dir, filepath.Join("wiki/big/facts", "p"+string(rune('a'+i))+".md"), + "---\ntitle: Filler\n---\nkubernetes cluster scheduling mentioned in passing among bookkeeping ledger entries\n") + } + s := wing.NewBM25Scorer(dir, []string{"small", "big"}, "") + got := s.Score("kubernetes cluster scheduling pods deployment nodes") + top, ok := wing.TopWing(got) + assert.True(t, ok) + assert.Equal(t, "small", top) +}