hyperguild/ingestion/cmd/wingbacktest/main.go

// Command wingbacktest measures how well the wing resolver would have placed
// pages whose wing a human already decided (hyperguild#205).
//
// It reads pages from the wings given in -wings, hides the wing, asks the
// local classifier and the BM25 scorer, and reports coverage/precision per
// confidence threshold. Page text is sent to -llm-url, which MUST be a local
// model (it is not checked here); sample only wings you would allow.
package main

import (
	"context"
	"flag"
	"fmt"
	"io/fs"
	"log/slog"
	"math/rand"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
	"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)

func main() {
	brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory (read only)")
	wingsFlag := flag.String("wings", "", "comma-separated wings to sample pages FROM (must be wings you would allow)")
	perWing := flag.Int("per-wing", 8, "pages sampled per wing")
	llmURL := flag.String("llm-url", "http://10.0.1.20:31234/v1", "OpenAI-compatible LOCAL endpoint")
	model := flag.String("model", "qwen35-9b-fast", "model name")
	seed := flag.Int64("seed", 1, "sampling seed")
	flag.Parse()

	if err := run(*brain, strings.Split(*wingsFlag, ","), *perWing, *llmURL, *model, *seed); err != nil {
		slog.Error("wingbacktest", "err", err)
		os.Exit(1)
	}
}

func run(brain string, sampleWings []string, perWing int, llmURL, model string, seed int64) error {
	known, err := wing.KnownWings(brain)
	if err != nil {
		return err
	}
	client := llm.New(llmURL, "", model, 3*time.Minute)
	classifier := wing.NewLLMClassifier(client.Complete)
	rng := rand.New(rand.NewSource(seed))

	var samples []wing.Sample
	perWingSamples := map[string][]wing.Sample{}
	for _, w := range sampleWings {
		w = strings.TrimSpace(w)
		if w == "" {
			continue
		}
		pages, err := listPages(brain, w)
		if err != nil {
			return err
		}
		rng.Shuffle(len(pages), func(i, j int) { pages[i], pages[j] = pages[j], pages[i] })
		if len(pages) > perWing {
			pages = pages[:perWing]
		}
		for _, rel := range pages {
			text, err := readBody(brain, rel)
			if err != nil || len(text) < 200 {
				continue
			}
			sugg, cerr := classifier.Classify(context.Background(), text, known)
			if cerr != nil {
				slog.Warn("classify failed", "page", rel, "err", cerr)
			}
			top, _ := wing.TopWing(wing.NewBM25Scorer(brain, known, rel).Score(text))
			s := wing.Sample{Truth: w, Suggested: strings.ToLower(sugg.Wing), Confidence: sugg.Confidence, ScorerTop: top}
			slog.Info("sample", "page", rel, "truth", w, "classifier", s.Suggested, "conf", s.Confidence, "scorer", top, "reason", sugg.Reason)
			samples = append(samples, s)
			perWingSamples[w] = append(perWingSamples[w], s)
		}
	}
	report(samples, perWingSamples)
	return nil
}

func report(samples []wing.Sample, per map[string][]wing.Sample) {
	fmt.Printf("# wing backtest\n\nn=%d pages. Classifier+scorer must agree; threshold applies to classifier confidence.\n\n", len(samples))
	fmt.Println("| threshold | auto | coverage | precision |")
	fmt.Println("|---|---|---|---|")
	for _, t := range []float64{0.5, 0.6, 0.7, 0.8, 0.9, 0.95} {
		r := wing.Evaluate(samples, t)
		fmt.Printf("| %.2f | %d/%d | %.0f%% | %.0f%% |\n", t, r.Auto, r.N, 100*r.Coverage, 100*r.Precision)
	}
	fmt.Println("\n## per wing (threshold 0.80)\n\n| wing | n | auto | precision |\n|---|---|---|---|")
	names := make([]string, 0, len(per))
	for w := range per {
		names = append(names, w)
	}
	sort.Strings(names)
	for _, w := range names {
		r := wing.Evaluate(per[w], 0.8)
		fmt.Printf("| %s | %d | %d | %.0f%% |\n", w, r.N, r.Auto, 100*r.Precision)
	}
	fmt.Println("\n## wrong auto-placements at 0.80")
	fmt.Println()
	for _, s := range samples {
		if s.Suggested != "" && s.Confidence >= 0.8 && s.ScorerTop == s.Suggested && s.Suggested != s.Truth {
			fmt.Printf("- truth=%s -> both said %s (conf %.2f)\n", s.Truth, s.Suggested, s.Confidence)
		}
	}
}

func listPages(brain, w string) ([]string, error) {
	var out []string
	root := filepath.Join(brain, "wiki", w)
	err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
		if err != nil || d.IsDir() || !d.Type().IsRegular() || !strings.HasSuffix(p, ".md") || d.Name() == "_index.md" {
			return err
		}
		rel, rerr := filepath.Rel(brain, p)
		if rerr != nil {
			return rerr
		}
		out = append(out, filepath.ToSlash(rel))
		return nil
	})
	if err != nil {
		return nil, fmt.Errorf("walk %s: %w", w, err)
	}
	return out, nil
}

// readBody returns the page text without YAML frontmatter, so the wing label
// (and anything else in frontmatter) cannot leak into the classification.
func readBody(brain, rel string) (string, error) {
	b, err := os.ReadFile(filepath.Join(brain, filepath.FromSlash(rel)))
	if err != nil {
		return "", fmt.Errorf("read %s: %w", rel, err)
	}
	s := string(b)
	if strings.HasPrefix(s, "---\n") {
		if end := strings.Index(s[4:], "\n---"); end >= 0 {
			s = s[4+end+4:]
		}
	}
	return strings.TrimSpace(s), nil
}