diff --git a/ingestion/internal/wing/resolve.go b/ingestion/internal/wing/resolve.go new file mode 100644 index 0000000..1dd35a2 --- /dev/null +++ b/ingestion/internal/wing/resolve.go @@ -0,0 +1,169 @@ +// Package wing decides which brain wing an incoming raw note belongs to +// without the human typing it (hyperguild#205). +// +// The contract is conservative: a note is placed automatically only when the +// decision is explicit, or when two independent opinions (an LLM and a +// deterministic scorer) agree with enough confidence on a wing that is both +// known and eligible. Everything else is an Exception for a human to approve. +// A wing is never created here, and a wing the caller does not mark eligible +// (client wings, anything not deliberately tagged) is never auto-assigned. +package wing + +import ( + "context" + "fmt" + "strings" +) + +// Source says how a Decision was reached. +type Source string + +const ( + Explicit Source = "explicit" // subfolder or frontmatter named the wing + Classified Source = "classified" // classifier + scorer agreed, confident + Exception Source = "exception" // needs a human +) + +// Note is what the resolver sees of an incoming raw note. +type Note struct { + Dir string // first subfolder under raw/, "" when dropped at the top + FrontmatterWing string // `wing:` from the note's frontmatter, "" if absent + Text string +} + +// Suggestion is a Classifier's answer. An empty Wing means "none fits". +type Suggestion struct { + Wing string + Confidence float64 // 0..1 + Reason string +} + +// Classifier proposes a wing from the candidates. Implementations must run on +// a local model; the resolver cannot enforce that. +type Classifier interface { + Classify(ctx context.Context, text string, candidates []string) (Suggestion, error) +} + +// Scorer is an independent, deterministic opinion (e.g. BM25 against each +// wing's index). Higher is better; the key is the wing name. +type Scorer interface { + Score(text string) map[string]float64 +} + +// Decision is the outcome. For an Exception, Wing holds the proposal when +// there was one (so a reviewer can approve it) and is empty otherwise. +type Decision struct { + Wing string + Source Source + Confidence float64 + Reason string +} + +// Resolver holds the policy. Known is every existing wing; Eligible reports +// whether a known wing may be assigned automatically. +type Resolver struct { + Known []string + Eligible func(wing string) bool + Classifier Classifier + Scorer Scorer + Threshold float64 +} + +// Resolve never returns an error: anything it cannot decide safely is an +// Exception with the reason spelled out. +func (r Resolver) Resolve(ctx context.Context, n Note) Decision { + dir := norm(n.Dir) + fm := norm(n.FrontmatterWing) + switch { + case dir != "" && fm != "" && dir != fm: + return exception("", 0, fmt.Sprintf("subfolder %q and frontmatter wing %q disagree", dir, fm)) + case dir != "": + return r.explicit(dir, "subfolder") + case fm != "": + return r.explicit(fm, "frontmatter") + } + return r.classify(ctx, n) +} + +func (r Resolver) explicit(w, via string) Decision { + if why := r.refuse(w); why != "" { + return exception(w, 0, fmt.Sprintf("%s names %q: %s", via, w, why)) + } + return Decision{Wing: w, Source: Explicit, Confidence: 1, Reason: "named by " + via} +} + +func (r Resolver) classify(ctx context.Context, n Note) Decision { + if r.Classifier == nil { + return exception("", 0, "no classifier configured") + } + s, err := r.Classifier.Classify(ctx, n.Text, r.Known) + if err != nil { + return exception("", 0, fmt.Sprintf("classifier failed: %v", err)) + } + w := norm(s.Wing) + if w == "" { + return exception("", s.Confidence, "classifier found no fitting wing") + } + if why := r.refuse(w); why != "" { + return exception(w, s.Confidence, fmt.Sprintf("classifier proposed %q: %s", w, why)) + } + if s.Confidence < r.Threshold { + return exception(w, s.Confidence, fmt.Sprintf("confidence %.2f below threshold %.2f", s.Confidence, r.Threshold)) + } + top, ok := r.scorerTop(n.Text) + if !ok { + return exception(w, s.Confidence, "scorer has no clear opinion to cross-check against") + } + if top != w { + return exception(w, s.Confidence, fmt.Sprintf("classifier says %q, scorer says %q", w, top)) + } + return Decision{Wing: w, Source: Classified, Confidence: s.Confidence, + Reason: fmt.Sprintf("classifier and scorer agree (%.2f): %s", s.Confidence, s.Reason)} +} + +// refuse returns why a wing may not be auto-assigned, or "" if it may. +func (r Resolver) refuse(w string) string { + if !r.known(w) { + return "not an existing wing (new wings are never created automatically)" + } + if r.Eligible == nil || !r.Eligible(w) { + return "not eligible for automatic assignment" + } + return "" +} + +func (r Resolver) known(w string) bool { + for _, k := range r.Known { + if norm(k) == w { + return true + } + } + return false +} + +// scorerTop returns the single best-scoring wing; a missing scorer, no scores, +// or a tie at the top is "no clear opinion". +func (r Resolver) scorerTop(text string) (string, bool) { + if r.Scorer == nil { + return "", false + } + best, bestScore, tied := "", 0.0, false + for k, v := range r.Scorer.Score(text) { + switch { + case v > bestScore: + best, bestScore, tied = norm(k), v, false + case v == bestScore && v > 0: + tied = true + } + } + if best == "" || tied { + return "", false + } + return best, true +} + +func exception(w string, conf float64, reason string) Decision { + return Decision{Wing: w, Source: Exception, Confidence: conf, Reason: reason} +} + +func norm(s string) string { return strings.ToLower(strings.TrimSpace(s)) } diff --git a/ingestion/internal/wing/resolve_test.go b/ingestion/internal/wing/resolve_test.go new file mode 100644 index 0000000..7d78e03 --- /dev/null +++ b/ingestion/internal/wing/resolve_test.go @@ -0,0 +1,112 @@ +package wing_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" +) + +type fakeClassifier struct { + s wing.Suggestion + err error + calls int +} + +func (f *fakeClassifier) Classify(_ context.Context, _ string, _ []string) (wing.Suggestion, error) { + f.calls++ + return f.s, f.err +} + +type fakeScorer map[string]float64 + +func (f fakeScorer) Score(string) map[string]float64 { return f } + +// eligibleExcept allows every name except the listed ones. Being permissive on +// names it has never heard of means only the "known wing" check can stop an +// invented wing, so a test of that check cannot pass by accident. +func eligibleExcept(names ...string) func(string) bool { + deny := map[string]bool{} + for _, n := range names { + deny[n] = true + } + return func(w string) bool { return !deny[w] } +} + +func newResolver(c wing.Classifier, s wing.Scorer) wing.Resolver { + return wing.Resolver{ + Known: []string{"homelab", "hyperguild", "tapir", "rikatillsammans"}, + Eligible: eligibleExcept("rikatillsammans"), // a client wing: known, never auto + Classifier: c, + Scorer: s, + Threshold: 0.8, + } +} + +func TestResolve_ExplicitSignal(t *testing.T) { + tests := []struct { + name string + note wing.Note + want wing.Source + wantWing string + }{ + {"subfolder of a known eligible wing", wing.Note{Dir: "homelab"}, wing.Explicit, "homelab"}, + {"frontmatter of a known eligible wing", wing.Note{FrontmatterWing: "tapir"}, wing.Explicit, "tapir"}, + {"subfolder and frontmatter agree", wing.Note{Dir: "tapir", FrontmatterWing: "tapir"}, wing.Explicit, "tapir"}, + {"subfolder and frontmatter disagree", wing.Note{Dir: "tapir", FrontmatterWing: "homelab"}, wing.Exception, ""}, + {"unknown wing is never created", wing.Note{Dir: "hoemlab"}, wing.Exception, "hoemlab"}, + {"known but not eligible (client) wing is never auto-assigned", wing.Note{Dir: "rikatillsammans"}, wing.Exception, "rikatillsammans"}, + {"subfolder is case-insensitive", wing.Note{Dir: "HomeLab"}, wing.Explicit, "homelab"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &fakeClassifier{} + d := newResolver(c, fakeScorer{}).Resolve(context.Background(), tt.note) + assert.Equal(t, tt.want, d.Source) + assert.Equal(t, tt.wantWing, d.Wing) + assert.Zero(t, c.calls, "an explicit signal must not spend an LLM call") + assert.NotEmpty(t, d.Reason) + }) + } +} + +func TestResolve_Classified(t *testing.T) { + tests := []struct { + name string + sugg wing.Suggestion + err error + scores fakeScorer + want wing.Source + wantWing string + }{ + {"confident and scorer agrees", wing.Suggestion{Wing: "tapir", Confidence: 0.9}, nil, fakeScorer{"tapir": 3, "homelab": 1}, wing.Classified, "tapir"}, + {"confidence exactly at threshold passes", wing.Suggestion{Wing: "tapir", Confidence: 0.8}, nil, fakeScorer{"tapir": 3}, wing.Classified, "tapir"}, + {"below threshold is an exception, proposal kept", wing.Suggestion{Wing: "tapir", Confidence: 0.79}, nil, fakeScorer{"tapir": 3}, wing.Exception, "tapir"}, + {"scorer disagrees", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{"homelab": 5, "tapir": 1}, wing.Exception, "tapir"}, + {"scorer has no opinion also needs review", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{}, wing.Exception, "tapir"}, + {"classifier proposes a client wing", wing.Suggestion{Wing: "rikatillsammans", Confidence: 0.99}, nil, fakeScorer{"rikatillsammans": 9}, wing.Exception, "rikatillsammans"}, + {"classifier invents a wing", wing.Suggestion{Wing: "brand-new-wing", Confidence: 0.99}, nil, fakeScorer{"brand-new-wing": 9}, wing.Exception, "brand-new-wing"}, + {"classifier abstains", wing.Suggestion{Wing: "", Confidence: 0}, nil, fakeScorer{"tapir": 3}, wing.Exception, ""}, + {"classifier error", wing.Suggestion{}, errors.New("llm down"), fakeScorer{"tapir": 3}, wing.Exception, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &fakeClassifier{s: tt.sugg, err: tt.err} + d := newResolver(c, tt.scores).Resolve(context.Background(), wing.Note{Text: "some note"}) + assert.Equal(t, tt.want, d.Source) + assert.Equal(t, tt.wantWing, d.Wing) + assert.NotEmpty(t, d.Reason, "every decision says why, so a spot-check can judge it") + assert.Equal(t, 1, c.calls) + }) + } +} + +func TestResolve_NilDependenciesFailClosed(t *testing.T) { + r := wing.Resolver{Known: []string{"tapir"}, Eligible: eligibleExcept(), Threshold: 0.8} + d := r.Resolve(context.Background(), wing.Note{Text: "x"}) + require.Equal(t, wing.Exception, d.Source, "no classifier configured must route to review, not panic or guess") +}