feat(wing): resolver that places a raw note in a wing without human typing (#205 S1)
Explicit subfolder/frontmatter wins; otherwise a classifier and an independent scorer must agree above a threshold on a known, eligible wing. Anything else is an Exception for a human. Never creates a wing, never assigns an ineligible (client/untagged) wing. Mutation-checked: dropping the eligibility, known-wing, scorer, threshold or conflict check fails a test. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,169 @@
|
||||
// Package wing decides which brain wing an incoming raw note belongs to
|
||||
// without the human typing it (hyperguild#205).
|
||||
//
|
||||
// The contract is conservative: a note is placed automatically only when the
|
||||
// decision is explicit, or when two independent opinions (an LLM and a
|
||||
// deterministic scorer) agree with enough confidence on a wing that is both
|
||||
// known and eligible. Everything else is an Exception for a human to approve.
|
||||
// A wing is never created here, and a wing the caller does not mark eligible
|
||||
// (client wings, anything not deliberately tagged) is never auto-assigned.
|
||||
package wing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Source says how a Decision was reached.
|
||||
type Source string
|
||||
|
||||
const (
|
||||
Explicit Source = "explicit" // subfolder or frontmatter named the wing
|
||||
Classified Source = "classified" // classifier + scorer agreed, confident
|
||||
Exception Source = "exception" // needs a human
|
||||
)
|
||||
|
||||
// Note is what the resolver sees of an incoming raw note.
|
||||
type Note struct {
|
||||
Dir string // first subfolder under raw/, "" when dropped at the top
|
||||
FrontmatterWing string // `wing:` from the note's frontmatter, "" if absent
|
||||
Text string
|
||||
}
|
||||
|
||||
// Suggestion is a Classifier's answer. An empty Wing means "none fits".
|
||||
type Suggestion struct {
|
||||
Wing string
|
||||
Confidence float64 // 0..1
|
||||
Reason string
|
||||
}
|
||||
|
||||
// Classifier proposes a wing from the candidates. Implementations must run on
|
||||
// a local model; the resolver cannot enforce that.
|
||||
type Classifier interface {
|
||||
Classify(ctx context.Context, text string, candidates []string) (Suggestion, error)
|
||||
}
|
||||
|
||||
// Scorer is an independent, deterministic opinion (e.g. BM25 against each
|
||||
// wing's index). Higher is better; the key is the wing name.
|
||||
type Scorer interface {
|
||||
Score(text string) map[string]float64
|
||||
}
|
||||
|
||||
// Decision is the outcome. For an Exception, Wing holds the proposal when
|
||||
// there was one (so a reviewer can approve it) and is empty otherwise.
|
||||
type Decision struct {
|
||||
Wing string
|
||||
Source Source
|
||||
Confidence float64
|
||||
Reason string
|
||||
}
|
||||
|
||||
// Resolver holds the policy. Known is every existing wing; Eligible reports
|
||||
// whether a known wing may be assigned automatically.
|
||||
type Resolver struct {
|
||||
Known []string
|
||||
Eligible func(wing string) bool
|
||||
Classifier Classifier
|
||||
Scorer Scorer
|
||||
Threshold float64
|
||||
}
|
||||
|
||||
// Resolve never returns an error: anything it cannot decide safely is an
|
||||
// Exception with the reason spelled out.
|
||||
func (r Resolver) Resolve(ctx context.Context, n Note) Decision {
|
||||
dir := norm(n.Dir)
|
||||
fm := norm(n.FrontmatterWing)
|
||||
switch {
|
||||
case dir != "" && fm != "" && dir != fm:
|
||||
return exception("", 0, fmt.Sprintf("subfolder %q and frontmatter wing %q disagree", dir, fm))
|
||||
case dir != "":
|
||||
return r.explicit(dir, "subfolder")
|
||||
case fm != "":
|
||||
return r.explicit(fm, "frontmatter")
|
||||
}
|
||||
return r.classify(ctx, n)
|
||||
}
|
||||
|
||||
func (r Resolver) explicit(w, via string) Decision {
|
||||
if why := r.refuse(w); why != "" {
|
||||
return exception(w, 0, fmt.Sprintf("%s names %q: %s", via, w, why))
|
||||
}
|
||||
return Decision{Wing: w, Source: Explicit, Confidence: 1, Reason: "named by " + via}
|
||||
}
|
||||
|
||||
func (r Resolver) classify(ctx context.Context, n Note) Decision {
|
||||
if r.Classifier == nil {
|
||||
return exception("", 0, "no classifier configured")
|
||||
}
|
||||
s, err := r.Classifier.Classify(ctx, n.Text, r.Known)
|
||||
if err != nil {
|
||||
return exception("", 0, fmt.Sprintf("classifier failed: %v", err))
|
||||
}
|
||||
w := norm(s.Wing)
|
||||
if w == "" {
|
||||
return exception("", s.Confidence, "classifier found no fitting wing")
|
||||
}
|
||||
if why := r.refuse(w); why != "" {
|
||||
return exception(w, s.Confidence, fmt.Sprintf("classifier proposed %q: %s", w, why))
|
||||
}
|
||||
if s.Confidence < r.Threshold {
|
||||
return exception(w, s.Confidence, fmt.Sprintf("confidence %.2f below threshold %.2f", s.Confidence, r.Threshold))
|
||||
}
|
||||
top, ok := r.scorerTop(n.Text)
|
||||
if !ok {
|
||||
return exception(w, s.Confidence, "scorer has no clear opinion to cross-check against")
|
||||
}
|
||||
if top != w {
|
||||
return exception(w, s.Confidence, fmt.Sprintf("classifier says %q, scorer says %q", w, top))
|
||||
}
|
||||
return Decision{Wing: w, Source: Classified, Confidence: s.Confidence,
|
||||
Reason: fmt.Sprintf("classifier and scorer agree (%.2f): %s", s.Confidence, s.Reason)}
|
||||
}
|
||||
|
||||
// refuse returns why a wing may not be auto-assigned, or "" if it may.
|
||||
func (r Resolver) refuse(w string) string {
|
||||
if !r.known(w) {
|
||||
return "not an existing wing (new wings are never created automatically)"
|
||||
}
|
||||
if r.Eligible == nil || !r.Eligible(w) {
|
||||
return "not eligible for automatic assignment"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (r Resolver) known(w string) bool {
|
||||
for _, k := range r.Known {
|
||||
if norm(k) == w {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// scorerTop returns the single best-scoring wing; a missing scorer, no scores,
|
||||
// or a tie at the top is "no clear opinion".
|
||||
func (r Resolver) scorerTop(text string) (string, bool) {
|
||||
if r.Scorer == nil {
|
||||
return "", false
|
||||
}
|
||||
best, bestScore, tied := "", 0.0, false
|
||||
for k, v := range r.Scorer.Score(text) {
|
||||
switch {
|
||||
case v > bestScore:
|
||||
best, bestScore, tied = norm(k), v, false
|
||||
case v == bestScore && v > 0:
|
||||
tied = true
|
||||
}
|
||||
}
|
||||
if best == "" || tied {
|
||||
return "", false
|
||||
}
|
||||
return best, true
|
||||
}
|
||||
|
||||
func exception(w string, conf float64, reason string) Decision {
|
||||
return Decision{Wing: w, Source: Exception, Confidence: conf, Reason: reason}
|
||||
}
|
||||
|
||||
func norm(s string) string { return strings.ToLower(strings.TrimSpace(s)) }
|
||||
@@ -0,0 +1,112 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
type fakeClassifier struct {
|
||||
s wing.Suggestion
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeClassifier) Classify(_ context.Context, _ string, _ []string) (wing.Suggestion, error) {
|
||||
f.calls++
|
||||
return f.s, f.err
|
||||
}
|
||||
|
||||
type fakeScorer map[string]float64
|
||||
|
||||
func (f fakeScorer) Score(string) map[string]float64 { return f }
|
||||
|
||||
// eligibleExcept allows every name except the listed ones. Being permissive on
|
||||
// names it has never heard of means only the "known wing" check can stop an
|
||||
// invented wing, so a test of that check cannot pass by accident.
|
||||
func eligibleExcept(names ...string) func(string) bool {
|
||||
deny := map[string]bool{}
|
||||
for _, n := range names {
|
||||
deny[n] = true
|
||||
}
|
||||
return func(w string) bool { return !deny[w] }
|
||||
}
|
||||
|
||||
func newResolver(c wing.Classifier, s wing.Scorer) wing.Resolver {
|
||||
return wing.Resolver{
|
||||
Known: []string{"homelab", "hyperguild", "tapir", "rikatillsammans"},
|
||||
Eligible: eligibleExcept("rikatillsammans"), // a client wing: known, never auto
|
||||
Classifier: c,
|
||||
Scorer: s,
|
||||
Threshold: 0.8,
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_ExplicitSignal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
note wing.Note
|
||||
want wing.Source
|
||||
wantWing string
|
||||
}{
|
||||
{"subfolder of a known eligible wing", wing.Note{Dir: "homelab"}, wing.Explicit, "homelab"},
|
||||
{"frontmatter of a known eligible wing", wing.Note{FrontmatterWing: "tapir"}, wing.Explicit, "tapir"},
|
||||
{"subfolder and frontmatter agree", wing.Note{Dir: "tapir", FrontmatterWing: "tapir"}, wing.Explicit, "tapir"},
|
||||
{"subfolder and frontmatter disagree", wing.Note{Dir: "tapir", FrontmatterWing: "homelab"}, wing.Exception, ""},
|
||||
{"unknown wing is never created", wing.Note{Dir: "hoemlab"}, wing.Exception, "hoemlab"},
|
||||
{"known but not eligible (client) wing is never auto-assigned", wing.Note{Dir: "rikatillsammans"}, wing.Exception, "rikatillsammans"},
|
||||
{"subfolder is case-insensitive", wing.Note{Dir: "HomeLab"}, wing.Explicit, "homelab"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &fakeClassifier{}
|
||||
d := newResolver(c, fakeScorer{}).Resolve(context.Background(), tt.note)
|
||||
assert.Equal(t, tt.want, d.Source)
|
||||
assert.Equal(t, tt.wantWing, d.Wing)
|
||||
assert.Zero(t, c.calls, "an explicit signal must not spend an LLM call")
|
||||
assert.NotEmpty(t, d.Reason)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_Classified(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sugg wing.Suggestion
|
||||
err error
|
||||
scores fakeScorer
|
||||
want wing.Source
|
||||
wantWing string
|
||||
}{
|
||||
{"confident and scorer agrees", wing.Suggestion{Wing: "tapir", Confidence: 0.9}, nil, fakeScorer{"tapir": 3, "homelab": 1}, wing.Classified, "tapir"},
|
||||
{"confidence exactly at threshold passes", wing.Suggestion{Wing: "tapir", Confidence: 0.8}, nil, fakeScorer{"tapir": 3}, wing.Classified, "tapir"},
|
||||
{"below threshold is an exception, proposal kept", wing.Suggestion{Wing: "tapir", Confidence: 0.79}, nil, fakeScorer{"tapir": 3}, wing.Exception, "tapir"},
|
||||
{"scorer disagrees", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{"homelab": 5, "tapir": 1}, wing.Exception, "tapir"},
|
||||
{"scorer has no opinion also needs review", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{}, wing.Exception, "tapir"},
|
||||
{"classifier proposes a client wing", wing.Suggestion{Wing: "rikatillsammans", Confidence: 0.99}, nil, fakeScorer{"rikatillsammans": 9}, wing.Exception, "rikatillsammans"},
|
||||
{"classifier invents a wing", wing.Suggestion{Wing: "brand-new-wing", Confidence: 0.99}, nil, fakeScorer{"brand-new-wing": 9}, wing.Exception, "brand-new-wing"},
|
||||
{"classifier abstains", wing.Suggestion{Wing: "", Confidence: 0}, nil, fakeScorer{"tapir": 3}, wing.Exception, ""},
|
||||
{"classifier error", wing.Suggestion{}, errors.New("llm down"), fakeScorer{"tapir": 3}, wing.Exception, ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &fakeClassifier{s: tt.sugg, err: tt.err}
|
||||
d := newResolver(c, tt.scores).Resolve(context.Background(), wing.Note{Text: "some note"})
|
||||
assert.Equal(t, tt.want, d.Source)
|
||||
assert.Equal(t, tt.wantWing, d.Wing)
|
||||
assert.NotEmpty(t, d.Reason, "every decision says why, so a spot-check can judge it")
|
||||
assert.Equal(t, 1, c.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_NilDependenciesFailClosed(t *testing.T) {
|
||||
r := wing.Resolver{Known: []string{"tapir"}, Eligible: eligibleExcept(), Threshold: 0.8}
|
||||
d := r.Resolve(context.Background(), wing.Note{Text: "x"})
|
||||
require.Equal(t, wing.Exception, d.Source, "no classifier configured must route to review, not panic or guess")
|
||||
}
|
||||
Reference in New Issue
Block a user