feat(server): INGEST_WING_ROUTING wires the Router; remove the classifier path (#208)
Removes the LLM classifier, the substring 'BM25' scorer, the agreement gate, the backtest and the old migrate/wingmigrate tooling: measured at 2 of 55 pages placed (#208), and the migration was built for the per-wing facts layout this branch replaces. wing now holds the decision types, the Router and KnownWings. INGEST_WING_ROUTING=1 builds the Router (default wing INGEST_DEFAULT_WING=web-clip, client pattern from INGEST_CLIENT_BLOCK or CLAUDE_INGEST_CLIENT_BLOCK); a bad default or pattern is fatal at startup. Watcher tests rewritten for the Router, including a deliberate subfolder into any existing wing, typo'd folders parked, forged-key and edit-while-placing cases. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -1,43 +0,0 @@
|
||||
package wing
|
||||
|
||||
// Sample is one backtest observation: the human-decided wing of an existing
|
||||
// page, and what the two independent opinions said about it with the wing
|
||||
// label removed.
|
||||
type Sample struct {
|
||||
Truth string
|
||||
Suggested string // classifier's wing, "" if it abstained or failed
|
||||
Confidence float64
|
||||
ScorerTop string // scorer's single best wing, "" if no clear opinion
|
||||
}
|
||||
|
||||
// Result is the policy outcome at one threshold.
|
||||
type Result struct {
|
||||
N int
|
||||
Auto int // placed without a human
|
||||
Correct int // of Auto, matching Truth
|
||||
Coverage float64 // Auto / N
|
||||
Precision float64 // Correct / Auto; 0 when Auto == 0
|
||||
}
|
||||
|
||||
// Evaluate applies the Resolver's classified-path rule (confident, and the
|
||||
// scorer agrees) to recorded samples. Eligibility is not modelled: callers
|
||||
// backtest only on wings they would allow.
|
||||
func Evaluate(samples []Sample, threshold float64) Result {
|
||||
r := Result{N: len(samples)}
|
||||
for _, s := range samples {
|
||||
if s.Suggested == "" || s.Confidence < threshold || s.ScorerTop != s.Suggested {
|
||||
continue
|
||||
}
|
||||
r.Auto++
|
||||
if s.Suggested == s.Truth {
|
||||
r.Correct++
|
||||
}
|
||||
}
|
||||
if r.N > 0 {
|
||||
r.Coverage = float64(r.Auto) / float64(r.N)
|
||||
}
|
||||
if r.Auto > 0 {
|
||||
r.Precision = float64(r.Correct) / float64(r.Auto)
|
||||
}
|
||||
return r
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
func TestEvaluate_PolicyAtAThreshold(t *testing.T) {
|
||||
samples := []wing.Sample{
|
||||
{Truth: "a", Suggested: "a", Confidence: 0.9, ScorerTop: "a"}, // auto, correct
|
||||
{Truth: "a", Suggested: "b", Confidence: 0.9, ScorerTop: "b"}, // auto, WRONG (both models agree on a wrong wing)
|
||||
{Truth: "a", Suggested: "a", Confidence: 0.5, ScorerTop: "a"}, // below threshold -> exception
|
||||
{Truth: "a", Suggested: "a", Confidence: 0.95, ScorerTop: "b"}, // scorer disagrees -> exception
|
||||
{Truth: "a", Suggested: "", Confidence: 0.99, ScorerTop: "a"}, // abstain -> exception
|
||||
}
|
||||
r := wing.Evaluate(samples, 0.8)
|
||||
assert.Equal(t, 5, r.N)
|
||||
assert.Equal(t, 2, r.Auto)
|
||||
assert.Equal(t, 1, r.Correct)
|
||||
assert.InDelta(t, 0.4, r.Coverage, 1e-9)
|
||||
assert.InDelta(t, 0.5, r.Precision, 1e-9)
|
||||
}
|
||||
|
||||
func TestEvaluate_NothingAutoHasZeroPrecisionNotNaN(t *testing.T) {
|
||||
r := wing.Evaluate([]wing.Sample{{Truth: "a", Suggested: "a", Confidence: 0.1, ScorerTop: "a"}}, 0.8)
|
||||
assert.Zero(t, r.Auto)
|
||||
assert.Zero(t, r.Precision)
|
||||
assert.Zero(t, r.Coverage)
|
||||
}
|
||||
|
||||
func TestEvaluate_EmptyInput(t *testing.T) {
|
||||
r := wing.Evaluate(nil, 0.8)
|
||||
assert.Zero(t, r.N)
|
||||
assert.Zero(t, r.Coverage)
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package wing
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
|
||||
)
|
||||
|
||||
const (
|
||||
maxQueryTerms = 12
|
||||
minTermLen = 5
|
||||
)
|
||||
|
||||
// BM25Scorer scores a note against each wing by running the brain's own BM25
|
||||
// search scoped to that wing. It is deliberately a different mechanism from
|
||||
// the LLM, so agreement between the two is evidence and not an echo.
|
||||
type BM25Scorer struct {
|
||||
brainDir string
|
||||
wings []string
|
||||
exclude string
|
||||
}
|
||||
|
||||
// NewBM25Scorer scores against wings under brainDir. exclude is a brain-
|
||||
// relative path ignored in results (the backtest sample itself); "" for none.
|
||||
func NewBM25Scorer(brainDir string, wings []string, exclude string) *BM25Scorer {
|
||||
return &BM25Scorer{brainDir: brainDir, wings: wings, exclude: exclude}
|
||||
}
|
||||
|
||||
// Score implements Scorer. A wing whose search fails scores 0 rather than
|
||||
// failing the note: the Resolver treats "no clear opinion" as an Exception.
|
||||
func (b *BM25Scorer) Score(text string) map[string]float64 {
|
||||
query := strings.Join(topTerms(text), " ")
|
||||
out := map[string]float64{}
|
||||
if query == "" {
|
||||
return out
|
||||
}
|
||||
for _, w := range b.wings {
|
||||
hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: 2})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, h := range hits {
|
||||
if b.exclude != "" && h.Path == b.exclude {
|
||||
continue
|
||||
}
|
||||
// Hits come best-first: a wing is scored by its single best
|
||||
// page, so a big wing cannot win on volume alone.
|
||||
out[w] = float64(h.Score)
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// topTerms picks the most frequent longer words, ties broken alphabetically
|
||||
// so the query is deterministic.
|
||||
func topTerms(text string) []string {
|
||||
if len(text) > maxNoteChars {
|
||||
text = text[:maxNoteChars]
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, f := range strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
}) {
|
||||
if len([]rune(f)) >= minTermLen {
|
||||
counts[f]++
|
||||
}
|
||||
}
|
||||
terms := make([]string, 0, len(counts))
|
||||
for t := range counts {
|
||||
terms = append(terms, t)
|
||||
}
|
||||
sort.Slice(terms, func(i, j int) bool {
|
||||
if counts[terms[i]] != counts[terms[j]] {
|
||||
return counts[terms[i]] > counts[terms[j]]
|
||||
}
|
||||
return terms[i] < terms[j]
|
||||
})
|
||||
if len(terms) > maxQueryTerms {
|
||||
terms = terms[:maxQueryTerms]
|
||||
}
|
||||
return terms
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
func writePage(t *testing.T, dir, rel, body string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, rel)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
||||
require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
|
||||
}
|
||||
|
||||
func scorerBrain(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
writePage(t, dir, "wiki/infra/facts/k3s.md", "---\ntitle: K3s cluster\n---\nkubernetes cluster nodes scheduling pods deployment\n")
|
||||
writePage(t, dir, "wiki/finance/facts/invoice.md", "---\ntitle: Invoices\n---\ninvoice payment bookkeeping ledger reconciliation\n")
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestBM25Scorer_RanksTheWingWhosePagesMatch(t *testing.T) {
|
||||
s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "")
|
||||
got := s.Score("notes about kubernetes cluster scheduling and pods")
|
||||
assert.Greater(t, got["infra"], got["finance"])
|
||||
assert.Greater(t, got["infra"], 0.0)
|
||||
}
|
||||
|
||||
func TestBM25Scorer_ExcludesThePageBeingClassified(t *testing.T) {
|
||||
// A page must not vouch for its own wing when it is the backtest sample.
|
||||
s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "wiki/infra/facts/k3s.md")
|
||||
got := s.Score("kubernetes cluster nodes scheduling pods deployment")
|
||||
assert.Zero(t, got["infra"], "the excluded page is the only infra match")
|
||||
}
|
||||
|
||||
func TestBM25Scorer_NoUsableTermsScoresNothing(t *testing.T) {
|
||||
s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "")
|
||||
assert.Empty(t, s.Score("a an to of"))
|
||||
}
|
||||
|
||||
// A big wing must not win just by having more pages that brush the query: a
|
||||
// wing is scored by its single best page, not by the sum of its top hits.
|
||||
func TestBM25Scorer_ABigWingDoesNotWinOnVolume(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePage(t, dir, "wiki/small/facts/target.md", "---\ntitle: Target\n---\nkubernetes cluster scheduling pods deployment nodes\n")
|
||||
for i := 0; i < 8; i++ {
|
||||
writePage(t, dir, filepath.Join("wiki/big/facts", "p"+string(rune('a'+i))+".md"),
|
||||
"---\ntitle: Filler\n---\nkubernetes cluster scheduling mentioned in passing among bookkeeping ledger entries\n")
|
||||
}
|
||||
s := wing.NewBM25Scorer(dir, []string{"small", "big"}, "")
|
||||
got := s.Score("kubernetes cluster scheduling pods deployment nodes")
|
||||
top, ok := wing.TopWing(got)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "small", top)
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package wing
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/classification"
|
||||
)
|
||||
|
||||
// NewConfiguredResolver builds the production Resolver for a brain: Known from
|
||||
// the wiki, Eligible from classification.yaml (a wing is auto-assignable only
|
||||
// when it classifies below Confidential, so an untagged wing never is), the
|
||||
// LLM classifier over complete, and BM25 over the known wings as the scorer.
|
||||
//
|
||||
// Known is read once. A wing created later is unknown to this Resolver until
|
||||
// the process restarts, which fails safe: notes aimed at it are parked.
|
||||
func NewConfiguredResolver(brainDir string, complete CompleteFunc, threshold float64) (*Resolver, error) {
|
||||
known, err := KnownWings(brainDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cls, err := classification.Load(brainDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load classification: %w", err)
|
||||
}
|
||||
return &Resolver{
|
||||
Known: known,
|
||||
Eligible: func(w string) bool { return cls.Wing(w) < classification.Confidential },
|
||||
Classifier: NewLLMClassifier(complete),
|
||||
Scorer: NewBM25Scorer(brainDir, known, ""),
|
||||
Threshold: threshold,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
func configuredBrain(t *testing.T, classification string) string {
|
||||
dir := t.TempDir()
|
||||
writePage(t, dir, "wiki/homelab/facts/a.md", "x")
|
||||
writePage(t, dir, "wiki/radar/facts/b.md", "x")
|
||||
writePage(t, dir, "wiki/client-x/facts/c.md", "x")
|
||||
if classification != "" {
|
||||
writePage(t, dir, "classification.yaml", classification)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestNewConfiguredResolver_EligibilityComesFromClassification(t *testing.T) {
|
||||
dir := configuredBrain(t, "wings:\n radar: internal\n")
|
||||
r, err := wing.NewConfiguredResolver(dir, func(context.Context, string, string) (string, error) { return "", nil }, 0.9)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.ElementsMatch(t, []string{"client-x", "homelab", "radar"}, r.Known)
|
||||
assert.True(t, r.Eligible("radar"), "tagged internal")
|
||||
assert.True(t, r.Eligible("homelab"), "internal by built-in default")
|
||||
assert.False(t, r.Eligible("client-x"), "an untagged wing fails safe to confidential")
|
||||
assert.InDelta(t, 0.9, r.Threshold, 1e-9)
|
||||
}
|
||||
|
||||
func TestNewConfiguredResolver_AMalformedClassificationFileIsAnError(t *testing.T) {
|
||||
dir := configuredBrain(t, "wings: [not, a, map")
|
||||
_, err := wing.NewConfiguredResolver(dir, nil, 0.9)
|
||||
assert.Error(t, err, "a classification source that cannot be trusted must not degrade to 'everything eligible'")
|
||||
}
|
||||
|
||||
func TestNewConfiguredResolver_AConfidentialTagIsNeverEligible(t *testing.T) {
|
||||
dir := configuredBrain(t, "wings:\n radar: confidential\n")
|
||||
r, err := wing.NewConfiguredResolver(dir, nil, 0.9)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, r.Eligible("radar"))
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func writePage(t *testing.T, dir, rel, body string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, rel)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
||||
require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package wing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CompleteFunc is a chat completion, matching llm.Client.Complete. It must be
|
||||
// backed by a local model: note content is sent to it.
|
||||
type CompleteFunc func(ctx context.Context, system, user string) (string, error)
|
||||
|
||||
// maxNoteChars bounds what is sent for classification; the start of a note
|
||||
// carries its subject, and a whole transcript is cost with no benefit.
|
||||
const maxNoteChars = 6000
|
||||
|
||||
const classifySystem = `You file notes into a personal knowledge base. Pick the single wing the note belongs to, from the candidate list only.
|
||||
Answer with ONLY a JSON object: {"wing": "<name or none>", "confidence": <0..1>, "reason": "<one short sentence>"}.
|
||||
Use "none" when no candidate clearly fits. Do not invent wings.`
|
||||
|
||||
// LLMClassifier asks a model to pick a wing.
|
||||
type LLMClassifier struct{ complete CompleteFunc }
|
||||
|
||||
// NewLLMClassifier wraps a chat completion as a Classifier.
|
||||
func NewLLMClassifier(c CompleteFunc) *LLMClassifier { return &LLMClassifier{complete: c} }
|
||||
|
||||
type classifyReply struct {
|
||||
Wing string `json:"wing"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// Classify implements Classifier. Any reply it cannot parse is an error, which
|
||||
// the Resolver turns into an Exception: an unreadable answer is not a guess.
|
||||
func (l *LLMClassifier) Classify(ctx context.Context, text string, candidates []string) (Suggestion, error) {
|
||||
if len(text) > maxNoteChars {
|
||||
text = text[:maxNoteChars]
|
||||
}
|
||||
user := "Candidate wings: " + strings.Join(candidates, ", ") + "\n\nNote:\n" + text
|
||||
out, err := l.complete(ctx, classifySystem, user)
|
||||
if err != nil {
|
||||
return Suggestion{}, fmt.Errorf("classify: %w", err)
|
||||
}
|
||||
raw, ok := firstJSONObject(out)
|
||||
if !ok {
|
||||
return Suggestion{}, errors.New("classify: reply contains no JSON object")
|
||||
}
|
||||
var r classifyReply
|
||||
if err := json.Unmarshal([]byte(raw), &r); err != nil {
|
||||
return Suggestion{}, fmt.Errorf("classify: parse reply: %w", err)
|
||||
}
|
||||
w := strings.TrimSpace(r.Wing)
|
||||
if strings.EqualFold(w, "none") {
|
||||
w = ""
|
||||
}
|
||||
return Suggestion{Wing: w, Confidence: clamp01(r.Confidence), Reason: r.Reason}, nil
|
||||
}
|
||||
|
||||
// firstJSONObject returns the first balanced {...} in s, ignoring braces
|
||||
// inside JSON strings, so prose or code fences around the reply are tolerated.
|
||||
func firstJSONObject(s string) (string, bool) {
|
||||
start := strings.IndexByte(s, '{')
|
||||
if start < 0 {
|
||||
return "", false
|
||||
}
|
||||
depth, inStr, esc := 0, false, false
|
||||
for i := start; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case esc:
|
||||
esc = false
|
||||
case inStr && c == '\\':
|
||||
esc = true
|
||||
case c == '"':
|
||||
inStr = !inStr
|
||||
case inStr:
|
||||
case c == '{':
|
||||
depth++
|
||||
case c == '}':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return s[start : i+1], true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func clamp01(f float64) float64 {
|
||||
switch {
|
||||
case f < 0:
|
||||
return 0
|
||||
case f > 1:
|
||||
return 1
|
||||
}
|
||||
return f
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
func completes(reply string, err error) wing.CompleteFunc {
|
||||
return func(_ context.Context, _, _ string) (string, error) { return reply, err }
|
||||
}
|
||||
|
||||
func TestLLMClassifier_ParsesTheReply(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
reply string
|
||||
want wing.Suggestion
|
||||
wantErr bool
|
||||
}{
|
||||
{"bare json", `{"wing":"tapir","confidence":0.9,"reason":"video summariser"}`, wing.Suggestion{Wing: "tapir", Confidence: 0.9, Reason: "video summariser"}, false},
|
||||
{"json inside prose and fences", "Sure!\n```json\n{\"wing\":\"homelab\",\"confidence\":0.7,\"reason\":\"k3s\"}\n```\nDone.", wing.Suggestion{Wing: "homelab", Confidence: 0.7, Reason: "k3s"}, false},
|
||||
{"none means abstain", `{"wing":"none","confidence":0.2,"reason":"generic"}`, wing.Suggestion{Wing: "", Confidence: 0.2, Reason: "generic"}, false},
|
||||
{"confidence above 1 is clamped", `{"wing":"tapir","confidence":7,"reason":"x"}`, wing.Suggestion{Wing: "tapir", Confidence: 1, Reason: "x"}, false},
|
||||
{"negative confidence is clamped", `{"wing":"tapir","confidence":-1,"reason":"x"}`, wing.Suggestion{Wing: "tapir", Confidence: 0, Reason: "x"}, false},
|
||||
{"empty reply (a reasoning model that ran out of tokens)", ``, wing.Suggestion{}, true},
|
||||
{"no json at all", `I think it is tapir`, wing.Suggestion{}, true},
|
||||
{"malformed json", `{"wing":"tapir","confidence":`, wing.Suggestion{}, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := wing.NewLLMClassifier(completes(tt.reply, nil))
|
||||
got, err := c.Classify(context.Background(), "note", []string{"tapir", "homelab"})
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMClassifier_PropagatesTheModelError(t *testing.T) {
|
||||
boom := errors.New("model down")
|
||||
_, err := wing.NewLLMClassifier(completes("", boom)).Classify(context.Background(), "n", []string{"tapir"})
|
||||
assert.ErrorIs(t, err, boom)
|
||||
}
|
||||
|
||||
func TestLLMClassifier_PromptListsCandidatesAndBoundsTheNote(t *testing.T) {
|
||||
var gotUser string
|
||||
c := wing.NewLLMClassifier(func(_ context.Context, _, user string) (string, error) {
|
||||
gotUser = user
|
||||
return `{"wing":"tapir","confidence":0.9,"reason":"x"}`, nil
|
||||
})
|
||||
long := strings.Repeat("word ", 5000)
|
||||
_, err := c.Classify(context.Background(), long, []string{"tapir", "homelab"})
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, gotUser, "tapir")
|
||||
assert.Contains(t, gotUser, "homelab")
|
||||
assert.Less(t, len(gotUser), 12000, "a whole transcript must not be sent as the classification input")
|
||||
}
|
||||
@@ -1,17 +1,17 @@
|
||||
// Package wing decides which brain wing an incoming raw note belongs to
|
||||
// without the human typing it (hyperguild#205).
|
||||
// without the human typing it (hyperguild#205, #208).
|
||||
//
|
||||
// The contract is conservative: a note is placed automatically only when the
|
||||
// decision is explicit, or when two independent opinions (an LLM and a
|
||||
// deterministic scorer) agree with enough confidence on a wing that is both
|
||||
// known and eligible. Everything else is an Exception for a human to approve.
|
||||
// A wing is never created here, and a wing the caller does not mark eligible
|
||||
// (client wings, anything not deliberately tagged) is never auto-assigned.
|
||||
// An earlier design classified each note with a local model and cross-checked
|
||||
// it with a text score. Measured against pages whose wing a human had chosen,
|
||||
// the model was right 60% of the time with a saturated self-reported
|
||||
// confidence, and the cross-check was right 11% of the time, so the gate it
|
||||
// formed placed 2 of 55 pages (hyperguild#208). The Router that replaced it
|
||||
// decides with rules only: a deliberate folder or frontmatter wing, else one
|
||||
// default wing; nothing it cannot decide safely is guessed.
|
||||
package wing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -19,38 +19,18 @@ import (
|
||||
type Source string
|
||||
|
||||
const (
|
||||
Explicit Source = "explicit" // subfolder or frontmatter named the wing
|
||||
Default Source = "default" // no signal; the configured default wing
|
||||
Classified Source = "classified" // classifier + scorer agreed, confident
|
||||
Exception Source = "exception" // needs a human
|
||||
Explicit Source = "explicit" // subfolder or frontmatter named the wing
|
||||
Default Source = "default" // no signal; the configured default wing
|
||||
Exception Source = "exception" // needs a human
|
||||
)
|
||||
|
||||
// Note is what the resolver sees of an incoming raw note.
|
||||
// Note is what the Placer sees of an incoming raw note.
|
||||
type Note struct {
|
||||
Dir string // first subfolder under raw/, "" when dropped at the top
|
||||
FrontmatterWing string // `wing:` from the note's frontmatter, "" if absent
|
||||
Text string
|
||||
}
|
||||
|
||||
// Suggestion is a Classifier's answer. An empty Wing means "none fits".
|
||||
type Suggestion struct {
|
||||
Wing string
|
||||
Confidence float64 // 0..1
|
||||
Reason string
|
||||
}
|
||||
|
||||
// Classifier proposes a wing from the candidates. Implementations must run on
|
||||
// a local model; the resolver cannot enforce that.
|
||||
type Classifier interface {
|
||||
Classify(ctx context.Context, text string, candidates []string) (Suggestion, error)
|
||||
}
|
||||
|
||||
// Scorer is an independent, deterministic opinion (e.g. BM25 against each
|
||||
// wing's index). Higher is better; the key is the wing name.
|
||||
type Scorer interface {
|
||||
Score(text string) map[string]float64
|
||||
}
|
||||
|
||||
// Decision is the outcome. For an Exception, Wing holds the proposal when
|
||||
// there was one (so a reviewer can approve it) and is empty otherwise.
|
||||
type Decision struct {
|
||||
@@ -60,114 +40,10 @@ type Decision struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
// Resolver holds the policy. Known is every existing wing; Eligible reports
|
||||
// whether a known wing may be assigned automatically.
|
||||
type Resolver struct {
|
||||
Known []string
|
||||
Eligible func(wing string) bool
|
||||
Classifier Classifier
|
||||
Scorer Scorer
|
||||
Threshold float64
|
||||
}
|
||||
|
||||
// Resolve never returns an error: anything it cannot decide safely is an
|
||||
// Exception with the reason spelled out.
|
||||
func (r Resolver) Resolve(ctx context.Context, n Note) Decision {
|
||||
dir := norm(n.Dir)
|
||||
fm := norm(n.FrontmatterWing)
|
||||
switch {
|
||||
case dir != "" && fm != "" && dir != fm:
|
||||
return exception("", 0, fmt.Sprintf("subfolder %q and frontmatter wing %q disagree", dir, fm))
|
||||
case dir != "":
|
||||
return r.explicit(dir, "subfolder")
|
||||
case fm != "":
|
||||
return r.explicit(fm, "frontmatter")
|
||||
}
|
||||
return r.classify(ctx, n)
|
||||
}
|
||||
|
||||
func (r Resolver) explicit(w, via string) Decision {
|
||||
if why := r.refuse(w); why != "" {
|
||||
return exception(w, 0, fmt.Sprintf("%s names %q: %s", via, w, why))
|
||||
}
|
||||
return Decision{Wing: w, Source: Explicit, Confidence: 1, Reason: "named by " + via}
|
||||
}
|
||||
|
||||
func (r Resolver) classify(ctx context.Context, n Note) Decision {
|
||||
if r.Classifier == nil {
|
||||
return exception("", 0, "no classifier configured")
|
||||
}
|
||||
s, err := r.Classifier.Classify(ctx, n.Text, r.Known)
|
||||
if err != nil {
|
||||
return exception("", 0, fmt.Sprintf("classifier failed: %v", err))
|
||||
}
|
||||
w := norm(s.Wing)
|
||||
if w == "" {
|
||||
return exception("", s.Confidence, "classifier found no fitting wing")
|
||||
}
|
||||
if why := r.refuse(w); why != "" {
|
||||
return exception(w, s.Confidence, fmt.Sprintf("classifier proposed %q: %s", w, why))
|
||||
}
|
||||
if s.Confidence < r.Threshold {
|
||||
return exception(w, s.Confidence, fmt.Sprintf("confidence %.2f below threshold %.2f", s.Confidence, r.Threshold))
|
||||
}
|
||||
top, ok := r.scorerTop(n.Text)
|
||||
if !ok {
|
||||
return exception(w, s.Confidence, "scorer has no clear opinion to cross-check against")
|
||||
}
|
||||
if top != w {
|
||||
return exception(w, s.Confidence, fmt.Sprintf("classifier says %q, scorer says %q", w, top))
|
||||
}
|
||||
return Decision{Wing: w, Source: Classified, Confidence: s.Confidence,
|
||||
Reason: fmt.Sprintf("classifier and scorer agree (%.2f): %s", s.Confidence, s.Reason)}
|
||||
}
|
||||
|
||||
// refuse returns why a wing may not be auto-assigned, or "" if it may.
|
||||
func (r Resolver) refuse(w string) string {
|
||||
if !r.known(w) {
|
||||
return "not an existing wing (new wings are never created automatically)"
|
||||
}
|
||||
if r.Eligible == nil || !r.Eligible(w) {
|
||||
return "not eligible for automatic assignment"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (r Resolver) known(w string) bool {
|
||||
for _, k := range r.Known {
|
||||
if norm(k) == w {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// scorerTop returns the single best-scoring wing; a missing scorer, no scores,
|
||||
// or a tie at the top is "no clear opinion".
|
||||
func (r Resolver) scorerTop(text string) (string, bool) {
|
||||
if r.Scorer == nil {
|
||||
return "", false
|
||||
}
|
||||
return TopWing(r.Scorer.Score(text))
|
||||
}
|
||||
|
||||
// TopWing returns the unique highest-scoring wing in scores, or false when
|
||||
// there is none or the top is tied. Shared by the Resolver and the backtest so
|
||||
// both apply the same notion of "the scorer has an opinion".
|
||||
func TopWing(scores map[string]float64) (string, bool) {
|
||||
best, bestScore, tied := "", 0.0, false
|
||||
for k, v := range scores {
|
||||
switch {
|
||||
case v > bestScore:
|
||||
best, bestScore, tied = norm(k), v, false
|
||||
case v == bestScore && v > 0:
|
||||
tied = true
|
||||
}
|
||||
}
|
||||
if best == "" || tied {
|
||||
return "", false
|
||||
}
|
||||
return best, true
|
||||
// Placer decides where a raw note goes. Router is the production
|
||||
// implementation; the interface lets the watcher stay ignorant of how.
|
||||
type Placer interface {
|
||||
Resolve(ctx context.Context, n Note) Decision
|
||||
}
|
||||
|
||||
func exception(w string, conf float64, reason string) Decision {
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
package wing_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
type fakeClassifier struct {
|
||||
s wing.Suggestion
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeClassifier) Classify(_ context.Context, _ string, _ []string) (wing.Suggestion, error) {
|
||||
f.calls++
|
||||
return f.s, f.err
|
||||
}
|
||||
|
||||
type fakeScorer map[string]float64
|
||||
|
||||
func (f fakeScorer) Score(string) map[string]float64 { return f }
|
||||
|
||||
// eligibleExcept allows every name except the listed ones. Being permissive on
|
||||
// names it has never heard of means only the "known wing" check can stop an
|
||||
// invented wing, so a test of that check cannot pass by accident.
|
||||
func eligibleExcept(names ...string) func(string) bool {
|
||||
deny := map[string]bool{}
|
||||
for _, n := range names {
|
||||
deny[n] = true
|
||||
}
|
||||
return func(w string) bool { return !deny[w] }
|
||||
}
|
||||
|
||||
func newResolver(c wing.Classifier, s wing.Scorer) wing.Resolver {
|
||||
return wing.Resolver{
|
||||
Known: []string{"homelab", "hyperguild", "tapir", "rikatillsammans"},
|
||||
Eligible: eligibleExcept("rikatillsammans"), // a client wing: known, never auto
|
||||
Classifier: c,
|
||||
Scorer: s,
|
||||
Threshold: 0.8,
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_ExplicitSignal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
note wing.Note
|
||||
want wing.Source
|
||||
wantWing string
|
||||
}{
|
||||
{"subfolder of a known eligible wing", wing.Note{Dir: "homelab"}, wing.Explicit, "homelab"},
|
||||
{"frontmatter of a known eligible wing", wing.Note{FrontmatterWing: "tapir"}, wing.Explicit, "tapir"},
|
||||
{"subfolder and frontmatter agree", wing.Note{Dir: "tapir", FrontmatterWing: "tapir"}, wing.Explicit, "tapir"},
|
||||
{"subfolder and frontmatter disagree", wing.Note{Dir: "tapir", FrontmatterWing: "homelab"}, wing.Exception, ""},
|
||||
{"unknown wing is never created", wing.Note{Dir: "hoemlab"}, wing.Exception, "hoemlab"},
|
||||
{"known but not eligible (client) wing is never auto-assigned", wing.Note{Dir: "rikatillsammans"}, wing.Exception, "rikatillsammans"},
|
||||
{"subfolder is case-insensitive", wing.Note{Dir: "HomeLab"}, wing.Explicit, "homelab"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &fakeClassifier{}
|
||||
d := newResolver(c, fakeScorer{}).Resolve(context.Background(), tt.note)
|
||||
assert.Equal(t, tt.want, d.Source)
|
||||
assert.Equal(t, tt.wantWing, d.Wing)
|
||||
assert.Zero(t, c.calls, "an explicit signal must not spend an LLM call")
|
||||
assert.NotEmpty(t, d.Reason)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_Classified(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sugg wing.Suggestion
|
||||
err error
|
||||
scores fakeScorer
|
||||
want wing.Source
|
||||
wantWing string
|
||||
}{
|
||||
{"confident and scorer agrees", wing.Suggestion{Wing: "tapir", Confidence: 0.9}, nil, fakeScorer{"tapir": 3, "homelab": 1}, wing.Classified, "tapir"},
|
||||
{"confidence exactly at threshold passes", wing.Suggestion{Wing: "tapir", Confidence: 0.8}, nil, fakeScorer{"tapir": 3}, wing.Classified, "tapir"},
|
||||
{"below threshold is an exception, proposal kept", wing.Suggestion{Wing: "tapir", Confidence: 0.79}, nil, fakeScorer{"tapir": 3}, wing.Exception, "tapir"},
|
||||
{"scorer disagrees", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{"homelab": 5, "tapir": 1}, wing.Exception, "tapir"},
|
||||
{"scorer has no opinion also needs review", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{}, wing.Exception, "tapir"},
|
||||
{"classifier proposes a client wing", wing.Suggestion{Wing: "rikatillsammans", Confidence: 0.99}, nil, fakeScorer{"rikatillsammans": 9}, wing.Exception, "rikatillsammans"},
|
||||
{"classifier invents a wing", wing.Suggestion{Wing: "brand-new-wing", Confidence: 0.99}, nil, fakeScorer{"brand-new-wing": 9}, wing.Exception, "brand-new-wing"},
|
||||
{"classifier abstains", wing.Suggestion{Wing: "", Confidence: 0}, nil, fakeScorer{"tapir": 3}, wing.Exception, ""},
|
||||
{"classifier error", wing.Suggestion{}, errors.New("llm down"), fakeScorer{"tapir": 3}, wing.Exception, ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &fakeClassifier{s: tt.sugg, err: tt.err}
|
||||
d := newResolver(c, tt.scores).Resolve(context.Background(), wing.Note{Text: "some note"})
|
||||
assert.Equal(t, tt.want, d.Source)
|
||||
assert.Equal(t, tt.wantWing, d.Wing)
|
||||
assert.NotEmpty(t, d.Reason, "every decision says why, so a spot-check can judge it")
|
||||
assert.Equal(t, 1, c.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_NilDependenciesFailClosed(t *testing.T) {
|
||||
r := wing.Resolver{Known: []string{"tapir"}, Eligible: eligibleExcept(), Threshold: 0.8}
|
||||
d := r.Resolve(context.Background(), wing.Note{Text: "x"})
|
||||
require.Equal(t, wing.Exception, d.Source, "no classifier configured must route to review, not panic or guess")
|
||||
}
|
||||
@@ -8,12 +8,6 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Placer decides where a raw note goes. Router is the production
|
||||
// implementation; the interface lets the watcher stay ignorant of how.
|
||||
type Placer interface {
|
||||
Resolve(ctx context.Context, n Note) Decision
|
||||
}
|
||||
|
||||
// Router places a note with rules only, no model (hyperguild#208):
|
||||
//
|
||||
// 1. A subfolder or frontmatter `wing:` is the owner's deliberate choice. It
|
||||
@@ -68,3 +62,23 @@ func (r Router) explicit(w, via string) Decision {
|
||||
}
|
||||
return Decision{Wing: w, Source: Explicit, Confidence: 1, Reason: "named by " + via}
|
||||
}
|
||||
|
||||
// NewRouter builds the production Router. defaultWing must be a plain wing
|
||||
// name; clientPattern, when set, is a case-insensitive regexp over note text
|
||||
// whose match parks an unsignalled note instead of filing it. Both fail at
|
||||
// startup rather than later: a client filter that silently does nothing is
|
||||
// worse than none.
|
||||
func NewRouter(brainDir, defaultWing, clientPattern string) (Router, error) {
|
||||
if defaultWing == "" || !plainWingName(defaultWing) {
|
||||
return Router{}, fmt.Errorf("default wing %q is not a plain wing name", defaultWing)
|
||||
}
|
||||
r := Router{BrainDir: brainDir, Default: defaultWing}
|
||||
if clientPattern != "" {
|
||||
re, err := regexp.Compile("(?i)" + clientPattern)
|
||||
if err != nil {
|
||||
return Router{}, fmt.Errorf("client-content pattern: %w", err)
|
||||
}
|
||||
r.ClientBlock = re
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
@@ -85,3 +85,30 @@ func TestRouter_ANameThatIsNotPlainIsRefusedAsSuch(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRouter(t *testing.T) {
|
||||
dir := routerBrain(t)
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
r, err := wing.NewRouter(dir, "web-clip", "acme-client|globex")
|
||||
assert.NoError(t, err)
|
||||
d := r.Resolve(context.Background(), wing.Note{Text: "kickoff with GLOBEX"})
|
||||
assert.Equal(t, wing.Exception, d.Source, "the marker is matched case-insensitively")
|
||||
})
|
||||
t.Run("no marker pattern means no content check", func(t *testing.T) {
|
||||
r, err := wing.NewRouter(dir, "web-clip", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: "anything"}).Source)
|
||||
})
|
||||
t.Run("a default that is not a plain wing name is a startup error", func(t *testing.T) {
|
||||
_, err := wing.NewRouter(dir, "Web Clip", "")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
t.Run("an empty default is a startup error", func(t *testing.T) {
|
||||
_, err := wing.NewRouter(dir, "", "")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
t.Run("a pattern that does not compile is a startup error", func(t *testing.T) {
|
||||
_, err := wing.NewRouter(dir, "web-clip", "([unclosed")
|
||||
assert.Error(t, err, "a client filter that silently does nothing is worse than none")
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user