mathias/hyperguild
1
0
Fork 0
Code Issues 35 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat(server): INGEST_WING_ROUTING wires the Router; remove the classifier path (#208)

Browse Source 
Removes the LLM classifier, the substring 'BM25' scorer, the agreement gate,
the backtest and the old migrate/wingmigrate tooling: measured at 2 of 55
pages placed (#208), and the migration was built for the per-wing facts layout
this branch replaces. wing now holds the decision types, the Router and
KnownWings. INGEST_WING_ROUTING=1 builds the Router (default wing
INGEST_DEFAULT_WING=web-clip, client pattern from INGEST_CLIENT_BLOCK or
CLAUDE_INGEST_CLIENT_BLOCK); a bad default or pattern is fatal at startup.
Watcher tests rewritten for the Router, including a deliberate subfolder into
any existing wing, typo'd folders parked, forged-key and edit-while-placing
cases.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
mathias
2026-10-20 20:18:39 +02:00
co-authored by Claude Sonnet 5.5
parent 409b85096b
commit 8988c291ca
20 changed files with 263 additions and 1752 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ingestion/cmd/server/main.go
+20 -23
View File
@@ -150,17 +150,6 @@ func envInt(key string, fallback int) int {
return fallback return fallback
} }
// envFloat reads a confidence-style value in [0,1]. Anything unparseable or
// outside the range falls back: a threshold of 0 would auto-place everything.
func envFloat(key string, fallback float64) float64 {
if v := os.Getenv(key); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil && f >= 0 && f <= 1 {
return f
}
}
return fallback
}
// buildNotifier returns the ntfy notifier, or a nil interface when // buildNotifier returns the ntfy notifier, or a nil interface when
// BRAIN_NTFY_URL is unset. A nil *interface* (not a typed-nil) is required so // BRAIN_NTFY_URL is unset. A nil *interface* (not a typed-nil) is required so
// DegradingSink/Reconcile and the governance observer skip it cleanly. // DegradingSink/Reconcile and the governance observer skip it cleanly.
@@ -406,26 +395,34 @@ func main() {
} }
if watchInterval > 0 { if watchInterval > 0 {
// INGEST_WING_RESOLVE=1 routes raw/ notes into wiki/<wing>/ by // INGEST_WING_ROUTING=1 files raw/ notes by rule (hyperguild#208): a
// subfolder, frontmatter or classifier+scorer, and parks the rest in // deliberate raw/<wing>/ folder or `wing:` frontmatter, else the default
// raw/needs-review/ (hyperguild#205). Off = the retired layout. Set // wing (INGEST_DEFAULT_WING, "web-clip"); a note whose text matches the
// explicitly, a failure to build the resolver is fatal: silently // client-content pattern (INGEST_CLIENT_BLOCK, else
// falling back would write into the layout this flag exists to leave. // CLAUDE_INGEST_CLIENT_BLOCK) or that names an unknown wing is parked in
var wingResolver *wing.Resolver // raw/needs-review/. Off = the retired layout. Set explicitly, a bad
if os.Getenv("INGEST_WING_RESOLVE") == "1" { // default or pattern is fatal: silently falling back would write into
var rerr error // the layout this flag exists to leave. The interface stays nil when off
wingResolver, rerr = wing.NewConfiguredResolver(brainDir, llmClient.Complete, envFloat("INGEST_WING_THRESHOLD", 0.9)) // (a nil *Router inside it would not be nil).
var placer wing.Placer
if os.Getenv("INGEST_WING_ROUTING") == "1" {
pattern := os.Getenv("INGEST_CLIENT_BLOCK")
if pattern == "" {
pattern = os.Getenv("CLAUDE_INGEST_CLIENT_BLOCK")
}
router, rerr := wing.NewRouter(brainDir, envOr("INGEST_DEFAULT_WING", "web-clip"), pattern)
if rerr != nil { if rerr != nil {
logger.Error("build wing resolver", "err", rerr) logger.Error("build wing router", "err", rerr)
os.Exit(1) os.Exit(1)
} }
logger.Info("wing resolver enabled", "wings", len(wingResolver.Known), "threshold", wingResolver.Threshold) placer = router
logger.Info("wing routing enabled", "default_wing", router.Default, "client_filter", router.ClientBlock != nil)
} }
watcher.Start(ctx, watcher.Config{ watcher.Start(ctx, watcher.Config{
BrainDir: brainDir, BrainDir: brainDir,
Interval: time.Duration(watchInterval) * time.Second, Interval: time.Duration(watchInterval) * time.Second,
Pipeline: pipelineCfg, Pipeline: pipelineCfg,
Resolver: wingResolver, Resolver: placer,
}) })
} }
ingestion/cmd/server/main_test.go
-20
View File
@@ -100,23 +100,3 @@ func TestClaudeSinkDoesNotWriteThroughASymlinkedArchiveDir(t *testing.T) {
}) })
} }
} }
func TestEnvFloat(t *testing.T) {
tests := []struct {
name string
set string
want float64
}{
{"unset falls back", "", 0.9},
{"valid value", "0.75", 0.75},
{"garbage falls back rather than becoming 0", "high", 0.9},
{"out of range is rejected, not clamped silently into auto-placing everything", "-1", 0.9},
{"above one is rejected", "1.5", 0.9},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("TEST_ENV_FLOAT", tt.set)
assert.InDelta(t, tt.want, envFloat("TEST_ENV_FLOAT", 0.9), 1e-9)
})
}
}
ingestion/cmd/wingbacktest/main.go
-147
View File
@@ -1,147 +0,0 @@
// Command wingbacktest measures how well the wing resolver would have placed
// pages whose wing a human already decided (hyperguild#205).
//
// It reads pages from the wings given in -wings, hides the wing, asks the
// local classifier and the BM25 scorer, and reports coverage/precision per
// confidence threshold. Page text is sent to -llm-url, which MUST be a local
// model (it is not checked here); sample only wings you would allow.
package main
import (
"context"
"flag"
"fmt"
"io/fs"
"log/slog"
"math/rand"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
func main() {
brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory (read only)")
wingsFlag := flag.String("wings", "", "comma-separated wings to sample pages FROM (must be wings you would allow)")
perWing := flag.Int("per-wing", 8, "pages sampled per wing")
llmURL := flag.String("llm-url", "http://10.0.1.20:31234/v1", "OpenAI-compatible LOCAL endpoint")
model := flag.String("model", "qwen35-9b-fast", "model name")
seed := flag.Int64("seed", 1, "sampling seed")
flag.Parse()
if err := run(*brain, strings.Split(*wingsFlag, ","), *perWing, *llmURL, *model, *seed); err != nil {
slog.Error("wingbacktest", "err", err)
os.Exit(1)
}
}
func run(brain string, sampleWings []string, perWing int, llmURL, model string, seed int64) error {
known, err := wing.KnownWings(brain)
if err != nil {
return err
}
client := llm.New(llmURL, "", model, 3*time.Minute)
classifier := wing.NewLLMClassifier(client.Complete)
rng := rand.New(rand.NewSource(seed))
var samples []wing.Sample
perWingSamples := map[string][]wing.Sample{}
for _, w := range sampleWings {
w = strings.TrimSpace(w)
if w == "" {
continue
}
pages, err := listPages(brain, w)
if err != nil {
return err
}
rng.Shuffle(len(pages), func(i, j int) { pages[i], pages[j] = pages[j], pages[i] })
if len(pages) > perWing {
pages = pages[:perWing]
}
for _, rel := range pages {
text, err := readBody(brain, rel)
if err != nil || len(text) < 200 {
continue
}
sugg, cerr := classifier.Classify(context.Background(), text, known)
if cerr != nil {
slog.Warn("classify failed", "page", rel, "err", cerr)
}
top, _ := wing.TopWing(wing.NewBM25Scorer(brain, known, rel).Score(text))
s := wing.Sample{Truth: w, Suggested: strings.ToLower(sugg.Wing), Confidence: sugg.Confidence, ScorerTop: top}
slog.Info("sample", "page", rel, "truth", w, "classifier", s.Suggested, "conf", s.Confidence, "scorer", top, "reason", sugg.Reason)
samples = append(samples, s)
perWingSamples[w] = append(perWingSamples[w], s)
}
}
report(samples, perWingSamples)
return nil
}
func report(samples []wing.Sample, per map[string][]wing.Sample) {
fmt.Printf("# wing backtest\n\nn=%d pages. Classifier+scorer must agree; threshold applies to classifier confidence.\n\n", len(samples))
fmt.Println("| threshold | auto | coverage | precision |")
fmt.Println("|---|---|---|---|")
for _, t := range []float64{0.5, 0.6, 0.7, 0.8, 0.9, 0.95} {
r := wing.Evaluate(samples, t)
fmt.Printf("| %.2f | %d/%d | %.0f%% | %.0f%% |\n", t, r.Auto, r.N, 100*r.Coverage, 100*r.Precision)
}
fmt.Println("\n## per wing (threshold 0.80)\n\n| wing | n | auto | precision |\n|---|---|---|---|")
names := make([]string, 0, len(per))
for w := range per {
names = append(names, w)
}
sort.Strings(names)
for _, w := range names {
r := wing.Evaluate(per[w], 0.8)
fmt.Printf("| %s | %d | %d | %.0f%% |\n", w, r.N, r.Auto, 100*r.Precision)
}
fmt.Println("\n## wrong auto-placements at 0.80")
fmt.Println()
for _, s := range samples {
if s.Suggested != "" && s.Confidence >= 0.8 && s.ScorerTop == s.Suggested && s.Suggested != s.Truth {
fmt.Printf("- truth=%s -> both said %s (conf %.2f)\n", s.Truth, s.Suggested, s.Confidence)
}
}
}
func listPages(brain, w string) ([]string, error) {
var out []string
root := filepath.Join(brain, "wiki", w)
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() || !d.Type().IsRegular() || !strings.HasSuffix(p, ".md") || d.Name() == "_index.md" {
return err
}
rel, rerr := filepath.Rel(brain, p)
if rerr != nil {
return rerr
}
out = append(out, filepath.ToSlash(rel))
return nil
})
if err != nil {
return nil, fmt.Errorf("walk %s: %w", w, err)
}
return out, nil
}
// readBody returns the page text without YAML frontmatter, so the wing label
// (and anything else in frontmatter) cannot leak into the classification.
func readBody(brain, rel string) (string, error) {
b, err := os.ReadFile(filepath.Join(brain, filepath.FromSlash(rel)))
if err != nil {
return "", fmt.Errorf("read %s: %w", rel, err)
}
s := string(b)
if strings.HasPrefix(s, "---\n") {
if end := strings.Index(s[4:], "\n---"); end >= 0 {
s = s[4+end+4:]
}
}
return strings.TrimSpace(s), nil
}
ingestion/cmd/wingmigrate/main.go
-92
View File
@@ -1,92 +0,0 @@
// Command wingmigrate moves pages out of the retired wiki/{sources,concepts,
// entities}/ layout into wiki/<wing>/ (hyperguild#205 S5).
//
// Default is a dry run that prints the plan. -apply performs the Auto items.
// Page text goes to -llm-url, which MUST be a local model.
package main
import (
"context"
"flag"
"fmt"
"log/slog"
"os"
"strings"
"time"
"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
"github.com/mathiasbq/hyperguild/ingestion/internal/migrate"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
func main() {
brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory")
llmURL := flag.String("llm-url", "http://10.0.1.20:31234/v1", "OpenAI-compatible LOCAL endpoint")
model := flag.String("model", "qwen35-9b-fast", "model name")
threshold := flag.Float64("threshold", 0.9, "classifier confidence threshold")
apply := flag.Bool("apply", false, "perform the Auto moves (default: dry run)")
what := flag.String("what-if-eligible", "", "dry run only: comma-separated wings to treat as eligible, to see what tagging them would allow")
flag.Parse()
if *apply && *what != "" {
slog.Error("-what-if-eligible is a dry-run aid and cannot be combined with -apply: eligibility is a sovereignty decision made in classification.yaml")
os.Exit(2)
}
if err := run(*brain, *llmURL, *model, *threshold, *apply, *what); err != nil {
slog.Error("wingmigrate", "err", err)
os.Exit(1)
}
}
func run(brain, llmURL, model string, threshold float64, apply bool, whatIf string) error {
client := llm.New(llmURL, "", model, 3*time.Minute)
base, err := wing.NewConfiguredResolver(brain, client.Complete, threshold)
if err != nil {
return err
}
if whatIf != "" {
allowed := map[string]bool{}
for _, w := range strings.Split(whatIf, ",") {
allowed[strings.TrimSpace(w)] = true
}
base.Eligible = func(w string) bool { return allowed[w] }
}
rf := func(exclude string) *wing.Resolver {
r := *base
r.Scorer = wing.NewBM25Scorer(brain, base.Known, exclude)
return &r
}
items, err := migrate.BuildPlan(context.Background(), brain, rf)
if err != nil {
return err
}
report(items)
if !apply {
fmt.Println("\n_dry run: nothing was changed_")
return nil
}
n, err := migrate.Apply(brain, items)
fmt.Printf("\napplied %d moves\n", n)
return err
}
func report(items []migrate.Item) {
auto, exc := 0, 0
for _, it := range items {
if it.Status == migrate.Auto {
auto++
} else {
exc++
}
}
fmt.Printf("# wing migration plan\n\n%d pages: %d auto, %d exceptions\n\n", len(items), auto, exc)
fmt.Println("| status | kind | from | to / proposal | how | why |")
fmt.Println("|---|---|---|---|---|---|")
for _, it := range items {
dest := it.To
if dest == "" {
dest = it.Wing
}
fmt.Printf("| %s | %s | %s | %s | %s | %s |\n", it.Status, it.Kind, it.From, dest, it.Method, strings.ReplaceAll(it.Reason, "|", "/"))
}
}
ingestion/internal/migrate/migrate.go
-309
View File
@@ -1,309 +0,0 @@
// Package migrate moves pages out of the retired wiki/{sources,concepts,
// entities}/ layout into wiki/<wing>/{sources,facts}/ (hyperguild#205).
//
// It is a two-step tool on purpose. BuildPlan only reads and decides; Apply
// performs exactly the items the plan marked Auto and nothing else. Sources are
// placed by the wing Resolver. Concepts and entities have no wing of their own:
// they inherit the strict-majority wing of the sources that cite them, which is
// deterministic and needs no model. Whatever cannot be placed safely is an
// Exception, left where it is for a human.
package migrate
import (
"context"
"errors"
"fmt"
"io/fs"
"path"
"regexp"
"sort"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
// Status is whether the plan moves a page by itself.
type Status string
const (
Auto Status = "auto"
Exception Status = "exception"
)
// Item is one planned move.
type Item struct {
From string // brain-relative, e.g. wiki/entities/jev.md
To string // brain-relative; empty for an Exception without a proposal
Slug string
Kind string // source | concept | entity
Wing string // decided wing, or the proposal for an Exception
Status Status
Method string // resolver | inherited | junk | blocked
Confidence float64
Reason string
}
// ResolverFor returns the Resolver to use for one page. It takes the page's
// own path so a scorer can exclude the page from its evidence.
type ResolverFor func(excludePath string) *wing.Resolver
var (
linkSlugRE = regexp.MustCompile(`\[\[([^|\]]+)`)
legacy = []struct{ dir, kind string }{
{"wiki/sources", "source"},
{"wiki/concepts", "concept"},
{"wiki/entities", "entity"},
}
)
// BuildPlan reads the legacy layout and decides, without writing anything.
func BuildPlan(ctx context.Context, brainDir string, rf ResolverFor) ([]Item, error) {
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
taken, err := slugsInWings(bfs)
if err != nil {
return nil, err
}
var items []Item
decided := map[string]Item{} // source slug -> its decision
for _, l := range legacy {
names, err := bfs.ReadDirRegular(l.dir)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return nil, fmt.Errorf("read %s: %w", l.dir, err)
}
for _, name := range names {
if !strings.HasSuffix(name, ".md") || name == "_index.md" {
continue
}
rel := l.dir + "/" + name
slug := strings.TrimSuffix(name, ".md")
it := Item{From: rel, Slug: slug, Kind: l.kind}
if strings.HasPrefix(name, "tunnel-candidates-") {
it.Status, it.Method, it.Reason = Exception, "junk", "auto-tunnel review log ingested as a source; not a source"
items = append(items, it)
continue
}
content, err := bfs.ReadRegular(rel)
if err != nil {
it.Status, it.Method, it.Reason = Exception, "blocked", fmt.Sprintf("unreadable: %v", err)
items = append(items, it)
continue
}
if l.kind == "source" {
d := rf(rel).Resolve(ctx, wing.Note{Text: string(content), FrontmatterWing: frontmatterValue(string(content), "wing")})
it.Method, it.Wing, it.Confidence, it.Reason = "resolver", d.Wing, d.Confidence, d.Reason
if d.Source == wing.Exception {
it.Status = Exception
} else {
it.Status, it.To = Auto, "wiki/"+d.Wing+"/sources/"+name
}
decided[slug] = it
}
items = append(items, it)
}
}
// Second pass: concepts and entities inherit from their sources.
for i, it := range items {
if it.Kind == "source" || it.Method == "junk" || it.Method == "blocked" {
continue
}
content, err := bfs.ReadRegular(it.From)
if err != nil {
continue
}
it.Method = "inherited"
w, reason := inheritedWing(string(content), decided)
if w == "" {
it.Status, it.Reason = Exception, reason
} else {
it.Status, it.Wing, it.To = Auto, w, "wiki/"+w+"/facts/"+it.Slug+".md"
it.Reason = fmt.Sprintf("strict majority of its sources are in %q", w)
}
items[i] = it
}
blockCollisions(bfs, items, taken)
sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
return items, nil
}
// inheritedWing returns the wing held by a strict majority of the legacy
// sources cited in the page's "## Sources" section, or "" with the reason.
func inheritedWing(content string, decided map[string]Item) (string, string) {
section := sourcesSection(content)
if section == "" {
return "", "page has no Sources section to inherit a wing from"
}
total := 0
votes := map[string]int{}
for _, m := range linkSlugRE.FindAllStringSubmatch(section, -1) {
d, ok := decided[strings.TrimSpace(m[1])]
if !ok {
continue // not a legacy source (already migrated, or not a source)
}
total++
if d.Status == Auto {
votes[d.Wing]++
}
}
if total == 0 {
return "", "none of its cited sources are in the legacy layout"
}
for w, n := range votes {
if n*2 > total {
return w, ""
}
}
return "", fmt.Sprintf("its %d cited sources disagree on a wing or were not placed", total)
}
// blockCollisions turns an Auto item into an Exception when its destination
// exists, its slug is already used in some wing (graph identity is the slug
// alone), or two planned items would land on the same path.
func blockCollisions(bfs *brainfs.Brain, items []Item, taken map[string]string) {
planned := map[string]int{}
for _, it := range items {
if it.Status == Auto {
planned[it.To]++
}
}
for i, it := range items {
if it.Status != Auto {
continue
}
switch {
case planned[it.To] > 1:
items[i].Status, items[i].Reason = Exception, fmt.Sprintf("another legacy page would land on %s", it.To)
case exists(bfs, it.To):
items[i].Status, items[i].Reason = Exception, fmt.Sprintf("%s already exists", it.To)
default:
if other, ok := taken[it.Slug]; ok {
items[i].Status, items[i].Reason = Exception, fmt.Sprintf("slug %q is already used by %s", it.Slug, other)
}
}
}
}
func exists(bfs *brainfs.Brain, rel string) bool {
_, err := bfs.Lstat(rel)
return err == nil
}
// slugsInWings maps every slug under wiki/<wing>/<hall>/ to its path.
func slugsInWings(bfs *brainfs.Brain) (map[string]string, error) {
out := map[string]string{}
wings, err := bfs.ReadDirDirs("wiki")
if errors.Is(err, fs.ErrNotExist) {
return out, nil
}
if err != nil {
return nil, fmt.Errorf("list wiki: %w", err)
}
for _, w := range wings {
if w == "sources" || w == "concepts" || w == "entities" {
continue
}
halls, err := bfs.ReadDirDirs("wiki/" + w)
if err != nil {
continue
}
for _, h := range halls {
names, err := bfs.ReadDirRegular("wiki/" + w + "/" + h)
if err != nil {
continue
}
for _, n := range names {
if strings.HasSuffix(n, ".md") && n != "_index.md" {
out[strings.TrimSuffix(n, ".md")] = "wiki/" + w + "/" + h + "/" + n
}
}
}
}
return out, nil
}
// Apply performs the Auto items, one at a time, and stops at the first
// failure. Each move writes the destination with O_EXCL, then removes the
// source, so a failure never loses a page. Exceptions are not touched.
func Apply(brainDir string, items []Item) (int, error) {
bfs, err := brainfs.Open(brainDir)
if err != nil {
return 0, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
n := 0
touched := map[string]bool{}
for _, it := range items {
if it.Status != Auto {
continue
}
content, err := bfs.ReadRegular(it.From)
if err != nil {
return n, fmt.Errorf("read %s: %w", it.From, err)
}
hall := path.Base(path.Dir(it.To))
if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing, hall, it.Kind))); err != nil {
return n, fmt.Errorf("write %s: %w", it.To, err)
}
if err := bfs.Remove(it.From); err != nil {
return n, fmt.Errorf("moved to %s but removing %s failed: %w", it.To, it.From, err)
}
touched[it.Wing] = true
n++
}
for w := range touched {
_ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved
}
return n, nil
}
// stamp inserts wing/hall (and kind for facts) after the opening fence, leaving
// every other byte of the page alone. A page without frontmatter gets a block.
func stamp(content, w, hall, kind string) string {
extra := "wing: " + w + "\nhall: " + hall + "\n"
if hall == "facts" {
extra += "kind: " + kind + "\n"
}
if strings.HasPrefix(content, "---\n") {
return "---\n" + extra + content[4:]
}
return "---\n" + extra + "---\n" + content
}
// sourcesSection returns the text under a "## Sources" heading up to the next
// "## " heading.
func sourcesSection(content string) string {
_, rest, ok := strings.Cut(content, "\n## Sources")
if !ok {
return ""
}
if i := strings.Index(rest, "\n## "); i >= 0 {
rest = rest[:i]
}
return rest
}
func frontmatterValue(content, key string) string {
if !strings.HasPrefix(content, "---\n") {
return ""
}
block, _, _ := strings.Cut(content[4:], "\n---")
for _, line := range strings.Split(block, "\n") {
if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key {
return strings.Trim(strings.TrimSpace(v), `"'`)
}
}
return ""
}
ingestion/internal/migrate/migrate_test.go
-243
View File
@@ -1,243 +0,0 @@
package migrate_test
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/migrate"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
// keywordClassifier picks a wing by a marker word in the text, and the scorer
// agrees by the same marker, so a page "belongs" to the wing whose marker it holds.
type keywordClassifier struct{}
func (keywordClassifier) Classify(_ context.Context, text string, _ []string) (wing.Suggestion, error) {
for _, w := range []string{"alpha", "beta"} {
if strings.Contains(text, strings.ToUpper(w)) {
return wing.Suggestion{Wing: w, Confidence: 0.95, Reason: "marker"}, nil
}
}
return wing.Suggestion{}, nil
}
type keywordScorer struct{}
func (keywordScorer) Score(text string) map[string]float64 {
out := map[string]float64{}
for _, w := range []string{"alpha", "beta"} {
if strings.Contains(text, strings.ToUpper(w)) {
out[w] = 1
}
}
return out
}
func resolverFor(string) *wing.Resolver {
return &wing.Resolver{
Known: []string{"alpha", "beta", "client"},
Eligible: func(w string) bool { return w != "client" },
Classifier: keywordClassifier{},
Scorer: keywordScorer{},
Threshold: 0.8,
}
}
func page(t *testing.T, dir, rel, body string) {
t.Helper()
full := filepath.Join(dir, rel)
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
}
func src(title, marker string) string {
return "---\ntitle: " + title + "\ntype: video\n---\n\n## Summary\n\n" + marker + " content.\n"
}
func entity(title string, sources ...string) string {
var b strings.Builder
b.WriteString("---\ntitle: " + title + "\ntype: tool\n---\n\n## Definition\n\nA thing.\n\n## Sources\n\n")
for _, s := range sources {
b.WriteString("- [[" + s + "|" + s + "]]\n")
}
return b.String()
}
func byFrom(items []migrate.Item) map[string]migrate.Item {
m := map[string]migrate.Item{}
for _, it := range items {
m[it.From] = it
}
return m
}
func TestPlan_AResolvedSourceGoesToItsWingsSourcesHall(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
it := byFrom(items)["wiki/sources/talk-a.md"]
assert.Equal(t, migrate.Auto, it.Status)
assert.Equal(t, "wiki/alpha/sources/talk-a.md", it.To)
}
func TestPlan_AutoTunnelLogsAreJunkNotSources(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("t", "ALPHA"))
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
it := byFrom(items)["wiki/sources/tunnel-candidates-2026-06-14.md"]
assert.Equal(t, migrate.Exception, it.Status, "never auto-moved, whatever the classifier says")
assert.Equal(t, "junk", it.Method)
}
func TestPlan_AnEntityFollowsTheMajorityWingOfItsSources(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/a1.md", src("A1", "ALPHA"))
page(t, dir, "wiki/sources/a2.md", src("A2", "ALPHA"))
page(t, dir, "wiki/sources/b1.md", src("B1", "BETA"))
page(t, dir, "wiki/entities/jev.md", entity("Jev", "a1", "a2", "b1"))
page(t, dir, "wiki/concepts/factory.md", "---\ntitle: Factory\n---\n\n## Definition\n\nx\n\n## Sources\n\n- [[a1|A1]]\n")
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
m := byFrom(items)
assert.Equal(t, "wiki/alpha/facts/jev.md", m["wiki/entities/jev.md"].To)
assert.Equal(t, "entity", m["wiki/entities/jev.md"].Kind)
assert.Equal(t, "wiki/alpha/facts/factory.md", m["wiki/concepts/factory.md"].To)
assert.Equal(t, "concept", m["wiki/concepts/factory.md"].Kind)
}
func TestPlan_ATieBetweenWingsIsAnException(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/a1.md", src("A1", "ALPHA"))
page(t, dir, "wiki/sources/b1.md", src("B1", "BETA"))
page(t, dir, "wiki/entities/jev.md", entity("Jev", "a1", "b1"))
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
it := byFrom(items)["wiki/entities/jev.md"]
assert.Equal(t, migrate.Exception, it.Status)
assert.Contains(t, it.Reason, "disagree")
}
func TestPlan_AnEntityWhoseSourcesWereNotPlacedIsAnException(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/mystery.md", src("Mystery", "nothing recognisable"))
page(t, dir, "wiki/entities/orphan.md", entity("Orphan", "mystery"))
page(t, dir, "wiki/entities/nosources.md", "---\ntitle: No Sources\n---\n\n## Definition\n\nx\n")
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
m := byFrom(items)
assert.Equal(t, migrate.Exception, m["wiki/entities/orphan.md"].Status)
assert.Equal(t, migrate.Exception, m["wiki/entities/nosources.md"].Status)
}
func TestPlan_ADestinationThatAlreadyExistsIsAnException(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
page(t, dir, "wiki/alpha/sources/talk-a.md", "someone else's page")
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
it := byFrom(items)["wiki/sources/talk-a.md"]
assert.Equal(t, migrate.Exception, it.Status)
assert.Contains(t, it.Reason, "already exists", "the occupied destination is named as the reason, not a generic slug clash")
}
func TestPlan_ASlugAlreadyUsedInAnotherWingIsAnException(t *testing.T) {
// Graph identity is the slug alone; two pages with one slug would merge.
dir := t.TempDir()
page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
page(t, dir, "wiki/beta/facts/talk-a.md", "a beta page with the same slug")
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
it := byFrom(items)["wiki/sources/talk-a.md"]
assert.Equal(t, migrate.Exception, it.Status)
assert.Contains(t, it.Reason, "slug")
}
func TestApply_MovesOnlyAutoItemsAndStampsFrontmatterWithoutTouchingTheBody(t *testing.T) {
dir := t.TempDir()
body := src("Talk A", "ALPHA") + "\nSee [[jev|Jev]] and [[unknown-slug]].\n"
page(t, dir, "wiki/sources/talk-a.md", body)
page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("t", "ALPHA"))
page(t, dir, "wiki/entities/jev.md", entity("Jev", "talk-a"))
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
n, err := migrate.Apply(dir, items)
require.NoError(t, err)
assert.Equal(t, 2, n)
moved, err := os.ReadFile(filepath.Join(dir, "wiki/alpha/sources/talk-a.md"))
require.NoError(t, err)
assert.Contains(t, string(moved), "wing: alpha")
assert.Contains(t, string(moved), "hall: sources")
assert.Contains(t, string(moved), "See [[jev|Jev]] and [[unknown-slug]].", "links and body are byte-preserved")
ent, err := os.ReadFile(filepath.Join(dir, "wiki/alpha/facts/jev.md"))
require.NoError(t, err)
assert.Contains(t, string(ent), "kind: entity")
_, err = os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md"))
assert.ErrorIs(t, err, os.ErrNotExist, "the original is removed after the copy is written")
_, err = os.Stat(filepath.Join(dir, "wiki/sources/tunnel-candidates-2026-06-14.md"))
assert.NoError(t, err, "an exception is left exactly where it is")
}
func TestApply_NeverOverwritesAndKeepsTheSourceWhenTheDestinationAppearsLate(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
page(t, dir, "wiki/alpha/sources/talk-a.md", "appeared after the plan")
n, err := migrate.Apply(dir, items)
require.Error(t, err)
assert.Zero(t, n)
got, _ := os.ReadFile(filepath.Join(dir, "wiki/alpha/sources/talk-a.md"))
assert.Equal(t, "appeared after the plan", string(got))
_, statErr := os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md"))
assert.NoError(t, statErr, "the source survives a failed write")
}
func TestPlan_IsEmptyOnceEverythingIsMigrated(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
_, err = migrate.Apply(dir, items)
require.NoError(t, err)
again, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
assert.Empty(t, again)
}
// An exception that blocked on a collision still carries a destination; Apply
// must treat Status, not the presence of a path, as the decision.
func TestApply_LeavesACollisionExceptionAloneEvenThoughItHasADestination(t *testing.T) {
dir := t.TempDir()
page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
page(t, dir, "wiki/alpha/sources/talk-a.md", "occupied")
items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
require.NoError(t, err)
require.Equal(t, migrate.Exception, byFrom(items)["wiki/sources/talk-a.md"].Status)
n, err := migrate.Apply(dir, items)
require.NoError(t, err)
assert.Zero(t, n)
_, statErr := os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md"))
assert.NoError(t, statErr)
}
ingestion/internal/watcher/watcher_wing_test.go
+164 -188
View File
@@ -3,9 +3,9 @@ package watcher
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
@@ -17,37 +17,11 @@ import (
"github.com/mathiasbq/hyperguild/ingestion/internal/wing" "github.com/mathiasbq/hyperguild/ingestion/internal/wing"
) )
type stubClassifier struct { func wingTestCfg(t *testing.T, mk func(brainDir string) wing.Placer, llmCalls *atomic.Int32) (Config, string) {
s wing.Suggestion
err error
calls atomic.Int32
}
func (c *stubClassifier) Classify(context.Context, string, []string) (wing.Suggestion, error) {
c.calls.Add(1)
return c.s, c.err
}
type stubScorer map[string]float64
func (s stubScorer) Score(string) map[string]float64 { return s }
func wingResolver(c wing.Classifier, s wing.Scorer) *wing.Resolver {
return &wing.Resolver{
Known: []string{"radar", "homelab", "client-x"},
Eligible: func(w string) bool { return w != "client-x" },
Classifier: c,
Scorer: s,
Threshold: 0.8,
}
}
func wingTestCfg(t *testing.T, r wing.Placer, llmCalls *atomic.Int32) (Config, string) {
t.Helper() t.Helper()
brainDir := setupBrainDir(t) brainDir := setupBrainDir(t)
return Config{ cfg := Config{
BrainDir: brainDir, BrainDir: brainDir,
Resolver: r,
Pipeline: pipeline.Config{ Pipeline: pipeline.Config{
Schema: "# Schema", Schema: "# Schema",
Complete: func(ctx context.Context, s, u string) (string, error) { Complete: func(ctx context.Context, s, u string) (string, error) {
@@ -55,7 +29,21 @@ func wingTestCfg(t *testing.T, r wing.Placer, llmCalls *atomic.Int32) (Config, s
return successComplete(pipeline.RawPage{Title: "Talk", Type: "source", Subtype: "video", Content: "## Summary\n\nx\n"})(ctx, s, u) return successComplete(pipeline.RawPage{Title: "Talk", Type: "source", Subtype: "video", Content: "## Summary\n\nx\n"})(ctx, s, u)
}, },
}, },
}, brainDir }
if mk != nil {
cfg.Resolver = mk(brainDir)
}
return cfg, brainDir
}
// router is the production Router with a client-content marker.
func router(brainDir string) wing.Placer {
return wing.Router{BrainDir: brainDir, Default: "web-clip", ClientBlock: regexp.MustCompile(`(?i)acme-client`)}
}
func makeWing(t *testing.T, brainDir, name string) {
t.Helper()
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, "wiki", name, "facts"), 0o755))
} }
func put(t *testing.T, brainDir, rel, body string) { func put(t *testing.T, brainDir, rel, body string) {
@@ -70,70 +58,91 @@ func has(brainDir, rel string) bool {
return err == nil return err == nil
} }
func mustJSONPages(pages []pipeline.RawPage) string {
b, err := json.Marshal(pages)
if err != nil {
panic(err)
}
return string(b)
}
func parkedBody(t *testing.T, brainDir, name string) string {
t.Helper()
b, err := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", name))
require.NoError(t, err)
return string(b)
}
func linesStartingWith(s, prefix string) int {
n := 0
for _, l := range strings.Split(s, "\n") {
if strings.HasPrefix(l, prefix) {
n++
}
}
return n
}
func TestProcessDir_ASubfolderNamesTheWing(t *testing.T) { func TestProcessDir_ASubfolderNamesTheWing(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
clf := &stubClassifier{} cfg, brainDir := wingTestCfg(t, router, &llmCalls)
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{}), &llmCalls) makeWing(t, brainDir, "radar")
put(t, brainDir, "radar/dark-factory.md", "A talk about factories.") put(t, brainDir, "radar/dark-factory.md", "A talk about factories.")
errs := processDir(context.Background(), cfg, "2026-10-20") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
require.Empty(t, errs)
assert.True(t, has(brainDir, "wiki/radar/sources/dark-factory.md"), "lands in the named wing") assert.True(t, has(brainDir, "wiki/radar/sources/dark-factory.md"), "lands in the named wing")
assert.False(t, has(brainDir, "raw/radar/dark-factory.md"), "inbox zero holds for subfolders too") assert.False(t, has(brainDir, "raw/radar/dark-factory.md"), "inbox zero holds for subfolders too")
assert.Zero(t, clf.calls.Load(), "an explicit wing costs no classification")
} }
func TestProcessDir_ANoteTheResolverCannotPlaceIsParkedNotIngested(t *testing.T) { func TestProcessDir_ADeliberateSubfolderIntoAnyExistingWingPlaces(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
clf := &stubClassifier{s: wing.Suggestion{Wing: "radar", Confidence: 0.5, Reason: "maybe"}} cfg, brainDir := wingTestCfg(t, router, &llmCalls)
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{"radar": 3}), &llmCalls) makeWing(t, brainDir, "rikatillsammans")
put(t, brainDir, "unsure.md", "Some note.") put(t, brainDir, "rikatillsammans/meeting.md", "Notes the owner put there on purpose.")
errs := processDir(context.Background(), cfg, "2026-10-20") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
assert.True(t, has(brainDir, "wiki/rikatillsammans/sources/meeting.md"),
"a folder is the owner's deliberate choice; extraction runs on the local model")
}
func TestProcessDir_ANoteTheRouterCannotPlaceIsParkedNotIngested(t *testing.T) {
var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, router, &llmCalls)
put(t, brainDir, "kickoff.md", "Some note about the ACME-CLIENT kickoff.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
require.Empty(t, errs)
assert.Zero(t, llmCalls.Load(), "no extraction runs for an unplaced note") assert.Zero(t, llmCalls.Load(), "no extraction runs for an unplaced note")
assert.False(t, has(brainDir, "raw/unsure.md"), "leaves the top of raw/") assert.False(t, has(brainDir, "raw/kickoff.md"), "leaves the top of raw/")
parked, err := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", "unsure.md")) parked := parkedBody(t, brainDir, "kickoff.md")
require.NoError(t, err) assert.Contains(t, parked, "proposal_reason:")
assert.Contains(t, string(parked), "proposed_wing: radar") assert.Contains(t, parked, "client-content marker")
assert.Contains(t, string(parked), "below threshold") assert.Contains(t, parked, "ACME-CLIENT kickoff", "the note body is preserved")
assert.Contains(t, string(parked), "Some note.", "the note body is preserved")
} }
func TestProcessDir_ANoteInAClientWingFolderIsParked(t *testing.T) { func TestProcessDir_ATypoedWingFolderIsParkedAndNeverCreated(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, wingResolver(&stubClassifier{}, stubScorer{}), &llmCalls) cfg, brainDir := wingTestCfg(t, router, &llmCalls)
put(t, brainDir, "client-x/secret.md", "Client material.") makeWing(t, brainDir, "homelab")
put(t, brainDir, "hoemlab/n.md", "Note.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
assert.Zero(t, llmCalls.Load()) assert.False(t, has(brainDir, "wiki/hoemlab"))
assert.False(t, has(brainDir, "wiki/client-x"), "nothing is ever written into an ineligible wing")
assert.True(t, has(brainDir, "raw/needs-review/secret.md"))
}
func TestProcessDir_ClassifierFailureParksTheNote(t *testing.T) {
var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, wingResolver(&stubClassifier{err: errors.New("llm down")}, stubScorer{}), &llmCalls)
put(t, brainDir, "n.md", "Note.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
assert.Zero(t, llmCalls.Load())
assert.True(t, has(brainDir, "raw/needs-review/n.md")) assert.True(t, has(brainDir, "raw/needs-review/n.md"))
assert.Zero(t, llmCalls.Load())
} }
func TestProcessDir_ParkedNotesAreNotReprocessed(t *testing.T) { func TestProcessDir_ParkedNotesAreNotReprocessed(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
clf := &stubClassifier{s: wing.Suggestion{Wing: "radar", Confidence: 0.99}} cfg, brainDir := wingTestCfg(t, router, &llmCalls)
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{"radar": 3}), &llmCalls)
put(t, brainDir, "needs-review/waiting.md", "---\nproposed_wing: radar\n---\nA parked note.") put(t, brainDir, "needs-review/waiting.md", "---\nproposed_wing: radar\n---\nA parked note.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
assert.Zero(t, clf.calls.Load(), "needs-review/ is a human queue, like rejected/") assert.Zero(t, llmCalls.Load(), "needs-review/ is a human queue, like rejected/")
assert.True(t, has(brainDir, "raw/needs-review/waiting.md")) assert.True(t, has(brainDir, "raw/needs-review/waiting.md"))
} }
@@ -147,141 +156,68 @@ func TestProcessDir_WithoutAResolverTheRetiredLayoutIsUnchanged(t *testing.T) {
assert.True(t, has(brainDir, "wiki/sources/plain.md")) assert.True(t, has(brainDir, "wiki/sources/plain.md"))
} }
// The reason comes from an LLM and ends up in frontmatter: a newline in it must // The reason comes from file and folder names and ends up in frontmatter: a
// not be able to start a new key (PR #147: a title once forged `origin:`). // newline in it must not be able to start a new key (PR #147: a title once
func TestProcessDir_AProposalReasonCannotForgeFrontmatterKeys(t *testing.T) { // forged `origin:`).
func TestProcessDir_AFolderNameCannotForgeFrontmatterKeys(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
// A classifier error is the path by which model text reaches the reason. cfg, brainDir := wingTestCfg(t, router, &llmCalls)
clf := &stubClassifier{err: errors.New("bad reply\norigin: trusted\nwing: homelab")} put(t, brainDir, "radar\nproposal_confidence: 0.99\nwing: homelab/forge.md", "Note.")
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{}), &llmCalls)
put(t, brainDir, "forge.md", "Note.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
parked, err := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", "forge.md")) entries, err := os.ReadDir(filepath.Join(brainDir, "raw", "needs-review"))
require.NoError(t, err) require.NoError(t, err)
assert.NotContains(t, string(parked), "\norigin: trusted", "no injected key at line start") require.Len(t, entries, 1)
assert.NotContains(t, string(parked), "\nwing: homelab") b := parkedBody(t, brainDir, entries[0].Name())
assert.Equal(t, 1, linesStartingWith(b, "proposal_confidence:"), "exactly one confidence key at line start")
assert.Equal(t, 0, linesStartingWith(b, "wing:"), "no injected wing key")
assert.Equal(t, 0, linesStartingWith(b, "proposed_wing:"), "a non-plain wing name is never a proposal")
} }
// Syncthing can bring a removed note back (an offline peer, a restored backup). func TestProcessDir_ANotesOwnProposalKeysDoNotOutrankTheRouters(t *testing.T) {
// The source-hash drift check must keep that from costing a second extraction.
func TestProcessDir_ANoteThatReappearsUnchangedIsNotExtractedAgain(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, wingResolver(&stubClassifier{}, stubScorer{}), &llmCalls) cfg, brainDir := wingTestCfg(t, router, &llmCalls)
put(t, brainDir, "n.md", "---\nproposed_wing: homelab\nproposal_confidence: 1.00\nproposal_reason: \"self-approved\"\n---\nACME-CLIENT body.")
put(t, brainDir, "radar/talk.md", "The same talk, byte for byte.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
require.Equal(t, int32(1), llmCalls.Load(), "positive control: the first drop is extracted")
require.False(t, has(brainDir, "raw/radar/talk.md"))
put(t, brainDir, "radar/talk.md", "The same talk, byte for byte.") // the peer sends it back
require.Empty(t, processDir(context.Background(), cfg, "2026-10-21"))
assert.Equal(t, int32(1), llmCalls.Load(), "an unchanged source is not extracted twice")
assert.False(t, has(brainDir, "raw/radar/talk.md"), "and inbox zero still holds")
}
func TestProcessDir_ANoteThatReappearsChangedIsExtractedAgain(t *testing.T) {
var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, wingResolver(&stubClassifier{}, stubScorer{}), &llmCalls)
put(t, brainDir, "radar/talk.md", "Version one.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
put(t, brainDir, "radar/talk.md", "Version two, edited.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-21"))
assert.Equal(t, int32(2), llmCalls.Load(), "a real edit must still be picked up")
}
// The parked copy is the only copy once the original is retired, so two notes
// that share a name must both survive.
func TestProcessDir_SameNamedNotesAreAllParkedNoneOverwritten(t *testing.T) {
var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, wingResolver(&stubClassifier{}, stubScorer{}), &llmCalls)
for i, body := range []string{"ONE", "TWO", "THREE", "FOUR"} {
put(t, brainDir, filepath.Join("u"+string(rune('1'+i)), "x.md"), body)
}
// Unknown subfolders: each is an exception, all named x.md.
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
files, err := filepath.Glob(filepath.Join(brainDir, "raw", "needs-review", "*.md"))
require.NoError(t, err)
var all string
for _, f := range files {
b, _ := os.ReadFile(f)
all += string(b)
}
for _, body := range []string{"ONE", "TWO", "THREE", "FOUR"} {
assert.Contains(t, all, body, "note %s must not have been overwritten", body)
}
assert.Len(t, files, 4)
}
// proposed_wing comes from a model reply or a folder name. It must never be
// able to start another key, and the note's own proposal keys must not outrank
// the resolver's (frontmatter.get is first-wins).
func TestProcessDir_AHostileWingCannotForgeTheProposal(t *testing.T) {
var llmCalls atomic.Int32
hostile := "radar\nproposal_confidence: 0.99\nproposal_reason: \"approved by operator\""
clf := &stubClassifier{s: wing.Suggestion{Wing: hostile, Confidence: 1}}
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{"radar": 3}), &llmCalls)
put(t, brainDir, "n.md", "Note.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
b, err := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", "n.md")) b := parkedBody(t, brainDir, "n.md")
require.NoError(t, err) assert.NotContains(t, b, "self-approved")
assert.NotContains(t, string(b), "\nproposal_confidence: 0.99") assert.Equal(t, 1, linesStartingWith(b, "proposal_reason:"))
assert.NotContains(t, string(b), "approved by operator\"\n") assert.Equal(t, 0, linesStartingWith(b, "proposed_wing:"))
assert.Equal(t, 1, linesStartingWith(string(b), "proposal_confidence:"), "exactly one confidence key at line start")
}
func TestProcessDir_ANotesOwnProposalKeysDoNotOutrankTheResolvers(t *testing.T) {
var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, wingResolver(&stubClassifier{}, stubScorer{}), &llmCalls)
put(t, brainDir, "n.md", "---\nproposed_wing: homelab\nproposal_confidence: 1.00\nproposal_reason: \"self-approved\"\n---\nBody.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
b, _ := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", "n.md"))
assert.NotContains(t, string(b), "self-approved")
assert.Equal(t, 1, linesStartingWith(string(b), "proposal_reason:"))
} }
func TestProcessDir_ACRLFNoteWithAWingKeyIsStillRead(t *testing.T) { func TestProcessDir_ACRLFNoteWithAWingKeyIsStillRead(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
clf := &stubClassifier{} cfg, brainDir := wingTestCfg(t, router, &llmCalls)
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{}), &llmCalls) makeWing(t, brainDir, "radar")
put(t, brainDir, "n.md", "---\r\nwing: radar\r\n---\r\nBody.\r\n") put(t, brainDir, "n.md", "---\r\nwing: radar\r\n---\r\nBody.\r\n")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
assert.True(t, has(brainDir, "wiki/radar/sources/n.md"), "the declared wing is honoured") assert.True(t, has(brainDir, "wiki/radar/sources/n.md"), "the declared wing is honoured")
assert.Zero(t, clf.calls.Load())
} }
// linesStartingWith counts frontmatter-style keys: a key quoted inside another // editingPlacer rewrites the note while the placement decision is being made,
// value is not at the start of a line and is not a key. // then parks it: the realistic window is a slow decision with Syncthing
func linesStartingWith(s, prefix string) int { // delivering a newer version meanwhile.
n := 0 type editingPlacer struct{ edit func() }
for _, l := range strings.Split(s, "\n") {
if strings.HasPrefix(l, prefix) { func (e editingPlacer) Resolve(context.Context, wing.Note) wing.Decision {
n++ e.edit()
} return wing.Decision{Source: wing.Exception, Reason: "test: parked after an edit"}
}
return n
} }
// Resolving costs an LLM call. If Syncthing delivers a newer version of the note func TestProcessDir_ANoteEditedWhileItIsBeingPlacedIsNotDeleted(t *testing.T) {
// meanwhile, deleting the original loses that version: the parked copy is stale.
func TestProcessDir_ANoteEditedWhileTheResolverRunsIsNotDeleted(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
var notePath string var notePath string
clf := &editingClassifier{} cfg, brainDir := wingTestCfg(t, func(string) wing.Placer {
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{}), &llmCalls) return editingPlacer{edit: func() {
require.NoError(t, os.WriteFile(notePath, []byte("NEWER and longer version"), 0o644))
}}
}, &llmCalls)
notePath = filepath.Join(brainDir, "raw", "n.md") notePath = filepath.Join(brainDir, "raw", "n.md")
clf.edit = func() { require.NoError(t, os.WriteFile(notePath, []byte("NEWER and longer version"), 0o644)) }
put(t, brainDir, "n.md", "ORIGINAL") put(t, brainDir, "n.md", "ORIGINAL")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
@@ -291,21 +227,14 @@ func TestProcessDir_ANoteEditedWhileTheResolverRunsIsNotDeleted(t *testing.T) {
assert.Equal(t, "NEWER and longer version", string(got)) assert.Equal(t, "NEWER and longer version", string(got))
} }
type editingClassifier struct{ edit func() }
func (c *editingClassifier) Classify(context.Context, string, []string) (wing.Suggestion, error) {
c.edit()
return wing.Suggestion{}, nil
}
// A planted symlink at raw/needs-review must not redirect a parked note out of // A planted symlink at raw/needs-review must not redirect a parked note out of
// the brain, nor lose it from the review queue. // the brain, nor lose it from the review queue.
func TestProcessDir_AParkedNoteIsNeverWrittenThroughASymlinkedNeedsReview(t *testing.T) { func TestProcessDir_AParkedNoteIsNeverWrittenThroughASymlinkedNeedsReview(t *testing.T) {
var llmCalls atomic.Int32 var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, wingResolver(&stubClassifier{}, stubScorer{}), &llmCalls) cfg, brainDir := wingTestCfg(t, router, &llmCalls)
outside := t.TempDir() outside := t.TempDir()
require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", "needs-review"))) require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", "needs-review")))
put(t, brainDir, "n.md", "Note.") put(t, brainDir, "n.md", "ACME-CLIENT note.")
errs := processDir(context.Background(), cfg, "2026-10-20") errs := processDir(context.Background(), cfg, "2026-10-20")
@@ -331,10 +260,57 @@ func TestWithProposal_ANonFenceDashLineDoesNotCloseTheBlock(t *testing.T) {
assert.NotContains(t, strings.Join(lines[:closing], "\n"), "foo: bar", "the unfenced note is body, not part of the block") assert.NotContains(t, strings.Join(lines[:closing], "\n"), "foo: bar", "the unfenced note is body, not part of the block")
} }
func mustJSONPages(pages []pipeline.RawPage) string { // The parked copy is the only copy once the original is retired, so two notes
b, err := json.Marshal(pages) // that share a name must both survive.
if err != nil { func TestProcessDir_SameNamedNotesAreAllParkedNoneOverwritten(t *testing.T) {
panic(err) var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, router, &llmCalls)
for i, body := range []string{"ONE", "TWO", "THREE", "FOUR"} {
put(t, brainDir, filepath.Join("typo"+string(rune('1'+i)), "x.md"), body)
} }
return string(b) require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
files, err := filepath.Glob(filepath.Join(brainDir, "raw", "needs-review", "*.md"))
require.NoError(t, err)
var all string
for _, f := range files {
b, _ := os.ReadFile(f)
all += string(b)
}
for _, body := range []string{"ONE", "TWO", "THREE", "FOUR"} {
assert.Contains(t, all, body, "note %s must not have been overwritten", body)
}
assert.Len(t, files, 4)
}
// Syncthing can bring a removed note back (an offline peer, a restored backup).
// The source-hash drift check must keep that from costing a second extraction.
func TestProcessDir_ANoteThatReappearsUnchangedIsNotExtractedAgain(t *testing.T) {
var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, router, &llmCalls)
makeWing(t, brainDir, "radar")
put(t, brainDir, "radar/talk.md", "The same talk, byte for byte.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
require.Equal(t, int32(1), llmCalls.Load(), "positive control: the first drop is extracted")
require.False(t, has(brainDir, "raw/radar/talk.md"))
put(t, brainDir, "radar/talk.md", "The same talk, byte for byte.") // the peer sends it back
require.Empty(t, processDir(context.Background(), cfg, "2026-10-21"))
assert.Equal(t, int32(1), llmCalls.Load(), "an unchanged source is not extracted twice")
assert.False(t, has(brainDir, "raw/radar/talk.md"), "and inbox zero still holds")
}
func TestProcessDir_ANoteThatReappearsChangedIsExtractedAgain(t *testing.T) {
var llmCalls atomic.Int32
cfg, brainDir := wingTestCfg(t, router, &llmCalls)
makeWing(t, brainDir, "radar")
put(t, brainDir, "radar/talk.md", "Version one.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
put(t, brainDir, "radar/talk.md", "Version two, edited.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-21"))
assert.Equal(t, int32(2), llmCalls.Load(), "a real edit must still be picked up")
} }
ingestion/internal/wing/backtest.go
-43
View File
@@ -1,43 +0,0 @@
package wing
// Sample is one backtest observation: the human-decided wing of an existing
// page, and what the two independent opinions said about it with the wing
// label removed.
type Sample struct {
Truth string
Suggested string // classifier's wing, "" if it abstained or failed
Confidence float64
ScorerTop string // scorer's single best wing, "" if no clear opinion
}
// Result is the policy outcome at one threshold.
type Result struct {
N int
Auto int // placed without a human
Correct int // of Auto, matching Truth
Coverage float64 // Auto / N
Precision float64 // Correct / Auto; 0 when Auto == 0
}
// Evaluate applies the Resolver's classified-path rule (confident, and the
// scorer agrees) to recorded samples. Eligibility is not modelled: callers
// backtest only on wings they would allow.
func Evaluate(samples []Sample, threshold float64) Result {
r := Result{N: len(samples)}
for _, s := range samples {
if s.Suggested == "" || s.Confidence < threshold || s.ScorerTop != s.Suggested {
continue
}
r.Auto++
if s.Suggested == s.Truth {
r.Correct++
}
}
if r.N > 0 {
r.Coverage = float64(r.Auto) / float64(r.N)
}
if r.Auto > 0 {
r.Precision = float64(r.Correct) / float64(r.Auto)
}
return r
}
ingestion/internal/wing/backtest_test.go
-38
View File
@@ -1,38 +0,0 @@
package wing_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
func TestEvaluate_PolicyAtAThreshold(t *testing.T) {
samples := []wing.Sample{
{Truth: "a", Suggested: "a", Confidence: 0.9, ScorerTop: "a"}, // auto, correct
{Truth: "a", Suggested: "b", Confidence: 0.9, ScorerTop: "b"}, // auto, WRONG (both models agree on a wrong wing)
{Truth: "a", Suggested: "a", Confidence: 0.5, ScorerTop: "a"}, // below threshold -> exception
{Truth: "a", Suggested: "a", Confidence: 0.95, ScorerTop: "b"}, // scorer disagrees -> exception
{Truth: "a", Suggested: "", Confidence: 0.99, ScorerTop: "a"}, // abstain -> exception
}
r := wing.Evaluate(samples, 0.8)
assert.Equal(t, 5, r.N)
assert.Equal(t, 2, r.Auto)
assert.Equal(t, 1, r.Correct)
assert.InDelta(t, 0.4, r.Coverage, 1e-9)
assert.InDelta(t, 0.5, r.Precision, 1e-9)
}
func TestEvaluate_NothingAutoHasZeroPrecisionNotNaN(t *testing.T) {
r := wing.Evaluate([]wing.Sample{{Truth: "a", Suggested: "a", Confidence: 0.1, ScorerTop: "a"}}, 0.8)
assert.Zero(t, r.Auto)
assert.Zero(t, r.Precision)
assert.Zero(t, r.Coverage)
}
func TestEvaluate_EmptyInput(t *testing.T) {
r := wing.Evaluate(nil, 0.8)
assert.Zero(t, r.N)
assert.Zero(t, r.Coverage)
}
ingestion/internal/wing/bm25scorer.go
-85
View File
@@ -1,85 +0,0 @@
package wing
import (
"sort"
"strings"
"unicode"
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
)
const (
maxQueryTerms = 12
minTermLen = 5
)
// BM25Scorer scores a note against each wing by running the brain's own BM25
// search scoped to that wing. It is deliberately a different mechanism from
// the LLM, so agreement between the two is evidence and not an echo.
type BM25Scorer struct {
brainDir string
wings []string
exclude string
}
// NewBM25Scorer scores against wings under brainDir. exclude is a brain-
// relative path ignored in results (the backtest sample itself); "" for none.
func NewBM25Scorer(brainDir string, wings []string, exclude string) *BM25Scorer {
return &BM25Scorer{brainDir: brainDir, wings: wings, exclude: exclude}
}
// Score implements Scorer. A wing whose search fails scores 0 rather than
// failing the note: the Resolver treats "no clear opinion" as an Exception.
func (b *BM25Scorer) Score(text string) map[string]float64 {
query := strings.Join(topTerms(text), " ")
out := map[string]float64{}
if query == "" {
return out
}
for _, w := range b.wings {
hits, err := search.Query(b.brainDir, search.QueryOptions{Query: query, Wing: w, Limit: 2})
if err != nil {
continue
}
for _, h := range hits {
if b.exclude != "" && h.Path == b.exclude {
continue
}
// Hits come best-first: a wing is scored by its single best
// page, so a big wing cannot win on volume alone.
out[w] = float64(h.Score)
break
}
}
return out
}
// topTerms picks the most frequent longer words, ties broken alphabetically
// so the query is deterministic.
func topTerms(text string) []string {
if len(text) > maxNoteChars {
text = text[:maxNoteChars]
}
counts := map[string]int{}
for _, f := range strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
}) {
if len([]rune(f)) >= minTermLen {
counts[f]++
}
}
terms := make([]string, 0, len(counts))
for t := range counts {
terms = append(terms, t)
}
sort.Slice(terms, func(i, j int) bool {
if counts[terms[i]] != counts[terms[j]] {
return counts[terms[i]] > counts[terms[j]]
}
return terms[i] < terms[j]
})
if len(terms) > maxQueryTerms {
terms = terms[:maxQueryTerms]
}
return terms
}
ingestion/internal/wing/bm25scorer_test.go
-61
View File
@@ -1,61 +0,0 @@
package wing_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
func writePage(t *testing.T, dir, rel, body string) {
t.Helper()
full := filepath.Join(dir, rel)
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
}
func scorerBrain(t *testing.T) string {
dir := t.TempDir()
writePage(t, dir, "wiki/infra/facts/k3s.md", "---\ntitle: K3s cluster\n---\nkubernetes cluster nodes scheduling pods deployment\n")
writePage(t, dir, "wiki/finance/facts/invoice.md", "---\ntitle: Invoices\n---\ninvoice payment bookkeeping ledger reconciliation\n")
return dir
}
func TestBM25Scorer_RanksTheWingWhosePagesMatch(t *testing.T) {
s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "")
got := s.Score("notes about kubernetes cluster scheduling and pods")
assert.Greater(t, got["infra"], got["finance"])
assert.Greater(t, got["infra"], 0.0)
}
func TestBM25Scorer_ExcludesThePageBeingClassified(t *testing.T) {
// A page must not vouch for its own wing when it is the backtest sample.
s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "wiki/infra/facts/k3s.md")
got := s.Score("kubernetes cluster nodes scheduling pods deployment")
assert.Zero(t, got["infra"], "the excluded page is the only infra match")
}
func TestBM25Scorer_NoUsableTermsScoresNothing(t *testing.T) {
s := wing.NewBM25Scorer(scorerBrain(t), []string{"infra", "finance"}, "")
assert.Empty(t, s.Score("a an to of"))
}
// A big wing must not win just by having more pages that brush the query: a
// wing is scored by its single best page, not by the sum of its top hits.
func TestBM25Scorer_ABigWingDoesNotWinOnVolume(t *testing.T) {
dir := t.TempDir()
writePage(t, dir, "wiki/small/facts/target.md", "---\ntitle: Target\n---\nkubernetes cluster scheduling pods deployment nodes\n")
for i := 0; i < 8; i++ {
writePage(t, dir, filepath.Join("wiki/big/facts", "p"+string(rune('a'+i))+".md"),
"---\ntitle: Filler\n---\nkubernetes cluster scheduling mentioned in passing among bookkeeping ledger entries\n")
}
s := wing.NewBM25Scorer(dir, []string{"small", "big"}, "")
got := s.Score("kubernetes cluster scheduling pods deployment nodes")
top, ok := wing.TopWing(got)
assert.True(t, ok)
assert.Equal(t, "small", top)
}
ingestion/internal/wing/configured.go
-32
View File
@@ -1,32 +0,0 @@
package wing
import (
"fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/classification"
)
// NewConfiguredResolver builds the production Resolver for a brain: Known from
// the wiki, Eligible from classification.yaml (a wing is auto-assignable only
// when it classifies below Confidential, so an untagged wing never is), the
// LLM classifier over complete, and BM25 over the known wings as the scorer.
//
// Known is read once. A wing created later is unknown to this Resolver until
// the process restarts, which fails safe: notes aimed at it are parked.
func NewConfiguredResolver(brainDir string, complete CompleteFunc, threshold float64) (*Resolver, error) {
known, err := KnownWings(brainDir)
if err != nil {
return nil, err
}
cls, err := classification.Load(brainDir)
if err != nil {
return nil, fmt.Errorf("load classification: %w", err)
}
return &Resolver{
Known: known,
Eligible: func(w string) bool { return cls.Wing(w) < classification.Confidential },
Classifier: NewLLMClassifier(complete),
Scorer: NewBM25Scorer(brainDir, known, ""),
Threshold: threshold,
}, nil
}
ingestion/internal/wing/configured_test.go
-47
View File
@@ -1,47 +0,0 @@
package wing_test
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
func configuredBrain(t *testing.T, classification string) string {
dir := t.TempDir()
writePage(t, dir, "wiki/homelab/facts/a.md", "x")
writePage(t, dir, "wiki/radar/facts/b.md", "x")
writePage(t, dir, "wiki/client-x/facts/c.md", "x")
if classification != "" {
writePage(t, dir, "classification.yaml", classification)
}
return dir
}
func TestNewConfiguredResolver_EligibilityComesFromClassification(t *testing.T) {
dir := configuredBrain(t, "wings:\n radar: internal\n")
r, err := wing.NewConfiguredResolver(dir, func(context.Context, string, string) (string, error) { return "", nil }, 0.9)
require.NoError(t, err)
assert.ElementsMatch(t, []string{"client-x", "homelab", "radar"}, r.Known)
assert.True(t, r.Eligible("radar"), "tagged internal")
assert.True(t, r.Eligible("homelab"), "internal by built-in default")
assert.False(t, r.Eligible("client-x"), "an untagged wing fails safe to confidential")
assert.InDelta(t, 0.9, r.Threshold, 1e-9)
}
func TestNewConfiguredResolver_AMalformedClassificationFileIsAnError(t *testing.T) {
dir := configuredBrain(t, "wings: [not, a, map")
_, err := wing.NewConfiguredResolver(dir, nil, 0.9)
assert.Error(t, err, "a classification source that cannot be trusted must not degrade to 'everything eligible'")
}
func TestNewConfiguredResolver_AConfidentialTagIsNeverEligible(t *testing.T) {
dir := configuredBrain(t, "wings:\n radar: confidential\n")
r, err := wing.NewConfiguredResolver(dir, nil, 0.9)
require.NoError(t, err)
assert.False(t, r.Eligible("radar"))
}
ingestion/internal/wing/helpers_test.go
+16
View File
@@ -0,0 +1,16 @@
package wing_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func writePage(t *testing.T, dir, rel, body string) {
t.Helper()
full := filepath.Join(dir, rel)
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
}
ingestion/internal/wing/llmclassifier.go
-99
View File
@@ -1,99 +0,0 @@
package wing
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
)
// CompleteFunc is a chat completion, matching llm.Client.Complete. It must be
// backed by a local model: note content is sent to it.
type CompleteFunc func(ctx context.Context, system, user string) (string, error)
// maxNoteChars bounds what is sent for classification; the start of a note
// carries its subject, and a whole transcript is cost with no benefit.
const maxNoteChars = 6000
const classifySystem = `You file notes into a personal knowledge base. Pick the single wing the note belongs to, from the candidate list only.
Answer with ONLY a JSON object: {"wing": "<name or none>", "confidence": <0..1>, "reason": "<one short sentence>"}.
Use "none" when no candidate clearly fits. Do not invent wings.`
// LLMClassifier asks a model to pick a wing.
type LLMClassifier struct{ complete CompleteFunc }
// NewLLMClassifier wraps a chat completion as a Classifier.
func NewLLMClassifier(c CompleteFunc) *LLMClassifier { return &LLMClassifier{complete: c} }
type classifyReply struct {
Wing string `json:"wing"`
Confidence float64 `json:"confidence"`
Reason string `json:"reason"`
}
// Classify implements Classifier. Any reply it cannot parse is an error, which
// the Resolver turns into an Exception: an unreadable answer is not a guess.
func (l *LLMClassifier) Classify(ctx context.Context, text string, candidates []string) (Suggestion, error) {
if len(text) > maxNoteChars {
text = text[:maxNoteChars]
}
user := "Candidate wings: " + strings.Join(candidates, ", ") + "\n\nNote:\n" + text
out, err := l.complete(ctx, classifySystem, user)
if err != nil {
return Suggestion{}, fmt.Errorf("classify: %w", err)
}
raw, ok := firstJSONObject(out)
if !ok {
return Suggestion{}, errors.New("classify: reply contains no JSON object")
}
var r classifyReply
if err := json.Unmarshal([]byte(raw), &r); err != nil {
return Suggestion{}, fmt.Errorf("classify: parse reply: %w", err)
}
w := strings.TrimSpace(r.Wing)
if strings.EqualFold(w, "none") {
w = ""
}
return Suggestion{Wing: w, Confidence: clamp01(r.Confidence), Reason: r.Reason}, nil
}
// firstJSONObject returns the first balanced {...} in s, ignoring braces
// inside JSON strings, so prose or code fences around the reply are tolerated.
func firstJSONObject(s string) (string, bool) {
start := strings.IndexByte(s, '{')
if start < 0 {
return "", false
}
depth, inStr, esc := 0, false, false
for i := start; i < len(s); i++ {
c := s[i]
switch {
case esc:
esc = false
case inStr && c == '\\':
esc = true
case c == '"':
inStr = !inStr
case inStr:
case c == '{':
depth++
case c == '}':
depth--
if depth == 0 {
return s[start : i+1], true
}
}
}
return "", false
}
func clamp01(f float64) float64 {
switch {
case f < 0:
return 0
case f > 1:
return 1
}
return f
}
ingestion/internal/wing/llmclassifier_test.go
-67
View File
@@ -1,67 +0,0 @@
package wing_test
import (
"context"
"errors"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
func completes(reply string, err error) wing.CompleteFunc {
return func(_ context.Context, _, _ string) (string, error) { return reply, err }
}
func TestLLMClassifier_ParsesTheReply(t *testing.T) {
tests := []struct {
name string
reply string
want wing.Suggestion
wantErr bool
}{
{"bare json", `{"wing":"tapir","confidence":0.9,"reason":"video summariser"}`, wing.Suggestion{Wing: "tapir", Confidence: 0.9, Reason: "video summariser"}, false},
{"json inside prose and fences", "Sure!\n```json\n{\"wing\":\"homelab\",\"confidence\":0.7,\"reason\":\"k3s\"}\n```\nDone.", wing.Suggestion{Wing: "homelab", Confidence: 0.7, Reason: "k3s"}, false},
{"none means abstain", `{"wing":"none","confidence":0.2,"reason":"generic"}`, wing.Suggestion{Wing: "", Confidence: 0.2, Reason: "generic"}, false},
{"confidence above 1 is clamped", `{"wing":"tapir","confidence":7,"reason":"x"}`, wing.Suggestion{Wing: "tapir", Confidence: 1, Reason: "x"}, false},
{"negative confidence is clamped", `{"wing":"tapir","confidence":-1,"reason":"x"}`, wing.Suggestion{Wing: "tapir", Confidence: 0, Reason: "x"}, false},
{"empty reply (a reasoning model that ran out of tokens)", ``, wing.Suggestion{}, true},
{"no json at all", `I think it is tapir`, wing.Suggestion{}, true},
{"malformed json", `{"wing":"tapir","confidence":`, wing.Suggestion{}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := wing.NewLLMClassifier(completes(tt.reply, nil))
got, err := c.Classify(context.Background(), "note", []string{"tapir", "homelab"})
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
}
func TestLLMClassifier_PropagatesTheModelError(t *testing.T) {
boom := errors.New("model down")
_, err := wing.NewLLMClassifier(completes("", boom)).Classify(context.Background(), "n", []string{"tapir"})
assert.ErrorIs(t, err, boom)
}
func TestLLMClassifier_PromptListsCandidatesAndBoundsTheNote(t *testing.T) {
var gotUser string
c := wing.NewLLMClassifier(func(_ context.Context, _, user string) (string, error) {
gotUser = user
return `{"wing":"tapir","confidence":0.9,"reason":"x"}`, nil
})
long := strings.Repeat("word ", 5000)
_, err := c.Classify(context.Background(), long, []string{"tapir", "homelab"})
require.NoError(t, err)
assert.Contains(t, gotUser, "tapir")
assert.Contains(t, gotUser, "homelab")
assert.Less(t, len(gotUser), 12000, "a whole transcript must not be sent as the classification input")
}
ingestion/internal/wing/resolve.go
+16 -140
View File
@@ -1,17 +1,17 @@
// Package wing decides which brain wing an incoming raw note belongs to // Package wing decides which brain wing an incoming raw note belongs to
// without the human typing it (hyperguild#205). // without the human typing it (hyperguild#205, #208).
// //
// The contract is conservative: a note is placed automatically only when the // An earlier design classified each note with a local model and cross-checked
// decision is explicit, or when two independent opinions (an LLM and a // it with a text score. Measured against pages whose wing a human had chosen,
// deterministic scorer) agree with enough confidence on a wing that is both // the model was right 60% of the time with a saturated self-reported
// known and eligible. Everything else is an Exception for a human to approve. // confidence, and the cross-check was right 11% of the time, so the gate it
// A wing is never created here, and a wing the caller does not mark eligible // formed placed 2 of 55 pages (hyperguild#208). The Router that replaced it
// (client wings, anything not deliberately tagged) is never auto-assigned. // decides with rules only: a deliberate folder or frontmatter wing, else one
// default wing; nothing it cannot decide safely is guessed.
package wing package wing
import ( import (
"context" "context"
"fmt"
"strings" "strings"
) )
@@ -19,38 +19,18 @@ import (
type Source string type Source string
const ( const (
Explicit Source = "explicit" // subfolder or frontmatter named the wing Explicit Source = "explicit" // subfolder or frontmatter named the wing
Default Source = "default" // no signal; the configured default wing Default Source = "default" // no signal; the configured default wing
Classified Source = "classified" // classifier + scorer agreed, confident Exception Source = "exception" // needs a human
Exception Source = "exception" // needs a human
) )
// Note is what the resolver sees of an incoming raw note. // Note is what the Placer sees of an incoming raw note.
type Note struct { type Note struct {
Dir string // first subfolder under raw/, "" when dropped at the top Dir string // first subfolder under raw/, "" when dropped at the top
FrontmatterWing string // `wing:` from the note's frontmatter, "" if absent FrontmatterWing string // `wing:` from the note's frontmatter, "" if absent
Text string Text string
} }
// Suggestion is a Classifier's answer. An empty Wing means "none fits".
type Suggestion struct {
Wing string
Confidence float64 // 0..1
Reason string
}
// Classifier proposes a wing from the candidates. Implementations must run on
// a local model; the resolver cannot enforce that.
type Classifier interface {
Classify(ctx context.Context, text string, candidates []string) (Suggestion, error)
}
// Scorer is an independent, deterministic opinion (e.g. BM25 against each
// wing's index). Higher is better; the key is the wing name.
type Scorer interface {
Score(text string) map[string]float64
}
// Decision is the outcome. For an Exception, Wing holds the proposal when // Decision is the outcome. For an Exception, Wing holds the proposal when
// there was one (so a reviewer can approve it) and is empty otherwise. // there was one (so a reviewer can approve it) and is empty otherwise.
type Decision struct { type Decision struct {
@@ -60,114 +40,10 @@ type Decision struct {
Reason string Reason string
} }
// Resolver holds the policy. Known is every existing wing; Eligible reports // Placer decides where a raw note goes. Router is the production
// whether a known wing may be assigned automatically. // implementation; the interface lets the watcher stay ignorant of how.
type Resolver struct { type Placer interface {
Known []string Resolve(ctx context.Context, n Note) Decision
Eligible func(wing string) bool
Classifier Classifier
Scorer Scorer
Threshold float64
}
// Resolve never returns an error: anything it cannot decide safely is an
// Exception with the reason spelled out.
func (r Resolver) Resolve(ctx context.Context, n Note) Decision {
dir := norm(n.Dir)
fm := norm(n.FrontmatterWing)
switch {
case dir != "" && fm != "" && dir != fm:
return exception("", 0, fmt.Sprintf("subfolder %q and frontmatter wing %q disagree", dir, fm))
case dir != "":
return r.explicit(dir, "subfolder")
case fm != "":
return r.explicit(fm, "frontmatter")
}
return r.classify(ctx, n)
}
func (r Resolver) explicit(w, via string) Decision {
if why := r.refuse(w); why != "" {
return exception(w, 0, fmt.Sprintf("%s names %q: %s", via, w, why))
}
return Decision{Wing: w, Source: Explicit, Confidence: 1, Reason: "named by " + via}
}
func (r Resolver) classify(ctx context.Context, n Note) Decision {
if r.Classifier == nil {
return exception("", 0, "no classifier configured")
}
s, err := r.Classifier.Classify(ctx, n.Text, r.Known)
if err != nil {
return exception("", 0, fmt.Sprintf("classifier failed: %v", err))
}
w := norm(s.Wing)
if w == "" {
return exception("", s.Confidence, "classifier found no fitting wing")
}
if why := r.refuse(w); why != "" {
return exception(w, s.Confidence, fmt.Sprintf("classifier proposed %q: %s", w, why))
}
if s.Confidence < r.Threshold {
return exception(w, s.Confidence, fmt.Sprintf("confidence %.2f below threshold %.2f", s.Confidence, r.Threshold))
}
top, ok := r.scorerTop(n.Text)
if !ok {
return exception(w, s.Confidence, "scorer has no clear opinion to cross-check against")
}
if top != w {
return exception(w, s.Confidence, fmt.Sprintf("classifier says %q, scorer says %q", w, top))
}
return Decision{Wing: w, Source: Classified, Confidence: s.Confidence,
Reason: fmt.Sprintf("classifier and scorer agree (%.2f): %s", s.Confidence, s.Reason)}
}
// refuse returns why a wing may not be auto-assigned, or "" if it may.
func (r Resolver) refuse(w string) string {
if !r.known(w) {
return "not an existing wing (new wings are never created automatically)"
}
if r.Eligible == nil || !r.Eligible(w) {
return "not eligible for automatic assignment"
}
return ""
}
func (r Resolver) known(w string) bool {
for _, k := range r.Known {
if norm(k) == w {
return true
}
}
return false
}
// scorerTop returns the single best-scoring wing; a missing scorer, no scores,
// or a tie at the top is "no clear opinion".
func (r Resolver) scorerTop(text string) (string, bool) {
if r.Scorer == nil {
return "", false
}
return TopWing(r.Scorer.Score(text))
}
// TopWing returns the unique highest-scoring wing in scores, or false when
// there is none or the top is tied. Shared by the Resolver and the backtest so
// both apply the same notion of "the scorer has an opinion".
func TopWing(scores map[string]float64) (string, bool) {
best, bestScore, tied := "", 0.0, false
for k, v := range scores {
switch {
case v > bestScore:
best, bestScore, tied = norm(k), v, false
case v == bestScore && v > 0:
tied = true
}
}
if best == "" || tied {
return "", false
}
return best, true
} }
func exception(w string, conf float64, reason string) Decision { func exception(w string, conf float64, reason string) Decision {
ingestion/internal/wing/resolve_test.go
-112
View File
@@ -1,112 +0,0 @@
package wing_test
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)
type fakeClassifier struct {
s wing.Suggestion
err error
calls int
}
func (f *fakeClassifier) Classify(_ context.Context, _ string, _ []string) (wing.Suggestion, error) {
f.calls++
return f.s, f.err
}
type fakeScorer map[string]float64
func (f fakeScorer) Score(string) map[string]float64 { return f }
// eligibleExcept allows every name except the listed ones. Being permissive on
// names it has never heard of means only the "known wing" check can stop an
// invented wing, so a test of that check cannot pass by accident.
func eligibleExcept(names ...string) func(string) bool {
deny := map[string]bool{}
for _, n := range names {
deny[n] = true
}
return func(w string) bool { return !deny[w] }
}
func newResolver(c wing.Classifier, s wing.Scorer) wing.Resolver {
return wing.Resolver{
Known: []string{"homelab", "hyperguild", "tapir", "rikatillsammans"},
Eligible: eligibleExcept("rikatillsammans"), // a client wing: known, never auto
Classifier: c,
Scorer: s,
Threshold: 0.8,
}
}
func TestResolve_ExplicitSignal(t *testing.T) {
tests := []struct {
name string
note wing.Note
want wing.Source
wantWing string
}{
{"subfolder of a known eligible wing", wing.Note{Dir: "homelab"}, wing.Explicit, "homelab"},
{"frontmatter of a known eligible wing", wing.Note{FrontmatterWing: "tapir"}, wing.Explicit, "tapir"},
{"subfolder and frontmatter agree", wing.Note{Dir: "tapir", FrontmatterWing: "tapir"}, wing.Explicit, "tapir"},
{"subfolder and frontmatter disagree", wing.Note{Dir: "tapir", FrontmatterWing: "homelab"}, wing.Exception, ""},
{"unknown wing is never created", wing.Note{Dir: "hoemlab"}, wing.Exception, "hoemlab"},
{"known but not eligible (client) wing is never auto-assigned", wing.Note{Dir: "rikatillsammans"}, wing.Exception, "rikatillsammans"},
{"subfolder is case-insensitive", wing.Note{Dir: "HomeLab"}, wing.Explicit, "homelab"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &fakeClassifier{}
d := newResolver(c, fakeScorer{}).Resolve(context.Background(), tt.note)
assert.Equal(t, tt.want, d.Source)
assert.Equal(t, tt.wantWing, d.Wing)
assert.Zero(t, c.calls, "an explicit signal must not spend an LLM call")
assert.NotEmpty(t, d.Reason)
})
}
}
func TestResolve_Classified(t *testing.T) {
tests := []struct {
name string
sugg wing.Suggestion
err error
scores fakeScorer
want wing.Source
wantWing string
}{
{"confident and scorer agrees", wing.Suggestion{Wing: "tapir", Confidence: 0.9}, nil, fakeScorer{"tapir": 3, "homelab": 1}, wing.Classified, "tapir"},
{"confidence exactly at threshold passes", wing.Suggestion{Wing: "tapir", Confidence: 0.8}, nil, fakeScorer{"tapir": 3}, wing.Classified, "tapir"},
{"below threshold is an exception, proposal kept", wing.Suggestion{Wing: "tapir", Confidence: 0.79}, nil, fakeScorer{"tapir": 3}, wing.Exception, "tapir"},
{"scorer disagrees", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{"homelab": 5, "tapir": 1}, wing.Exception, "tapir"},
{"scorer has no opinion also needs review", wing.Suggestion{Wing: "tapir", Confidence: 0.95}, nil, fakeScorer{}, wing.Exception, "tapir"},
{"classifier proposes a client wing", wing.Suggestion{Wing: "rikatillsammans", Confidence: 0.99}, nil, fakeScorer{"rikatillsammans": 9}, wing.Exception, "rikatillsammans"},
{"classifier invents a wing", wing.Suggestion{Wing: "brand-new-wing", Confidence: 0.99}, nil, fakeScorer{"brand-new-wing": 9}, wing.Exception, "brand-new-wing"},
{"classifier abstains", wing.Suggestion{Wing: "", Confidence: 0}, nil, fakeScorer{"tapir": 3}, wing.Exception, ""},
{"classifier error", wing.Suggestion{}, errors.New("llm down"), fakeScorer{"tapir": 3}, wing.Exception, ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &fakeClassifier{s: tt.sugg, err: tt.err}
d := newResolver(c, tt.scores).Resolve(context.Background(), wing.Note{Text: "some note"})
assert.Equal(t, tt.want, d.Source)
assert.Equal(t, tt.wantWing, d.Wing)
assert.NotEmpty(t, d.Reason, "every decision says why, so a spot-check can judge it")
assert.Equal(t, 1, c.calls)
})
}
}
func TestResolve_NilDependenciesFailClosed(t *testing.T) {
r := wing.Resolver{Known: []string{"tapir"}, Eligible: eligibleExcept(), Threshold: 0.8}
d := r.Resolve(context.Background(), wing.Note{Text: "x"})
require.Equal(t, wing.Exception, d.Source, "no classifier configured must route to review, not panic or guess")
}
ingestion/internal/wing/router.go
+20 -6
View File
@@ -8,12 +8,6 @@ import (
"strings" "strings"
) )
// Placer decides where a raw note goes. Router is the production
// implementation; the interface lets the watcher stay ignorant of how.
type Placer interface {
Resolve(ctx context.Context, n Note) Decision
}
// Router places a note with rules only, no model (hyperguild#208): // Router places a note with rules only, no model (hyperguild#208):
// //
// 1. A subfolder or frontmatter `wing:` is the owner's deliberate choice. It // 1. A subfolder or frontmatter `wing:` is the owner's deliberate choice. It
@@ -68,3 +62,23 @@ func (r Router) explicit(w, via string) Decision {
} }
return Decision{Wing: w, Source: Explicit, Confidence: 1, Reason: "named by " + via} return Decision{Wing: w, Source: Explicit, Confidence: 1, Reason: "named by " + via}
} }
// NewRouter builds the production Router. defaultWing must be a plain wing
// name; clientPattern, when set, is a case-insensitive regexp over note text
// whose match parks an unsignalled note instead of filing it. Both fail at
// startup rather than later: a client filter that silently does nothing is
// worse than none.
func NewRouter(brainDir, defaultWing, clientPattern string) (Router, error) {
if defaultWing == "" || !plainWingName(defaultWing) {
return Router{}, fmt.Errorf("default wing %q is not a plain wing name", defaultWing)
}
r := Router{BrainDir: brainDir, Default: defaultWing}
if clientPattern != "" {
re, err := regexp.Compile("(?i)" + clientPattern)
if err != nil {
return Router{}, fmt.Errorf("client-content pattern: %w", err)
}
r.ClientBlock = re
}
return r, nil
}
ingestion/internal/wing/router_test.go
+27
View File
@@ -85,3 +85,30 @@ func TestRouter_ANameThatIsNotPlainIsRefusedAsSuch(t *testing.T) {
}) })
} }
} }
func TestNewRouter(t *testing.T) {
dir := routerBrain(t)
t.Run("valid", func(t *testing.T) {
r, err := wing.NewRouter(dir, "web-clip", "acme-client|globex")
assert.NoError(t, err)
d := r.Resolve(context.Background(), wing.Note{Text: "kickoff with GLOBEX"})
assert.Equal(t, wing.Exception, d.Source, "the marker is matched case-insensitively")
})
t.Run("no marker pattern means no content check", func(t *testing.T) {
r, err := wing.NewRouter(dir, "web-clip", "")
assert.NoError(t, err)
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: "anything"}).Source)
})
t.Run("a default that is not a plain wing name is a startup error", func(t *testing.T) {
_, err := wing.NewRouter(dir, "Web Clip", "")
assert.Error(t, err)
})
t.Run("an empty default is a startup error", func(t *testing.T) {
_, err := wing.NewRouter(dir, "", "")
assert.Error(t, err)
})
t.Run("a pattern that does not compile is a startup error", func(t *testing.T) {
_, err := wing.NewRouter(dir, "web-clip", "([unclosed")
assert.Error(t, err, "a client filter that silently does nothing is worse than none")
})
}