From 20d874a23db726d712aefe937ceaa02416035ff0 Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 20 Oct 2026 06:09:17 +0200 Subject: [PATCH] feat(server): INGEST_WING_RESOLVE=1 wires the wing resolver into the watcher (#205 S3) Off by default (retired layout unchanged). On: eligibility from classification.yaml, local ingest LLM as classifier, BM25 as cross-check, INGEST_WING_THRESHOLD (default 0.9, rejected outside [0,1]). Build failure is fatal when explicitly enabled. Co-Authored-By: Claude Sonnet 5.5 --- ingestion/cmd/server/main.go | 28 +++++++++++++ ingestion/cmd/server/main_test.go | 20 +++++++++ ingestion/cmd/wingbacktest/main.go | 28 +------------ ingestion/internal/wing/configured.go | 32 +++++++++++++++ ingestion/internal/wing/configured_test.go | 47 ++++++++++++++++++++++ ingestion/internal/wing/known.go | 39 ++++++++++++++++++ ingestion/internal/wing/known_test.go | 29 +++++++++++++ 7 files changed, 196 insertions(+), 27 deletions(-) create mode 100644 ingestion/internal/wing/configured.go create mode 100644 ingestion/internal/wing/configured_test.go create mode 100644 ingestion/internal/wing/known.go create mode 100644 ingestion/internal/wing/known_test.go diff --git a/ingestion/cmd/server/main.go b/ingestion/cmd/server/main.go index 5b2fbb8..a08322c 100644 --- a/ingestion/cmd/server/main.go +++ b/ingestion/cmd/server/main.go @@ -36,6 +36,7 @@ import ( "github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore" "github.com/mathiasbq/hyperguild/ingestion/internal/watcher" "github.com/mathiasbq/hyperguild/ingestion/internal/webhook" + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -144,6 +145,17 @@ func envInt(key string, fallback int) int { return fallback } +// envFloat reads a confidence-style value in [0,1]. Anything unparseable or +// outside the range falls back: a threshold of 0 would auto-place everything. +func envFloat(key string, fallback float64) float64 { + if v := os.Getenv(key); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil && f >= 0 && f <= 1 { + return f + } + } + return fallback +} + // buildNotifier returns the ntfy notifier, or a nil interface when // BRAIN_NTFY_URL is unset. A nil *interface* (not a typed-nil) is required so // DegradingSink/Reconcile and the governance observer skip it cleanly. @@ -389,10 +401,26 @@ func main() { } if watchInterval > 0 { + // INGEST_WING_RESOLVE=1 routes raw/ notes into wiki// by + // subfolder, frontmatter or classifier+scorer, and parks the rest in + // raw/needs-review/ (hyperguild#205). Off = the retired layout. Set + // explicitly, a failure to build the resolver is fatal: silently + // falling back would write into the layout this flag exists to leave. + var wingResolver *wing.Resolver + if os.Getenv("INGEST_WING_RESOLVE") == "1" { + var rerr error + wingResolver, rerr = wing.NewConfiguredResolver(brainDir, llmClient.Complete, envFloat("INGEST_WING_THRESHOLD", 0.9)) + if rerr != nil { + logger.Error("build wing resolver", "err", rerr) + os.Exit(1) + } + logger.Info("wing resolver enabled", "wings", len(wingResolver.Known), "threshold", wingResolver.Threshold) + } watcher.Start(ctx, watcher.Config{ BrainDir: brainDir, Interval: time.Duration(watchInterval) * time.Second, Pipeline: pipelineCfg, + Resolver: wingResolver, }) } diff --git a/ingestion/cmd/server/main_test.go b/ingestion/cmd/server/main_test.go index 4d1236d..8f6f125 100644 --- a/ingestion/cmd/server/main_test.go +++ b/ingestion/cmd/server/main_test.go @@ -76,3 +76,23 @@ func TestAlertingWarnings(t *testing.T) { }) } } + +func TestEnvFloat(t *testing.T) { + tests := []struct { + name string + set string + want float64 + }{ + {"unset falls back", "", 0.9}, + {"valid value", "0.75", 0.75}, + {"garbage falls back rather than becoming 0", "high", 0.9}, + {"out of range is rejected, not clamped silently into auto-placing everything", "-1", 0.9}, + {"above one is rejected", "1.5", 0.9}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("TEST_ENV_FLOAT", tt.set) + assert.InDelta(t, tt.want, envFloat("TEST_ENV_FLOAT", 0.9), 1e-9) + }) + } +} diff --git a/ingestion/cmd/wingbacktest/main.go b/ingestion/cmd/wingbacktest/main.go index 08a73e7..cab4638 100644 --- a/ingestion/cmd/wingbacktest/main.go +++ b/ingestion/cmd/wingbacktest/main.go @@ -24,8 +24,6 @@ import ( "github.com/mathiasbq/hyperguild/ingestion/internal/wing" ) -var legacyDirs = map[string]bool{"concepts": true, "entities": true, "sources": true} - func main() { brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory (read only)") wingsFlag := flag.String("wings", "", "comma-separated wings to sample pages FROM (must be wings you would allow)") @@ -42,7 +40,7 @@ func main() { } func run(brain string, sampleWings []string, perWing int, llmURL, model string, seed int64) error { - known, err := knownWings(brain) + known, err := wing.KnownWings(brain) if err != nil { return err } @@ -112,30 +110,6 @@ func report(samples []wing.Sample, per map[string][]wing.Sample) { } } -// knownWings lists wiki/ dirs that hold hall subdirectories, skipping -// the retired legacy layout. -func knownWings(brain string) ([]string, error) { - entries, err := os.ReadDir(filepath.Join(brain, "wiki")) - if err != nil { - return nil, fmt.Errorf("list wings: %w", err) - } - var out []string - for _, e := range entries { - if !e.IsDir() || legacyDirs[e.Name()] { - continue - } - subs, _ := os.ReadDir(filepath.Join(brain, "wiki", e.Name())) - for _, s := range subs { - if s.IsDir() { - out = append(out, e.Name()) - break - } - } - } - sort.Strings(out) - return out, nil -} - func listPages(brain, w string) ([]string, error) { var out []string root := filepath.Join(brain, "wiki", w) diff --git a/ingestion/internal/wing/configured.go b/ingestion/internal/wing/configured.go new file mode 100644 index 0000000..c27236c --- /dev/null +++ b/ingestion/internal/wing/configured.go @@ -0,0 +1,32 @@ +package wing + +import ( + "fmt" + + "github.com/mathiasbq/hyperguild/ingestion/internal/classification" +) + +// NewConfiguredResolver builds the production Resolver for a brain: Known from +// the wiki, Eligible from classification.yaml (a wing is auto-assignable only +// when it classifies below Confidential, so an untagged wing never is), the +// LLM classifier over complete, and BM25 over the known wings as the scorer. +// +// Known is read once. A wing created later is unknown to this Resolver until +// the process restarts, which fails safe: notes aimed at it are parked. +func NewConfiguredResolver(brainDir string, complete CompleteFunc, threshold float64) (*Resolver, error) { + known, err := KnownWings(brainDir) + if err != nil { + return nil, err + } + cls, err := classification.Load(brainDir) + if err != nil { + return nil, fmt.Errorf("load classification: %w", err) + } + return &Resolver{ + Known: known, + Eligible: func(w string) bool { return cls.Wing(w) < classification.Confidential }, + Classifier: NewLLMClassifier(complete), + Scorer: NewBM25Scorer(brainDir, known, ""), + Threshold: threshold, + }, nil +} diff --git a/ingestion/internal/wing/configured_test.go b/ingestion/internal/wing/configured_test.go new file mode 100644 index 0000000..16173ef --- /dev/null +++ b/ingestion/internal/wing/configured_test.go @@ -0,0 +1,47 @@ +package wing_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" +) + +func configuredBrain(t *testing.T, classification string) string { + dir := t.TempDir() + writePage(t, dir, "wiki/homelab/facts/a.md", "x") + writePage(t, dir, "wiki/radar/facts/b.md", "x") + writePage(t, dir, "wiki/client-x/facts/c.md", "x") + if classification != "" { + writePage(t, dir, "classification.yaml", classification) + } + return dir +} + +func TestNewConfiguredResolver_EligibilityComesFromClassification(t *testing.T) { + dir := configuredBrain(t, "wings:\n radar: internal\n") + r, err := wing.NewConfiguredResolver(dir, func(context.Context, string, string) (string, error) { return "", nil }, 0.9) + require.NoError(t, err) + + assert.ElementsMatch(t, []string{"client-x", "homelab", "radar"}, r.Known) + assert.True(t, r.Eligible("radar"), "tagged internal") + assert.True(t, r.Eligible("homelab"), "internal by built-in default") + assert.False(t, r.Eligible("client-x"), "an untagged wing fails safe to confidential") + assert.InDelta(t, 0.9, r.Threshold, 1e-9) +} + +func TestNewConfiguredResolver_AMalformedClassificationFileIsAnError(t *testing.T) { + dir := configuredBrain(t, "wings: [not, a, map") + _, err := wing.NewConfiguredResolver(dir, nil, 0.9) + assert.Error(t, err, "a classification source that cannot be trusted must not degrade to 'everything eligible'") +} + +func TestNewConfiguredResolver_AConfidentialTagIsNeverEligible(t *testing.T) { + dir := configuredBrain(t, "wings:\n radar: confidential\n") + r, err := wing.NewConfiguredResolver(dir, nil, 0.9) + require.NoError(t, err) + assert.False(t, r.Eligible("radar")) +} diff --git a/ingestion/internal/wing/known.go b/ingestion/internal/wing/known.go new file mode 100644 index 0000000..dc9229c --- /dev/null +++ b/ingestion/internal/wing/known.go @@ -0,0 +1,39 @@ +package wing + +import ( + "fmt" + "os" + "path/filepath" + "sort" +) + +// legacyDirs are the retired wiki/{concepts,entities,sources}/ layout, not wings. +var legacyDirs = map[string]bool{"concepts": true, "entities": true, "sources": true} + +// KnownWings lists wiki// directories that contain at least one hall +// subdirectory, sorted. The retired layout is not a wing. A missing wiki/ is +// an error: an unreadable brain must not look like "no wings". +func KnownWings(brainDir string) ([]string, error) { + entries, err := os.ReadDir(filepath.Join(brainDir, "wiki")) + if err != nil { + return nil, fmt.Errorf("list wings: %w", err) + } + var out []string + for _, e := range entries { + if !e.IsDir() || legacyDirs[e.Name()] { + continue + } + subs, err := os.ReadDir(filepath.Join(brainDir, "wiki", e.Name())) + if err != nil { + continue + } + for _, s := range subs { + if s.IsDir() { + out = append(out, e.Name()) + break + } + } + } + sort.Strings(out) + return out, nil +} diff --git a/ingestion/internal/wing/known_test.go b/ingestion/internal/wing/known_test.go new file mode 100644 index 0000000..e9c1f44 --- /dev/null +++ b/ingestion/internal/wing/known_test.go @@ -0,0 +1,29 @@ +package wing_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" +) + +func TestKnownWings_ListsWingsWithHallsAndSkipsTheRetiredLayout(t *testing.T) { + dir := t.TempDir() + writePage(t, dir, "wiki/homelab/facts/a.md", "x") + writePage(t, dir, "wiki/tapir/decisions/b.md", "x") + writePage(t, dir, "wiki/concepts/c.md", "x") // retired layout, a file directly under a legacy dir + writePage(t, dir, "wiki/sources/d.md", "x") // retired layout + writePage(t, dir, "wiki/entities/e.md", "x") // retired layout + writePage(t, dir, "wiki/loose-note.md", "x") // a file, not a wing + writePage(t, dir, "wiki/emptywing/_index.md", "x") // a dir with no hall subdirectory + got, err := wing.KnownWings(dir) + require.NoError(t, err) + assert.Equal(t, []string{"homelab", "tapir"}, got) +} + +func TestKnownWings_NoWikiDirIsAnError(t *testing.T) { + _, err := wing.KnownWings(t.TempDir()) + assert.Error(t, err, "a brain without wiki/ must not look like a brain with no wings") +}