package watcher import ( "context" "encoding/json" "os" "path/filepath" "regexp" "strings" "sync/atomic" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/mathiasbq/hyperguild/ingestion/internal/pipeline" "github.com/mathiasbq/hyperguild/ingestion/internal/wing" ) func wingTestCfg(t *testing.T, mk func(brainDir string) wing.Placer, llmCalls *atomic.Int32) (Config, string) { t.Helper() brainDir := setupBrainDir(t) cfg := Config{ BrainDir: brainDir, Pipeline: pipeline.Config{ Schema: "# Schema", Complete: func(ctx context.Context, s, u string) (string, error) { llmCalls.Add(1) return successComplete(pipeline.RawPage{Title: "Talk", Type: "source", Subtype: "video", Content: "## Summary\n\nx\n"})(ctx, s, u) }, }, } if mk != nil { cfg.Resolver = mk(brainDir) } return cfg, brainDir } // router is the production Router with a client-content marker. func router(brainDir string) wing.Placer { return wing.Router{BrainDir: brainDir, Default: "web-clip", ClientBlock: regexp.MustCompile(`(?i)acme-client`)} } func makeWing(t *testing.T, brainDir, name string) { t.Helper() require.NoError(t, os.MkdirAll(filepath.Join(brainDir, "wiki", name, "facts"), 0o755)) } func put(t *testing.T, brainDir, rel, body string) { t.Helper() full := filepath.Join(brainDir, "raw", rel) require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) require.NoError(t, os.WriteFile(full, []byte(body), 0o644)) } func has(brainDir, rel string) bool { _, err := os.Stat(filepath.Join(brainDir, rel)) return err == nil } func mustJSONPages(pages []pipeline.RawPage) string { b, err := json.Marshal(pages) if err != nil { panic(err) } return string(b) } func parkedBody(t *testing.T, brainDir, name string) string { t.Helper() b, err := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", name)) require.NoError(t, err) return string(b) } func linesStartingWith(s, prefix string) int { n := 0 for _, l := range strings.Split(s, "\n") { if strings.HasPrefix(l, prefix) { n++ } } return n } func TestProcessDir_ASubfolderNamesTheWing(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) makeWing(t, brainDir, "radar") put(t, brainDir, "radar/dark-factory.md", "A talk about factories.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.True(t, has(brainDir, "wiki/radar/sources/dark-factory.md"), "lands in the named wing") assert.False(t, has(brainDir, "raw/radar/dark-factory.md"), "inbox zero holds for subfolders too") } func TestProcessDir_ADeliberateSubfolderIntoAnyExistingWingPlaces(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) makeWing(t, brainDir, "rikatillsammans") put(t, brainDir, "rikatillsammans/meeting.md", "Notes the owner put there on purpose.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.True(t, has(brainDir, "wiki/rikatillsammans/sources/meeting.md"), "a folder is the owner's deliberate choice; extraction runs on the local model") } func TestProcessDir_ANoteTheRouterCannotPlaceIsParkedNotIngested(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) put(t, brainDir, "kickoff.md", "Some note about the ACME-CLIENT kickoff.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.Zero(t, llmCalls.Load(), "no extraction runs for an unplaced note") assert.False(t, has(brainDir, "raw/kickoff.md"), "leaves the top of raw/") parked := parkedBody(t, brainDir, "kickoff.md") assert.Contains(t, parked, "proposal_reason:") assert.Contains(t, parked, "client-content marker") assert.Contains(t, parked, "ACME-CLIENT kickoff", "the note body is preserved") } func TestProcessDir_ATypoedWingFolderIsParkedAndNeverCreated(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) makeWing(t, brainDir, "homelab") put(t, brainDir, "hoemlab/n.md", "Note.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.False(t, has(brainDir, "wiki/hoemlab")) assert.True(t, has(brainDir, "raw/needs-review/n.md")) assert.Zero(t, llmCalls.Load()) } func TestProcessDir_ParkedNotesAreNotReprocessed(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) put(t, brainDir, "needs-review/waiting.md", "---\nproposed_wing: radar\n---\nA parked note.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.Zero(t, llmCalls.Load(), "needs-review/ is a human queue, like rejected/") assert.True(t, has(brainDir, "raw/needs-review/waiting.md")) } func TestProcessDir_WithoutAResolverTheRetiredLayoutIsUnchanged(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, nil, &llmCalls) put(t, brainDir, "plain.md", "Plain note.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.True(t, has(brainDir, "wiki/sources/plain.md")) } // The reason comes from file and folder names and ends up in frontmatter: a // newline in it must not be able to start a new key (PR #147: a title once // forged `origin:`). func TestProcessDir_AFolderNameCannotForgeFrontmatterKeys(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) put(t, brainDir, "radar\nproposal_confidence: 0.99\nwing: homelab/forge.md", "Note.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) entries, err := os.ReadDir(filepath.Join(brainDir, "raw", "needs-review")) require.NoError(t, err) require.Len(t, entries, 1) b := parkedBody(t, brainDir, entries[0].Name()) assert.Equal(t, 1, linesStartingWith(b, "proposal_confidence:"), "exactly one confidence key at line start") assert.Equal(t, 0, linesStartingWith(b, "wing:"), "no injected wing key") assert.Equal(t, 0, linesStartingWith(b, "proposed_wing:"), "a non-plain wing name is never a proposal") } func TestProcessDir_ANotesOwnProposalKeysDoNotOutrankTheRouters(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) put(t, brainDir, "n.md", "---\nproposed_wing: homelab\nproposal_confidence: 1.00\nproposal_reason: \"self-approved\"\n---\nACME-CLIENT body.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) b := parkedBody(t, brainDir, "n.md") assert.NotContains(t, b, "self-approved") assert.Equal(t, 1, linesStartingWith(b, "proposal_reason:")) assert.Equal(t, 0, linesStartingWith(b, "proposed_wing:")) } func TestProcessDir_ACRLFNoteWithAWingKeyIsStillRead(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) makeWing(t, brainDir, "radar") put(t, brainDir, "n.md", "---\r\nwing: radar\r\n---\r\nBody.\r\n") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.True(t, has(brainDir, "wiki/radar/sources/n.md"), "the declared wing is honoured") } // editingPlacer rewrites the note while the placement decision is being made, // then parks it: the realistic window is a slow decision with Syncthing // delivering a newer version meanwhile. type editingPlacer struct{ edit func() } func (e editingPlacer) Resolve(context.Context, wing.Note) wing.Decision { e.edit() return wing.Decision{Source: wing.Exception, Reason: "test: parked after an edit"} } func TestProcessDir_ANoteEditedWhileItIsBeingPlacedIsNotDeleted(t *testing.T) { var llmCalls atomic.Int32 var notePath string cfg, brainDir := wingTestCfg(t, func(string) wing.Placer { return editingPlacer{edit: func() { require.NoError(t, os.WriteFile(notePath, []byte("NEWER and longer version"), 0o644)) }} }, &llmCalls) notePath = filepath.Join(brainDir, "raw", "n.md") put(t, brainDir, "n.md", "ORIGINAL") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) got, err := os.ReadFile(notePath) require.NoError(t, err, "the edited note must still be in raw/") assert.Equal(t, "NEWER and longer version", string(got)) } // A planted symlink at raw/needs-review must not redirect a parked note out of // the brain, nor lose it from the review queue. func TestProcessDir_AParkedNoteIsNeverWrittenThroughASymlinkedNeedsReview(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) outside := t.TempDir() require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", "needs-review"))) put(t, brainDir, "n.md", "ACME-CLIENT note.") errs := processDir(context.Background(), cfg, "2026-10-20") assert.NotEmpty(t, errs, "parking is refused, loudly") entries, _ := os.ReadDir(outside) assert.Empty(t, entries, "nothing was written through the symlink") assert.True(t, has(brainDir, "raw/n.md"), "the note stays where it was") } func TestWithProposal_ANonFenceDashLineDoesNotCloseTheBlock(t *testing.T) { out := withProposal("---\nfoo: bar\n----\nBody", wing.Decision{Wing: "radar", Confidence: 0.4, Reason: "r"}) lines := strings.Split(out, "\n") require.Equal(t, "---", lines[0]) closing := -1 for i := 1; i < len(lines); i++ { if lines[i] == "---" { closing = i break } } require.Positive(t, closing, "the proposal block is closed by a real fence") assert.Contains(t, strings.Join(lines[:closing], "\n"), "proposal_reason:") assert.NotContains(t, strings.Join(lines[:closing], "\n"), "foo: bar", "the unfenced note is body, not part of the block") } // The parked copy is the only copy once the original is retired, so two notes // that share a name must both survive. func TestProcessDir_SameNamedNotesAreAllParkedNoneOverwritten(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) for i, body := range []string{"ONE", "TWO", "THREE", "FOUR"} { put(t, brainDir, filepath.Join("typo"+string(rune('1'+i)), "x.md"), body) } require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) files, err := filepath.Glob(filepath.Join(brainDir, "raw", "needs-review", "*.md")) require.NoError(t, err) var all string for _, f := range files { b, _ := os.ReadFile(f) all += string(b) } for _, body := range []string{"ONE", "TWO", "THREE", "FOUR"} { assert.Contains(t, all, body, "note %s must not have been overwritten", body) } assert.Len(t, files, 4) } // Syncthing can bring a removed note back (an offline peer, a restored backup). // The source-hash drift check must keep that from costing a second extraction. func TestProcessDir_ANoteThatReappearsUnchangedIsNotExtractedAgain(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) makeWing(t, brainDir, "radar") put(t, brainDir, "radar/talk.md", "The same talk, byte for byte.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) require.Equal(t, int32(1), llmCalls.Load(), "positive control: the first drop is extracted") require.False(t, has(brainDir, "raw/radar/talk.md")) put(t, brainDir, "radar/talk.md", "The same talk, byte for byte.") // the peer sends it back require.Empty(t, processDir(context.Background(), cfg, "2026-10-21")) assert.Equal(t, int32(1), llmCalls.Load(), "an unchanged source is not extracted twice") assert.False(t, has(brainDir, "raw/radar/talk.md"), "and inbox zero still holds") } func TestProcessDir_ANoteThatReappearsChangedIsExtractedAgain(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) makeWing(t, brainDir, "radar") put(t, brainDir, "radar/talk.md", "Version one.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) put(t, brainDir, "radar/talk.md", "Version two, edited.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-21")) assert.Equal(t, int32(2), llmCalls.Load(), "a real edit must still be picked up") } func TestFrontmatterWing(t *testing.T) { tests := []struct { name, in, want string }{ {"plain key", "---\nwing: radar\n---\nbody", "radar"}, {"quoted", "---\nwing: \"radar\"\n---\nbody", "radar"}, {"CRLF", "---\r\nwing: radar\r\n---\r\nbody", "radar"}, {"no frontmatter", "wing: radar\nbody", ""}, {"an indented key belongs to another mapping", "---\nmeta:\n wing: radar\n---\nbody", ""}, {"a key before a non-fence dash line is not frontmatter", "---\nwing: radar\n----\nbody", ""}, {"no closing fence", "---\nwing: radar\nbody", ""}, {"closing fence at end of input", "---\nwing: radar\n---", "radar"}, {"another key first", "---\ntitle: x\nwing: radar\n---\nbody", "radar"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert.Equal(t, tt.want, frontmatterWing(tt.in)) }) } } func TestFrontmatterWing_ABOMDoesNotHideTheKey(t *testing.T) { assert.Equal(t, "homelab", frontmatterWing("\ufeff---\nwing: homelab\n---\nbody")) } // A note that produced nothing must not leave the inbox as "processed". func TestProcessDir_AWingIngestThatWritesNothingFailsTheNoteVisibly(t *testing.T) { var llmCalls atomic.Int32 cfg, brainDir := wingTestCfg(t, router, &llmCalls) cfg.Pipeline.Complete = func(context.Context, string, string) (string, error) { return "[]", nil } put(t, brainDir, "empty-result.md", "A note the model returned nothing for.") require.Empty(t, processDir(context.Background(), cfg, "2026-10-20")) assert.True(t, has(brainDir, "raw/failed/empty-result.md"), "kept where a human looks") processed, _ := filepath.Glob(filepath.Join(brainDir, "raw", "processed", "*", "*")) assert.Empty(t, processed, "never archived as processed") logb, err := os.ReadFile(filepath.Join(brainDir, "log.md")) require.NoError(t, err) assert.Contains(t, string(logb), "no source page") }