From da020d1e6c40e43de7354fc69b57add8c044de11 Mon Sep 17 00:00:00 2001 From: Mathias Date: Mon, 5 Oct 2026 20:51:31 +0200 Subject: [PATCH 1/2] fix(tunnel): DetectTunnels walks and reads through brainfs (#174, #157 slice 5b) DetectTunnels runs inside every brain_write and brain_update (wikiUpkeep -> AutoTunnel). It walked wiki/ with filepath.WalkDir and opened each note with a blocking os.Open: a FIFO named *.md anywhere under wiki/ hung the write with no deadline, and a symlinked note's title became a tunnel candidate (171k leaks in the swap race). It now walks with brainfs.WalkRegular (regular files only) and reads titles with readTitleAndCreatedIn (ReadRegular: O_NONBLOCK, handle and SameFile checks). A wiki/ that is not a real directory yields no candidates, as filepath.WalkDir did; other walk errors still abort. The path-based readTitleAndCreated, now unused, is removed. Co-Authored-By: Claude Opus 5.5 --- ingestion/internal/brain/detect_root_test.go | 79 ++++++++++++++++++++ ingestion/internal/brain/index.go | 16 ---- ingestion/internal/brain/tunnel.go | 45 +++++++---- 3 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 ingestion/internal/brain/detect_root_test.go diff --git a/ingestion/internal/brain/detect_root_test.go b/ingestion/internal/brain/detect_root_test.go new file mode 100644 index 0000000..93d5162 --- /dev/null +++ b/ingestion/internal/brain/detect_root_test.go @@ -0,0 +1,79 @@ +package brain_test + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brain" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest" +) + +// hyperguild#174 / #157 slice 5b: DetectTunnels walked wiki/ and opened +// every note with a blocking os.Open, so a FIFO named *.md anywhere under +// wiki/ hung every brain_write and brain_update (wikiUpkeep -> AutoTunnel), +// and a symlinked note's title became a tunnel candidate. +func TestDetectTunnelsReturnsPromptlyWithAFIFOUnderWiki(t *testing.T) { + b := t.TempDir() + facts := filepath.Join(b, "wiki", "w", "facts") + require.NoError(t, os.MkdirAll(facts, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644)) + require.NoError(t, syscall.Mkfifo(filepath.Join(facts, "f.md"), 0o644)) + done := make(chan []brain.TunnelCandidate, 1) + go func() { + cs, _ := brain.DetectTunnels(b, "text mentioning zqinside phrase here") + done <- cs + }() + select { + case cs := <-done: + assert.Contains(t, fmt.Sprintf("%+v", cs), "Zqinside Phrase", "positive control") + case <-time.After(3 * time.Second): + if r, err := os.OpenFile(filepath.Join(facts, "f.md"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil { + _ = r.Close() + } + t.Fatal("DetectTunnels blocked on a FIFO") + } +} + +func TestDetectTunnelsRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) { + b := t.TempDir() + facts := filepath.Join(b, "wiki", "w", "facts") + require.NoError(t, os.MkdirAll(facts, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644)) + require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "out.md"))) + inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(b, "raw", "s.md")) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "in.md"))) + content := "zqinside phrase and " + brainfstest.Sentinel + + cs, err := brain.DetectTunnels(b, content) + require.NoError(t, err) + got := fmt.Sprintf("%+v", cs) + assert.Contains(t, got, "Zqinside Phrase", "positive control") + assert.NotContains(t, got, brainfstest.Sentinel) + + p := filepath.Join(facts, "r.md") + read := func() string { + cs, _ := brain.DetectTunnels(b, content) + return fmt.Sprintf("%+v", cs) + } + brainfstest.AssertNoLeak(t, p, "---\ntitle: Zqinside Phrase\n---\n", 1500*time.Millisecond, read) + brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: Zqinside Phrase\n---\n", inBrain, 1500*time.Millisecond, read) +} + +// A wiki/ that is not a real directory yields no candidates and no error: +// AutoTunnel runs inside brain_write. +func TestDetectTunnelsSkipsAWikiThatIsNotARealDirectory(t *testing.T) { + b := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(b, "raw", "w", "facts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(b, "raw", "w", "facts", "r.md"), []byte("---\ntitle: Zqrawonly Phrase\n---\n"), 0o644)) + require.NoError(t, os.Symlink("raw", filepath.Join(b, "wiki"))) + cs, err := brain.DetectTunnels(b, "zqrawonly phrase") + require.NoError(t, err) + assert.Empty(t, cs) +} diff --git a/ingestion/internal/brain/index.go b/ingestion/internal/brain/index.go index 4c029e1..020b164 100644 --- a/ingestion/internal/brain/index.go +++ b/ingestion/internal/brain/index.go @@ -116,22 +116,6 @@ func collectWingEntries(bfs *brainfs.Brain, wingRel string) ([]noteEntry, error) return out, nil } -// readTitleAndCreated reads YAML frontmatter for title + created_at; falls -// back to slug and file mtime when absent. -func readTitleAndCreated(path, slug string) (string, string) { - f, err := os.Open(path) - if err != nil { - return slug, "" - } - defer func() { _ = f.Close() }() - title, created := parseTitleAndCreated(f) - if created == "" { - if info, err := os.Stat(path); err == nil { - created = info.ModTime().UTC().Format("2006-01-02") - } - } - return titleOrSlug(title, slug), created -} // readTitleAndCreatedIn is readTitleAndCreated through the brain root, // regular files only (hyperguild#157). diff --git a/ingestion/internal/brain/tunnel.go b/ingestion/internal/brain/tunnel.go index f445e0f..db3be1b 100644 --- a/ingestion/internal/brain/tunnel.go +++ b/ingestion/internal/brain/tunnel.go @@ -2,7 +2,9 @@ package brain import ( "bufio" + "errors" "fmt" + "io/fs" "os" "path" "path/filepath" @@ -39,24 +41,29 @@ type TunnelCandidate struct { // A note's title is read from YAML frontmatter `title:`; failing that, // the filename slug (sans `.md`, hyphens → spaces) is used. func DetectTunnels(brainDir, content string) ([]TunnelCandidate, error) { - wikiDir := filepath.Join(brainDir, "wiki") - if _, err := os.Stat(wikiDir); os.IsNotExist(err) { + // Walked and read through the brain root, regular files only + // (hyperguild#174, #157): this runs inside every brain_write, so a FIFO + // must not block it and a symlinked note's title must not become a + // candidate. + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + if _, err := bfs.Lstat("wiki"); errors.Is(err, fs.ErrNotExist) { return nil, nil - } else if err != nil { - return nil, fmt.Errorf("stat wiki: %w", err) } lowerContent := strings.ToLower(content) var out []TunnelCandidate - err := filepath.WalkDir(wikiDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(path, ".md") || d.Name() == "_index.md" { + var walkErr error + err = bfs.WalkRegular("wiki", func(rel string) error { + name := path.Base(rel) + if !strings.HasSuffix(name, ".md") || name == "_index.md" { return nil } - title, _ := readTitleAndCreated(path, strings.TrimSuffix(d.Name(), ".md")) + title, _ := readTitleAndCreatedIn(bfs, rel, strings.TrimSuffix(name, ".md")) title = strings.TrimSpace(title) if title == "" { return nil @@ -78,17 +85,25 @@ func DetectTunnels(brainDir, content string) ([]TunnelCandidate, error) { if idx == -1 { return nil } - rel, err := filepath.Rel(brainDir, path) - if err != nil { - return err - } out = append(out, TunnelCandidate{ - TargetPath: filepath.ToSlash(rel), + TargetPath: rel, MatchedTerm: title, Exact: isWholeWord(haystack, idx, len(needle)), }) return nil + }, func(rel string, err error) { + // A wiki/ that is not a real directory yields no candidates, as + // filepath.WalkDir did; other walk errors still abort. + if rel == "wiki" && errors.Is(err, brainfs.ErrNotRegular) { + return + } + if walkErr == nil { + walkErr = fmt.Errorf("%s: %w", rel, err) + } }) + if err == nil { + err = walkErr + } if err != nil { return nil, err } -- 2.54.0 From 1c9c218dbf96b6dc3dda5c4728b0e2ecaaee5596 Mon Sep 17 00:00:00 2001 From: Mathias Date: Mon, 5 Oct 2026 21:03:31 +0200 Subject: [PATCH 2/2] test(tunnel): pin the raced read, the walk error and the _index.md skip (#174) Review round 1 on PR #186: - The swap test's static note had the same title as the raced note, so the harness's "inside was read" floor passed without reading r.md (skipping r.md stayed green). The static note now has its own title. - An unreadable directory under wiki/ reaches the walk-error path, and swallowing that error stayed green. New test asserts the error. - The FIFO test now runs with and without a writer held open. - _index.md skip had no test; new test. - index.go: the double blank line left by removing readTitleAndCreated. Co-Authored-By: Claude Opus 5.5 --- ingestion/internal/brain/detect_root_test.go | 90 +++++++++++++++----- ingestion/internal/brain/index.go | 1 - 2 files changed, 68 insertions(+), 23 deletions(-) diff --git a/ingestion/internal/brain/detect_root_test.go b/ingestion/internal/brain/detect_root_test.go index 93d5162..c5f451c 100644 --- a/ingestion/internal/brain/detect_root_test.go +++ b/ingestion/internal/brain/detect_root_test.go @@ -19,25 +19,37 @@ import ( // every note with a blocking os.Open, so a FIFO named *.md anywhere under // wiki/ hung every brain_write and brain_update (wikiUpkeep -> AutoTunnel), // and a symlinked note's title became a tunnel candidate. -func TestDetectTunnelsReturnsPromptlyWithAFIFOUnderWiki(t *testing.T) { - b := t.TempDir() - facts := filepath.Join(b, "wiki", "w", "facts") - require.NoError(t, os.MkdirAll(facts, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644)) - require.NoError(t, syscall.Mkfifo(filepath.Join(facts, "f.md"), 0o644)) - done := make(chan []brain.TunnelCandidate, 1) - go func() { - cs, _ := brain.DetectTunnels(b, "text mentioning zqinside phrase here") - done <- cs - }() - select { - case cs := <-done: - assert.Contains(t, fmt.Sprintf("%+v", cs), "Zqinside Phrase", "positive control") - case <-time.After(3 * time.Second): - if r, err := os.OpenFile(filepath.Join(facts, "f.md"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil { - _ = r.Close() - } - t.Fatal("DetectTunnels blocked on a FIFO") +func TestDetectTunnelsReturnsPromptlyWithAFIFOUnderWikiWithAndWithoutAWriter(t *testing.T) { + for _, withWriter := range []bool{false, true} { + t.Run(fmt.Sprintf("writer=%v", withWriter), func(t *testing.T) { + b := t.TempDir() + facts := filepath.Join(b, "wiki", "w", "facts") + require.NoError(t, os.MkdirAll(facts, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644)) + fifo := filepath.Join(facts, "f.md") + require.NoError(t, syscall.Mkfifo(fifo, 0o644)) + if withWriter { + // A writer held open: a reader that opens the FIFO gets no + // EOF and parks, unless the FIFO is never opened. + w, err := os.OpenFile(fifo, os.O_RDWR, 0) + require.NoError(t, err) + t.Cleanup(func() { _ = w.Close() }) + } + done := make(chan []brain.TunnelCandidate, 1) + go func() { + cs, _ := brain.DetectTunnels(b, "text mentioning zqinside phrase here") + done <- cs + }() + select { + case cs := <-done: + assert.Contains(t, fmt.Sprintf("%+v", cs), "Zqinside Phrase", "positive control") + case <-time.After(3 * time.Second): + if r, err := os.OpenFile(fifo, os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil { + _ = r.Close() + } + t.Fatal("DetectTunnels blocked on a FIFO") + } + }) } } @@ -45,16 +57,18 @@ func TestDetectTunnelsRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) { b := t.TempDir() facts := filepath.Join(b, "wiki", "w", "facts") require.NoError(t, os.MkdirAll(facts, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644)) + // The static note's title differs from the raced note's, so the + // harness's "inside was read" floor can only be met by reading r.md. + require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqstatic Words\n---\n"), 0o644)) require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "out.md"))) inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(b, "raw", "s.md")) require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "in.md"))) - content := "zqinside phrase and " + brainfstest.Sentinel + content := "zqstatic words, zqinside phrase and " + brainfstest.Sentinel cs, err := brain.DetectTunnels(b, content) require.NoError(t, err) got := fmt.Sprintf("%+v", cs) - assert.Contains(t, got, "Zqinside Phrase", "positive control") + assert.Contains(t, got, "Zqstatic Words", "positive control") assert.NotContains(t, got, brainfstest.Sentinel) p := filepath.Join(facts, "r.md") @@ -77,3 +91,35 @@ func TestDetectTunnelsSkipsAWikiThatIsNotARealDirectory(t *testing.T) { require.NoError(t, err) assert.Empty(t, cs) } + +// An unreadable directory under wiki/ is a walk error and DetectTunnels +// reports it, as filepath.WalkDir did; it is not silently dropped. +func TestDetectTunnelsReportsAnUnreadableDirectoryUnderWiki(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores directory permissions") + } + b := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(b, "wiki", "w", "facts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(b, "wiki", "w", "facts", "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644)) + locked := filepath.Join(b, "wiki", "other", "locked") + require.NoError(t, os.MkdirAll(locked, 0o755)) + require.NoError(t, os.Chmod(locked, 0o000)) + t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) + _, err := brain.DetectTunnels(b, "zqinside phrase") + assert.ErrorIs(t, err, os.ErrPermission) +} + +// _index.md files are generated, not notes: their title never becomes a +// candidate. +func TestDetectTunnelsSkipsIndexFiles(t *testing.T) { + b := t.TempDir() + w := filepath.Join(b, "wiki", "w") + require.NoError(t, os.MkdirAll(filepath.Join(w, "facts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(w, "_index.md"), []byte("---\ntitle: Zqindex Words\n---\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(w, "facts", "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644)) + cs, err := brain.DetectTunnels(b, "zqindex words and zqinside phrase") + require.NoError(t, err) + got := fmt.Sprintf("%+v", cs) + assert.Contains(t, got, "Zqinside Phrase", "positive control") + assert.NotContains(t, got, "Zqindex") +} diff --git a/ingestion/internal/brain/index.go b/ingestion/internal/brain/index.go index 020b164..d700b74 100644 --- a/ingestion/internal/brain/index.go +++ b/ingestion/internal/brain/index.go @@ -116,7 +116,6 @@ func collectWingEntries(bfs *brainfs.Brain, wingRel string) ([]noteEntry, error) return out, nil } - // readTitleAndCreatedIn is readTitleAndCreated through the brain root, // regular files only (hyperguild#157). func readTitleAndCreatedIn(bfs *brainfs.Brain, rel, slug string) (string, string) { -- 2.54.0