diff --git a/ingestion/internal/brain/index.go b/ingestion/internal/brain/index.go index 3317bf7..4c029e1 100644 --- a/ingestion/internal/brain/index.go +++ b/ingestion/internal/brain/index.go @@ -2,12 +2,18 @@ package brain import ( "bufio" + "bytes" + "errors" "fmt" + "io" + "io/fs" "os" "path/filepath" "sort" "strings" "time" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" ) // noteEntry is one row in a Wing _index.md. @@ -27,13 +33,18 @@ func BuildWingIndex(brainDir, wing string) error { return fmt.Errorf("invalid wing %q", wing) } wingDir := filepath.Join(brainDir, "wiki", w) - if _, err := os.Stat(wingDir); os.IsNotExist(err) { + // Listed and read through the brain root (hyperguild#157): a wing or + // hall that is a symlink is not followed, and notes are regular files. + bfs, err := brainfs.Open(brainDir) + if err != nil { + return fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + if _, err := bfs.Lstat("wiki/" + w); errors.Is(err, fs.ErrNotExist) { return nil - } else if err != nil { - return fmt.Errorf("stat wing: %w", err) } - entries, err := collectWingEntries(wingDir) + entries, err := collectWingEntries(bfs, "wiki/"+w) if err != nil { return err } @@ -58,51 +69,47 @@ func BuildWingIndex(brainDir, wing string) error { // BuildAllWingIndexes regenerates _index.md for every wing under brain/wiki/. func BuildAllWingIndexes(brainDir string) error { - wikiDir := filepath.Join(brainDir, "wiki") - ents, err := os.ReadDir(wikiDir) - if os.IsNotExist(err) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return fmt.Errorf("open brain: %w", err) + } + wings, err := bfs.ReadDirDirs("wiki") + _ = bfs.Close() + if errors.Is(err, fs.ErrNotExist) { return nil } if err != nil { return fmt.Errorf("read wiki: %w", err) } - for _, e := range ents { - if !e.IsDir() { - continue - } - if err := BuildWingIndex(brainDir, e.Name()); err != nil { - return fmt.Errorf("index %s: %w", e.Name(), err) + for _, w := range wings { + if err := BuildWingIndex(brainDir, w); err != nil { + return fmt.Errorf("index %s: %w", w, err) } } return nil } -func collectWingEntries(wingDir string) ([]noteEntry, error) { +func collectWingEntries(bfs *brainfs.Brain, wingRel string) ([]noteEntry, error) { var out []noteEntry - ents, err := os.ReadDir(wingDir) + halls, err := bfs.ReadDirDirs(wingRel) if err != nil { return nil, fmt.Errorf("read wing: %w", err) } - for _, hallEnt := range ents { - if !hallEnt.IsDir() { - continue - } - hall := hallEnt.Name() + for _, hall := range halls { if !IsValidHall(hall) { continue } - hallDir := filepath.Join(wingDir, hall) - notes, err := os.ReadDir(hallDir) + hallRel := wingRel + "/" + hall + names, err := bfs.ReadDirRegular(hallRel) if err != nil { return nil, fmt.Errorf("read hall %s: %w", hall, err) } - for _, n := range notes { - if n.IsDir() || !strings.HasSuffix(n.Name(), ".md") || n.Name() == "_index.md" { + for _, name := range names { + if !strings.HasSuffix(name, ".md") || name == "_index.md" { continue } - slug := strings.TrimSuffix(n.Name(), ".md") - full := filepath.Join(hallDir, n.Name()) - title, created := readTitleAndCreated(full, slug) + slug := strings.TrimSuffix(name, ".md") + title, created := readTitleAndCreatedIn(bfs, hallRel+"/"+name, slug) out = append(out, noteEntry{Hall: hall, Slug: slug, Title: title, Created: created}) } } @@ -117,9 +124,40 @@ func readTitleAndCreated(path, slug string) (string, string) { return slug, "" } defer func() { _ = f.Close() }() + title, created := parseTitleAndCreated(f) + if created == "" { + if info, err := os.Stat(path); err == nil { + created = info.ModTime().UTC().Format("2006-01-02") + } + } + return titleOrSlug(title, slug), created +} - title, created := "", "" - scanner := bufio.NewScanner(f) +// readTitleAndCreatedIn is readTitleAndCreated through the brain root, +// regular files only (hyperguild#157). +func readTitleAndCreatedIn(bfs *brainfs.Brain, rel, slug string) (string, string) { + content, err := bfs.ReadRegular(rel) + if err != nil { + return slug, "" + } + title, created := parseTitleAndCreated(bytes.NewReader(content)) + if created == "" { + if info, err := bfs.Lstat(rel); err == nil { + created = info.ModTime().UTC().Format("2006-01-02") + } + } + return titleOrSlug(title, slug), created +} + +func titleOrSlug(title, slug string) string { + if title == "" { + return strings.ReplaceAll(slug, "-", " ") + } + return title +} + +func parseTitleAndCreated(r io.Reader) (title, created string) { + scanner := bufio.NewScanner(r) inFrontmatter := false for scanner.Scan() { line := scanner.Text() @@ -149,13 +187,5 @@ func readTitleAndCreated(path, slug string) (string, string) { } } } - if title == "" { - title = strings.ReplaceAll(slug, "-", " ") - } - if created == "" { - if info, err := os.Stat(path); err == nil { - created = info.ModTime().UTC().Format("2006-01-02") - } - } return title, created } diff --git a/ingestion/internal/brain/index_root_test.go b/ingestion/internal/brain/index_root_test.go new file mode 100644 index 0000000..1a794e8 --- /dev/null +++ b/ingestion/internal/brain/index_root_test.go @@ -0,0 +1,83 @@ +package brain_test + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brain" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest" +) + +func readIndex(t *testing.T, brainDir, wing string) string { + t.Helper() + b, _ := os.ReadFile(filepath.Join(brainDir, "wiki", wing, "_index.md")) + return string(b) +} + +// hyperguild#157 slice 5: BuildWingIndex listed wiki/// with +// os.ReadDir and opened each note by path, so a symlinked note put the +// target's title into _index.md, and a FIFO hung every brain_write. +func TestBuildWingIndexRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) { + b := t.TempDir() + facts := filepath.Join(b, "wiki", "w", "facts") + require.NoError(t, os.MkdirAll(facts, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644)) + require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md"))) + inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(b, "raw", "s.md")) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md"))) + + require.NoError(t, brain.BuildWingIndex(b, "w")) + idx := readIndex(t, b, "w") + assert.Contains(t, idx, "zqinside", "positive control") + assert.NotContains(t, idx, brainfstest.Sentinel) + assert.NotContains(t, idx, "leak") + assert.NotContains(t, idx, "inlink") + + p := filepath.Join(facts, "r.md") + read := func() string { + _ = brain.BuildWingIndex(b, "w") + return readIndex(t, b, "w") + } + brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read) + brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: zqinside\n---\n", inBrain, 1500*time.Millisecond, read) + brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() }) +} + +// A wing or hall directory that is a symlink is not listed, and nothing is +// written into its target. +func TestBuildWingIndexDoesNotFollowASymlinkedWingOrHall(t *testing.T) { + b := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(b, "raw", "facts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(b, "raw", "facts", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(b, "wiki", "w"), 0o755)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "facts"), filepath.Join(b, "wiki", "w", "facts"))) + require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(b, "wiki", "alias"))) + + _ = brain.BuildWingIndex(b, "w") + assert.NotContains(t, readIndex(t, b, "w"), "zqrawonly", "symlinked hall") + + assert.Error(t, brain.BuildWingIndex(b, "alias")) + _, err := os.Stat(filepath.Join(b, "raw", "_index.md")) + assert.True(t, os.IsNotExist(err), "an index was written into the symlinked wing's target") +} + +// A link to a symlinked note does not resolve: only regular files count as +// entries, so StripUnresolvedLinks unwraps it. +func TestStripUnresolvedLinksDoesNotCountASymlinkedNoteAsAnEntry(t *testing.T) { + b := t.TempDir() + facts := filepath.Join(b, "wiki", "w", "facts") + require.NoError(t, os.MkdirAll(facts, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(facts, "real.md"), []byte("x"), 0o644)) + brainfstest.WriteSentinelAt(t, filepath.Join(b, "raw", "s.md")) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "ghost.md"))) + out, unlinked, err := brain.StripUnresolvedLinks(b, "see [[real]] and [[ghost]]") + require.NoError(t, err) + assert.Contains(t, out, "[[real]]", "positive control") + assert.NotContains(t, out, "[[ghost]]") + assert.Contains(t, unlinked, "ghost") +} diff --git a/ingestion/internal/brain/linkcheck.go b/ingestion/internal/brain/linkcheck.go index d81d08b..7f44817 100644 --- a/ingestion/internal/brain/linkcheck.go +++ b/ingestion/internal/brain/linkcheck.go @@ -1,12 +1,15 @@ package brain import ( + "errors" "fmt" - "os" - "path/filepath" + "io/fs" + "path" "regexp" "sort" "strings" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" ) // wikilinkRE matches [[target]] and [[target|Display Text]]. @@ -98,29 +101,33 @@ var linkedTrees = []string{"knowledge", "wiki"} // one the audit will also resolve. func entryIndex(brainDir string) (map[string]struct{}, error) { index := map[string]struct{}{} + // Walked through the brain root (hyperguild#157): only regular files + // count as entries, so a symlinked .md name cannot make a link resolve. + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() for _, dir := range linkedTrees { - root := filepath.Join(brainDir, dir) - if _, err := os.Stat(root); os.IsNotExist(err) { + if _, err := bfs.Lstat(dir); errors.Is(err, fs.ErrNotExist) { continue - } else if err != nil { - return nil, fmt.Errorf("stat %s: %w", dir, err) } - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(path, ".md") { + var walkErr error + err := bfs.WalkRegular(dir, func(rel string) error { + if !strings.HasSuffix(rel, ".md") { return nil } - rel, err := filepath.Rel(brainDir, path) - if err != nil { - return err - } - rel = filepath.ToSlash(rel) - index[strings.TrimSuffix(filepath.Base(rel), ".md")] = struct{}{} + index[strings.TrimSuffix(path.Base(rel), ".md")] = struct{}{} index[strings.TrimPrefix(strings.TrimSuffix(rel, ".md"), "wiki/")] = struct{}{} return nil + }, func(rel string, err error) { + if walkErr == nil { + walkErr = fmt.Errorf("%s: %w", rel, err) + } }) + if err == nil { + err = walkErr + } if err != nil { return nil, fmt.Errorf("walk %s: %w", dir, err) } diff --git a/ingestion/internal/brainfs/brainfs.go b/ingestion/internal/brainfs/brainfs.go index fb1b76c..4feca7c 100644 --- a/ingestion/internal/brainfs/brainfs.go +++ b/ingestion/internal/brainfs/brainfs.go @@ -104,6 +104,27 @@ func (b *Brain) ReadDirRegular(relDir string) ([]string, error) { return names, nil } +// ReadDirDirs lists the real subdirectories directly in a brain-relative +// directory, by name. Every component of relDir must be a real directory +// (ErrNotRegular otherwise); symlinks to directories are left out. +func (b *Brain) ReadDirDirs(relDir string) ([]string, error) { + relDir = filepath.ToSlash(relDir) + if err := b.realDir(relDir); err != nil { + return nil, err + } + ents, err := fs.ReadDir(b.root.FS(), relDir) + if err != nil { + return nil, err + } + var names []string + for _, e := range ents { + if e.Type().IsDir() { + names = append(names, e.Name()) + } + } + return names, nil +} + // ReadRegular reads a brain-relative path, refusing anything that is not a // regular file, both before the open and on the opened handle. func (b *Brain) ReadRegular(rel string) ([]byte, error) { diff --git a/ingestion/internal/brainfs/brainfs_test.go b/ingestion/internal/brainfs/brainfs_test.go index 7146af8..76ea7d1 100644 --- a/ingestion/internal/brainfs/brainfs_test.go +++ b/ingestion/internal/brainfs/brainfs_test.go @@ -262,3 +262,21 @@ func TestReadRegularNeverReturnsAnInBrainFileSwappedInBehindASymlink(t *testing. assert.Zero(t, leaks) assert.Positive(t, ok, "positive control: the regular file is read") } + +func TestReadDirDirsListsOnlyRealSubdirsAndRefusesSymlinkedComponents(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "a", "real"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "other"), 0o755)) + require.NoError(t, os.Symlink(filepath.Join("..", "other"), filepath.Join(dir, "a", "linked"))) + require.NoError(t, os.WriteFile(filepath.Join(dir, "a", "f.md"), []byte("f"), 0o644)) + require.NoError(t, os.Symlink("a", filepath.Join(dir, "alias"))) + b, err := Open(dir) + require.NoError(t, err) + defer func() { _ = b.Close() }() + + names, err := b.ReadDirDirs("a") + require.NoError(t, err) + assert.Equal(t, []string{"real"}, names) + _, err = b.ReadDirDirs("alias") + assert.ErrorIs(t, err, ErrNotRegular) +}