fix(vectorstore): walk and read the brain through brainfs (#157, slice 3) (#166)

vectorstore.Sync walks and reads via brainfs (regular files only, handle-checked), so symlinked notes are no longer embedded and their old chunks are purged. brainfs.WalkRegular requires every component of its root to be a real directory (fixes in-brain root aliasing in Sync and search Wing/Wing+Hall). Walk errors, including a symlinked scan root, abort Sync before the delete pass. brainfs gains Lstat. Directory-swap race recorded as accepted residual (#167).

Refs #157. Adversarial review: BLOCKED, BLOCKED, PASS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/brainfs/brainfs.go

@@ -13,7 +13,9 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
@@ -53,6 +55,33 @@ func (b *Brain) Rel(path string) (string, error) {
 	return filepath.ToSlash(rel), nil
 }
 
+// Lstat stats a brain-relative path through the root without following a
+// final symlink.
+func (b *Brain) Lstat(rel string) (fs.FileInfo, error) {
+	return b.root.Lstat(filepath.ToSlash(rel))
+}
+
+// realDir returns ErrNotRegular unless every component of rel is a real
+// directory. A swap after the check is not caught (accepted residual,
+// hyperguild#157).
+func (b *Brain) realDir(rel string) error {
+	if rel == "." {
+		return nil
+	}
+	prefix := ""
+	for _, part := range strings.Split(rel, "/") {
+		prefix = path.Join(prefix, part)
+		fi, err := b.root.Lstat(prefix)
+		if err != nil {
+			return err
+		}
+		if !fi.IsDir() {
+			return fmt.Errorf("%s: walk root component %s is not a directory: %w", rel, prefix, ErrNotRegular)
+		}
+	}
+	return nil
+}
+
 // ReadRegular reads a brain-relative path, refusing anything that is not a
 // regular file, both before the open and on the opened handle.
 func (b *Brain) ReadRegular(rel string) ([]byte, error) {
@@ -127,15 +156,34 @@ func (b *Brain) readOpenedMax(rel string, max int64) ([]byte, error) {
 }
 
 // wrapAfterStat is a test seam: it runs between the size check and the read,
-// and wraps the reader the read consumes.
+// and wraps the reader the read consumes. A package global: brainfs tests
+// must not use t.Parallel.
 var wrapAfterStat func(io.Reader) io.Reader
 
 // WalkRegular walks a brain-relative directory through the root and calls fn
-// for each regular file (directories are descended; symlinks, FIFOs and
-// devices are skipped, never followed). An entry error is passed to onErr and
-// the walk continues; a nil onErr ignores it.
+// for each regular file. Directories are descended; symlinks, FIFOs and
+// devices found by the walk are skipped. A walk root with any component that
+// is not a real directory goes to onErr as ErrNotRegular and nothing is
+// yielded. An entry error is passed to onErr and the walk continues; a nil
+// onErr ignores it.
+//
+// Accepted residual (hyperguild#157): a directory swapped for a symlink
+// between the listing and the read is followed. os.Root still keeps the read
+// inside the brain, but not inside relDir.
 func (b *Brain) WalkRegular(relDir string, fn func(rel string) error, onErr func(rel string, err error)) error {
-	return fs.WalkDir(b.root.FS(), filepath.ToSlash(relDir), func(p string, d fs.DirEntry, err error) error {
+	relDir = filepath.ToSlash(relDir)
+	// fs.WalkDir stats its root with a follow, and os.Root allows a symlink
+	// that stays inside the brain: knowledge -> . would alias the whole
+	// brain. Require every component of the root to be a real directory
+	// (Lstat checks only the last one).
+	err := b.realDir(relDir)
+	if err != nil {
+		if onErr != nil {
+			onErr(relDir, err)
+		}
+		return nil
+	}
+	return fs.WalkDir(b.root.FS(), relDir, func(p string, d fs.DirEntry, err error) error {
 		if err != nil {
 			if onErr != nil {
 				onErr(p, err)

ingestion/internal/brainfs/brainfs_test.go

@@ -3,6 +3,7 @@ package brainfs
 import (
 	"errors"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"syscall"
@@ -139,3 +140,52 @@ func (c *countingReader) Read(p []byte) (int, error) {
 	c.n += n
 	return n, err
 }
+
+func TestLstatDoesNotFollowSymlinksAndRefusesEscapes(t *testing.T) {
+	dir := t.TempDir()
+	outside := filepath.Join(t.TempDir(), "o.md")
+	require.NoError(t, os.WriteFile(outside, []byte("o"), 0o644))
+	require.NoError(t, os.Symlink(outside, filepath.Join(dir, "l.md")))
+	b, err := Open(dir)
+	require.NoError(t, err)
+	defer func() { _ = b.Close() }()
+	fi, err := b.Lstat("l.md")
+	require.NoError(t, err)
+	assert.Equal(t, fs.ModeSymlink, fi.Mode().Type())
+	_, err = b.Lstat("../" + filepath.Base(filepath.Dir(outside)) + "/o.md")
+	assert.Error(t, err)
+}
+
+func TestWalkRegularReportsASymlinkedWalkRootAndYieldsNothing(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "real"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "real", "n.md"), []byte("n"), 0o644))
+	require.NoError(t, os.Symlink("real", filepath.Join(dir, "alias")))
+	b, err := Open(dir)
+	require.NoError(t, err)
+	defer func() { _ = b.Close() }()
+	var got []string
+	var errs []error
+	require.NoError(t, b.WalkRegular("alias", func(rel string) error { got = append(got, rel); return nil },
+		func(_ string, err error) { errs = append(errs, err) }))
+	assert.Empty(t, got)
+	require.Len(t, errs, 1)
+	assert.ErrorIs(t, errs[0], ErrNotRegular)
+}
+
+func TestWalkRegularReportsASymlinkedIntermediateComponentOfTheWalkRoot(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "real", "sub"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "real", "sub", "n.md"), []byte("n"), 0o644))
+	require.NoError(t, os.Symlink("real", filepath.Join(dir, "alias")))
+	b, err := Open(dir)
+	require.NoError(t, err)
+	defer func() { _ = b.Close() }()
+	var got []string
+	var errs []error
+	require.NoError(t, b.WalkRegular("alias/sub", func(rel string) error { got = append(got, rel); return nil },
+		func(_ string, err error) { errs = append(errs, err) }))
+	assert.Empty(t, got)
+	require.Len(t, errs, 1)
+	assert.ErrorIs(t, errs[0], ErrNotRegular)
+}