diff --git a/ingestion/internal/brainfs/brainfs.go b/ingestion/internal/brainfs/brainfs.go new file mode 100644 index 0000000..8cbb477 --- /dev/null +++ b/ingestion/internal/brainfs/brainfs.go @@ -0,0 +1,98 @@ +// Package brainfs reads the brain directory through an os.Root (hyperguild#157). +// The brain dir is a Syncthing sendreceive folder and Syncthing syncs +// symlinks; a reader that walks and reads by path follows a planted symlink +// out of the brain, and blocks forever on a planted FIFO. Readers that go +// through a Brain only ever read regular files inside the brain: os.Root +// refuses to leave the root even if an entry is swapped mid-read, and every +// open is non-blocking with the type checked on the opened handle. +package brainfs + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "syscall" +) + +// ErrNotRegular is returned for an entry that is not a regular file. +var ErrNotRegular = errors.New("not a regular file") + +// Brain is an open brain directory. +type Brain struct { + root *os.Root + dir string +} + +// Open opens brainDir (which may itself be a mount symlink) as a root. +func Open(brainDir string) (*Brain, error) { + r, err := os.OpenRoot(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain root: %w", err) + } + return &Brain{root: r, dir: brainDir}, nil +} + +// Close releases the root. +func (b *Brain) Close() error { return b.root.Close() } + +// Rel converts a path under the brain dir to a root-relative path. +func (b *Brain) Rel(path string) (string, error) { + rel, err := filepath.Rel(b.dir, path) + if err != nil { + return "", err + } + if rel == ".." || len(rel) > 2 && rel[:3] == ".."+string(filepath.Separator) { + return "", fmt.Errorf("%s is outside the brain dir", path) + } + return filepath.ToSlash(rel), nil +} + +// ReadRegular reads a brain-relative path, refusing anything that is not a +// regular file, both before the open and on the opened handle. +func (b *Brain) ReadRegular(rel string) ([]byte, error) { + fi, err := b.root.Lstat(rel) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular) + } + f, err := b.root.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + hfi, err := f.Stat() + if err != nil { + return nil, err + } + if !hfi.Mode().IsRegular() { + return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular) + } + return io.ReadAll(f) +} + +// WalkRegular walks a brain-relative directory through the root and calls fn +// for each regular file (directories are descended; symlinks, FIFOs and +// devices are skipped, never followed). An entry error is passed to onErr and +// the walk continues; a nil onErr ignores it. +func (b *Brain) WalkRegular(relDir string, fn func(rel string) error, onErr func(rel string, err error)) error { + return fs.WalkDir(b.root.FS(), filepath.ToSlash(relDir), func(p string, d fs.DirEntry, err error) error { + if err != nil { + if onErr != nil { + onErr(p, err) + } + if d != nil && d.IsDir() { + return fs.SkipDir + } + return nil + } + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + return fn(p) + }) +} diff --git a/ingestion/internal/search/brainroot_test.go b/ingestion/internal/search/brainroot_test.go new file mode 100644 index 0000000..77715d4 --- /dev/null +++ b/ingestion/internal/search/brainroot_test.go @@ -0,0 +1,124 @@ +package search + +import ( + "os" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// hyperguild#157 option 1: search walked and read the brain by path, so a +// symlink anywhere in the Syncthing-shared brain dir (wiki///leak.md -> +// /any/file) put the target's contents in brain_query results (reproduced in +// the PR #156 review). Search now walks and reads through an os.Root on the +// brain dir and only reads regular files. + +const sentinel157 = "zqsentinelonefiftyseven" + +func searchBrain(t *testing.T) (brain, facts, outside string) { + t.Helper() + brain = t.TempDir() + facts = filepath.Join(brain, "wiki", "w", "facts") + require.NoError(t, os.MkdirAll(facts, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(brain, "knowledge"), 0o755)) + outside = filepath.Join(t.TempDir(), "secret.md") + require.NoError(t, os.WriteFile(outside, []byte("# secret\n\n"+sentinel157), 0o644)) + return brain, facts, outside +} + +func leaked(rs []Result) int { + n := 0 + for _, r := range rs { + n += strings.Count(r.Excerpt+r.Title, sentinel157) + } + return n +} + +func TestQueryDoesNotFollowASymlinkInTheWiki(t *testing.T) { + brain, facts, outside := searchBrain(t) + require.NoError(t, os.Symlink(outside, filepath.Join(facts, "leak.md"))) + require.NoError(t, os.WriteFile(filepath.Join(facts, "real.md"), []byte("# real\n\nzqrealnote"), 0o644)) + rs, err := Query(brain, QueryOptions{Query: sentinel157, Limit: 10}) + require.NoError(t, err) + assert.Zero(t, leaked(rs)) + rs, err = Query(brain, QueryOptions{Query: "zqrealnote", Limit: 10}) + require.NoError(t, err) + assert.Len(t, rs, 1, "regular notes are still found") +} + +func TestQueryDoesNotFollowASymlinkedDirectory(t *testing.T) { + brain, _, outside := searchBrain(t) + require.NoError(t, os.Symlink(filepath.Dir(outside), filepath.Join(brain, "wiki", "linked"))) + rs, err := Query(brain, QueryOptions{Query: sentinel157, Limit: 10}) + require.NoError(t, err) + assert.Zero(t, leaked(rs)) +} + +func swapInto(path, outside string, fifo bool, stop *atomic.Bool) { + tmp := path + ".swap" + for i := 0; !stop.Load(); i++ { + _ = os.Remove(tmp) + switch { + case i%2 == 0: + _ = os.WriteFile(tmp, []byte("# r\n\nzqinside "+"zqracenote"), 0o644) + case fifo: + _ = syscall.Mkfifo(tmp, 0o644) + default: + _ = os.Symlink(outside, tmp) + } + _ = os.Rename(tmp, path) + } +} + +func TestQueryLeaksNothingUnderASymlinkSwapRace(t *testing.T) { + brain, facts, outside := searchBrain(t) + var stop atomic.Bool + go swapInto(filepath.Join(facts, "r.md"), outside, false, &stop) + defer stop.Store(true) + leaks, iters := 0, 0 + for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); iters++ { + rs, _ := Query(brain, QueryOptions{Query: sentinel157, Limit: 10}) + leaks += leaked(rs) + } + if leaks > 0 { + t.Fatalf("outside file leaked %d times in %d iterations", leaks, iters) + } +} + +func TestQueryDoesNotHangUnderAFIFOSwapRace(t *testing.T) { + brain, facts, outside := searchBrain(t) + var stop atomic.Bool + go swapInto(filepath.Join(facts, "r.md"), outside, true, &stop) + done := make(chan struct{}) + go func() { + for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); { + _, _ = Query(brain, QueryOptions{Query: "zqracenote", Limit: 10}) + } + close(done) + }() + select { + case <-done: + stop.Store(true) + case <-time.After(6 * time.Second): + stop.Store(true) + if f, err := os.OpenFile(filepath.Join(facts, "r.md"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil { + _ = f.Close() + } + t.Fatal("search hung on a swapped-in FIFO") + } +} + +// hydrate (vector-only hits) reads by relative path; it must refuse a symlink. +func TestHydrateRefusesASymlink(t *testing.T) { + brain, facts, outside := searchBrain(t) + require.NoError(t, os.Symlink(outside, filepath.Join(facts, "leak.md"))) + r, err := hydrate(brain, "wiki/w/facts/leak.md") + assert.Error(t, err) + assert.NotContains(t, r.Excerpt, sentinel157) +} diff --git a/ingestion/internal/search/search.go b/ingestion/internal/search/search.go index c272a61..842fd7c 100644 --- a/ingestion/internal/search/search.go +++ b/ingestion/internal/search/search.go @@ -5,6 +5,7 @@ import ( "bufio" "context" "fmt" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter" "log/slog" "os" @@ -119,20 +120,30 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re return nil, err } + // Walk and read through an os.Root on the brain dir, regular files only + // (hyperguild#157): a symlink planted in the Syncthing-shared brain must + // not put an outside file in results, nor a FIFO hang the query. + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("search: %w", err) + } + defer func() { _ = bfs.Close() }() + var results []Result for _, dir := range roots { if _, statErr := os.Stat(dir); os.IsNotExist(statErr) { continue } - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil { - slog.Warn("search: skipping path", "path", path, "err", err) + relDir, relErr := bfs.Rel(dir) + if relErr != nil { + return nil, fmt.Errorf("search: %w", relErr) + } + err := bfs.WalkRegular(relDir, func(relPath string) error { + if !strings.HasSuffix(relPath, ".md") { return nil } - if d.IsDir() || !strings.HasSuffix(path, ".md") { - return nil - } - content, err := os.ReadFile(path) + path := filepath.Join(brainDir, filepath.FromSlash(relPath)) + content, err := bfs.ReadRegular(relPath) if err != nil { slog.Warn("search: skipping unreadable file", "path", path, "err", err) return nil @@ -155,7 +166,7 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re origin, writtenBy := extractProvenance(string(content)) results = append(results, Result{ Path: rel, - Title: extractTitle(string(content), d.Name()), + Title: extractTitle(string(content), filepath.Base(relPath)), Excerpt: excerpt(string(content), 300), Score: score, Wing: wing, @@ -166,6 +177,8 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re WrittenBy: writtenBy, }) return nil + }, func(p string, err error) { + slog.Warn("search: skipping path", "path", p, "err", err) }) if err != nil { return nil, err @@ -275,8 +288,13 @@ func pathInScope(relPath, wing, hall string) bool { // excerpt, wing, and hall populated. Used for paths that surface only // via vector search. func hydrate(brainDir, relPath string) (Result, error) { - full := filepath.Join(brainDir, filepath.FromSlash(relPath)) - content, err := os.ReadFile(full) + // Through the brain root, regular files only (hyperguild#157). + bfs, err := brainfs.Open(brainDir) + if err != nil { + return Result{}, err + } + defer func() { _ = bfs.Close() }() + content, err := bfs.ReadRegular(filepath.ToSlash(relPath)) if err != nil { return Result{}, err }