Compare commits

..
Author SHA1 Message Date
mathiasandClaude Opus 5.5 31071bc9b1 fix(brainfs): check every component of the walk root (#157)
CI / Lint / Test / Vet (pull_request) Successful in 41s
CI / Mirror to GitHub (pull_request) Skipped
Review round 2 on PR #166: Lstat on the walk root checks only its last
component, so search with Wing+Hall (root wiki/<wing>/<hall>) walked a
symlinked <wing> whose target had a real <hall> directory.

WalkRegular now Lstats every prefix of the root and requires each to be
a real directory. New brainfs and search (Wing+Hall) tests; the Wing-only
search test is renamed to say what it covers.

The doc now records the accepted residual: a directory swapped for a
symlink between listing and read is still followed; os.Root keeps the
read inside the brain but not inside the walk root.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 08:16:50 +02:00
mathiasandClaude Opus 5.5 66b6cf8a77 fix(brainfs): refuse a symlinked walk root (#157)
CI / Lint / Test / Vet (pull_request) Successful in 40s
CI / Mirror to GitHub (pull_request) Skipped
Review round 1 on PR #166: fs.WalkDir stats its root with a follow, and
os.Root allows a symlink that stays inside the brain, so knowledge -> .
aliased raw/, sessions/ and root files into the embeddings (and a
wiki/<w> -> .. root into search results). On main, filepath.WalkDir
Lstat'ed the root and did not descend.

- WalkRegular Lstats its root and requires a real directory; anything
  else goes to onErr as ErrNotRegular and nothing is yielded.
- Sync skips a missing scan root via bfs.Lstat before walking, and every
  other walk error, including a symlinked root, aborts before the delete
  pass. This removes the unpinned rel == sub special case.
- Search warns and skips a symlinked root (new test).
- Test renamed to TestSyncRefusesSymlinkedNotesAndSurvivesSwapRaces.
- wrapAfterStat comment: brainfs tests must not use t.Parallel.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 08:11:16 +02:00
mathiasandClaude Opus 5.5 74a9621ba6 fix(vectorstore): walk and read the brain through brainfs (#157, slice 3)
CI / Lint / Test / Vet (pull_request) Successful in 40s
CI / Mirror to GitHub (pull_request) Skipped
Sync walked wiki/ and knowledge/ with filepath.WalkDir and read each .md
with os.ReadFile. A symlinked note (Syncthing syncs symlinks) had its
target embedded into pgvector and served by semantic search; a FIFO
stalled the sync. The walk and read now go through brainfs: regular
files only, type checked on the opened handle.

- Chunks already embedded from a symlinked note are deleted on the next
  sync, because the note is no longer seen.
- A walk error still aborts Sync before the delete pass, so an unreadable
  subdir does not delete chunks of files that exist. A missing scan dir
  is skipped as before.
- brainfs gains Lstat, used for the mtime staleness check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 08:02:16 +02:00
8 changed files with 49 additions and 397 deletions
+4 -33
View File
@@ -82,28 +82,6 @@ func (b *Brain) realDir(rel string) error {
return nil
}
// ReadDirRegular lists the regular files directly in a brain-relative
// directory, by name. Every component of relDir must be a real directory
// (ErrNotRegular otherwise); symlinks, FIFOs, devices and subdirectories in
// it are left out.
func (b *Brain) ReadDirRegular(relDir string) ([]string, error) {
relDir = filepath.ToSlash(relDir)
if err := b.realDir(relDir); err != nil {
return nil, err
}
ents, err := fs.ReadDir(b.root.FS(), relDir)
if err != nil {
return nil, err
}
var names []string
for _, e := range ents {
if e.Type().IsRegular() {
names = append(names, e.Name())
}
}
return names, nil
}
// ReadRegular reads a brain-relative path, refusing anything that is not a
// regular file, both before the open and on the opened handle.
func (b *Brain) ReadRegular(rel string) ([]byte, error) {
@@ -114,7 +92,7 @@ func (b *Brain) ReadRegular(rel string) ([]byte, error) {
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular)
}
return b.readOpenedMax(rel, -1, fi)
return b.readOpened(rel)
}
// readOpened opens rel non-blocking and reads it only if the OPENED handle is
@@ -128,7 +106,7 @@ func (b *Brain) ReadRegular(rel string) ([]byte, error) {
// cannot create one; fs.protected_hardlinks=1 and the pod's secrets are on a
// different filesystem from the brain volume, so a local caller cannot link
// them in. Refusing Nlink > 1 is the next step if either changes.
func (b *Brain) readOpened(rel string) ([]byte, error) { return b.readOpenedMax(rel, -1, nil) }
func (b *Brain) readOpened(rel string) ([]byte, error) { return b.readOpenedMax(rel, -1) }
// ReadRegularMax is ReadRegular with a size limit, checked on the opened
// handle before reading (max < 0 means no limit).
@@ -140,14 +118,10 @@ func (b *Brain) ReadRegularMax(rel string, max int64) ([]byte, error) {
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular)
}
return b.readOpenedMax(rel, max, fi)
return b.readOpenedMax(rel, max)
}
// readOpenedMax opens rel and reads it if the opened handle is a regular file
// and, when lfi is set, the same file the caller Lstat-ed: os.Root follows an
// in-root final symlink even with O_NOFOLLOW, so a symlink to another brain
// file swapped in between the Lstat and the open is refused only here.
func (b *Brain) readOpenedMax(rel string, max int64, lfi fs.FileInfo) ([]byte, error) {
func (b *Brain) readOpenedMax(rel string, max int64) ([]byte, error) {
f, err := b.root.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, err
@@ -160,9 +134,6 @@ func (b *Brain) readOpenedMax(rel string, max int64, lfi fs.FileInfo) ([]byte, e
if !hfi.Mode().IsRegular() {
return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular)
}
if lfi != nil && !os.SameFile(lfi, hfi) {
return nil, fmt.Errorf("%s: changed while opening: %w", rel, ErrNotRegular)
}
if max >= 0 && hfi.Size() > max {
return nil, fmt.Errorf("%s is %d bytes (limit %d): %w", rel, hfi.Size(), max, ErrTooLarge)
}
@@ -6,8 +6,6 @@ import (
"io/fs"
"os"
"path/filepath"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
@@ -191,74 +189,3 @@ func TestWalkRegularReportsASymlinkedIntermediateComponentOfTheWalkRoot(t *testi
require.Len(t, errs, 1)
assert.ErrorIs(t, errs[0], ErrNotRegular)
}
func TestReadDirRegularListsOnlyRegularFilesAndRefusesSymlinkedComponents(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "a", "b", "sub"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "a", "b", "n.md"), []byte("n"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "other.md"), []byte("o"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "other.md"), filepath.Join(dir, "a", "b", "l.md")))
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "a", "b", "f.md"), 0o644))
require.NoError(t, os.Symlink("a", filepath.Join(dir, "alias")))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "x"), 0o755))
require.NoError(t, os.Symlink(filepath.Join("..", "a"), filepath.Join(dir, "x", "mid")))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
names, err := b.ReadDirRegular("a/b")
require.NoError(t, err)
assert.Equal(t, []string{"n.md"}, names)
for _, rel := range []string{"alias/b", "x/mid/b"} {
_, err := b.ReadDirRegular(rel)
assert.ErrorIs(t, err, ErrNotRegular, rel)
}
}
// A final component swapped for a symlink to another regular file in the
// brain between the Lstat and the open: os.Root follows in-root links, so
// only comparing the two (os.SameFile) refuses it.
func TestReadRegularNeverReturnsAnInBrainFileSwappedInBehindASymlink(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "s.md"), []byte("zqsamefilesentinel"), 0o644))
p := filepath.Join(dir, "r.md")
require.NoError(t, os.WriteFile(p, []byte("inside"), 0o644))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
var stop atomic.Bool
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
tmp := p + ".swap"
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
if i%2 == 0 {
_ = os.WriteFile(tmp, []byte("inside"), 0o644)
} else {
_ = os.Symlink("s.md", tmp)
}
_ = os.Rename(tmp, p)
}
}()
leaks, ok := 0, 0
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
got, err := b.ReadRegular("r.md")
if err == nil {
ok++
}
if string(got) == "zqsamefilesentinel" {
leaks++
}
got, _ = b.ReadRegularMax("r.md", 1<<20)
if string(got) == "zqsamefilesentinel" {
leaks++
}
}
stop.Store(true)
wg.Wait()
assert.Zero(t, leaks)
assert.Positive(t, ok, "positive control: the regular file is read")
}
@@ -7,7 +7,6 @@
package brainfstest
import (
"fmt"
"os"
"path/filepath"
"strings"
@@ -21,23 +20,16 @@ import (
// Sentinel marks the outside file's contents.
const Sentinel = "zqbrainfssentinelonefiftyseven"
// outsideContent carries Sentinel in every field a reader might extract: the
// frontmatter title and aliases, a heading, a wikilink and the body.
const outsideContent = "---\ntitle: " + Sentinel + "\naliases:\n - " + Sentinel + "\n---\n# " + Sentinel +
"\n\n" + Sentinel + " [[" + Sentinel + "]]\n"
// Outside writes a file outside any brain dir containing Sentinel.
func Outside(t *testing.T) string {
t.Helper()
p := filepath.Join(t.TempDir(), "secret.md")
if err := os.WriteFile(p, []byte(outsideContent), 0o644); err != nil {
if err := os.WriteFile(p, []byte("# secret\n\n"+Sentinel), 0o644); err != nil {
t.Fatal(err)
}
return p
}
var swapSeq atomic.Int64
// StartSwap renames, in a loop, a regular note (content inside) and either
// a symlink to outside or a FIFO into path, until the test ends. FIFOs get a
// writer held open until the swapper exits, so a reader that skips the
@@ -50,15 +42,7 @@ func StartSwap(t *testing.T, path, inside, outside string, fifo bool) {
wg.Add(1)
go func() {
defer wg.Done()
// A name of its own: swappers on one path share a test, and one
// opening another's FIFO for write would block forever.
tmp := fmt.Sprintf("%s.swap%d", path, swapSeq.Add(1))
// A relative link: os.Root refuses every absolute link on its own, so
// an absolute in-brain target would test nothing.
link := outside
if rel, err := filepath.Rel(filepath.Dir(path), outside); err == nil {
link = rel
}
tmp := path + ".swap"
var writers []*os.File
defer func() {
for _, w := range writers {
@@ -77,7 +61,7 @@ func StartSwap(t *testing.T, path, inside, outside string, fifo bool) {
}
}
default:
_ = os.Symlink(link, tmp)
_ = os.Symlink(outside, tmp)
}
_ = os.Rename(tmp, path)
}
@@ -85,33 +69,11 @@ func StartSwap(t *testing.T, path, inside, outside string, fifo bool) {
t.Cleanup(func() { stop.Store(true); wg.Wait() })
}
// WriteSentinelAt writes a file carrying Sentinel at p (creating its
// directory), for an in-brain symlink target.
func WriteSentinelAt(t *testing.T, p string) string {
t.Helper()
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(outsideContent), 0o644); err != nil {
t.Fatal(err)
}
return p
}
// AssertNoLeak loops read for d while a symlink swapper runs on path, and
// fails if any output contains Sentinel. The symlink points outside the
// brain; os.Root refuses that on its own, so pair it with AssertNoLeakFrom
// and an in-brain target.
// fails if any output contains Sentinel.
func AssertNoLeak(t *testing.T, path, inside string, d time.Duration, read func() string) {
t.Helper()
AssertNoLeakFrom(t, path, inside, Outside(t), d, read)
}
// AssertNoLeakFrom is AssertNoLeak with the symlink pointing at target, e.g.
// a file elsewhere in the brain written with WriteSentinelAt.
func AssertNoLeakFrom(t *testing.T, path, inside, target string, d time.Duration, read func() string) {
t.Helper()
StartSwap(t, path, inside, target, false)
StartSwap(t, path, inside, Outside(t), false)
leaks, iters := 0, 0
for deadline := time.Now().Add(d); time.Now().Before(deadline); iters++ {
leaks += strings.Count(read(), Sentinel)
+21 -43
View File
@@ -12,14 +12,10 @@ package graphsync
import (
"context"
"errors"
"fmt"
"io/fs"
"path"
"os"
"path/filepath"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/graph"
"github.com/mathiasbq/hyperguild/ingestion/internal/graphstore"
)
@@ -50,25 +46,8 @@ func IndexDoc(ctx context.Context, store Store, brainDir, relPath string) error
if relPath == "" {
return nil
}
// Callers pass caller-supplied paths (brain_tunnel, brain_update): index
// only a clean path under wiki/ or knowledge/.
if path.Clean(relPath) != relPath || !filepath.IsLocal(relPath) ||
(!strings.HasPrefix(relPath, "wiki/") && !strings.HasPrefix(relPath, "knowledge/")) {
return fmt.Errorf("index %q: not a clean wiki/ or knowledge/ path", relPath)
}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
return indexDoc(ctx, store, bfs, relPath)
}
// indexDoc reads relPath through the brain root, regular files only
// (hyperguild#157): the brain is a Syncthing folder, and os.ReadFile
// followed a synced symlink into the graph and blocked on a FIFO.
func indexDoc(ctx context.Context, store Store, bfs *brainfs.Brain, relPath string) error {
content, err := bfs.ReadRegular(relPath)
abs := filepath.Join(brainDir, filepath.FromSlash(relPath))
content, err := os.ReadFile(abs)
if err != nil {
return fmt.Errorf("read %q: %w", relPath, err)
}
@@ -98,34 +77,33 @@ func BackfillFromBrainDir(ctx context.Context, store Store, brainDir string) (in
if store == nil {
return 0, nil
}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return 0, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
for _, root := range []string{"wiki", "knowledge"} {
if _, err := bfs.Lstat(root); errors.Is(err, fs.ErrNotExist) {
roots := []string{"wiki", "knowledge"}
for _, root := range roots {
base := filepath.Join(brainDir, root)
if _, err := os.Stat(base); os.IsNotExist(err) {
continue
}
// Any walk error, including a root that is a symlink, aborts.
var walkErr error
err := bfs.WalkRegular(root, func(rel string) error {
if path.Ext(rel) != ".md" {
err := filepath.WalkDir(base, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
if err := indexDoc(ctx, store, bfs, rel); err != nil {
if filepath.Ext(path) != ".md" {
return nil
}
rel, relErr := filepath.Rel(brainDir, path)
if relErr != nil {
return fmt.Errorf("rel %q: %w", path, relErr)
}
rel = filepath.ToSlash(rel)
if err := IndexDoc(ctx, store, brainDir, rel); err != nil {
return fmt.Errorf("index %q: %w", rel, err)
}
indexed++
return nil
}, func(rel string, err error) {
if walkErr == nil {
walkErr = fmt.Errorf("%s: %w", rel, err)
}
})
if err == nil {
err = walkErr
}
if err != nil {
return indexed, fmt.Errorf("walk %s: %w", root, err)
}
@@ -1,92 +0,0 @@
package graphsync
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
func (f *fakeStore) dump() string {
f.mu.Lock()
defer f.mu.Unlock()
return fmt.Sprintf("%+v %+v", f.upserts, f.replaces)
}
// hyperguild#157 slice 4: IndexDoc and BackfillFromBrainDir read notes with
// os.ReadFile, so a symlinked note put the target's title and wikilinks into
// the graph, and a FIFO hung the read.
func TestIndexDocRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: zqinside\n---\nbody [[other]]\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
store := newFakeStore()
require.NoError(t, IndexDoc(context.Background(), store, brain, "wiki/w/facts/ok.md"))
assert.Contains(t, store.dump(), "zqinside", "positive control: a regular note is indexed")
assert.Error(t, IndexDoc(context.Background(), store, brain, "wiki/w/facts/leak.md"))
// A link to a file elsewhere in the brain: os.Root allows it, only the
// regular-file check refuses it.
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
assert.Error(t, IndexDoc(context.Background(), store, brain, "wiki/w/facts/inlink.md"))
assert.NotContains(t, store.dump(), brainfstest.Sentinel)
p := filepath.Join(facts, "r.md")
read := func() string {
s := newFakeStore()
_ = IndexDoc(context.Background(), s, brain, "wiki/w/facts/r.md")
return s.dump()
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read)
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: zqinside\n---\n", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
func TestBackfillSkipsSymlinkedNotesAndAbortsOnASymlinkedRoot(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "r.md"), filepath.Join(facts, "inlink.md")))
require.NoError(t, os.Symlink(".", filepath.Join(brain, "knowledge")))
store := newFakeStore()
_, err := BackfillFromBrainDir(context.Background(), store, brain)
assert.Error(t, err, "a symlinked root aborts the backfill")
assert.NotContains(t, store.dump(), brainfstest.Sentinel)
assert.NotContains(t, store.dump(), "zqrawonly")
require.NoError(t, os.Remove(filepath.Join(brain, "knowledge")))
store = newFakeStore()
n, err := BackfillFromBrainDir(context.Background(), store, brain)
require.NoError(t, err)
assert.Equal(t, 1, n, "only the regular note is indexed")
assert.Contains(t, store.dump(), "zqinside")
assert.NotContains(t, store.dump(), brainfstest.Sentinel)
}
// IndexDoc gets caller-supplied paths (brain_tunnel, brain_update): only a
// clean path under wiki/ or knowledge/ is indexed.
func TestIndexDocRefusesPathsOutsideWikiAndKnowledgeAndUncleanPaths(t *testing.T) {
brain := t.TempDir()
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "secret.md"))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki", "w"), 0o755))
for _, p := range []string{"raw/secret.md", "wiki/w/../../raw/secret.md", "./raw/secret.md", "/raw/secret.md"} {
store := newFakeStore()
assert.Error(t, IndexDoc(context.Background(), store, brain, p), p)
assert.NotContains(t, store.dump(), brainfstest.Sentinel, p)
}
}
+4 -11
View File
@@ -7,7 +7,6 @@ import (
"path/filepath"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/textutil"
)
@@ -17,11 +16,6 @@ func RebuildIndex(brainDir, date string) error {
if err != nil {
return fmt.Errorf("load inventory: %w", err)
}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
total := len(inv[PageTypeConcept]) + len(inv[PageTypeEntity]) + len(inv[PageTypeSource])
var sb strings.Builder
@@ -40,7 +34,7 @@ func RebuildIndex(brainDir, date string) error {
label := strings.ToUpper(string(pt)[:1]) + string(pt)[1:]
fmt.Fprintf(&sb, "## %s\n\n", label)
for _, e := range entries {
summary := pageFirstSentence(bfs, e)
summary := pageFirstSentence(brainDir, e)
if summary != "" {
fmt.Fprintf(&sb, "- [[%s|%s]] — %s\n", e.Slug, e.Title, summary)
} else {
@@ -54,10 +48,9 @@ func RebuildIndex(brainDir, date string) error {
return os.WriteFile(dest, []byte(sb.String()), 0o644)
}
// pageFirstSentence reads the page through the brain root, regular files
// only (hyperguild#157).
func pageFirstSentence(bfs *brainfs.Brain, e Entry) string {
content, err := bfs.ReadRegular("wiki/" + string(e.Type) + "/" + e.Slug + ".md")
func pageFirstSentence(brainDir string, e Entry) string {
path := filepath.Join(brainDir, "wiki", string(e.Type), e.Slug+".md")
content, err := os.ReadFile(path)
if err != nil {
return ""
}
+15 -23
View File
@@ -3,45 +3,36 @@ package wiki
import (
"bufio"
"bytes"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
)
// LoadInventory walks brain/wiki/ and returns all pages grouped by type.
// Missing subdirectories are silently skipped. Pages are listed and read
// through the brain root, regular files only (hyperguild#157); a type
// directory that is a symlink is an error.
// Missing subdirectories are silently skipped.
func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
result := map[PageType][]Entry{
PageTypeConcept: {},
PageTypeEntity: {},
PageTypeSource: {},
}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
for pt := range result {
dir := "wiki/" + string(pt)
names, err := bfs.ReadDirRegular(dir)
if errors.Is(err, fs.ErrNotExist) {
dir := filepath.Join(brainDir, "wiki", string(pt))
entries, err := os.ReadDir(dir)
if os.IsNotExist(err) {
continue
}
if err != nil {
return nil, fmt.Errorf("read dir %s: %w", dir, err)
}
for _, name := range names {
if !strings.HasSuffix(name, ".md") {
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") {
continue
}
slug := strings.TrimSuffix(name, ".md")
title, aliases := readFrontmatter(bfs, dir+"/"+name, slug)
slug := strings.TrimSuffix(e.Name(), ".md")
path := filepath.Join(dir, e.Name())
title, aliases := readFrontmatter(path, slug)
result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt})
}
}
@@ -50,14 +41,15 @@ func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
// readFrontmatter extracts title and aliases from YAML frontmatter.
// Falls back to slug for title and empty aliases on any error.
func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string, aliases []string) {
func readFrontmatter(path, fallbackSlug string) (title string, aliases []string) {
title = fallbackSlug
content, err := bfs.ReadRegular(rel)
f, err := os.Open(path)
if err != nil {
return
}
defer f.Close() //nolint:errcheck
scanner := bufio.NewScanner(bytes.NewReader(content))
scanner := bufio.NewScanner(f)
inFM := false
inAliases := false
for scanner.Scan() {
-79
View File
@@ -1,79 +0,0 @@
package wiki
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
// hyperguild#157 slice 4: LoadInventory listed wiki/<type>/ with os.ReadDir
// and opened each .md by path, so a symlinked page put the target's title and
// aliases into the inventory (and the LLM prompts built from it), and a FIFO
// hung it. RebuildIndex read first sentences the same way.
func TestLoadInventoryRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
dir := filepath.Join(brain, "wiki", "concepts")
require.NoError(t, os.MkdirAll(dir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(dir, "leak.md")))
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(dir, "inlink.md")))
inv, err := LoadInventory(brain)
require.NoError(t, err)
got := fmt.Sprintf("%+v", inv)
assert.Contains(t, got, "zqinside", "positive control")
assert.NotContains(t, got, brainfstest.Sentinel)
assert.NotContains(t, got, "leak", "a symlinked page is not listed")
assert.NotContains(t, got, "inlink", "a page linked to another brain file is not listed")
p := filepath.Join(dir, "r.md")
read := func() string {
inv, _ := LoadInventory(brain)
return fmt.Sprintf("%+v", inv)
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read)
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: zqinside\n---\n", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
func TestLoadInventoryReturnsErrNotRegularForASymlinkedTypeDir(t *testing.T) {
brain := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki"), 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(brain, "wiki", "concepts")))
inv, err := LoadInventory(brain)
require.ErrorIs(t, err, brainfs.ErrNotRegular)
assert.NotContains(t, fmt.Sprintf("%+v", inv), "zqrawonly")
}
func TestRebuildIndexDoesNotReadAPageSwappedForASymlink(t *testing.T) {
brain := t.TempDir()
dir := filepath.Join(brain, "wiki", "concepts")
require.NoError(t, os.MkdirAll(dir, 0o755))
p := filepath.Join(dir, "r.md")
require.NoError(t, os.WriteFile(p, []byte("---\ntitle: R\n---\nzqinside sentence.\n"), 0o644))
require.NoError(t, RebuildIndex(brain, "2026-10-04"))
idx, err := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
require.NoError(t, err)
require.Contains(t, string(idx), "zqinside", "positive control: the summary is read")
read := func() string {
_ = RebuildIndex(brain, "2026-10-04")
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
return string(b)
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, read)
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}