Compare commits

..
Author SHA1 Message Date
mathiasandClaude Opus 5.5 8fdf7284aa fix(linkcheck): skip a linked tree that is not a real directory (#157)
CI / Lint / Test / Vet (pull_request) Successful in 56s
CI / Mirror to GitHub (pull_request) Skipped
Review round 1 on PR #175:

- entryIndex returned WalkRegular's walk-root ErrNotRegular as an error,
  and StripUnresolvedLinks runs before every brain_write: a symlinked,
  dangling or regular-file knowledge/ (or wiki/) made every write fail
  and lose the learning. main's filepath.WalkDir skipped such a root. It
  is now skipped with a warning; other walk errors still abort, as on
  main. New table test over the three shapes.
- The symlinked-hall assertion passed with no index written at all. It
  now indexes a real hall next to the symlinked one and asserts no
  error, the index exists and lists the real note; a new test does the
  same for BuildAllWingIndexes with a symlinked wing next to a real one.
- The test comment no longer says this PR fixed the FIFO hang on every
  brain_write: that path is tunnel.DetectTunnels (slice 5b).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 07:17:43 +02:00
mathiasandClaude Opus 5.5 6e3c51c952 fix(brain): wing indexes and the link check read through brainfs (#157, slice 5a)
CI / Lint / Test / Vet (pull_request) Successful in 56s
CI / Mirror to GitHub (pull_request) Skipped
BuildWingIndex/BuildAllWingIndexes listed wiki/<wing>/<hall>/ with
os.ReadDir and opened each note by path: a symlinked note put the
target's title into _index.md (110k leaks in the swap race), a symlinked
wing was followed and its _index.md written into the target, and a FIFO
hung the build (it runs on every brain_write).

- New brainfs.ReadDirDirs: real subdirectories only, every component of
  the dir a real directory. Wings and halls are listed with it and
  ReadDirRegular; titles are read with ReadRegular
  (readTitleAndCreatedIn). A symlinked wing is an error; a symlinked hall
  is skipped.
- linkcheck entryIndex walks with WalkRegular: a symlinked .md name no
  longer makes a [[link]] resolve.
- readTitleAndCreated (path-based) stays for tunnel.DetectTunnels until
  PR #170 lands; that is slice 5b.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 07:09:03 +02:00
8 changed files with 26 additions and 254 deletions
@@ -23,13 +23,6 @@ func TestReadNoteRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
_, body, _, err := api.ReadNote(brain, "wiki/w/facts/leak.md")
assert.Error(t, err)
assert.NotContains(t, body, brainfstest.Sentinel)
// A link to a file elsewhere in the brain: os.Root allows it, only the
// regular-file check refuses it (hyperguild#171).
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
_, body, _, err = api.ReadNote(brain, "wiki/w/facts/inlink.md")
assert.Error(t, err)
assert.NotContains(t, body, brainfstest.Sentinel)
p := filepath.Join(facts, "r.md")
require.NoError(t, os.WriteFile(p, []byte("---\nx: y\n---\ninside"), 0o644))
@@ -46,7 +39,5 @@ func TestReadNoteRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
return s
}
brainfstest.AssertNoLeak(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, read)
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
brainfstest.AssertNoLeakFrom(t, p, "---\nx: y\n---\ninside", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
+1 -1
View File
@@ -218,7 +218,7 @@ func TestReadDirRegularListsOnlyRegularFilesAndRefusesSymlinkedComponents(t *tes
// A final component swapped for a symlink to another regular file in the
// brain between the Lstat and the open: os.Root follows in-root links, so
// only comparing the two (os.SameFile) refuses it.
func TestReadRegularDoesNotReturnAnInBrainFileSwappedInBehindASymlink(t *testing.T) {
func TestReadRegularNeverReturnsAnInBrainFileSwappedInBehindASymlink(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "s.md"), []byte("zqsamefilesentinel"), 0o644))
p := filepath.Join(dir, "r.md")
@@ -1,10 +1,8 @@
// Package brainfstest is the shared harness for hyperguild#157's swap-race
// tests: it keeps swapping a regular note, a symlink to a sentinel target file
// (outside the brain, or inside it for hyperguild#171), or a FIFO (with a
// writer held open) into a path while a reader under test loops.
// It observes two things, over a sampled window only: no sentinel byte from
// the target reaches the reader's output, and the reader returns within a
// timeout. It does not cover
// tests: it keeps swapping a regular note, a symlink to an outside file, or a
// FIFO (with a writer held open) into a path while a reader under test loops.
// It observes two things: no sentinel byte from the outside file reaches the
// reader's output, and the reader returns within a timeout. It does not cover
// hard links, other paths into the reader, or races it does not generate.
package brainfstest
@@ -20,7 +18,7 @@ import (
"time"
)
// Sentinel marks the target file's contents.
// Sentinel marks the outside file's contents.
const Sentinel = "zqbrainfssentinelonefiftyseven"
// outsideContent carries Sentinel in every field a reader might extract: the
@@ -29,7 +27,7 @@ const outsideContent = "---\ntitle: " + Sentinel + "\naliases:\n - " + Sentinel
"\n\n" + Sentinel + " [[" + Sentinel + "]]\n"
// Outside writes a file outside any brain dir containing Sentinel.
func Outside(t testing.TB) string {
func Outside(t *testing.T) string {
t.Helper()
p := filepath.Join(t.TempDir(), "secret.md")
if err := os.WriteFile(p, []byte(outsideContent), 0o644); err != nil {
@@ -44,12 +42,9 @@ var swapSeq atomic.Int64
// a symlink to outside or a FIFO into path, until the test ends. FIFOs get a
// writer held open until the swapper exits, so a reader that skips the
// handle check parks forever instead of reading EOF. The swapper is stopped
// and awaited before t.TempDir cleanup. It returns the number of symlinks
// or FIFOs actually renamed into path so far, so a caller can tell a race
// that ran from one whose every swap failed (hyperguild#177).
func StartSwap(t testing.TB, path, inside, outside string, fifo bool) *atomic.Int64 {
// and awaited before t.TempDir cleanup.
func StartSwap(t *testing.T, path, inside, outside string, fifo bool) {
t.Helper()
var swaps atomic.Int64
var stop atomic.Bool
var wg sync.WaitGroup
wg.Add(1)
@@ -72,26 +67,22 @@ func StartSwap(t testing.TB, path, inside, outside string, fifo bool) *atomic.In
}()
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
var made error
switch {
case i%2 == 0:
_ = os.WriteFile(tmp, []byte(inside), 0o644)
case fifo:
if made = syscall.Mkfifo(tmp, 0o644); made == nil {
if err := syscall.Mkfifo(tmp, 0o644); err == nil {
if w, err := os.OpenFile(tmp, os.O_RDWR, 0); err == nil {
writers = append(writers, w)
}
}
default:
made = os.Symlink(link, tmp)
}
if err := os.Rename(tmp, path); err == nil && made == nil && i%2 == 1 {
swaps.Add(1)
_ = os.Symlink(link, tmp)
}
_ = os.Rename(tmp, path)
}
}()
t.Cleanup(func() { stop.Store(true); wg.Wait() })
return &swaps
}
// WriteSentinelAt writes a file carrying Sentinel at p (creating its
@@ -116,40 +107,17 @@ func AssertNoLeak(t *testing.T, path, inside string, d time.Duration, read func(
AssertNoLeakFrom(t, path, inside, Outside(t), d, read)
}
// InsideMarker must appear in the inside note's content and in read's
// output whenever read saw that note: the floor below counts such reads.
const InsideMarker = "inside"
// AssertNoLeakFrom is AssertNoLeak with the symlink pointing at target, e.g.
// a file elsewhere in the brain written with WriteSentinelAt.
//
// Beyond "no Sentinel in any output" it sets two floors, because without
// them the assertion passes when nothing was tested (hyperguild#177): at
// least one symlink must have been swapped into path, and at least one read
// must have returned InsideMarker (a reader that always errors returns
// nothing, and so never leaks).
func AssertNoLeakFrom(t testing.TB, path, inside, target string, d time.Duration, read func() string) {
func AssertNoLeakFrom(t *testing.T, path, inside, target string, d time.Duration, read func() string) {
t.Helper()
if !strings.Contains(inside, InsideMarker) {
t.Fatalf("inside content must contain %q", InsideMarker)
}
swaps := StartSwap(t, path, inside, target, false)
leaks, iters, insideReads := 0, 0, 0
StartSwap(t, path, inside, target, false)
leaks, iters := 0, 0
for deadline := time.Now().Add(d); time.Now().Before(deadline); iters++ {
out := read()
leaks += strings.Count(out, Sentinel)
if strings.Contains(out, InsideMarker) {
insideReads++
}
leaks += strings.Count(read(), Sentinel)
}
if leaks > 0 {
t.Fatalf("symlink target %s leaked %d times in %d iterations", target, leaks, iters)
}
if n := swaps.Load(); n == 0 {
t.Fatalf("no symlink was swapped into %s in %d iterations: the race did not run", path, iters)
}
if insideReads == 0 {
t.Fatalf("no read in %d iterations returned the inside note (%q): the reader read nothing", iters, InsideMarker)
t.Fatalf("outside file leaked %d times in %d iterations", leaks, iters)
}
}
@@ -1,79 +0,0 @@
package brainfstest
import (
"fmt"
"os"
"path/filepath"
"runtime"
"syscall"
"testing"
"time"
)
// recordingTB stands in for the test under AssertNoLeakFrom so a fatal
// there can be observed instead of failing this test. Fatalf ends the
// calling goroutine the way testing.T does.
type recordingTB struct {
testing.TB
failed bool
msg string
}
func (r *recordingTB) Fatalf(format string, args ...any) {
r.failed, r.msg = true, fmt.Sprintf(format, args...)
runtime.Goexit()
}
func runAssert(t *testing.T, path, target string, read func() string) *recordingTB {
t.Helper()
r := &recordingTB{TB: t}
done := make(chan struct{})
go func() {
defer close(done)
AssertNoLeakFrom(r, path, "inside", target, 200*time.Millisecond, read)
}()
<-done
return r
}
// hyperguild#177: with every swap failing, no leak can occur, so the leak
// assertion passed without testing anything.
func TestAssertNoLeakFromFailsWhenNoSymlinkSwapSucceeds(t *testing.T) {
path := filepath.Join(t.TempDir(), "missing-dir", "r.md")
r := runAssert(t, path, Outside(t), func() string { return "inside" })
if !r.failed {
t.Fatal("AssertNoLeakFrom passed with zero successful symlink swaps")
}
}
// hyperguild#177: a reader that always errors returns nothing, so it never
// leaks either.
func TestAssertNoLeakFromFailsWhenNoReadReturnsTheInsideNote(t *testing.T) {
path := filepath.Join(t.TempDir(), "r.md")
r := runAssert(t, path, Outside(t), func() string { return "" })
if !r.failed {
t.Fatal("AssertNoLeakFrom passed with zero reads returning the inside note")
}
}
// Positive control: a reader that refuses symlinks and reads regular files
// passes, so the floors do not fail a working reader.
func TestAssertNoLeakFromPassesAReaderThatRefusesSymlinks(t *testing.T) {
path := filepath.Join(t.TempDir(), "r.md")
read := func() string {
f, err := os.OpenFile(path, os.O_RDONLY|noFollow, 0)
if err != nil {
return ""
}
defer func() { _ = f.Close() }()
b := make([]byte, 64)
n, _ := f.Read(b)
return string(b[:n])
}
r := runAssert(t, path, Outside(t), read)
if r.failed {
t.Fatalf("a symlink-refusing reader failed: %s", r.msg)
}
}
const noFollow = syscall.O_NOFOLLOW
@@ -21,11 +21,6 @@ func TestReadExcerptRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
assert.NotContains(t, readExcerpt(brain, "wiki/w/facts/leak.md", 500), brainfstest.Sentinel)
// A link to a file elsewhere in the brain: os.Root allows it, only the
// regular-file check refuses it (hyperguild#171).
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
assert.NotContains(t, readExcerpt(brain, "wiki/w/facts/inlink.md", 500), brainfstest.Sentinel)
p := filepath.Join(facts, "r.md")
require.NoError(t, os.WriteFile(p, []byte("---\nx: y\n---\ninside"), 0o644))
@@ -33,9 +28,6 @@ func TestReadExcerptRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brainfstest.AssertNoLeak(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, func() string {
return readExcerpt(brain, "wiki/w/facts/r.md", 500)
})
brainfstest.AssertNoLeakFrom(t, p, "---\nx: y\n---\ninside", inBrain, 1500*time.Millisecond, func() string {
return readExcerpt(brain, "wiki/w/facts/r.md", 500)
})
brainfstest.AssertNoHang(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, 6*time.Second, func() {
_ = readExcerpt(brain, "wiki/w/facts/r.md", 500)
})
@@ -38,31 +38,3 @@ func TestReadScopedRefusesANoteOverMaxNoteBytes(t *testing.T) {
_, err := ReadScoped(brain, "wiki/w/facts/big.md", ScopeNotes)
assert.ErrorIs(t, err, ErrTooLarge)
}
// hyperguild#171 (review of #172, finding 1): os.Root alone refuses links that
// leave the brain, so the outside-target race above stays green for a reader
// that skips brainfs's Lstat and SameFile checks. An in-brain target (raw/,
// outside ScopeNotes) is what tells them apart.
func TestReadScopedDoesNotReturnAnInBrainSymlinkTargetStaticallyOrDuringASampledSwapRace(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
target := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
b, err := ReadScoped(brain, "wiki/w/facts/inlink.md", ScopeNotes)
assert.Error(t, err, "static in-brain link")
assert.NotContains(t, string(b), brainfstest.Sentinel, "static in-brain link")
p := filepath.Join(facts, "r.md")
require.NoError(t, os.WriteFile(p, []byte("inside"), 0o644))
b, err = ReadScoped(brain, "wiki/w/facts/r.md", ScopeNotes)
require.NoError(t, err)
assert.Equal(t, "inside", string(b), "positive control: a regular note reads")
read := func() string {
b, _ := ReadScoped(brain, "wiki/w/facts/r.md", ScopeNotes)
return string(b)
}
brainfstest.AssertNoLeakFrom(t, p, "inside", target, 1500*time.Millisecond, read)
}
+9 -74
View File
@@ -12,8 +12,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
// hyperguild#157 option 1: search walked and read the brain by path, so a
@@ -95,20 +93,16 @@ func TestQueryDoesNotFollowASymlinkedWingUnderAWingAndHallRoot(t *testing.T) {
// bypass of readOpened hung Query after 5 iterations).
// startSwap runs swapInto until the test ends and waits for it before
// t.TempDir's cleanup (hyperguild#162).
// It returns the count of symlinks or FIFOs actually renamed into path
// (hyperguild#177: a swapper whose every swap fails tests nothing).
func startSwap(t *testing.T, path, outside string, fifo bool) *atomic.Int64 {
func startSwap(t *testing.T, path, outside string, fifo bool) {
t.Helper()
var stop atomic.Bool
var swaps atomic.Int64
var wg sync.WaitGroup
wg.Add(1)
go func() { defer wg.Done(); swapInto(path, outside, fifo, &stop, &swaps) }()
go func() { defer wg.Done(); swapInto(path, outside, fifo, &stop) }()
t.Cleanup(func() { stop.Store(true); wg.Wait() })
return &swaps
}
func swapInto(path, outside string, fifo bool, stop *atomic.Bool, swaps *atomic.Int64) {
func swapInto(path, outside string, fifo bool, stop *atomic.Bool) {
tmp := path + ".swap"
// Writers stay open until the swapper exits, so a reader parked on a
// FIFO stays parked (closing one would hand the reader EOF and hide the
@@ -121,46 +115,33 @@ func swapInto(path, outside string, fifo bool, stop *atomic.Bool, swaps *atomic.
}()
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
var made error
switch {
case i%2 == 0:
_ = os.WriteFile(tmp, []byte("# r\n\nzqinside "+"zqracenote"), 0o644)
case fifo:
if made = syscall.Mkfifo(tmp, 0o644); made == nil {
if err := syscall.Mkfifo(tmp, 0o644); err == nil {
if w, err := os.OpenFile(tmp, os.O_RDWR, 0); err == nil {
writers = append(writers, w)
}
}
default:
made = os.Symlink(outside, tmp)
}
if err := os.Rename(tmp, path); err == nil && made == nil && i%2 == 1 {
swaps.Add(1)
_ = os.Symlink(outside, tmp)
}
_ = os.Rename(tmp, path)
}
}
func TestQueryLeaksNothingUnderASymlinkSwapRace(t *testing.T) {
brain, facts, outside := searchBrain(t)
swaps := startSwap(t, filepath.Join(facts, "r.md"), outside, false)
leaks, iters, insideReads := 0, 0, 0
startSwap(t, filepath.Join(facts, "r.md"), outside, false)
leaks, iters := 0, 0
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); iters++ {
rs, err := Query(brain, QueryOptions{Query: sentinel157, Limit: 10})
require.NoError(t, err)
rs, _ := Query(brain, QueryOptions{Query: sentinel157, Limit: 10})
leaks += leaked(rs)
// Positive control (hyperguild#177): a Query that always errors
// returns nothing and so never leaks.
rs, err = Query(brain, QueryOptions{Query: "zqracenote", Limit: 10})
require.NoError(t, err)
if len(rs) > 0 {
insideReads++
}
}
if leaks > 0 {
t.Fatalf("outside file leaked %d times in %d iterations", leaks, iters)
}
require.Positive(t, swaps.Load(), "no symlink was swapped in: the race did not run")
require.Positive(t, insideReads, "no Query returned the inside note")
}
func TestQueryDoesNotHangUnderAFIFOSwapRace(t *testing.T) {
@@ -191,49 +172,3 @@ func TestHydrateRefusesASymlink(t *testing.T) {
assert.Error(t, err)
assert.NotContains(t, r.Excerpt, sentinel157)
}
// hyperguild#171: os.Root refuses links that leave the brain on its own, so
// the outside-target cases above would stay green for a caller that read
// without brainfs's Lstat and SameFile checks. These targets are inside the
// brain (raw/, which is not a search root).
func inBrainTarget(t *testing.T, brain string) string {
t.Helper()
return brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
}
func TestQueryDoesNotReturnAnInBrainSymlinkTargetDuringASampledSwapRace(t *testing.T) {
brain, facts, _ := searchBrain(t)
target := inBrainTarget(t, brain)
read := func() string {
// Both terms, so the regular note is a possible hit: a reader that
// never returns the inside note would pass vacuously (#177 floor).
rs, _ := Query(brain, QueryOptions{Query: brainfstest.Sentinel + " zqinside", Limit: 10})
s := ""
for _, r := range rs {
s += r.Title + r.Excerpt
}
return s
}
brainfstest.AssertNoLeakFrom(t, filepath.Join(facts, "r.md"), "# r\n\nzqinside", target, 1500*time.Millisecond, read)
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
assert.NotContains(t, read(), brainfstest.Sentinel, "static in-brain link")
}
func TestHydrateRefusesAnInBrainSymlinkAndDoesNotReturnItsTargetDuringASampledSwapRace(t *testing.T) {
brain, facts, _ := searchBrain(t)
target := inBrainTarget(t, brain)
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
r, err := hydrate(brain, "wiki/w/facts/inlink.md")
assert.Error(t, err)
assert.NotContains(t, r.Excerpt+r.Title, brainfstest.Sentinel)
p := filepath.Join(facts, "r.md")
require.NoError(t, os.WriteFile(p, []byte("# r\n\nzqinside"), 0o644))
r, err = hydrate(brain, "wiki/w/facts/r.md")
require.NoError(t, err)
assert.Contains(t, r.Excerpt, "zqinside", "positive control: a regular note hydrates")
brainfstest.AssertNoLeakFrom(t, p, "# r\n\nzqinside", target, 1500*time.Millisecond, func() string {
r, _ := hydrate(brain, "wiki/w/facts/r.md")
return r.Title + r.Excerpt
})
}
@@ -42,11 +42,6 @@ func TestSyncRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
// A link to a file elsewhere in the brain: os.Root allows it, only the
// regular-file checks refuse it (hyperguild#171). raw/ is not a scan root,
// so its sentinel can only arrive through the link.
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "w", "facts", "inlink.md")))
emb := &recordingEmbedder{}
store := &stubStore{}
@@ -56,7 +51,6 @@ func TestSyncRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
assert.NotContains(t, emb.all(), brainfstest.Sentinel)
for p := range store.upserts {
assert.NotContains(t, p, "leak.md")
assert.NotContains(t, p, "inlink.md")
}
p := filepath.Join(brain, "knowledge", "r.md")
@@ -67,7 +61,6 @@ func TestSyncRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
return e.all()
}
brainfstest.AssertNoLeak(t, p, "inside", 1500*time.Millisecond, sync)
brainfstest.AssertNoLeakFrom(t, p, "inside", inBrain, 1500*time.Millisecond, sync)
brainfstest.AssertNoHang(t, p, "inside", 1500*time.Millisecond, 6*time.Second, func() { _ = sync() })
}