fix(vectorstore): walk and read the brain through brainfs (#157, slice 3) (#166)

vectorstore.Sync walks and reads via brainfs (regular files only, handle-checked), so symlinked notes are no longer embedded and their old chunks are purged. brainfs.WalkRegular requires every component of its root to be a real directory (fixes in-brain root aliasing in Sync and search Wing/Wing+Hall). Walk errors, including a symlinked scan root, abort Sync before the delete pass. brainfs gains Lstat. Directory-swap race recorded as accepted residual (#167).

Refs #157. Adversarial review: BLOCKED, BLOCKED, PASS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/vectorstore/sync_root_test.go

@@ -0,0 +1,126 @@
+package vectorstore_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
+)
+
+// recordingEmbedder keeps every text it was asked to embed.
+type recordingEmbedder struct {
+	mu    sync.Mutex
+	texts []string
+}
+
+func (e *recordingEmbedder) Embed(_ context.Context, text string) ([]float32, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.texts = append(e.texts, text)
+	return []float32{1}, nil
+}
+
+func (e *recordingEmbedder) all() string {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return strings.Join(e.texts, "\n")
+}
+
+// hyperguild#157 slice 3: Sync walked wiki/ and knowledge/ and read each .md
+// with os.ReadFile, so a symlinked note had its target embedded and served by
+// semantic search, and a FIFO stalled the sync.
+func TestSyncRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
+	brain := t.TempDir()
+	writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
+	require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
+
+	emb := &recordingEmbedder{}
+	store := &stubStore{}
+	_, err := vectorstore.Sync(context.Background(), brain, store, emb)
+	require.NoError(t, err)
+	assert.Contains(t, emb.all(), "inside note", "positive control: the regular note is embedded")
+	assert.NotContains(t, emb.all(), brainfstest.Sentinel)
+	for p := range store.upserts {
+		assert.NotContains(t, p, "leak.md")
+	}
+
+	p := filepath.Join(brain, "knowledge", "r.md")
+	require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
+	sync := func() string {
+		e := &recordingEmbedder{}
+		_, _ = vectorstore.Sync(context.Background(), brain, &stubStore{}, e)
+		return e.all()
+	}
+	brainfstest.AssertNoLeak(t, p, "inside", 1500*time.Millisecond, sync)
+	brainfstest.AssertNoHang(t, p, "inside", 1500*time.Millisecond, 6*time.Second, func() { _ = sync() })
+}
+
+// Chunks embedded from a symlinked note before this fix are deleted: the
+// note is no longer seen, so the delete pass drops its rows.
+func TestSyncDeletesChunksOfASymlinkedNote(t *testing.T) {
+	brain := t.TempDir()
+	writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
+	require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
+	store := &stubStore{known: map[string]time.Time{
+		"wiki/w/facts/leak.md#0001": {},
+		"wiki/w/facts/ok.md#0001":   {},
+	}}
+	_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"wiki/w/facts/leak.md#0001"}, store.deletes)
+}
+
+// A walk error aborts Sync before the delete pass, so chunks of files the
+// walk could not see are kept.
+func TestSyncAbortsBeforeDeletingWhenASubdirIsUnreadable(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root ignores directory permissions")
+	}
+	brain := t.TempDir()
+	writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
+	locked := filepath.Join(brain, "wiki", "w")
+	require.NoError(t, os.Chmod(locked, 0o000))
+	t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
+	store := &stubStore{known: map[string]time.Time{"wiki/w/facts/ok.md#0001": {}}}
+	_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
+	assert.Error(t, err)
+	assert.Empty(t, store.deletes)
+}
+
+// A scan root that is a symlink inside the brain (knowledge -> .) was
+// followed by fs.WalkDir, aliasing raw/, sessions/ and root files into the
+// index. Sync now aborts on it, before embedding or deleting anything.
+func TestSyncAbortsOnASymlinkedScanRoot(t *testing.T) {
+	brain := t.TempDir()
+	writeNote(t, brain, "wiki/w/a.md", "wiki note")
+	writeNote(t, brain, "sessions/s1.md", "session note")
+	require.NoError(t, os.Symlink(".", filepath.Join(brain, "knowledge")))
+	emb := &recordingEmbedder{}
+	store := &stubStore{known: map[string]time.Time{"wiki/w/a.md#0001": {}}}
+	_, err := vectorstore.Sync(context.Background(), brain, store, emb)
+	assert.Error(t, err)
+	assert.NotContains(t, emb.all(), "session note")
+	for p := range store.upserts {
+		assert.False(t, strings.HasPrefix(p, "knowledge/"), "upserted %s", p)
+	}
+	assert.Empty(t, store.deletes)
+}
+
+// A missing scan root is skipped; the other root still syncs.
+func TestSyncSkipsAMissingScanRoot(t *testing.T) {
+	brain := t.TempDir()
+	writeNote(t, brain, "wiki/w/a.md", "wiki note")
+	emb := &recordingEmbedder{}
+	_, err := vectorstore.Sync(context.Background(), brain, &stubStore{}, emb)
+	require.NoError(t, err)
+	assert.Contains(t, emb.all(), "wiki note")
+}