mathiasandClaude Opus 5.5
74a9621ba6
Sync walked wiki/ and knowledge/ with filepath.WalkDir and read each .md with os.ReadFile. A symlinked note (Syncthing syncs symlinks) had its target embedded into pgvector and served by semantic search; a FIFO stalled the sync. The walk and read now go through brainfs: regular files only, type checked on the opened handle. - Chunks already embedded from a symlinked note are deleted on the next sync, because the note is no longer seen. - A walk error still aborts Sync before the delete pass, so an unreadable subdir does not delete chunks of files that exist. A missing scan dir is skipped as before. - brainfs gains Lstat, used for the mtime staleness check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
98 lines
3.3 KiB
Go
98 lines
3.3 KiB
Go
package vectorstore_test
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
|
)
|
|
|
|
// recordingEmbedder keeps every text it was asked to embed.
|
|
type recordingEmbedder struct {
|
|
mu sync.Mutex
|
|
texts []string
|
|
}
|
|
|
|
func (e *recordingEmbedder) Embed(_ context.Context, text string) ([]float32, error) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
e.texts = append(e.texts, text)
|
|
return []float32{1}, nil
|
|
}
|
|
|
|
func (e *recordingEmbedder) all() string {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
return strings.Join(e.texts, "\n")
|
|
}
|
|
|
|
// hyperguild#157 slice 3: Sync walked wiki/ and knowledge/ and read each .md
|
|
// with os.ReadFile, so a symlinked note had its target embedded and served by
|
|
// semantic search, and a FIFO stalled the sync.
|
|
func TestSyncRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
|
|
brain := t.TempDir()
|
|
writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
|
|
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
|
|
|
|
emb := &recordingEmbedder{}
|
|
store := &stubStore{}
|
|
_, err := vectorstore.Sync(context.Background(), brain, store, emb)
|
|
require.NoError(t, err)
|
|
assert.Contains(t, emb.all(), "inside note", "positive control: the regular note is embedded")
|
|
assert.NotContains(t, emb.all(), brainfstest.Sentinel)
|
|
for p := range store.upserts {
|
|
assert.NotContains(t, p, "leak.md")
|
|
}
|
|
|
|
p := filepath.Join(brain, "knowledge", "r.md")
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
|
|
sync := func() string {
|
|
e := &recordingEmbedder{}
|
|
_, _ = vectorstore.Sync(context.Background(), brain, &stubStore{}, e)
|
|
return e.all()
|
|
}
|
|
brainfstest.AssertNoLeak(t, p, "inside", 1500*time.Millisecond, sync)
|
|
brainfstest.AssertNoHang(t, p, "inside", 1500*time.Millisecond, 6*time.Second, func() { _ = sync() })
|
|
}
|
|
|
|
// Chunks embedded from a symlinked note before this fix are deleted: the
|
|
// note is no longer seen, so the delete pass drops its rows.
|
|
func TestSyncDeletesChunksOfASymlinkedNote(t *testing.T) {
|
|
brain := t.TempDir()
|
|
writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
|
|
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
|
|
store := &stubStore{known: map[string]time.Time{
|
|
"wiki/w/facts/leak.md#0001": {},
|
|
"wiki/w/facts/ok.md#0001": {},
|
|
}}
|
|
_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"wiki/w/facts/leak.md#0001"}, store.deletes)
|
|
}
|
|
|
|
// A walk error aborts Sync before the delete pass, so chunks of files the
|
|
// walk could not see are kept.
|
|
func TestSyncAbortsBeforeDeletingWhenASubdirIsUnreadable(t *testing.T) {
|
|
if os.Geteuid() == 0 {
|
|
t.Skip("root ignores directory permissions")
|
|
}
|
|
brain := t.TempDir()
|
|
writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
|
|
locked := filepath.Join(brain, "wiki", "w")
|
|
require.NoError(t, os.Chmod(locked, 0o000))
|
|
t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
|
|
store := &stubStore{known: map[string]time.Time{"wiki/w/facts/ok.md#0001": {}}}
|
|
_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
|
|
assert.Error(t, err)
|
|
assert.Empty(t, store.deletes)
|
|
}
|