hyperguild/ingestion/internal/vectorstore/sync_root_test.go

package vectorstore_test

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
	"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
)

// recordingEmbedder keeps every text it was asked to embed.
type recordingEmbedder struct {
	mu    sync.Mutex
	texts []string
}

func (e *recordingEmbedder) Embed(_ context.Context, text string) ([]float32, error) {
	e.mu.Lock()
	defer e.mu.Unlock()
	e.texts = append(e.texts, text)
	return []float32{1}, nil
}

func (e *recordingEmbedder) all() string {
	e.mu.Lock()
	defer e.mu.Unlock()
	return strings.Join(e.texts, "\n")
}

// hyperguild#157 slice 3: Sync walked wiki/ and knowledge/ and read each .md
// with os.ReadFile, so a symlinked note had its target embedded and served by
// semantic search, and a FIFO stalled the sync.
func TestSyncRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
	brain := t.TempDir()
	writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
	require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))

	emb := &recordingEmbedder{}
	store := &stubStore{}
	_, err := vectorstore.Sync(context.Background(), brain, store, emb)
	require.NoError(t, err)
	assert.Contains(t, emb.all(), "inside note", "positive control: the regular note is embedded")
	assert.NotContains(t, emb.all(), brainfstest.Sentinel)
	for p := range store.upserts {
		assert.NotContains(t, p, "leak.md")
	}

	p := filepath.Join(brain, "knowledge", "r.md")
	require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
	sync := func() string {
		e := &recordingEmbedder{}
		_, _ = vectorstore.Sync(context.Background(), brain, &stubStore{}, e)
		return e.all()
	}
	brainfstest.AssertNoLeak(t, p, "inside", 1500*time.Millisecond, sync)
	brainfstest.AssertNoHang(t, p, "inside", 1500*time.Millisecond, 6*time.Second, func() { _ = sync() })
}

// Chunks embedded from a symlinked note before this fix are deleted: the
// note is no longer seen, so the delete pass drops its rows.
func TestSyncDeletesChunksOfASymlinkedNote(t *testing.T) {
	brain := t.TempDir()
	writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
	require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
	store := &stubStore{known: map[string]time.Time{
		"wiki/w/facts/leak.md#0001": {},
		"wiki/w/facts/ok.md#0001":   {},
	}}
	_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
	require.NoError(t, err)
	assert.Equal(t, []string{"wiki/w/facts/leak.md#0001"}, store.deletes)
}

// A walk error aborts Sync before the delete pass, so chunks of files the
// walk could not see are kept.
func TestSyncAbortsBeforeDeletingWhenASubdirIsUnreadable(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("root ignores directory permissions")
	}
	brain := t.TempDir()
	writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
	locked := filepath.Join(brain, "wiki", "w")
	require.NoError(t, os.Chmod(locked, 0o000))
	t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
	store := &stubStore{known: map[string]time.Time{"wiki/w/facts/ok.md#0001": {}}}
	_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
	assert.Error(t, err)
	assert.Empty(t, store.deletes)
}

// A scan root that is a symlink inside the brain (knowledge -> .) was
// followed by fs.WalkDir, aliasing raw/, sessions/ and root files into the
// index. Sync now aborts on it, before embedding or deleting anything.
func TestSyncAbortsOnASymlinkedScanRoot(t *testing.T) {
	brain := t.TempDir()
	writeNote(t, brain, "wiki/w/a.md", "wiki note")
	writeNote(t, brain, "sessions/s1.md", "session note")
	require.NoError(t, os.Symlink(".", filepath.Join(brain, "knowledge")))
	emb := &recordingEmbedder{}
	store := &stubStore{known: map[string]time.Time{"wiki/w/a.md#0001": {}}}
	_, err := vectorstore.Sync(context.Background(), brain, store, emb)
	assert.Error(t, err)
	assert.NotContains(t, emb.all(), "session note")
	for p := range store.upserts {
		assert.False(t, strings.HasPrefix(p, "knowledge/"), "upserted %s", p)
	}
	assert.Empty(t, store.deletes)
}

// A missing scan root is skipped; the other root still syncs.
func TestSyncSkipsAMissingScanRoot(t *testing.T) {
	brain := t.TempDir()
	writeNote(t, brain, "wiki/w/a.md", "wiki note")
	emb := &recordingEmbedder{}
	_, err := vectorstore.Sync(context.Background(), brain, &stubStore{}, emb)
	require.NoError(t, err)
	assert.Contains(t, emb.all(), "wiki note")
}