package pipeline import ( "context" "fmt" "io" "os" "path/filepath" "syscall" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest" "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" ) // hyperguild#157 slice 6: the pipeline read the source page, schema.md, the // pages it revises and the pages it adds source refs to with os.ReadFile, so // a symlinked file's content reached the LLM prompt and the written pages, // and a FIFO hung the ingest. func mkdirs(t *testing.T, brain string, dirs ...string) { t.Helper() for _, d := range dirs { require.NoError(t, os.MkdirAll(filepath.Join(brain, filepath.FromSlash(d)), 0o755)) } } func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T) { brain := t.TempDir() mkdirs(t, brain, "wiki/sources") inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md")) require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644)) assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control") assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control") require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md"))) require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md"))) assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel) assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel) assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel) src := filepath.Join(brain, "wiki", "sources", "r.md") read := func() string { return readSourcePage(brain, "r") } brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read) brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read) brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() }) schema := filepath.Join(brain, "schema.md") rs := func() string { return loadSchema(brain) } brainfstest.AssertNoLeakFrom(t, schema, "zqinside schema", inBrain, 1500*time.Millisecond, rs) brainfstest.AssertNoHang(t, schema, "zqinside schema", 1500*time.Millisecond, 6*time.Second, func() { _ = rs() }) } func TestReviseOrCreateRefusesASymlinkedExistingPage(t *testing.T) { brain := t.TempDir() mkdirs(t, brain, "wiki/concepts") brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md")) require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md"))) page := wiki.Page{Path: "wiki/concepts/c.md", Content: "---\ntitle: C\n---\nnew\n"} out, _, err := reviseOrCreate(brain, page) assert.Error(t, err) assert.NotContains(t, out, brainfstest.Sentinel) require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("---\ntitle: D\n---\nzqold\n"), 0o644)) out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/d.md", Content: "---\ntitle: D\n---\nnew\n"}) require.NoError(t, err) assert.Contains(t, out, "zqold", "positive control: the existing page is read") out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/new.md", Content: "fresh"}) require.NoError(t, err) assert.Equal(t, "fresh", out, "positive control: a missing page is created") } func TestInjectSourceRefsDoesNotReadASymlinkedPage(t *testing.T) { brain := t.TempDir() mkdirs(t, brain, "wiki/concepts") brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md")) require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md"))) require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("zqinside d"), 0o644)) inv := map[wiki.PageType][]wiki.Entry{wiki.PageTypeConcept: { {Slug: "c", Type: wiki.PageTypeConcept}, {Slug: "d", Type: wiki.PageTypeConcept}, }} src := wiki.Page{Path: "wiki/sources/src.md", Content: "---\ntitle: Src\n---\nsee [[c|C]] and [[d|D]]\n"} out := injectSourceRefs([]wiki.Page{src}, inv, brain) got := fmt.Sprintf("%+v", out) assert.Contains(t, got, "zqinside d", "positive control") assert.NotContains(t, got, brainfstest.Sentinel) } func TestBackfillRefsSkipsSymlinkedSourcesAndPages(t *testing.T) { brain := t.TempDir() mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw") require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "real.md"), []byte("---\ntitle: R\n---\nbody\n"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("---\ntitle: zqinside\n---\n[[real|R]]\n"), 0o644)) // A source elsewhere in the brain whose title is the sentinel and which // links the real concept with the [[slug|title]] form the pipeline follows. require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "src.md"), []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644)) require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "src.md"), filepath.Join(brain, "wiki", "sources", "in.md"))) // A concept page linked by the regular source that is itself a symlink. require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok2.md"), []byte("---\ntitle: Two\n---\n[[lnk|L]]\n"), 0o644)) brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "page.md")) require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "page.md"), filepath.Join(brain, "wiki", "concepts", "lnk.md"))) _, err := BackfillRefs(context.Background(), brain) require.NoError(t, err) b, _ := os.ReadFile(filepath.Join(brain, "wiki", "concepts", "real.md")) assert.Contains(t, string(b), "zqinside", "positive control: the regular source's ref is added") assert.NotContains(t, string(b), brainfstest.Sentinel, "a symlinked source's title was written as a ref") raw, _ := os.ReadFile(filepath.Join(brain, "raw", "page.md")) assert.NotContains(t, string(raw), "[[ok2", "the symlinked concept page's target was rewritten") } // A source swapped between a regular file and a symlink to an in-brain source // whose title is the sentinel: the read refuses the swapped-in link, so the // sentinel never lands in the concept page as a ref. func TestBackfillRefsDoesNotWriteARefFromASourceSwappedInBehindASymlink(t *testing.T) { brain := t.TempDir() mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw") realPage := filepath.Join(brain, "wiki", "concepts", "real.md") target := filepath.Join(brain, "raw", "src.md") require.NoError(t, os.WriteFile(target, []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644)) src := filepath.Join(brain, "wiki", "sources", "r.md") inside := "---\ntitle: zqinside\n---\n[[real|R]]\n" read := func() string { _ = os.WriteFile(realPage, []byte("---\ntitle: R\n---\nbody\n"), 0o644) _, _ = BackfillRefs(context.Background(), brain) b, _ := os.ReadFile(realPage) return string(b) } require.NoError(t, os.WriteFile(src, []byte(inside), 0o644)) require.Contains(t, read(), "zqinside", "positive control") brainfstest.AssertNoLeakFrom(t, src, inside, target, 1500*time.Millisecond, read) } // A concept page that is regular when the inventory lists it, then swapped // for a symlink to an in-brain file before BackfillRefs reads it: only the // page read's own check refuses it (a static symlink never gets this far, // because LoadInventory does not list it). func TestBackfillRefsDoesNotCopyAPageSwappedInBehindASymlink(t *testing.T) { brain := t.TempDir() mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw") require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "s.md"), []byte("---\ntitle: S\n---\n[[c|C]]\n"), 0o644)) page := filepath.Join(brain, "wiki", "concepts", "c.md") target := filepath.Join(brain, "raw", "secret.md") inside := "---\ntitle: C\n---\nzqinside\n" read := func() string { // The write path still follows a swapped-in link (#158) and can // overwrite the target, so restore it each round. _ = os.WriteFile(target, []byte("---\ntitle: X\n---\n"+brainfstest.Sentinel+"\n"), 0o644) _, _ = BackfillRefs(context.Background(), brain) f, err := os.OpenFile(page, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0) if err != nil { return "" } defer func() { _ = f.Close() }() b, _ := io.ReadAll(f) return string(b) } require.NoError(t, os.WriteFile(page, []byte(inside), 0o644)) require.Contains(t, read(), "[[s|S]]", "positive control: the regular page gets the source ref") brainfstest.AssertNoLeakFrom(t, page, inside, target, 3*time.Second, read) } // hyperguild#158: BackfillRefs read a regular concept page, a symlink could // be swapped in, and os.WriteFile then followed it and overwrote the link's // target. The target is not restored between rounds here, so any // write-through shows. func TestBackfillRefsNeverWritesThroughAPageSwappedForASymlink(t *testing.T) { brain := t.TempDir() mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw") require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "s.md"), []byte("---\ntitle: S\n---\n[[c|C]]\n"), 0o644)) page := filepath.Join(brain, "wiki", "concepts", "c.md") target := filepath.Join(brain, "raw", "secret.md") require.NoError(t, os.WriteFile(target, []byte("untouched"), 0o644)) inside := "---\ntitle: C\n---\nbody\n" require.NoError(t, os.WriteFile(page, []byte(inside), 0o644)) brainfstest.StartSwap(t, page, inside, target, false) ok := 0 for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); { if _, err := BackfillRefs(context.Background(), brain); err == nil { ok++ } } got, _ := os.ReadFile(target) assert.Equal(t, "untouched", string(got), "BackfillRefs wrote through a swapped-in symlink") assert.Positive(t, ok, "positive control: some backfills ran") }