mathiasandClaude Opus 5.5
eedb110be5
Ingest pipeline readers (readSourcePage and loadSchema feed the LLM prompt; reviseOrCreate; injectSourceRefs; BackfillRefs list and reads) and classification.Load read through brainfs: symlinked or special files no longer reach the prompt, written pages or the I1 sovereignty config, and a FIFO no longer hangs ingest or startup. A symlinked/special classification.yaml is a hard error; an absent one still yields defaults. Refs #157. Adversarial review: BLOCKED, BLOCKED, PASS. Follow-ups: #158 (write paths), #179 (FIFO tripwire). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Mathias <mthbqv@gmail.com>
168 lines
8.6 KiB
Go
168 lines
8.6 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
|
)
|
|
|
|
// hyperguild#157 slice 6: the pipeline read the source page, schema.md, the
|
|
// pages it revises and the pages it adds source refs to with os.ReadFile, so
|
|
// a symlinked file's content reached the LLM prompt and the written pages,
|
|
// and a FIFO hung the ingest.
|
|
|
|
func mkdirs(t *testing.T, brain string, dirs ...string) {
|
|
t.Helper()
|
|
for _, d := range dirs {
|
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, filepath.FromSlash(d)), 0o755))
|
|
}
|
|
}
|
|
|
|
func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources")
|
|
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644))
|
|
assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control")
|
|
assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control")
|
|
|
|
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md")))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel)
|
|
assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel)
|
|
assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel)
|
|
|
|
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
read := func() string { return readSourcePage(brain, "r") }
|
|
brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read)
|
|
brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read)
|
|
brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
|
|
|
|
schema := filepath.Join(brain, "schema.md")
|
|
rs := func() string { return loadSchema(brain) }
|
|
brainfstest.AssertNoLeakFrom(t, schema, "zqinside schema", inBrain, 1500*time.Millisecond, rs)
|
|
brainfstest.AssertNoHang(t, schema, "zqinside schema", 1500*time.Millisecond, 6*time.Second, func() { _ = rs() })
|
|
}
|
|
|
|
func TestReviseOrCreateRefusesASymlinkedExistingPage(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/concepts")
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
page := wiki.Page{Path: "wiki/concepts/c.md", Content: "---\ntitle: C\n---\nnew\n"}
|
|
out, _, err := reviseOrCreate(brain, page)
|
|
assert.Error(t, err)
|
|
assert.NotContains(t, out, brainfstest.Sentinel)
|
|
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("---\ntitle: D\n---\nzqold\n"), 0o644))
|
|
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/d.md", Content: "---\ntitle: D\n---\nnew\n"})
|
|
require.NoError(t, err)
|
|
assert.Contains(t, out, "zqold", "positive control: the existing page is read")
|
|
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/new.md", Content: "fresh"})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "fresh", out, "positive control: a missing page is created")
|
|
}
|
|
|
|
func TestInjectSourceRefsDoesNotReadASymlinkedPage(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/concepts")
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("zqinside d"), 0o644))
|
|
inv := map[wiki.PageType][]wiki.Entry{wiki.PageTypeConcept: {
|
|
{Slug: "c", Type: wiki.PageTypeConcept}, {Slug: "d", Type: wiki.PageTypeConcept},
|
|
}}
|
|
src := wiki.Page{Path: "wiki/sources/src.md", Content: "---\ntitle: Src\n---\nsee [[c|C]] and [[d|D]]\n"}
|
|
out := injectSourceRefs([]wiki.Page{src}, inv, brain)
|
|
got := fmt.Sprintf("%+v", out)
|
|
assert.Contains(t, got, "zqinside d", "positive control")
|
|
assert.NotContains(t, got, brainfstest.Sentinel)
|
|
}
|
|
|
|
func TestBackfillRefsSkipsSymlinkedSourcesAndPages(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "real.md"), []byte("---\ntitle: R\n---\nbody\n"), 0o644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("---\ntitle: zqinside\n---\n[[real|R]]\n"), 0o644))
|
|
// A source elsewhere in the brain whose title is the sentinel and which
|
|
// links the real concept with the [[slug|title]] form the pipeline follows.
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "src.md"),
|
|
[]byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "src.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
// A concept page linked by the regular source that is itself a symlink.
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok2.md"), []byte("---\ntitle: Two\n---\n[[lnk|L]]\n"), 0o644))
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "page.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "page.md"), filepath.Join(brain, "wiki", "concepts", "lnk.md")))
|
|
|
|
_, err := BackfillRefs(context.Background(), brain)
|
|
require.NoError(t, err)
|
|
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "concepts", "real.md"))
|
|
assert.Contains(t, string(b), "zqinside", "positive control: the regular source's ref is added")
|
|
assert.NotContains(t, string(b), brainfstest.Sentinel, "a symlinked source's title was written as a ref")
|
|
raw, _ := os.ReadFile(filepath.Join(brain, "raw", "page.md"))
|
|
assert.NotContains(t, string(raw), "[[ok2", "the symlinked concept page's target was rewritten")
|
|
}
|
|
|
|
// A source swapped between a regular file and a symlink to an in-brain source
|
|
// whose title is the sentinel: the read refuses the swapped-in link, so the
|
|
// sentinel never lands in the concept page as a ref.
|
|
func TestBackfillRefsDoesNotWriteARefFromASourceSwappedInBehindASymlink(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
realPage := filepath.Join(brain, "wiki", "concepts", "real.md")
|
|
target := filepath.Join(brain, "raw", "src.md")
|
|
require.NoError(t, os.WriteFile(target, []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
inside := "---\ntitle: zqinside\n---\n[[real|R]]\n"
|
|
read := func() string {
|
|
_ = os.WriteFile(realPage, []byte("---\ntitle: R\n---\nbody\n"), 0o644)
|
|
_, _ = BackfillRefs(context.Background(), brain)
|
|
b, _ := os.ReadFile(realPage)
|
|
return string(b)
|
|
}
|
|
require.NoError(t, os.WriteFile(src, []byte(inside), 0o644))
|
|
require.Contains(t, read(), "zqinside", "positive control")
|
|
brainfstest.AssertNoLeakFrom(t, src, inside, target, 1500*time.Millisecond, read)
|
|
}
|
|
|
|
// A concept page that is regular when the inventory lists it, then swapped
|
|
// for a symlink to an in-brain file before BackfillRefs reads it: only the
|
|
// page read's own check refuses it (a static symlink never gets this far,
|
|
// because LoadInventory does not list it).
|
|
func TestBackfillRefsDoesNotCopyAPageSwappedInBehindASymlink(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "s.md"), []byte("---\ntitle: S\n---\n[[c|C]]\n"), 0o644))
|
|
page := filepath.Join(brain, "wiki", "concepts", "c.md")
|
|
target := filepath.Join(brain, "raw", "secret.md")
|
|
inside := "---\ntitle: C\n---\nzqinside\n"
|
|
read := func() string {
|
|
// The write path still follows a swapped-in link (#158) and can
|
|
// overwrite the target, so restore it each round.
|
|
_ = os.WriteFile(target, []byte("---\ntitle: X\n---\n"+brainfstest.Sentinel+"\n"), 0o644)
|
|
_, _ = BackfillRefs(context.Background(), brain)
|
|
f, err := os.OpenFile(page, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
b, _ := io.ReadAll(f)
|
|
return string(b)
|
|
}
|
|
require.NoError(t, os.WriteFile(page, []byte(inside), 0o644))
|
|
require.Contains(t, read(), "[[s|S]]", "positive control: the regular page gets the source ref")
|
|
brainfstest.AssertNoLeakFrom(t, page, inside, target, 3*time.Second, read)
|
|
}
|