mathiasandClaude Opus 5.5
aee5f66398
The ingest pipeline read wiki/sources/<slug>.md and schema.md (both go into the LLM prompt), the pages it revises, and the pages it adds source refs to, with os.ReadFile; BackfillRefs listed wiki/sources with os.ReadDir. A symlinked file's content reached the prompt and the written pages (1.6M leaks in the race), and a FIFO hung the ingest. classification.Load (the I1 sovereignty levels) read classification.yaml the same way and hung startup on a FIFO. - pipeline: readSourcePage, loadSchema, reviseOrCreate, injectSourceRefs read through readBrainFile (brainfs ReadRegular); reviseOrCreate takes the brain dir and reads page.Path, a missing page is still created and a non-regular one is an error. BackfillRefs lists with ReadDirRegular and reads with ReadRegular. - classification.Load: read through brainfs; a symlinked or special classification.yaml is a hard error (as a malformed one is); an absent file or brain dir still yields the default config. Tests use in-brain targets and the [[slug|title]] link form the pipeline actually follows (bare [[slug]] links are ignored by it, which made a first draft of the backfill test vacuous). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
137 lines
7.2 KiB
Go
137 lines
7.2 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
|
)
|
|
|
|
// hyperguild#157 slice 6: the pipeline read the source page, schema.md, the
|
|
// pages it revises and the pages it adds source refs to with os.ReadFile, so
|
|
// a symlinked file's content reached the LLM prompt and the written pages,
|
|
// and a FIFO hung the ingest.
|
|
|
|
func mkdirs(t *testing.T, brain string, dirs ...string) {
|
|
t.Helper()
|
|
for _, d := range dirs {
|
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, filepath.FromSlash(d)), 0o755))
|
|
}
|
|
}
|
|
|
|
func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources")
|
|
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644))
|
|
assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control")
|
|
assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control")
|
|
|
|
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md")))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel)
|
|
assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel)
|
|
assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel)
|
|
|
|
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
read := func() string { return readSourcePage(brain, "r") }
|
|
brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read)
|
|
brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read)
|
|
brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
|
|
|
|
schema := filepath.Join(brain, "schema.md")
|
|
rs := func() string { return loadSchema(brain) }
|
|
brainfstest.AssertNoLeakFrom(t, schema, "zqinside schema", inBrain, 1500*time.Millisecond, rs)
|
|
brainfstest.AssertNoHang(t, schema, "zqinside schema", 1500*time.Millisecond, 6*time.Second, func() { _ = rs() })
|
|
}
|
|
|
|
func TestReviseOrCreateRefusesASymlinkedExistingPage(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/concepts")
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
page := wiki.Page{Path: "wiki/concepts/c.md", Content: "---\ntitle: C\n---\nnew\n"}
|
|
out, _, err := reviseOrCreate(brain, page)
|
|
assert.Error(t, err)
|
|
assert.NotContains(t, out, brainfstest.Sentinel)
|
|
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("---\ntitle: D\n---\nzqold\n"), 0o644))
|
|
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/d.md", Content: "---\ntitle: D\n---\nnew\n"})
|
|
require.NoError(t, err)
|
|
assert.Contains(t, out, "zqold", "positive control: the existing page is read")
|
|
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/new.md", Content: "fresh"})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "fresh", out, "positive control: a missing page is created")
|
|
}
|
|
|
|
func TestInjectSourceRefsDoesNotReadASymlinkedPage(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/concepts")
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("zqinside d"), 0o644))
|
|
inv := map[wiki.PageType][]wiki.Entry{wiki.PageTypeConcept: {
|
|
{Slug: "c", Type: wiki.PageTypeConcept}, {Slug: "d", Type: wiki.PageTypeConcept},
|
|
}}
|
|
src := wiki.Page{Path: "wiki/sources/src.md", Content: "---\ntitle: Src\n---\nsee [[c|C]] and [[d|D]]\n"}
|
|
out := injectSourceRefs([]wiki.Page{src}, inv, brain)
|
|
got := fmt.Sprintf("%+v", out)
|
|
assert.Contains(t, got, "zqinside d", "positive control")
|
|
assert.NotContains(t, got, brainfstest.Sentinel)
|
|
}
|
|
|
|
func TestBackfillRefsSkipsSymlinkedSourcesAndPages(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "real.md"), []byte("---\ntitle: R\n---\nbody\n"), 0o644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("---\ntitle: zqinside\n---\n[[real|R]]\n"), 0o644))
|
|
// A source elsewhere in the brain whose title is the sentinel and which
|
|
// links the real concept with the [[slug|title]] form the pipeline follows.
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "src.md"),
|
|
[]byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "src.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
// A concept page linked by the regular source that is itself a symlink.
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok2.md"), []byte("---\ntitle: Two\n---\n[[lnk|L]]\n"), 0o644))
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "page.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "page.md"), filepath.Join(brain, "wiki", "concepts", "lnk.md")))
|
|
|
|
_, err := BackfillRefs(context.Background(), brain)
|
|
require.NoError(t, err)
|
|
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "concepts", "real.md"))
|
|
assert.Contains(t, string(b), "zqinside", "positive control: the regular source's ref is added")
|
|
assert.NotContains(t, string(b), brainfstest.Sentinel, "a symlinked source's title was written as a ref")
|
|
raw, _ := os.ReadFile(filepath.Join(brain, "raw", "page.md"))
|
|
assert.NotContains(t, string(raw), "[[ok2", "the symlinked concept page's target was rewritten")
|
|
}
|
|
|
|
// A source swapped between a regular file and a symlink to an in-brain source
|
|
// whose title is the sentinel: the read refuses the swapped-in link, so the
|
|
// sentinel never lands in the concept page as a ref.
|
|
func TestBackfillRefsDoesNotWriteARefFromASourceSwappedInBehindASymlink(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
realPage := filepath.Join(brain, "wiki", "concepts", "real.md")
|
|
target := filepath.Join(brain, "raw", "src.md")
|
|
require.NoError(t, os.WriteFile(target, []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
inside := "---\ntitle: zqinside\n---\n[[real|R]]\n"
|
|
read := func() string {
|
|
_ = os.WriteFile(realPage, []byte("---\ntitle: R\n---\nbody\n"), 0o644)
|
|
_, _ = BackfillRefs(context.Background(), brain)
|
|
b, _ := os.ReadFile(realPage)
|
|
return string(b)
|
|
}
|
|
require.NoError(t, os.WriteFile(src, []byte(inside), 0o644))
|
|
require.Contains(t, read(), "zqinside", "positive control")
|
|
brainfstest.AssertNoLeakFrom(t, src, inside, target, 1500*time.Millisecond, read)
|
|
}
|