diff --git a/ingestion/internal/classification/classification.go b/ingestion/internal/classification/classification.go index cbca9f8..1a72b90 100644 --- a/ingestion/internal/classification/classification.go +++ b/ingestion/internal/classification/classification.go @@ -16,9 +16,10 @@ package classification import ( + "errors" "fmt" - "os" - "path/filepath" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" + "io/fs" "strings" "gopkg.in/yaml.v3" @@ -110,9 +111,20 @@ type rawConfig struct { func Load(brainDir string) (*Config, error) { cfg := &Config{wings: map[string]Level{}, repos: map[string]Level{}} - data, err := os.ReadFile(filepath.Join(brainDir, "classification.yaml")) + // Read through the brain root, regular files only (hyperguild#157): a + // symlinked or special file is not trusted as the config, which is a hard + // error like a malformed one. + bfs, err := brainfs.Open(brainDir) + if errors.Is(err, fs.ErrNotExist) { + return cfg, nil + } if err != nil { - if os.IsNotExist(err) { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + data, err := bfs.ReadRegular("classification.yaml") + if err != nil { + if errors.Is(err, fs.ErrNotExist) { return cfg, nil } return nil, fmt.Errorf("read classification.yaml: %w", err) diff --git a/ingestion/internal/classification/root_test.go b/ingestion/internal/classification/root_test.go new file mode 100644 index 0000000..036abea --- /dev/null +++ b/ingestion/internal/classification/root_test.go @@ -0,0 +1,46 @@ +package classification + +import ( + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// hyperguild#157 slice 6: Load read classification.yaml with os.ReadFile. It +// sets the I1 sovereignty levels; a symlinked file must not be trusted as the +// config (a hard error, as for a malformed file), and a FIFO must not hang +// startup. +func TestLoadRefusesASymlinkedOrFIFOClassificationFile(t *testing.T) { + ok := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(ok, "classification.yaml"), []byte("wings:\n w: confidential\n"), 0o644)) + _, err := Load(ok) + require.NoError(t, err, "positive control") + + linked := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(linked, "real.yaml"), []byte("wings:\n w: public\n"), 0o644)) + require.NoError(t, os.Symlink("real.yaml", filepath.Join(linked, "classification.yaml"))) + _, err = Load(linked) + assert.Error(t, err, "an in-brain symlink is not trusted as the config") + + fifo := t.TempDir() + require.NoError(t, syscall.Mkfifo(filepath.Join(fifo, "classification.yaml"), 0o644)) + w, err := os.OpenFile(filepath.Join(fifo, "classification.yaml"), os.O_RDWR, 0) + require.NoError(t, err) + defer func() { _ = w.Close() }() + done := make(chan error, 1) + go func() { _, err := Load(fifo); done <- err }() + select { + case err := <-done: + assert.Error(t, err) + case <-time.After(3 * time.Second): + t.Fatal("Load hung on a FIFO") + } + + _, err = Load(t.TempDir()) + assert.NoError(t, err, "an absent file is still not an error") +} diff --git a/ingestion/internal/pipeline/backfill.go b/ingestion/internal/pipeline/backfill.go index dc20c94..1217f57 100644 --- a/ingestion/internal/pipeline/backfill.go +++ b/ingestion/internal/pipeline/backfill.go @@ -3,7 +3,10 @@ package pipeline import ( "context" + "errors" "fmt" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" + "io/fs" "os" "path/filepath" "strings" @@ -23,10 +26,16 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) { return 0, fmt.Errorf("load inventory: %w", err) } - sourcesDir := filepath.Join(brainDir, "wiki", "sources") - entries, err := os.ReadDir(sourcesDir) + // Listed and read through the brain root, regular files only + // (hyperguild#157). + bfs, err := brainfs.Open(brainDir) if err != nil { - if os.IsNotExist(err) { + return 0, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + names, err := bfs.ReadDirRegular("wiki/sources") + if err != nil { + if errors.Is(err, fs.ErrNotExist) { return 0, nil } return 0, fmt.Errorf("read sources dir: %w", err) @@ -37,20 +46,20 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) { // get both refs merged before a single write. pending := make(map[string]wiki.Page) - for _, e := range entries { + for _, name := range names { if ctx.Err() != nil { return 0, ctx.Err() } - if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") { + if !strings.HasSuffix(name, ".md") { continue } - b, err := os.ReadFile(filepath.Join(sourcesDir, e.Name())) + b, err := bfs.ReadRegular("wiki/sources/" + name) if err != nil { continue } sourceContent := string(b) - sourceSlug := strings.TrimSuffix(e.Name(), ".md") + sourceSlug := strings.TrimSuffix(name, ".md") sourceTitle := extractTitle(sourceContent) if sourceTitle == "" { sourceTitle = sourceSlug @@ -70,7 +79,7 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) { // Start from already-accumulated version if we've seen this page. page, seen := pending[relPath] if !seen { - raw, err := os.ReadFile(filepath.Join(brainDir, filepath.FromSlash(relPath))) + raw, err := bfs.ReadRegular(relPath) if err != nil { continue } diff --git a/ingestion/internal/pipeline/pipeline.go b/ingestion/internal/pipeline/pipeline.go index e3a326a..c368d7f 100644 --- a/ingestion/internal/pipeline/pipeline.go +++ b/ingestion/internal/pipeline/pipeline.go @@ -5,7 +5,10 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" + "io/fs" "os" "path/filepath" "strings" @@ -129,7 +132,7 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err) } - content, conflicts, err := reviseOrCreate(dest, page) + content, conflicts, err := reviseOrCreate(brainDir, page) if err != nil { return Result{}, err } @@ -188,14 +191,27 @@ func hashBody(content string) string { // readSourcePage returns the full on-disk content of the source page at // wiki/sources/.md, or "" when it doesn't exist yet. +// It reads through the brain root, regular files only (hyperguild#157): the +// content goes into the LLM prompt. func readSourcePage(brainDir, sourceSlug string) string { - raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md")) + raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md") if err != nil { return "" } return string(raw) } +// readBrainFile reads one brain-relative file through the brain root, +// regular files only (hyperguild#157). +func readBrainFile(brainDir, rel string) ([]byte, error) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, err + } + defer func() { _ = bfs.Close() }() + return bfs.ReadRegular(rel) +} + // storedSourceHash reads source_hash from the source page a previous ingest // wrote, or "" when there is no such page or no such field. // @@ -230,10 +246,10 @@ func storedSourceHash(brainDir, sourceSlug string) string { // its first-ingest date on a revised page. Left deliberately — the alternative // is this function editing frontmatter it does not own, and no lint reads that // field today. -func reviseOrCreate(dest string, page wiki.Page) (string, []string, error) { - existing, err := os.ReadFile(dest) +func reviseOrCreate(brainDir string, page wiki.Page) (string, []string, error) { + existing, err := readBrainFile(brainDir, page.Path) switch { - case os.IsNotExist(err): + case errors.Is(err, fs.ErrNotExist): return page.Content, nil, nil case err != nil: return "", nil, fmt.Errorf("read existing %s: %w", page.Path, err) @@ -267,7 +283,7 @@ See brain/schema.md for the full schema. ` func loadSchema(brainDir string) string { - b, err := os.ReadFile(filepath.Join(brainDir, "schema.md")) + b, err := readBrainFile(brainDir, "schema.md") if err != nil { return defaultSchema } diff --git a/ingestion/internal/pipeline/refs.go b/ingestion/internal/pipeline/refs.go index ae633a2..ba6c08a 100644 --- a/ingestion/internal/pipeline/refs.go +++ b/ingestion/internal/pipeline/refs.go @@ -2,7 +2,6 @@ package pipeline import ( - "os" "path/filepath" "regexp" "strings" @@ -53,8 +52,7 @@ func injectSourceRefs(pages []wiki.Page, inventory map[wiki.PageType][]wiki.Entr if !ok { continue } - diskPath := filepath.Join(brainDir, "wiki", string(pt), slug+".md") - b, err := os.ReadFile(diskPath) + b, err := readBrainFile(brainDir, "wiki/"+string(pt)+"/"+slug+".md") if err != nil { continue } diff --git a/ingestion/internal/pipeline/root_test.go b/ingestion/internal/pipeline/root_test.go new file mode 100644 index 0000000..185cb48 --- /dev/null +++ b/ingestion/internal/pipeline/root_test.go @@ -0,0 +1,136 @@ +package pipeline + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest" + "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" +) + +// hyperguild#157 slice 6: the pipeline read the source page, schema.md, the +// pages it revises and the pages it adds source refs to with os.ReadFile, so +// a symlinked file's content reached the LLM prompt and the written pages, +// and a FIFO hung the ingest. + +func mkdirs(t *testing.T, brain string, dirs ...string) { + t.Helper() + for _, d := range dirs { + require.NoError(t, os.MkdirAll(filepath.Join(brain, filepath.FromSlash(d)), 0o755)) + } +} + +func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T) { + brain := t.TempDir() + mkdirs(t, brain, "wiki/sources") + inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md")) + require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644)) + assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control") + assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control") + + require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md"))) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md"))) + assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel) + assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel) + assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel) + + src := filepath.Join(brain, "wiki", "sources", "r.md") + read := func() string { return readSourcePage(brain, "r") } + brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read) + brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read) + brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() }) + + schema := filepath.Join(brain, "schema.md") + rs := func() string { return loadSchema(brain) } + brainfstest.AssertNoLeakFrom(t, schema, "zqinside schema", inBrain, 1500*time.Millisecond, rs) + brainfstest.AssertNoHang(t, schema, "zqinside schema", 1500*time.Millisecond, 6*time.Second, func() { _ = rs() }) +} + +func TestReviseOrCreateRefusesASymlinkedExistingPage(t *testing.T) { + brain := t.TempDir() + mkdirs(t, brain, "wiki/concepts") + brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md")) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md"))) + page := wiki.Page{Path: "wiki/concepts/c.md", Content: "---\ntitle: C\n---\nnew\n"} + out, _, err := reviseOrCreate(brain, page) + assert.Error(t, err) + assert.NotContains(t, out, brainfstest.Sentinel) + + require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("---\ntitle: D\n---\nzqold\n"), 0o644)) + out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/d.md", Content: "---\ntitle: D\n---\nnew\n"}) + require.NoError(t, err) + assert.Contains(t, out, "zqold", "positive control: the existing page is read") + out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/new.md", Content: "fresh"}) + require.NoError(t, err) + assert.Equal(t, "fresh", out, "positive control: a missing page is created") +} + +func TestInjectSourceRefsDoesNotReadASymlinkedPage(t *testing.T) { + brain := t.TempDir() + mkdirs(t, brain, "wiki/concepts") + brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md")) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md"))) + require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("zqinside d"), 0o644)) + inv := map[wiki.PageType][]wiki.Entry{wiki.PageTypeConcept: { + {Slug: "c", Type: wiki.PageTypeConcept}, {Slug: "d", Type: wiki.PageTypeConcept}, + }} + src := wiki.Page{Path: "wiki/sources/src.md", Content: "---\ntitle: Src\n---\nsee [[c|C]] and [[d|D]]\n"} + out := injectSourceRefs([]wiki.Page{src}, inv, brain) + got := fmt.Sprintf("%+v", out) + assert.Contains(t, got, "zqinside d", "positive control") + assert.NotContains(t, got, brainfstest.Sentinel) +} + +func TestBackfillRefsSkipsSymlinkedSourcesAndPages(t *testing.T) { + brain := t.TempDir() + mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw") + require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "real.md"), []byte("---\ntitle: R\n---\nbody\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("---\ntitle: zqinside\n---\n[[real|R]]\n"), 0o644)) + // A source elsewhere in the brain whose title is the sentinel and which + // links the real concept with the [[slug|title]] form the pipeline follows. + require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "src.md"), + []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "src.md"), filepath.Join(brain, "wiki", "sources", "in.md"))) + // A concept page linked by the regular source that is itself a symlink. + require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok2.md"), []byte("---\ntitle: Two\n---\n[[lnk|L]]\n"), 0o644)) + brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "page.md")) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "page.md"), filepath.Join(brain, "wiki", "concepts", "lnk.md"))) + + _, err := BackfillRefs(context.Background(), brain) + require.NoError(t, err) + b, _ := os.ReadFile(filepath.Join(brain, "wiki", "concepts", "real.md")) + assert.Contains(t, string(b), "zqinside", "positive control: the regular source's ref is added") + assert.NotContains(t, string(b), brainfstest.Sentinel, "a symlinked source's title was written as a ref") + raw, _ := os.ReadFile(filepath.Join(brain, "raw", "page.md")) + assert.NotContains(t, string(raw), "[[ok2", "the symlinked concept page's target was rewritten") +} + +// A source swapped between a regular file and a symlink to an in-brain source +// whose title is the sentinel: the read refuses the swapped-in link, so the +// sentinel never lands in the concept page as a ref. +func TestBackfillRefsDoesNotWriteARefFromASourceSwappedInBehindASymlink(t *testing.T) { + brain := t.TempDir() + mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw") + realPage := filepath.Join(brain, "wiki", "concepts", "real.md") + target := filepath.Join(brain, "raw", "src.md") + require.NoError(t, os.WriteFile(target, []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644)) + src := filepath.Join(brain, "wiki", "sources", "r.md") + inside := "---\ntitle: zqinside\n---\n[[real|R]]\n" + read := func() string { + _ = os.WriteFile(realPage, []byte("---\ntitle: R\n---\nbody\n"), 0o644) + _, _ = BackfillRefs(context.Background(), brain) + b, _ := os.ReadFile(realPage) + return string(b) + } + require.NoError(t, os.WriteFile(src, []byte(inside), 0o644)) + require.Contains(t, read(), "zqinside", "positive control") + brainfstest.AssertNoLeakFrom(t, src, inside, target, 1500*time.Millisecond, read) +}