fix(pipeline,classification): read the brain through brainfs (#157, slice 6)

The ingest pipeline read wiki/sources/<slug>.md and schema.md (both go
into the LLM prompt), the pages it revises, and the pages it adds
source refs to, with os.ReadFile; BackfillRefs listed wiki/sources with
os.ReadDir. A symlinked file's content reached the prompt and the
written pages (1.6M leaks in the race), and a FIFO hung the ingest.
classification.Load (the I1 sovereignty levels) read
classification.yaml the same way and hung startup on a FIFO.

- pipeline: readSourcePage, loadSchema, reviseOrCreate, injectSourceRefs
  read through readBrainFile (brainfs ReadRegular); reviseOrCreate takes
  the brain dir and reads page.Path, a missing page is still created and
  a non-regular one is an error. BackfillRefs lists with ReadDirRegular
  and reads with ReadRegular.
- classification.Load: read through brainfs; a symlinked or special
  classification.yaml is a hard error (as a malformed one is); an absent
  file or brain dir still yields the default config.

Tests use in-brain targets and the [[slug|title]] link form the pipeline
actually follows (bare [[slug]] links are ignored by it, which made a
first draft of the backfill test vacuous).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-05 20:47:57 +02:00
co-authored by Claude Opus 5.5
parent febab3e45d
commit aee5f66398
6 changed files with 238 additions and 21 deletions
+22 -6
View File
@@ -5,7 +5,10 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"io/fs"
"os"
"path/filepath"
"strings"
@@ -129,7 +132,7 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
}
content, conflicts, err := reviseOrCreate(dest, page)
content, conflicts, err := reviseOrCreate(brainDir, page)
if err != nil {
return Result{}, err
}
@@ -188,14 +191,27 @@ func hashBody(content string) string {
// readSourcePage returns the full on-disk content of the source page at
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
// It reads through the brain root, regular files only (hyperguild#157): the
// content goes into the LLM prompt.
func readSourcePage(brainDir, sourceSlug string) string {
raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md")
if err != nil {
return ""
}
return string(raw)
}
// readBrainFile reads one brain-relative file through the brain root,
// regular files only (hyperguild#157).
func readBrainFile(brainDir, rel string) ([]byte, error) {
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, err
}
defer func() { _ = bfs.Close() }()
return bfs.ReadRegular(rel)
}
// storedSourceHash reads source_hash from the source page a previous ingest
// wrote, or "" when there is no such page or no such field.
//
@@ -230,10 +246,10 @@ func storedSourceHash(brainDir, sourceSlug string) string {
// its first-ingest date on a revised page. Left deliberately — the alternative
// is this function editing frontmatter it does not own, and no lint reads that
// field today.
func reviseOrCreate(dest string, page wiki.Page) (string, []string, error) {
existing, err := os.ReadFile(dest)
func reviseOrCreate(brainDir string, page wiki.Page) (string, []string, error) {
existing, err := readBrainFile(brainDir, page.Path)
switch {
case os.IsNotExist(err):
case errors.Is(err, fs.ErrNotExist):
return page.Content, nil, nil
case err != nil:
return "", nil, fmt.Errorf("read existing %s: %w", page.Path, err)
@@ -267,7 +283,7 @@ See brain/schema.md for the full schema.
`
func loadSchema(brainDir string) string {
b, err := os.ReadFile(filepath.Join(brainDir, "schema.md"))
b, err := readBrainFile(brainDir, "schema.md")
if err != nil {
return defaultSchema
}