fix(pipeline,classification): read the brain through brainfs (#157, slice 6)
The ingest pipeline read wiki/sources/<slug>.md and schema.md (both go into the LLM prompt), the pages it revises, and the pages it adds source refs to, with os.ReadFile; BackfillRefs listed wiki/sources with os.ReadDir. A symlinked file's content reached the prompt and the written pages (1.6M leaks in the race), and a FIFO hung the ingest. classification.Load (the I1 sovereignty levels) read classification.yaml the same way and hung startup on a FIFO. - pipeline: readSourcePage, loadSchema, reviseOrCreate, injectSourceRefs read through readBrainFile (brainfs ReadRegular); reviseOrCreate takes the brain dir and reads page.Path, a missing page is still created and a non-regular one is an error. BackfillRefs lists with ReadDirRegular and reads with ReadRegular. - classification.Load: read through brainfs; a symlinked or special classification.yaml is a hard error (as a malformed one is); an absent file or brain dir still yields the default config. Tests use in-brain targets and the [[slug|title]] link form the pipeline actually follows (bare [[slug]] links are ignored by it, which made a first draft of the backfill test vacuous). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,10 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -129,7 +132,7 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
||||
return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
|
||||
}
|
||||
content, conflicts, err := reviseOrCreate(dest, page)
|
||||
content, conflicts, err := reviseOrCreate(brainDir, page)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
@@ -188,14 +191,27 @@ func hashBody(content string) string {
|
||||
|
||||
// readSourcePage returns the full on-disk content of the source page at
|
||||
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
|
||||
// It reads through the brain root, regular files only (hyperguild#157): the
|
||||
// content goes into the LLM prompt.
|
||||
func readSourcePage(brainDir, sourceSlug string) string {
|
||||
raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
|
||||
raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// readBrainFile reads one brain-relative file through the brain root,
|
||||
// regular files only (hyperguild#157).
|
||||
func readBrainFile(brainDir, rel string) ([]byte, error) {
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = bfs.Close() }()
|
||||
return bfs.ReadRegular(rel)
|
||||
}
|
||||
|
||||
// storedSourceHash reads source_hash from the source page a previous ingest
|
||||
// wrote, or "" when there is no such page or no such field.
|
||||
//
|
||||
@@ -230,10 +246,10 @@ func storedSourceHash(brainDir, sourceSlug string) string {
|
||||
// its first-ingest date on a revised page. Left deliberately — the alternative
|
||||
// is this function editing frontmatter it does not own, and no lint reads that
|
||||
// field today.
|
||||
func reviseOrCreate(dest string, page wiki.Page) (string, []string, error) {
|
||||
existing, err := os.ReadFile(dest)
|
||||
func reviseOrCreate(brainDir string, page wiki.Page) (string, []string, error) {
|
||||
existing, err := readBrainFile(brainDir, page.Path)
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return page.Content, nil, nil
|
||||
case err != nil:
|
||||
return "", nil, fmt.Errorf("read existing %s: %w", page.Path, err)
|
||||
@@ -267,7 +283,7 @@ See brain/schema.md for the full schema.
|
||||
`
|
||||
|
||||
func loadSchema(brainDir string) string {
|
||||
b, err := os.ReadFile(filepath.Join(brainDir, "schema.md"))
|
||||
b, err := readBrainFile(brainDir, "schema.md")
|
||||
if err != nil {
|
||||
return defaultSchema
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user