test(pipeline): pin BackfillRefs' page read with a swap race (#157)

Review round 1 on PR #176: reverting the BackfillRefs concept-page read
to os.ReadFile left every test green. The static symlinked page never
reaches that read, because LoadInventory does not list it; the read is
reachable only when a page listed as regular is swapped for a symlink
before it is read. New test does that against an in-brain target and
reads the page back with O_NOFOLLOW; red under the flip (2/2), green
3/3 on head.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ingestion/cmd/server/jwt_validator.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+
+	chassisauth "git.d-ma.be/mathias/mcp-chassis/auth"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/startupretry"
+)
+
+// buildJWTValidator runs OIDC discovery under policy, so a 502 from an auth
+// upstream that is not serving yet on a cold boot is retried instead of
+// exiting (#83). It honours ctx between attempts.
+func buildJWTValidator(
+	ctx context.Context, logger *slog.Logger, issuer, audience string, policy startupretry.Policy,
+) (*chassisauth.JWTValidator, error) {
+	return startupretry.Do(ctx, policy, func(ctx context.Context) (*chassisauth.JWTValidator, error) {
+		v, err := chassisauth.NewJWTValidator(ctx, issuer, audience)
+		if err != nil {
+			logger.Warn("oidc discovery not ready, retrying", "err", err)
+		}
+		return v, err
+	})
+}

ingestion/cmd/server/jwt_validator_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/startupretry"
+)
+
+// fakeIssuer answers OIDC discovery with 502 for the first fail requests, as
+// ingress does while dex is not yet serving on a cold boot (#83), then with a
+// real discovery document and an empty JWKS.
+func fakeIssuer(t *testing.T, fail int32) (*httptest.Server, *atomic.Int32) {
+	t.Helper()
+	var discoveries atomic.Int32
+	mux := http.NewServeMux()
+	var srv *httptest.Server
+	mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
+		if discoveries.Add(1) <= fail {
+			w.WriteHeader(http.StatusBadGateway)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.WriteString(w, `{"issuer":"`+srv.URL+`","jwks_uri":"`+srv.URL+`/keys"}`)
+	})
+	mux.HandleFunc("/keys", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.WriteString(w, `{"keys":[]}`)
+	})
+	srv = httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv, &discoveries
+}
+
+var fastPolicy = startupretry.Policy{Attempts: 8, Base: time.Millisecond, Max: 5 * time.Millisecond}
+
+func TestBuildJWTValidatorRetriesDiscovery502sThenSucceeds(t *testing.T) {
+	srv, discoveries := fakeIssuer(t, 3)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	v, err := buildJWTValidator(ctx, slog.New(slog.DiscardHandler), srv.URL, "brain", fastPolicy)
+	require.NoError(t, err)
+	assert.NotNil(t, v)
+	assert.Equal(t, int32(4), discoveries.Load(), "three 502s, then the successful discovery")
+}
+
+func TestBuildJWTValidatorGivesUpAfterTheBudgetWithTheLastError(t *testing.T) {
+	srv, discoveries := fakeIssuer(t, 1000)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	_, err := buildJWTValidator(ctx, slog.New(slog.DiscardHandler), srv.URL, "brain", fastPolicy)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "status 502")
+	assert.Equal(t, int32(8), discoveries.Load(), "bounded: exactly the attempt budget")
+}
+
+func TestBuildJWTValidatorStopsRetryingWhenTheContextIsCancelled(t *testing.T) {
+	srv, _ := fakeIssuer(t, 1000)
+	ctx, cancel := context.WithCancel(context.Background())
+	slow := startupretry.Policy{Attempts: 8, Base: time.Hour, Max: time.Hour}
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := buildJWTValidator(ctx, slog.New(slog.DiscardHandler), srv.URL, "brain", slow)
+		done <- err
+	}()
+	time.Sleep(50 * time.Millisecond)
+	cancel()
+	select {
+	case err := <-done:
+		assert.ErrorIs(t, err, context.Canceled)
+	case <-time.After(3 * time.Second):
+		t.Fatal("did not return after the context was cancelled")
+	}
+}

ingestion/cmd/server/main.go

@@ -381,15 +381,9 @@ func main() {
 	//
 	// The budget stays bounded: retrying forever would turn a misconfiguration
 	// into a process that looks alive and never serves.
-	jwtValidator, err := startupretry.Do(ctx,
-		startupretry.Policy{Attempts: 8, Base: time.Second, Max: 30 * time.Second},
-		func(ctx context.Context) (*chassisauth.JWTValidator, error) {
-			v, err := chassisauth.NewJWTValidator(ctx, os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE"))
-			if err != nil {
-				logger.Warn("oidc discovery not ready, retrying", "err", err)
-			}
-			return v, err
-		})
+	jwtValidator, err := buildJWTValidator(ctx, logger,
+		os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE"),
+		startupretry.Policy{Attempts: 8, Base: time.Second, Max: 30 * time.Second})
 	if err != nil {
 		logger.Error("build jwt validator", "err", err)
 		os.Exit(1)

ingestion/go.mod

@@ -34,9 +34,9 @@ require (
 	github.com/rogpeppe/go-internal v1.15.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/term v0.43.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
@@ -66,9 +66,9 @@ require (
 	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
-	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/sync v0.17.0 // indirect
-	golang.org/x/sys v0.31.0 // indirect
-	golang.org/x/text v0.29.0 // indirect
+	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/sync v0.21.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/text v0.39.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1
 )

ingestion/go.sum

@@ -117,42 +117,42 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
-golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
+golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
 golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
 golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
-golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
-golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
-golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
+golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
-golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

ingestion/internal/classification/classification.go

@@ -16,9 +16,10 @@
 package classification
 
 import (
+	"errors"
 	"fmt"
-	"os"
-	"path/filepath"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
+	"io/fs"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -110,9 +111,20 @@ type rawConfig struct {
 func Load(brainDir string) (*Config, error) {
 	cfg := &Config{wings: map[string]Level{}, repos: map[string]Level{}}
 
-	data, err := os.ReadFile(filepath.Join(brainDir, "classification.yaml"))
+	// Read through the brain root, regular files only (hyperguild#157): a
+	// symlinked or special file is not trusted as the config, which is a hard
+	// error like a malformed one.
+	bfs, err := brainfs.Open(brainDir)
+	if errors.Is(err, fs.ErrNotExist) {
+		return cfg, nil
+	}
 	if err != nil {
-		if os.IsNotExist(err) {
+		return nil, fmt.Errorf("open brain: %w", err)
+	}
+	defer func() { _ = bfs.Close() }()
+	data, err := bfs.ReadRegular("classification.yaml")
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
 			return cfg, nil
 		}
 		return nil, fmt.Errorf("read classification.yaml: %w", err)

ingestion/internal/classification/root_test.go

@@ -0,0 +1,46 @@
+package classification
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// hyperguild#157 slice 6: Load read classification.yaml with os.ReadFile. It
+// sets the I1 sovereignty levels; a symlinked file must not be trusted as the
+// config (a hard error, as for a malformed file), and a FIFO must not hang
+// startup.
+func TestLoadRefusesASymlinkedOrFIFOClassificationFile(t *testing.T) {
+	ok := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(ok, "classification.yaml"), []byte("wings:\n  w: confidential\n"), 0o644))
+	_, err := Load(ok)
+	require.NoError(t, err, "positive control")
+
+	linked := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(linked, "real.yaml"), []byte("wings:\n  w: public\n"), 0o644))
+	require.NoError(t, os.Symlink("real.yaml", filepath.Join(linked, "classification.yaml")))
+	_, err = Load(linked)
+	assert.Error(t, err, "an in-brain symlink is not trusted as the config")
+
+	fifo := t.TempDir()
+	require.NoError(t, syscall.Mkfifo(filepath.Join(fifo, "classification.yaml"), 0o644))
+	w, err := os.OpenFile(filepath.Join(fifo, "classification.yaml"), os.O_RDWR, 0)
+	require.NoError(t, err)
+	defer func() { _ = w.Close() }()
+	done := make(chan error, 1)
+	go func() { _, err := Load(fifo); done <- err }()
+	select {
+	case err := <-done:
+		assert.Error(t, err)
+	case <-time.After(3 * time.Second):
+		t.Fatal("Load hung on a FIFO")
+	}
+
+	_, err = Load(t.TempDir())
+	assert.NoError(t, err, "an absent file is still not an error")
+}

ingestion/internal/pipeline/backfill.go

@@ -3,7 +3,10 @@ package pipeline
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -23,10 +26,16 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) {
 		return 0, fmt.Errorf("load inventory: %w", err)
 	}
 
-	sourcesDir := filepath.Join(brainDir, "wiki", "sources")
-	entries, err := os.ReadDir(sourcesDir)
+	// Listed and read through the brain root, regular files only
+	// (hyperguild#157).
+	bfs, err := brainfs.Open(brainDir)
 	if err != nil {
-		if os.IsNotExist(err) {
+		return 0, fmt.Errorf("open brain: %w", err)
+	}
+	defer func() { _ = bfs.Close() }()
+	names, err := bfs.ReadDirRegular("wiki/sources")
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
 			return 0, nil
 		}
 		return 0, fmt.Errorf("read sources dir: %w", err)
@@ -37,20 +46,20 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) {
 	// get both refs merged before a single write.
 	pending := make(map[string]wiki.Page)
 
-	for _, e := range entries {
+	for _, name := range names {
 		if ctx.Err() != nil {
 			return 0, ctx.Err()
 		}
-		if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") {
+		if !strings.HasSuffix(name, ".md") {
 			continue
 		}
 
-		b, err := os.ReadFile(filepath.Join(sourcesDir, e.Name()))
+		b, err := bfs.ReadRegular("wiki/sources/" + name)
 		if err != nil {
 			continue
 		}
 		sourceContent := string(b)
-		sourceSlug := strings.TrimSuffix(e.Name(), ".md")
+		sourceSlug := strings.TrimSuffix(name, ".md")
 		sourceTitle := extractTitle(sourceContent)
 		if sourceTitle == "" {
 			sourceTitle = sourceSlug
@@ -70,7 +79,7 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) {
 			// Start from already-accumulated version if we've seen this page.
 			page, seen := pending[relPath]
 			if !seen {
-				raw, err := os.ReadFile(filepath.Join(brainDir, filepath.FromSlash(relPath)))
+				raw, err := bfs.ReadRegular(relPath)
 				if err != nil {
 					continue
 				}

ingestion/internal/pipeline/pipeline.go

@@ -5,7 +5,10 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"fmt"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -129,7 +132,7 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br
 			if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
 				return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
 			}
-			content, conflicts, err := reviseOrCreate(dest, page)
+			content, conflicts, err := reviseOrCreate(brainDir, page)
 			if err != nil {
 				return Result{}, err
 			}
@@ -188,14 +191,27 @@ func hashBody(content string) string {
 
 // readSourcePage returns the full on-disk content of the source page at
 // wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
+// It reads through the brain root, regular files only (hyperguild#157): the
+// content goes into the LLM prompt.
 func readSourcePage(brainDir, sourceSlug string) string {
-	raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
+	raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md")
 	if err != nil {
 		return ""
 	}
 	return string(raw)
 }
 
+// readBrainFile reads one brain-relative file through the brain root,
+// regular files only (hyperguild#157).
+func readBrainFile(brainDir, rel string) ([]byte, error) {
+	bfs, err := brainfs.Open(brainDir)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = bfs.Close() }()
+	return bfs.ReadRegular(rel)
+}
+
 // storedSourceHash reads source_hash from the source page a previous ingest
 // wrote, or "" when there is no such page or no such field.
 //
@@ -230,10 +246,10 @@ func storedSourceHash(brainDir, sourceSlug string) string {
 // its first-ingest date on a revised page. Left deliberately — the alternative
 // is this function editing frontmatter it does not own, and no lint reads that
 // field today.
-func reviseOrCreate(dest string, page wiki.Page) (string, []string, error) {
-	existing, err := os.ReadFile(dest)
+func reviseOrCreate(brainDir string, page wiki.Page) (string, []string, error) {
+	existing, err := readBrainFile(brainDir, page.Path)
 	switch {
-	case os.IsNotExist(err):
+	case errors.Is(err, fs.ErrNotExist):
 		return page.Content, nil, nil
 	case err != nil:
 		return "", nil, fmt.Errorf("read existing %s: %w", page.Path, err)
@@ -267,7 +283,7 @@ See brain/schema.md for the full schema.
 `
 
 func loadSchema(brainDir string) string {
-	b, err := os.ReadFile(filepath.Join(brainDir, "schema.md"))
+	b, err := readBrainFile(brainDir, "schema.md")
 	if err != nil {
 		return defaultSchema
 	}

ingestion/internal/pipeline/refs.go

@@ -2,7 +2,6 @@
 package pipeline
 
 import (
-	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -53,8 +52,7 @@ func injectSourceRefs(pages []wiki.Page, inventory map[wiki.PageType][]wiki.Entr
 		if !ok {
 			continue
 		}
-		diskPath := filepath.Join(brainDir, "wiki", string(pt), slug+".md")
-		b, err := os.ReadFile(diskPath)
+		b, err := readBrainFile(brainDir, "wiki/"+string(pt)+"/"+slug+".md")
 		if err != nil {
 			continue
 		}

ingestion/internal/pipeline/root_test.go

@@ -0,0 +1,167 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
+)
+
+// hyperguild#157 slice 6: the pipeline read the source page, schema.md, the
+// pages it revises and the pages it adds source refs to with os.ReadFile, so
+// a symlinked file's content reached the LLM prompt and the written pages,
+// and a FIFO hung the ingest.
+
+func mkdirs(t *testing.T, brain string, dirs ...string) {
+	t.Helper()
+	for _, d := range dirs {
+		require.NoError(t, os.MkdirAll(filepath.Join(brain, filepath.FromSlash(d)), 0o755))
+	}
+}
+
+func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T) {
+	brain := t.TempDir()
+	mkdirs(t, brain, "wiki/sources")
+	inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644))
+	assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control")
+	assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control")
+
+	require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md")))
+	require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
+	assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel)
+	assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel)
+	assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel)
+
+	src := filepath.Join(brain, "wiki", "sources", "r.md")
+	read := func() string { return readSourcePage(brain, "r") }
+	brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read)
+	brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read)
+	brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
+
+	schema := filepath.Join(brain, "schema.md")
+	rs := func() string { return loadSchema(brain) }
+	brainfstest.AssertNoLeakFrom(t, schema, "zqinside schema", inBrain, 1500*time.Millisecond, rs)
+	brainfstest.AssertNoHang(t, schema, "zqinside schema", 1500*time.Millisecond, 6*time.Second, func() { _ = rs() })
+}
+
+func TestReviseOrCreateRefusesASymlinkedExistingPage(t *testing.T) {
+	brain := t.TempDir()
+	mkdirs(t, brain, "wiki/concepts")
+	brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
+	require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
+	page := wiki.Page{Path: "wiki/concepts/c.md", Content: "---\ntitle: C\n---\nnew\n"}
+	out, _, err := reviseOrCreate(brain, page)
+	assert.Error(t, err)
+	assert.NotContains(t, out, brainfstest.Sentinel)
+
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("---\ntitle: D\n---\nzqold\n"), 0o644))
+	out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/d.md", Content: "---\ntitle: D\n---\nnew\n"})
+	require.NoError(t, err)
+	assert.Contains(t, out, "zqold", "positive control: the existing page is read")
+	out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/new.md", Content: "fresh"})
+	require.NoError(t, err)
+	assert.Equal(t, "fresh", out, "positive control: a missing page is created")
+}
+
+func TestInjectSourceRefsDoesNotReadASymlinkedPage(t *testing.T) {
+	brain := t.TempDir()
+	mkdirs(t, brain, "wiki/concepts")
+	brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
+	require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("zqinside d"), 0o644))
+	inv := map[wiki.PageType][]wiki.Entry{wiki.PageTypeConcept: {
+		{Slug: "c", Type: wiki.PageTypeConcept}, {Slug: "d", Type: wiki.PageTypeConcept},
+	}}
+	src := wiki.Page{Path: "wiki/sources/src.md", Content: "---\ntitle: Src\n---\nsee [[c|C]] and [[d|D]]\n"}
+	out := injectSourceRefs([]wiki.Page{src}, inv, brain)
+	got := fmt.Sprintf("%+v", out)
+	assert.Contains(t, got, "zqinside d", "positive control")
+	assert.NotContains(t, got, brainfstest.Sentinel)
+}
+
+func TestBackfillRefsSkipsSymlinkedSourcesAndPages(t *testing.T) {
+	brain := t.TempDir()
+	mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "real.md"), []byte("---\ntitle: R\n---\nbody\n"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("---\ntitle: zqinside\n---\n[[real|R]]\n"), 0o644))
+	// A source elsewhere in the brain whose title is the sentinel and which
+	// links the real concept with the [[slug|title]] form the pipeline follows.
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "src.md"),
+		[]byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
+	require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "src.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
+	// A concept page linked by the regular source that is itself a symlink.
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok2.md"), []byte("---\ntitle: Two\n---\n[[lnk|L]]\n"), 0o644))
+	brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "page.md"))
+	require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "page.md"), filepath.Join(brain, "wiki", "concepts", "lnk.md")))
+
+	_, err := BackfillRefs(context.Background(), brain)
+	require.NoError(t, err)
+	b, _ := os.ReadFile(filepath.Join(brain, "wiki", "concepts", "real.md"))
+	assert.Contains(t, string(b), "zqinside", "positive control: the regular source's ref is added")
+	assert.NotContains(t, string(b), brainfstest.Sentinel, "a symlinked source's title was written as a ref")
+	raw, _ := os.ReadFile(filepath.Join(brain, "raw", "page.md"))
+	assert.NotContains(t, string(raw), "[[ok2", "the symlinked concept page's target was rewritten")
+}
+
+// A source swapped between a regular file and a symlink to an in-brain source
+// whose title is the sentinel: the read refuses the swapped-in link, so the
+// sentinel never lands in the concept page as a ref.
+func TestBackfillRefsDoesNotWriteARefFromASourceSwappedInBehindASymlink(t *testing.T) {
+	brain := t.TempDir()
+	mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
+	realPage := filepath.Join(brain, "wiki", "concepts", "real.md")
+	target := filepath.Join(brain, "raw", "src.md")
+	require.NoError(t, os.WriteFile(target, []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
+	src := filepath.Join(brain, "wiki", "sources", "r.md")
+	inside := "---\ntitle: zqinside\n---\n[[real|R]]\n"
+	read := func() string {
+		_ = os.WriteFile(realPage, []byte("---\ntitle: R\n---\nbody\n"), 0o644)
+		_, _ = BackfillRefs(context.Background(), brain)
+		b, _ := os.ReadFile(realPage)
+		return string(b)
+	}
+	require.NoError(t, os.WriteFile(src, []byte(inside), 0o644))
+	require.Contains(t, read(), "zqinside", "positive control")
+	brainfstest.AssertNoLeakFrom(t, src, inside, target, 1500*time.Millisecond, read)
+}
+
+// A concept page that is regular when the inventory lists it, then swapped
+// for a symlink to an in-brain file before BackfillRefs reads it: only the
+// page read's own check refuses it (a static symlink never gets this far,
+// because LoadInventory does not list it).
+func TestBackfillRefsDoesNotCopyAPageSwappedInBehindASymlink(t *testing.T) {
+	brain := t.TempDir()
+	mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
+	require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "s.md"), []byte("---\ntitle: S\n---\n[[c|C]]\n"), 0o644))
+	page := filepath.Join(brain, "wiki", "concepts", "c.md")
+	target := filepath.Join(brain, "raw", "secret.md")
+	inside := "---\ntitle: C\n---\nzqinside\n"
+	read := func() string {
+		// The write path still follows a swapped-in link (#158) and can
+		// overwrite the target, so restore it each round.
+		_ = os.WriteFile(target, []byte("---\ntitle: X\n---\n"+brainfstest.Sentinel+"\n"), 0o644)
+		_, _ = BackfillRefs(context.Background(), brain)
+		f, err := os.OpenFile(page, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
+		if err != nil {
+			return ""
+		}
+		defer func() { _ = f.Close() }()
+		b, _ := io.ReadAll(f)
+		return string(b)
+	}
+	require.NoError(t, os.WriteFile(page, []byte(inside), 0o644))
+	require.Contains(t, read(), "zqinside", "positive control: the regular page gets the ref")
+	brainfstest.AssertNoLeakFrom(t, page, inside, target, 3*time.Second, read)
+}