fix: pipeline, backfill, watcher and archive writes go through the brain root (#158 part B)

The remaining writers created or rewrote brain files by path:
- pipeline buildAndWrite: os.MkdirAll + os.WriteFile of every page.
- BackfillRefs: os.WriteFile of each concept page; a page swapped for a
  symlink after the read was written through (the race test overwrote an
  in-brain target before this change).
- watcher: the raw/processed and raw/failed archive copies (os.Create),
  the .processed/.failed markers and the log.md entry (os.OpenFile).
- cmd/server: the claude-session archive.

All now go through brainfs PutRegular/AppendRegular (new package-level
PutFile/AppendFile for one-off writers: the watcher's call-site lint bans
.Open calls in that file so raw/ reads stay on OpenRegular).

Stacked on #189 (needs PutRegular/AppendRegular).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ingestion/internal/watcher/watcher.go

@@ -12,6 +12,7 @@ import (
 	"time"
 	"unicode"
 
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
 )
@@ -145,16 +146,12 @@ func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, d
 
 	_, runErr := pipeline.Run(ctx, cfg.Pipeline, cfg.BrainDir, content, source, false)
 	if runErr != nil {
-		// Copy to failed/ and leave a .failed marker so we don't retry.
-		failedDir := filepath.Join(cfg.BrainDir, "raw", "failed")
-		if mkErr := os.MkdirAll(failedDir, 0o755); mkErr != nil {
-			return fmt.Errorf("mkdir failed dir: %w", mkErr)
-		}
-		dest := filepath.Join(failedDir, filename)
-		if cpErr := copyFromRoot(root, rel, dest); cpErr != nil {
+		// Copy to failed/ and leave a .failed marker so we don't retry. Both
+		// written through the brain root (hyperguild#158).
+		if cpErr := copyFromRoot(root, rel, cfg.BrainDir, "raw/failed/"+filename); cpErr != nil {
 			return fmt.Errorf("copy to failed: %w", cpErr)
 		}
-		if mkErr := os.WriteFile(path+".failed", []byte(runErr.Error()), 0o644); mkErr != nil {
+		if mkErr := putBrainFile(cfg.BrainDir, "raw/"+filepath.ToSlash(rel)+".failed", []byte(runErr.Error())); mkErr != nil {
 			slog.Error("watcher: failed to write .failed marker", "error", mkErr)
 		}
 
@@ -168,21 +165,18 @@ func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, d
 	}
 
 	// Copy to processed/YYYY-MM-DD/ and leave a .processed marker so we don't re-ingest.
-	processedDir := filepath.Join(cfg.BrainDir, "raw", "processed", date)
-	if err := os.MkdirAll(processedDir, 0o755); err != nil {
-		return fmt.Errorf("mkdir processed dir: %w", err)
-	}
-	dest := filepath.Join(processedDir, filename)
-	if _, err := os.Stat(dest); err == nil {
+	// Written through the brain root (hyperguild#158).
+	dest := "raw/processed/" + date + "/" + filename
+	if _, err := os.Lstat(filepath.Join(cfg.BrainDir, filepath.FromSlash(dest))); err == nil {
 		// Archive copy already exists; append timestamp to avoid overwriting.
 		ext := filepath.Ext(filename)
 		base := strings.TrimSuffix(filename, ext)
-		dest = filepath.Join(processedDir, base+"-"+time.Now().UTC().Format("150405")+ext)
+		dest = "raw/processed/" + date + "/" + base + "-" + time.Now().UTC().Format("150405") + ext
 	}
-	if err := copyFromRoot(root, rel, dest); err != nil {
+	if err := copyFromRoot(root, rel, cfg.BrainDir, dest); err != nil {
 		return fmt.Errorf("copy to processed: %w", err)
 	}
-	if err := os.WriteFile(path+".processed", []byte(date), 0o644); err != nil {
+	if err := putBrainFile(cfg.BrainDir, "raw/"+filepath.ToSlash(rel)+".processed", []byte(date)); err != nil {
 		slog.Error("watcher: failed to write .processed marker", "error", err)
 	}
 
@@ -214,23 +208,22 @@ func appendWatcherLog(brainDir, filename string, runErr error, date string) erro
 	entry := fmt.Sprintf("## %s — watcher error\n\n- **File:** %s\n- **Error:** %s\n\n",
 		date, filename, runErr.Error())
 
-	logPath := filepath.Join(brainDir, "log.md")
-	f, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
-	if err != nil {
-		return fmt.Errorf("open log: %w", err)
-	}
-
-	if _, err = f.WriteString(entry); err != nil {
-		f.Close() //nolint:errcheck
+	if err := brainfs.AppendFile(brainDir, "log.md", []byte(entry)); err != nil {
 		return fmt.Errorf("write log: %w", err)
 	}
-	return f.Close()
+	return nil
+}
+
+// putBrainFile creates or replaces a brain-relative file through the brain
+// root, never through a symlinked directory or file (hyperguild#158).
+func putBrainFile(brainDir, rel string, data []byte) error {
+	return brainfs.PutFile(brainDir, rel, data)
 }
 
 // copyFromRoot archives a raw/ file read through the root, refusing anything
 // that is not a regular file, so a swapped-in symlink cannot copy an outside
 // file into raw/processed/ or raw/failed/ (hyperguild#153).
-func copyFromRoot(root *os.Root, rel, dst string) error {
+func copyFromRoot(root *os.Root, rel, brainDir, dstRel string) error {
 	fi, err := root.Lstat(rel)
 	if err != nil {
 		return err
@@ -243,13 +236,9 @@ func copyFromRoot(root *os.Root, rel, dst string) error {
 		return err
 	}
 	defer func() { _ = in.Close() }()
-	out, err := os.Create(dst)
+	data, err := io.ReadAll(in)
 	if err != nil {
 		return err
 	}
-	if _, err := io.Copy(out, in); err != nil {
-		_ = out.Close()
-		return err
-	}
-	return out.Close()
+	return putBrainFile(brainDir, dstRel, data)
 }

ingestion/internal/watcher/writes_redirect_test.go

@@ -0,0 +1,66 @@
+package watcher
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
+)
+
+func okPipeline() pipeline.Config {
+	return pipeline.Config{Complete: func(context.Context, string, string) (string, error) {
+		b, _ := json.Marshal([]pipeline.RawPage{{Title: "X", Type: "source", Subtype: "article", Content: "## S\n\nx\n"}})
+		return string(b), nil
+	}, Schema: "# Schema"}
+}
+
+func failingPipeline() pipeline.Config {
+	return pipeline.Config{Complete: func(context.Context, string, string) (string, error) {
+		return "", errors.New("model down")
+	}, Schema: "# Schema"}
+}
+
+// hyperguild#158: the watcher wrote the archive copy under raw/processed or
+// raw/failed, and its log.md entry, by path, so a symlinked archive
+// directory or log redirected the write elsewhere in the brain or outside
+// it. (A pre-planted marker symlink makes the watcher skip the note, so the
+// marker write itself is reachable only by a race.)
+func TestProcessDirDoesNotWriteThroughASymlinkedArchiveDirOrLog(t *testing.T) {
+	for name, pcfg := range map[string]pipeline.Config{"processed": okPipeline(), "failed": failingPipeline()} {
+		t.Run(name, func(t *testing.T) {
+			brainDir := setupBrainDir(t)
+			outside := t.TempDir()
+			victim := filepath.Join(brainDir, "wiki", "concepts", "v.md")
+			require.NoError(t, os.WriteFile(victim, []byte("untouched"), 0o644))
+			require.NoError(t, os.WriteFile(filepath.Join(brainDir, "raw", "n.md"), []byte("note"), 0o644))
+			require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", name)))
+			if name == "failed" {
+				require.NoError(t, os.Symlink(filepath.Join("wiki", "concepts", "v.md"), filepath.Join(brainDir, "log.md")))
+			}
+			_ = processDir(context.Background(), Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pcfg}, "2026-10-05")
+			got, _ := os.ReadFile(victim)
+			assert.Equal(t, "untouched", string(got), "the log was written through a symlink")
+			ents, _ := os.ReadDir(outside)
+			assert.Empty(t, ents, "the archive copy was written outside the brain")
+		})
+	}
+}
+
+func TestProcessDirWritesMarkersAndArchiveInRealDirectories(t *testing.T) {
+	brainDir := setupBrainDir(t)
+	require.NoError(t, os.WriteFile(filepath.Join(brainDir, "raw", "n.md"), []byte("note"), 0o644))
+	errs := processDir(context.Background(), Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: okPipeline()}, "2026-10-05")
+	assert.Empty(t, errs)
+	_, err := os.Stat(filepath.Join(brainDir, "raw", "n.md.processed"))
+	assert.NoError(t, err, "positive control: marker written")
+	_, err = os.Stat(filepath.Join(brainDir, "raw", "processed", "2026-10-05", "n.md"))
+	assert.NoError(t, err, "positive control: archive copy written")
+}