fix(ingestion): brain/raw/ readers go through an os.Root; symlinks are inert (#153) (#156)

Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/watcher/watcher_test.go

@@ -7,6 +7,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"syscall"
 	"testing"
 	"time"
 
@@ -282,3 +285,102 @@ func TestProcessDir_SkipsTunnelCandidateFiles(t *testing.T) {
 	_, err = os.Stat(filepath.Join(brainDir, "raw", "processed", date, "tunnel-candidates-2026-07-19.md"))
 	assert.True(t, os.IsNotExist(err), "tunnel-candidates file should not be copied to processed/")
 }
+
+// hyperguild#153: raw/ is writable via Syncthing, and the watcher ingests
+// whatever lands there with no API call at all. WalkDir yields symlinked
+// files, and extract.Text followed them, so a symlink to any pod-readable
+// file would be extracted into the brain. The watcher now ingests only
+// regular files, read through an os.Root on raw/.
+func TestProcessDir_IngestsTheRealNoteAndSkipsTheSymlinkBesideIt(t *testing.T) {
+	brainDir := setupBrainDir(t)
+	outside := filepath.Join(t.TempDir(), "secret.md")
+	require.NoError(t, os.WriteFile(outside, []byte("SENTINEL-153-WATCHER"), 0o644))
+	// "a-" sorts before "real": a walk that stopped at the first symlink
+	// (filepath.SkipAll — one synced link halting all ingestion) would fail
+	// the call count below (PR #156 review).
+	require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", "a-leak.md")))
+	require.NoError(t, os.WriteFile(filepath.Join(brainDir, "raw", "real.md"), []byte("real note"), 0o644))
+
+	var sawSentinel bool
+	calls := 0
+	cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{
+		Complete: func(_ context.Context, _, user string) (string, error) {
+			calls++
+			if strings.Contains(user, "SENTINEL-153-WATCHER") {
+				sawSentinel = true
+			}
+			b, _ := json.Marshal([]pipeline.RawPage{{Title: "X", Type: "source", Subtype: "article", Content: "## S\n\nx\n"}})
+			return string(b), nil
+		},
+		Schema: "# Schema",
+	}}
+	errs := processDir(context.Background(), cfg, time.Now().UTC().Format("2006-01-02"))
+	assert.Empty(t, errs)
+	assert.False(t, sawSentinel, "the symlink target must never reach the pipeline")
+	assert.Equal(t, 1, calls, "exactly the real note is ingested; the symlink is not a note")
+}
+
+// A symlinked raw/ is refused outright: os.OpenRoot would otherwise follow
+// it and confine reads to wherever it points.
+func TestProcessDir_RefusesASymlinkedRawDir(t *testing.T) {
+	brainDir := t.TempDir()
+	elsewhere := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(elsewhere, "x.md"), []byte("SENTINEL-153-RAWLINK"), 0o644))
+	require.NoError(t, os.Symlink(elsewhere, filepath.Join(brainDir, "raw")))
+	calls := 0
+	cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{
+		Complete: func(context.Context, string, string) (string, error) { calls++; return "[]", nil },
+		Schema:   "# Schema",
+	}}
+	errs := processDir(context.Background(), cfg, time.Now().UTC().Format("2006-01-02"))
+	assert.NotEmpty(t, errs, "a symlinked raw/ must be reported, not silently ingested")
+	assert.Zero(t, calls)
+}
+
+// PR #156: a FIFO swapped into raw/ between the watcher's checks and its
+// read or archive copy must not hang the watcher, which runs unattended.
+func TestProcessDir_DoesNotHangUnderAFIFOSwapRace(t *testing.T) {
+	brainDir := setupBrainDir(t)
+	raw := filepath.Join(brainDir, "raw")
+	var stop atomic.Bool
+	go func() {
+		for i := 0; !stop.Load(); i++ {
+			tmp := filepath.Join(raw, ".swap")
+			_ = os.Remove(tmp)
+			if i%2 == 0 {
+				_ = os.WriteFile(tmp, []byte("x"), 0o644)
+			} else {
+				_ = syscall.Mkfifo(tmp, 0o644)
+			}
+			_ = os.Rename(tmp, filepath.Join(raw, "r.md"))
+		}
+	}()
+	cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{
+		Complete: func(context.Context, string, string) (string, error) {
+			b, _ := json.Marshal([]pipeline.RawPage{{Title: "R", Type: "source", Subtype: "article", Content: "## S\n\nx\n"}})
+			return string(b), nil
+		},
+		Schema: "# Schema",
+	}}
+	var iters atomic.Int64
+	done := make(chan struct{})
+	go func() {
+		for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
+			_ = os.Remove(filepath.Join(raw, "r.md.processed"))
+			_ = os.Remove(filepath.Join(raw, "r.md.failed"))
+			_ = processDir(context.Background(), cfg, "2026-10-03")
+			iters.Add(1)
+		}
+		close(done)
+	}()
+	select {
+	case <-done:
+		stop.Store(true)
+	case <-time.After(8 * time.Second):
+		stop.Store(true)
+		if f, err := os.OpenFile(filepath.Join(raw, "r.md"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil {
+			_ = f.Close()
+		}
+		t.Fatalf("the watcher hung on a swapped-in FIFO after %d iterations", iters.Load())
+	}
+}