From 8a91feb02bb8de95764fea48e61439fb270b877c Mon Sep 17 00:00:00 2001 From: Mathias Date: Sat, 3 Oct 2026 16:47:43 +0200 Subject: [PATCH] fix(ingestion): every brain/raw/ reader goes through an os.Root; symlinks are inert raw/ is writable via Syncthing (a sendreceive folder shared with flamingo, symlinks synced) and by the brain CronJobs, and several readers followed symlinks out of it: - /pending and brain_pending showed a symlink target's contents; brain_promote would copy it into wiki/ (then brain-sync to Gitea); brain_reject would copy it into raw/rejected/. - The watcher, which needs no API call at all, ingested symlinked files (WalkDir yields them; extract.Text followed them). Not in the issue; found while fixing it. - Ingest checked a path then reopened it by name (TOCTOU). OpenRawRoot opens brain/raw as an os.Root and refuses a raw/ that is itself a symlink. ListPending, PromoteNote and RejectNote read through it and refuse anything not a regular file. extract.TextIn reads through a root (temp copy for pdf/docmark); /ingest-path, brain_ingest and the watcher use it, and the watcher archives via the root too, so a file swapped for a symlink after a check still cannot lead out. Refs #153 Co-Authored-By: Claude Opus 5.5 --- ingestion/internal/api/handler.go | 5 +- ingestion/internal/api/ingestpath_root.go | 29 ++++++++ ingestion/internal/api/promote.go | 57 +++++++++------ ingestion/internal/api/raw_symlink_test.go | 74 +++++++++++++++++++ ingestion/internal/api/rawroot.go | 59 +++++++++++++++ ingestion/internal/extract/extract.go | 46 ++++++++++++ ingestion/internal/extract/root_test.go | 36 ++++++++++ ingestion/internal/mcp/handlers.go | 3 +- ingestion/internal/watcher/watcher.go | 84 +++++++++++++++------- ingestion/internal/watcher/watcher_test.go | 30 ++++++++ 10 files changed, 372 insertions(+), 51 deletions(-) create mode 100644 ingestion/internal/api/raw_symlink_test.go create mode 100644 ingestion/internal/api/rawroot.go create mode 100644 ingestion/internal/extract/root_test.go diff --git a/ingestion/internal/api/handler.go b/ingestion/internal/api/handler.go index ad49182..3921e89 100644 --- a/ingestion/internal/api/handler.go +++ b/ingestion/internal/api/handler.go @@ -13,7 +13,6 @@ import ( "github.com/mathiasbq/hyperguild/ingestion/internal/brain" "github.com/mathiasbq/hyperguild/ingestion/internal/claim" - "github.com/mathiasbq/hyperguild/ingestion/internal/extract" "github.com/mathiasbq/hyperguild/ingestion/internal/pipeline" "github.com/mathiasbq/hyperguild/ingestion/internal/search" "github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore" @@ -481,7 +480,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) { allWarnings = append(allWarnings, fmt.Sprintf("skipped %s: %v", path, rerr)) return nil } - content, readErr := extract.Text(path) + content, readErr := ReadIngestFile(h.brainDir, path) if readErr != nil { allWarnings = append(allWarnings, fmt.Sprintf("extract %s: %v", path, readErr)) return nil @@ -510,7 +509,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext)) return } - content, readErr := extract.Text(req.Path) + content, readErr := ReadIngestFile(h.brainDir, req.Path) if readErr != nil { writeError(w, http.StatusInternalServerError, fmt.Sprintf("extract text: %v", readErr)) return diff --git a/ingestion/internal/api/ingestpath_root.go b/ingestion/internal/api/ingestpath_root.go index fe36a90..2de2b92 100644 --- a/ingestion/internal/api/ingestpath_root.go +++ b/ingestion/internal/api/ingestpath_root.go @@ -2,6 +2,9 @@ package api import ( "errors" + "os" + + "github.com/mathiasbq/hyperguild/ingestion/internal/extract" "path/filepath" "strings" ) @@ -19,6 +22,11 @@ func ResolveIngestPath(brainDir, p string) (string, error) { if strings.TrimSpace(p) == "" { return "", ErrOutsideIngestRoot } + // raw/ itself must be a real directory: if it were a symlink, containment + // inside it would mean nothing (hyperguild#153 item 4). + if fi, lerr := os.Lstat(filepath.Join(brainDir, "raw")); lerr != nil || fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() { + return "", ErrOutsideIngestRoot + } root, err := filepath.EvalSymlinks(filepath.Join(brainDir, "raw")) if err != nil { return "", ErrOutsideIngestRoot @@ -36,3 +44,24 @@ func ResolveIngestPath(brainDir, p string) (string, error) { } return real, nil } + +// ReadIngestFile extracts text from an ingest path already confined by +// ResolveIngestPath, reading it through the raw/ root rather than reopening +// the path by name (hyperguild#153 item 3): if the file is swapped for a +// symlink after the check, the read still cannot leave raw/. +func ReadIngestFile(brainDir, resolved string) (string, error) { + root, rawPath, err := OpenRawRoot(brainDir) + if err != nil { + return "", ErrOutsideIngestRoot + } + defer func() { _ = root.Close() }() + realRoot, err := filepath.EvalSymlinks(rawPath) + if err != nil { + return "", ErrOutsideIngestRoot + } + rel, err := filepath.Rel(realRoot, resolved) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return "", ErrOutsideIngestRoot + } + return extract.TextIn(root, rel) +} diff --git a/ingestion/internal/api/promote.go b/ingestion/internal/api/promote.go index 7b96b53..c6ec563 100644 --- a/ingestion/internal/api/promote.go +++ b/ingestion/internal/api/promote.go @@ -1,7 +1,9 @@ package api import ( + "errors" "fmt" + "io/fs" "os" "path/filepath" "regexp" @@ -28,25 +30,25 @@ var datePrefix = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}-`) // (natural review order). An absent raw/ dir yields an empty slice, not an // error. Only .md files are listed; tunnel-candidate files are skipped. func ListPending(brainDir string) ([]PendingNote, error) { - dir := filepath.Join(brainDir, "raw") - entries, err := os.ReadDir(dir) + root, _, err := OpenRawRoot(brainDir) if err != nil { - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { return []PendingNote{}, nil } return nil, fmt.Errorf("read raw dir: %w", err) } + defer func() { _ = root.Close() }() + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + return nil, fmt.Errorf("read raw dir: %w", err) + } out := make([]PendingNote, 0, len(entries)) for _, e := range entries { - if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") { - continue + if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") { + continue // directories, and symlinks whose target may be anywhere (#153) } - info, statErr := e.Info() - if statErr != nil { - continue - } - raw, readErr := os.ReadFile(filepath.Join(dir, e.Name())) + raw, info, readErr := readRawNote(root, e.Name()) if readErr != nil { continue } @@ -105,12 +107,9 @@ func PromoteNote(brainDir string, opts PromoteOptions) (string, error) { } src := filepath.Join(brainDir, "raw", base) - raw, err := os.ReadFile(src) + raw, err := readRawSource(brainDir, base) if err != nil { - if os.IsNotExist(err) { - return "", fmt.Errorf("pending note %q does not exist in raw/", base) - } - return "", fmt.Errorf("read source: %w", err) + return "", err } // Collision: never silently overwrite an existing promoted note. @@ -180,12 +179,9 @@ func RejectNote(brainDir string, opts RejectOptions) (string, error) { } src := filepath.Join(brainDir, "raw", base) - raw, err := os.ReadFile(src) + raw, err := readRawSource(brainDir, base) if err != nil { - if os.IsNotExist(err) { - return "", fmt.Errorf("pending note %q does not exist in raw/", base) - } - return "", fmt.Errorf("read source: %w", err) + return "", err } dest := filepath.Join(brainDir, "raw", "rejected", base) @@ -224,3 +220,24 @@ func excerpt(s string, n int) string { } return strings.TrimSpace(string(r)) } + +// readRawSource reads a pending note for promote/reject through the raw root, +// refusing symlinks and a symlinked raw/ (hyperguild#153). +func readRawSource(brainDir, base string) ([]byte, error) { + root, _, err := OpenRawRoot(brainDir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("pending note %q does not exist in raw/", base) + } + return nil, fmt.Errorf("read source: %w", err) + } + defer func() { _ = root.Close() }() + raw, _, err := readRawNote(root, base) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("pending note %q does not exist in raw/", base) + } + return nil, fmt.Errorf("read source: %w", err) + } + return raw, nil +} diff --git a/ingestion/internal/api/raw_symlink_test.go b/ingestion/internal/api/raw_symlink_test.go new file mode 100644 index 0000000..3842e29 --- /dev/null +++ b/ingestion/internal/api/raw_symlink_test.go @@ -0,0 +1,74 @@ +package api_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/api" +) + +// hyperguild#153: readers of brain/raw/ followed symlinks. /pending showed the +// target's contents and promote would copy it into wiki/ (and brain-sync to +// Gitea). raw/ is writable via Syncthing, so a symlink there is reachable +// input. Every raw/ reader now treats a symlink as not-a-note. +const sentinel153 = "SENTINEL-153-OUTSIDE" + +func rawWithSymlink(t *testing.T) (brain string) { + t.Helper() + brain = t.TempDir() + raw := filepath.Join(brain, "raw") + require.NoError(t, os.MkdirAll(raw, 0o755)) + outside := filepath.Join(t.TempDir(), "secret.md") + require.NoError(t, os.WriteFile(outside, []byte("---\ncreated_at: 2026-01-01T00:00:00Z\n---\n# s\n\n"+sentinel153), 0o644)) + require.NoError(t, os.Symlink(outside, filepath.Join(raw, "2026-01-01-leak.md"))) + require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-01-02-real.md"), []byte("# real\n\nreal body"), 0o644)) + return brain +} + +func TestListPendingSkipsSymlinkedNotes(t *testing.T) { + notes, err := api.ListPending(rawWithSymlink(t)) + require.NoError(t, err) + var names []string + for _, n := range notes { + names = append(names, n.Filename) + assert.NotContains(t, n.Excerpt, sentinel153) + } + assert.Equal(t, []string{"2026-01-02-real.md"}, names) +} + +func TestPromoteNoteRefusesASymlinkedSource(t *testing.T) { + brain := rawWithSymlink(t) + _, err := api.PromoteNote(brain, api.PromoteOptions{Filename: "2026-01-01-leak.md", Wing: "testing", Hall: "facts"}) + require.Error(t, err) + _, statErr := os.Stat(filepath.Join(brain, "wiki", "testing", "facts", "leak.md")) + assert.True(t, os.IsNotExist(statErr), "nothing may be promoted from a symlink") +} + +func TestRejectNoteRefusesASymlinkedSource(t *testing.T) { + brain := rawWithSymlink(t) + _, err := api.RejectNote(brain, api.RejectOptions{Filename: "2026-01-01-leak.md"}) + require.Error(t, err) + b, _ := os.ReadFile(filepath.Join(brain, "raw", "rejected", "2026-01-01-leak.md")) + assert.False(t, strings.Contains(string(b), sentinel153)) +} + +// Item 4: if raw/ itself is a symlink, containment inside it means nothing. +func TestRawReadersRefuseASymlinkedRawRoot(t *testing.T) { + brain := t.TempDir() + elsewhere := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(elsewhere, "x.md"), []byte("# x\n\n"+sentinel153), 0o644)) + require.NoError(t, os.Symlink(elsewhere, filepath.Join(brain, "raw"))) + + _, err := api.ResolveIngestPath(brain, "x.md") + assert.Error(t, err, "ingest must refuse a symlinked raw/") + notes, err := api.ListPending(brain) + assert.Error(t, err, "pending must refuse a symlinked raw/") + assert.Empty(t, notes) + _, err = api.PromoteNote(brain, api.PromoteOptions{Filename: "x.md", Wing: "testing", Hall: "facts"}) + assert.Error(t, err, "promote must refuse a symlinked raw/") +} diff --git a/ingestion/internal/api/rawroot.go b/ingestion/internal/api/rawroot.go new file mode 100644 index 0000000..e41d993 --- /dev/null +++ b/ingestion/internal/api/rawroot.go @@ -0,0 +1,59 @@ +package api + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +// errNotANote is returned for a raw/ entry that is not a regular file — in +// particular a symlink, whose target may be anywhere (hyperguild#153). +var errNotANote = errors.New("not a regular note file in raw/ (symlinks are refused)") + +// errRawNotDir means brain/raw is not a real directory (e.g. a symlink), so +// containment inside it would mean nothing (hyperguild#153 item 4). +var errRawNotDir = errors.New("brain/raw is not a real directory") + +// OpenRawRoot opens brain/raw as an os.Root, refusing a raw/ that is itself a +// symlink. Reads through the root cannot leave it, even if an entry is +// swapped for a symlink between a check and the read. It returns the root +// and raw/'s path. A missing raw/ returns an error wrapping os.ErrNotExist. +func OpenRawRoot(brainDir string) (*os.Root, string, error) { + p := filepath.Join(brainDir, "raw") + fi, err := os.Lstat(p) + if err != nil { + return nil, "", err + } + if fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() { + return nil, "", errRawNotDir + } + r, err := os.OpenRoot(p) + if err != nil { + return nil, "", fmt.Errorf("open raw root: %w", err) + } + return r, p, nil +} + +// readRawNote reads one raw/ entry by basename through the root, refusing +// anything that is not a regular file. +func readRawNote(r *os.Root, base string) ([]byte, os.FileInfo, error) { + fi, err := r.Lstat(base) + if err != nil { + return nil, nil, err + } + if !fi.Mode().IsRegular() { + return nil, nil, errNotANote + } + f, err := r.Open(base) + if err != nil { + return nil, nil, err + } + defer func() { _ = f.Close() }() + b, err := io.ReadAll(f) + if err != nil { + return nil, nil, err + } + return b, fi, nil +} diff --git a/ingestion/internal/extract/extract.go b/ingestion/internal/extract/extract.go index f7512ec..6c8ce5e 100644 --- a/ingestion/internal/extract/extract.go +++ b/ingestion/internal/extract/extract.go @@ -3,6 +3,7 @@ package extract import ( "fmt" + "io" "os" "strings" ) @@ -41,3 +42,48 @@ func fileExt(path string) string { } return "" } + +// TextIn is Text for a file read through an os.Root (hyperguild#153): the +// read cannot leave the root, so a path checked earlier cannot be swapped +// for a symlink pointing elsewhere before it is read. Formats that need an +// external tool (pdf, docmark) are copied to a temp file from the handle +// opened through the root, then extracted from the copy. +func TextIn(r *os.Root, rel string) (string, error) { + fi, err := r.Lstat(rel) + if err != nil { + return "", fmt.Errorf("stat %s: %w", rel, err) + } + if !fi.Mode().IsRegular() { + return "", fmt.Errorf("%s: not a regular file", rel) + } + f, err := r.Open(rel) + if err != nil { + return "", fmt.Errorf("open %s: %w", rel, err) + } + defer func() { _ = f.Close() }() + ext := strings.ToLower(fileExt(rel)) + switch ext { + case ".md", ".txt": + b, err := io.ReadAll(f) + if err != nil { + return "", fmt.Errorf("read %s: %w", rel, err) + } + return string(b), nil + case ".pdf", ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg": + tmp, err := os.CreateTemp("", "ingest-*"+ext) + if err != nil { + return "", fmt.Errorf("temp copy: %w", err) + } + defer func() { _ = os.Remove(tmp.Name()) }() + if _, err := io.Copy(tmp, f); err != nil { + _ = tmp.Close() + return "", fmt.Errorf("copy %s: %w", rel, err) + } + if err := tmp.Close(); err != nil { + return "", fmt.Errorf("copy %s: %w", rel, err) + } + return Text(tmp.Name()) + default: + return "", fmt.Errorf("unsupported file extension: %s", ext) + } +} diff --git a/ingestion/internal/extract/root_test.go b/ingestion/internal/extract/root_test.go new file mode 100644 index 0000000..a55ccd2 --- /dev/null +++ b/ingestion/internal/extract/root_test.go @@ -0,0 +1,36 @@ +package extract + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// hyperguild#153 item 3: ingest checked a path, then extract.Text reopened it +// by name, so a file swapped for a symlink in between was followed out of +// raw/. TextIn reads through an os.Root, which refuses to leave the root. +func TestTextInReadsInsideTheRootAndRefusesEscapes(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "ok.md"), []byte("inside"), 0o644)) + outside := filepath.Join(t.TempDir(), "secret.md") + require.NoError(t, os.WriteFile(outside, []byte("SENTINEL-153-TEXTIN"), 0o644)) + require.NoError(t, os.Symlink(outside, filepath.Join(root, "leak.md"))) + + r, err := os.OpenRoot(root) + require.NoError(t, err) + defer func() { _ = r.Close() }() + + got, err := TextIn(r, "ok.md") + require.NoError(t, err) + assert.Equal(t, "inside", got) + + got, err = TextIn(r, "leak.md") + assert.Error(t, err) + assert.NotContains(t, got, "SENTINEL-153-TEXTIN") + + _, err = TextIn(r, "../"+filepath.Base(outside)) + assert.Error(t, err) +} diff --git a/ingestion/internal/mcp/handlers.go b/ingestion/internal/mcp/handlers.go index e4eeb59..62872e1 100644 --- a/ingestion/internal/mcp/handlers.go +++ b/ingestion/internal/mcp/handlers.go @@ -12,7 +12,6 @@ import ( "github.com/mathiasbq/hyperguild/ingestion/internal/api" "github.com/mathiasbq/hyperguild/ingestion/internal/brain" "github.com/mathiasbq/hyperguild/ingestion/internal/capture" - "github.com/mathiasbq/hyperguild/ingestion/internal/extract" "github.com/mathiasbq/hyperguild/ingestion/internal/graphsync" "github.com/mathiasbq/hyperguild/ingestion/internal/notes" "github.com/mathiasbq/hyperguild/ingestion/internal/pipeline" @@ -549,7 +548,7 @@ func (s *Server) brainIngest(ctx context.Context, args json.RawMessage) (json.Ra return nil, err } a.Path = resolved - text, err := extract.Text(a.Path) + text, err := api.ReadIngestFile(s.brainDir, a.Path) if err != nil { return nil, fmt.Errorf("extract: %w", err) } diff --git a/ingestion/internal/watcher/watcher.go b/ingestion/internal/watcher/watcher.go index dea92f9..64c8d87 100644 --- a/ingestion/internal/watcher/watcher.go +++ b/ingestion/internal/watcher/watcher.go @@ -48,8 +48,25 @@ func Start(ctx context.Context, cfg Config) { func processDir(ctx context.Context, cfg Config, date string) []error { rawDir := filepath.Join(cfg.BrainDir, "raw") + // Every read goes through an os.Root on raw/ (hyperguild#153): raw/ is + // writable via Syncthing, and a file swapped for a symlink between the walk + // and the read must not lead out of it. A symlinked raw/ is refused. + if fi, lerr := os.Lstat(rawDir); lerr != nil { + if os.IsNotExist(lerr) { + return nil + } + return []error{fmt.Errorf("stat raw dir: %w", lerr)} + } else if fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() { + return []error{fmt.Errorf("raw dir %s is not a real directory; refusing to ingest from it", rawDir)} + } + root, err := os.OpenRoot(rawDir) + if err != nil { + return []error{fmt.Errorf("open raw root: %w", err)} + } + defer func() { _ = root.Close() }() + var errs []error - err := filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error { + err = filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error { if err != nil { return err } @@ -86,6 +103,10 @@ func processDir(ctx context.Context, cfg Config, date string) []error { if strings.HasPrefix(d.Name(), "tunnel-candidates-") { return nil } + // Only regular files are notes; a symlink's target may be anywhere. + if !d.Type().IsRegular() { + return nil + } // Skip files that have already been processed or permanently failed. if _, err := os.Stat(path + ".processed"); err == nil { @@ -95,7 +116,7 @@ func processDir(ctx context.Context, cfg Config, date string) []error { return nil } - if err := processFile(ctx, cfg, path, date); err != nil { + if err := processFile(ctx, cfg, root, rawDir, path, date); err != nil { errs = append(errs, fmt.Errorf("process %s: %w", filepath.Base(path), err)) } return nil @@ -109,11 +130,15 @@ func processDir(ctx context.Context, cfg Config, date string) []error { // processFile reads a file, calls pipeline.Run, copies it to processed/ or failed/, // and writes a marker file next to the original so the watcher skips it next poll. // The original file is never deleted, keeping Syncthing-connected vaults (e.g. Obsidian) intact. -func processFile(ctx context.Context, cfg Config, path, date string) error { +func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, date string) error { filename := filepath.Base(path) source := deriveSource(filename) + rel, err := filepath.Rel(rawDir, path) + if err != nil { + return fmt.Errorf("relative path: %w", err) + } - content, err := extract.Text(path) + content, err := extract.TextIn(root, rel) if err != nil { return fmt.Errorf("extract text: %w", err) } @@ -126,7 +151,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error { return fmt.Errorf("mkdir failed dir: %w", mkErr) } dest := filepath.Join(failedDir, filename) - if cpErr := copyFile(path, dest); cpErr != nil { + if cpErr := copyFromRoot(root, rel, dest); cpErr != nil { return fmt.Errorf("copy to failed: %w", cpErr) } if mkErr := os.WriteFile(path+".failed", []byte(runErr.Error()), 0o644); mkErr != nil { @@ -154,7 +179,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error { base := strings.TrimSuffix(filename, ext) dest = filepath.Join(processedDir, base+"-"+time.Now().UTC().Format("150405")+ext) } - if err := copyFile(path, dest); err != nil { + if err := copyFromRoot(root, rel, dest); err != nil { return fmt.Errorf("copy to processed: %w", err) } if err := os.WriteFile(path+".processed", []byte(date), 0o644); err != nil { @@ -165,26 +190,6 @@ func processFile(ctx context.Context, cfg Config, path, date string) error { return nil } -// copyFile copies src to dst, creating dst if it doesn't exist. -func copyFile(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return fmt.Errorf("open src: %w", err) - } - defer in.Close() //nolint:errcheck - - out, err := os.Create(dst) - if err != nil { - return fmt.Errorf("create dst: %w", err) - } - - if _, err := io.Copy(out, in); err != nil { - out.Close() //nolint:errcheck - return fmt.Errorf("copy: %w", err) - } - return out.Close() -} - // deriveSource turns a filename into a human-readable source name. // "shape-up-book.md" → "Shape Up Book" func deriveSource(filename string) string { @@ -221,3 +226,30 @@ func appendWatcherLog(brainDir, filename string, runErr error, date string) erro } return f.Close() } + +// copyFromRoot archives a raw/ file read through the root, refusing anything +// that is not a regular file, so a swapped-in symlink cannot copy an outside +// file into raw/processed/ or raw/failed/ (hyperguild#153). +func copyFromRoot(root *os.Root, rel, dst string) error { + fi, err := root.Lstat(rel) + if err != nil { + return err + } + if !fi.Mode().IsRegular() { + return fmt.Errorf("%s: not a regular file", rel) + } + in, err := root.Open(rel) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.Create(dst) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() +} diff --git a/ingestion/internal/watcher/watcher_test.go b/ingestion/internal/watcher/watcher_test.go index 7581194..eeccbe8 100644 --- a/ingestion/internal/watcher/watcher_test.go +++ b/ingestion/internal/watcher/watcher_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -282,3 +283,32 @@ func TestProcessDir_SkipsTunnelCandidateFiles(t *testing.T) { _, err = os.Stat(filepath.Join(brainDir, "raw", "processed", date, "tunnel-candidates-2026-07-19.md")) assert.True(t, os.IsNotExist(err), "tunnel-candidates file should not be copied to processed/") } + +// hyperguild#153: raw/ is writable via Syncthing, and the watcher ingests +// whatever lands there with no API call at all. WalkDir yields symlinked +// files, and extract.Text followed them, so a symlink to any pod-readable +// file would be extracted into the brain. The watcher now ingests only +// regular files, read through an os.Root on raw/. +func TestProcessDir_DoesNotIngestSymlinks(t *testing.T) { + brainDir := setupBrainDir(t) + outside := filepath.Join(t.TempDir(), "secret.md") + require.NoError(t, os.WriteFile(outside, []byte("SENTINEL-153-WATCHER"), 0o644)) + require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", "leak.md"))) + + var sawSentinel bool + calls := 0 + cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{ + Complete: func(_ context.Context, _, user string) (string, error) { + calls++ + if strings.Contains(user, "SENTINEL-153-WATCHER") { + sawSentinel = true + } + b, _ := json.Marshal([]pipeline.RawPage{{Title: "X", Type: "source", Subtype: "article", Content: "## S\n\nx\n"}}) + return string(b), nil + }, + Schema: "# Schema", + }} + _ = processDir(context.Background(), cfg, time.Now().UTC().Format("2006-01-02")) + assert.False(t, sawSentinel, "the symlink target must never reach the pipeline") + assert.Zero(t, calls, "a symlink is not a note") +}