fix(ingestion): every brain/raw/ reader goes through an os.Root; symlinks are inert
CI / Lint / Test / Vet (pull_request) Successful in 34s
CI / Mirror to GitHub (pull_request) Skipped

raw/ is writable via Syncthing (a sendreceive folder shared with
flamingo, symlinks synced) and by the brain CronJobs, and several
readers followed symlinks out of it:

- /pending and brain_pending showed a symlink target's contents;
  brain_promote would copy it into wiki/ (then brain-sync to Gitea);
  brain_reject would copy it into raw/rejected/.
- The watcher, which needs no API call at all, ingested symlinked files
  (WalkDir yields them; extract.Text followed them). Not in the issue;
  found while fixing it.
- Ingest checked a path then reopened it by name (TOCTOU).

OpenRawRoot opens brain/raw as an os.Root and refuses a raw/ that is
itself a symlink. ListPending, PromoteNote and RejectNote read through
it and refuse anything not a regular file. extract.TextIn reads through
a root (temp copy for pdf/docmark); /ingest-path, brain_ingest and the
watcher use it, and the watcher archives via the root too, so a file
swapped for a symlink after a check still cannot lead out.

Refs #153

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-03 16:47:43 +02:00
co-authored by Claude Opus 5.5
parent cfe3f24e25
commit 8a91feb02b
10 changed files with 372 additions and 51 deletions
+58 -26
View File
@@ -48,8 +48,25 @@ func Start(ctx context.Context, cfg Config) {
func processDir(ctx context.Context, cfg Config, date string) []error {
rawDir := filepath.Join(cfg.BrainDir, "raw")
// Every read goes through an os.Root on raw/ (hyperguild#153): raw/ is
// writable via Syncthing, and a file swapped for a symlink between the walk
// and the read must not lead out of it. A symlinked raw/ is refused.
if fi, lerr := os.Lstat(rawDir); lerr != nil {
if os.IsNotExist(lerr) {
return nil
}
return []error{fmt.Errorf("stat raw dir: %w", lerr)}
} else if fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() {
return []error{fmt.Errorf("raw dir %s is not a real directory; refusing to ingest from it", rawDir)}
}
root, err := os.OpenRoot(rawDir)
if err != nil {
return []error{fmt.Errorf("open raw root: %w", err)}
}
defer func() { _ = root.Close() }()
var errs []error
err := filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error {
err = filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
@@ -86,6 +103,10 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
if strings.HasPrefix(d.Name(), "tunnel-candidates-") {
return nil
}
// Only regular files are notes; a symlink's target may be anywhere.
if !d.Type().IsRegular() {
return nil
}
// Skip files that have already been processed or permanently failed.
if _, err := os.Stat(path + ".processed"); err == nil {
@@ -95,7 +116,7 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
return nil
}
if err := processFile(ctx, cfg, path, date); err != nil {
if err := processFile(ctx, cfg, root, rawDir, path, date); err != nil {
errs = append(errs, fmt.Errorf("process %s: %w", filepath.Base(path), err))
}
return nil
@@ -109,11 +130,15 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
// processFile reads a file, calls pipeline.Run, copies it to processed/ or failed/,
// and writes a marker file next to the original so the watcher skips it next poll.
// The original file is never deleted, keeping Syncthing-connected vaults (e.g. Obsidian) intact.
func processFile(ctx context.Context, cfg Config, path, date string) error {
func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, date string) error {
filename := filepath.Base(path)
source := deriveSource(filename)
rel, err := filepath.Rel(rawDir, path)
if err != nil {
return fmt.Errorf("relative path: %w", err)
}
content, err := extract.Text(path)
content, err := extract.TextIn(root, rel)
if err != nil {
return fmt.Errorf("extract text: %w", err)
}
@@ -126,7 +151,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
return fmt.Errorf("mkdir failed dir: %w", mkErr)
}
dest := filepath.Join(failedDir, filename)
if cpErr := copyFile(path, dest); cpErr != nil {
if cpErr := copyFromRoot(root, rel, dest); cpErr != nil {
return fmt.Errorf("copy to failed: %w", cpErr)
}
if mkErr := os.WriteFile(path+".failed", []byte(runErr.Error()), 0o644); mkErr != nil {
@@ -154,7 +179,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
base := strings.TrimSuffix(filename, ext)
dest = filepath.Join(processedDir, base+"-"+time.Now().UTC().Format("150405")+ext)
}
if err := copyFile(path, dest); err != nil {
if err := copyFromRoot(root, rel, dest); err != nil {
return fmt.Errorf("copy to processed: %w", err)
}
if err := os.WriteFile(path+".processed", []byte(date), 0o644); err != nil {
@@ -165,26 +190,6 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
return nil
}
// copyFile copies src to dst, creating dst if it doesn't exist.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("open src: %w", err)
}
defer in.Close() //nolint:errcheck
out, err := os.Create(dst)
if err != nil {
return fmt.Errorf("create dst: %w", err)
}
if _, err := io.Copy(out, in); err != nil {
out.Close() //nolint:errcheck
return fmt.Errorf("copy: %w", err)
}
return out.Close()
}
// deriveSource turns a filename into a human-readable source name.
// "shape-up-book.md" → "Shape Up Book"
func deriveSource(filename string) string {
@@ -221,3 +226,30 @@ func appendWatcherLog(brainDir, filename string, runErr error, date string) erro
}
return f.Close()
}
// copyFromRoot archives a raw/ file read through the root, refusing anything
// that is not a regular file, so a swapped-in symlink cannot copy an outside
// file into raw/processed/ or raw/failed/ (hyperguild#153).
func copyFromRoot(root *os.Root, rel, dst string) error {
fi, err := root.Lstat(rel)
if err != nil {
return err
}
if !fi.Mode().IsRegular() {
return fmt.Errorf("%s: not a regular file", rel)
}
in, err := root.Open(rel)
if err != nil {
return err
}
defer func() { _ = in.Close() }()
out, err := os.Create(dst)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
_ = out.Close()
return err
}
return out.Close()
}