fix(ingestion): brain/raw/ readers go through an os.Root; symlinks are inert (#153) #156
@@ -13,7 +13,6 @@ import (
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/claim"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
||||
@@ -481,7 +480,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
|
||||
allWarnings = append(allWarnings, fmt.Sprintf("skipped %s: %v", path, rerr))
|
||||
return nil
|
||||
}
|
||||
content, readErr := extract.Text(path)
|
||||
content, readErr := ReadIngestFile(h.brainDir, path)
|
||||
if readErr != nil {
|
||||
allWarnings = append(allWarnings, fmt.Sprintf("extract %s: %v", path, readErr))
|
||||
return nil
|
||||
@@ -510,7 +509,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext))
|
||||
return
|
||||
}
|
||||
content, readErr := extract.Text(req.Path)
|
||||
content, readErr := ReadIngestFile(h.brainDir, req.Path)
|
||||
if readErr != nil {
|
||||
writeError(w, http.StatusInternalServerError, fmt.Sprintf("extract text: %v", readErr))
|
||||
return
|
||||
|
||||
@@ -2,6 +2,9 @@ package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
@@ -19,6 +22,11 @@ func ResolveIngestPath(brainDir, p string) (string, error) {
|
||||
if strings.TrimSpace(p) == "" {
|
||||
return "", ErrOutsideIngestRoot
|
||||
}
|
||||
// raw/ itself must be a real directory: if it were a symlink, containment
|
||||
// inside it would mean nothing (hyperguild#153 item 4).
|
||||
if fi, lerr := os.Lstat(filepath.Join(brainDir, "raw")); lerr != nil || fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() {
|
||||
return "", ErrOutsideIngestRoot
|
||||
}
|
||||
root, err := filepath.EvalSymlinks(filepath.Join(brainDir, "raw"))
|
||||
if err != nil {
|
||||
return "", ErrOutsideIngestRoot
|
||||
@@ -36,3 +44,24 @@ func ResolveIngestPath(brainDir, p string) (string, error) {
|
||||
}
|
||||
return real, nil
|
||||
}
|
||||
|
||||
// ReadIngestFile extracts text from an ingest path already confined by
|
||||
// ResolveIngestPath, reading it through the raw/ root rather than reopening
|
||||
// the path by name (hyperguild#153 item 3): if the file is swapped for a
|
||||
// symlink after the check, the read still cannot leave raw/.
|
||||
func ReadIngestFile(brainDir, resolved string) (string, error) {
|
||||
root, rawPath, err := OpenRawRoot(brainDir)
|
||||
if err != nil {
|
||||
return "", ErrOutsideIngestRoot
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
realRoot, err := filepath.EvalSymlinks(rawPath)
|
||||
if err != nil {
|
||||
return "", ErrOutsideIngestRoot
|
||||
}
|
||||
rel, err := filepath.Rel(realRoot, resolved)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||
return "", ErrOutsideIngestRoot
|
||||
}
|
||||
return extract.TextIn(root, rel)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@@ -28,25 +30,25 @@ var datePrefix = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}-`)
|
||||
// (natural review order). An absent raw/ dir yields an empty slice, not an
|
||||
// error. Only .md files are listed; tunnel-candidate files are skipped.
|
||||
func ListPending(brainDir string) ([]PendingNote, error) {
|
||||
dir := filepath.Join(brainDir, "raw")
|
||||
entries, err := os.ReadDir(dir)
|
||||
root, _, err := OpenRawRoot(brainDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return []PendingNote{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("read raw dir: %w", err)
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
entries, err := fs.ReadDir(root.FS(), ".")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read raw dir: %w", err)
|
||||
}
|
||||
|
||||
out := make([]PendingNote, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
|
||||
continue
|
||||
if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
|
||||
continue // directories, and symlinks whose target may be anywhere (#153)
|
||||
}
|
||||
info, statErr := e.Info()
|
||||
if statErr != nil {
|
||||
continue
|
||||
}
|
||||
raw, readErr := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
raw, info, readErr := readRawNote(root, e.Name())
|
||||
if readErr != nil {
|
||||
continue
|
||||
}
|
||||
@@ -105,12 +107,9 @@ func PromoteNote(brainDir string, opts PromoteOptions) (string, error) {
|
||||
}
|
||||
|
||||
src := filepath.Join(brainDir, "raw", base)
|
||||
raw, err := os.ReadFile(src)
|
||||
raw, err := readRawSource(brainDir, base)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("pending note %q does not exist in raw/", base)
|
||||
}
|
||||
return "", fmt.Errorf("read source: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Collision: never silently overwrite an existing promoted note.
|
||||
@@ -180,12 +179,9 @@ func RejectNote(brainDir string, opts RejectOptions) (string, error) {
|
||||
}
|
||||
|
||||
src := filepath.Join(brainDir, "raw", base)
|
||||
raw, err := os.ReadFile(src)
|
||||
raw, err := readRawSource(brainDir, base)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("pending note %q does not exist in raw/", base)
|
||||
}
|
||||
return "", fmt.Errorf("read source: %w", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
dest := filepath.Join(brainDir, "raw", "rejected", base)
|
||||
@@ -224,3 +220,24 @@ func excerpt(s string, n int) string {
|
||||
}
|
||||
return strings.TrimSpace(string(r))
|
||||
}
|
||||
|
||||
// readRawSource reads a pending note for promote/reject through the raw root,
|
||||
// refusing symlinks and a symlinked raw/ (hyperguild#153).
|
||||
func readRawSource(brainDir, base string) ([]byte, error) {
|
||||
root, _, err := OpenRawRoot(brainDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("pending note %q does not exist in raw/", base)
|
||||
}
|
||||
return nil, fmt.Errorf("read source: %w", err)
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
raw, _, err := readRawNote(root, base)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("pending note %q does not exist in raw/", base)
|
||||
}
|
||||
return nil, fmt.Errorf("read source: %w", err)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
||||
)
|
||||
|
||||
// hyperguild#153: readers of brain/raw/ followed symlinks. /pending showed the
|
||||
// target's contents and promote would copy it into wiki/ (and brain-sync to
|
||||
// Gitea). raw/ is writable via Syncthing, so a symlink there is reachable
|
||||
// input. Every raw/ reader now treats a symlink as not-a-note.
|
||||
const sentinel153 = "SENTINEL-153-OUTSIDE"
|
||||
|
||||
func rawWithSymlink(t *testing.T) (brain string) {
|
||||
t.Helper()
|
||||
brain = t.TempDir()
|
||||
raw := filepath.Join(brain, "raw")
|
||||
require.NoError(t, os.MkdirAll(raw, 0o755))
|
||||
outside := filepath.Join(t.TempDir(), "secret.md")
|
||||
require.NoError(t, os.WriteFile(outside, []byte("---\ncreated_at: 2026-01-01T00:00:00Z\n---\n# s\n\n"+sentinel153), 0o644))
|
||||
require.NoError(t, os.Symlink(outside, filepath.Join(raw, "2026-01-01-leak.md")))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-01-02-real.md"), []byte("# real\n\nreal body"), 0o644))
|
||||
return brain
|
||||
}
|
||||
|
||||
func TestListPendingSkipsSymlinkedNotes(t *testing.T) {
|
||||
notes, err := api.ListPending(rawWithSymlink(t))
|
||||
require.NoError(t, err)
|
||||
var names []string
|
||||
for _, n := range notes {
|
||||
names = append(names, n.Filename)
|
||||
assert.NotContains(t, n.Excerpt, sentinel153)
|
||||
}
|
||||
assert.Equal(t, []string{"2026-01-02-real.md"}, names)
|
||||
}
|
||||
|
||||
func TestPromoteNoteRefusesASymlinkedSource(t *testing.T) {
|
||||
brain := rawWithSymlink(t)
|
||||
_, err := api.PromoteNote(brain, api.PromoteOptions{Filename: "2026-01-01-leak.md", Wing: "testing", Hall: "facts"})
|
||||
require.Error(t, err)
|
||||
_, statErr := os.Stat(filepath.Join(brain, "wiki", "testing", "facts", "leak.md"))
|
||||
assert.True(t, os.IsNotExist(statErr), "nothing may be promoted from a symlink")
|
||||
}
|
||||
|
||||
func TestRejectNoteRefusesASymlinkedSource(t *testing.T) {
|
||||
brain := rawWithSymlink(t)
|
||||
_, err := api.RejectNote(brain, api.RejectOptions{Filename: "2026-01-01-leak.md"})
|
||||
require.Error(t, err)
|
||||
b, _ := os.ReadFile(filepath.Join(brain, "raw", "rejected", "2026-01-01-leak.md"))
|
||||
assert.False(t, strings.Contains(string(b), sentinel153))
|
||||
}
|
||||
|
||||
// Item 4: if raw/ itself is a symlink, containment inside it means nothing.
|
||||
func TestRawReadersRefuseASymlinkedRawRoot(t *testing.T) {
|
||||
brain := t.TempDir()
|
||||
elsewhere := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(elsewhere, "x.md"), []byte("# x\n\n"+sentinel153), 0o644))
|
||||
require.NoError(t, os.Symlink(elsewhere, filepath.Join(brain, "raw")))
|
||||
|
||||
_, err := api.ResolveIngestPath(brain, "x.md")
|
||||
assert.Error(t, err, "ingest must refuse a symlinked raw/")
|
||||
notes, err := api.ListPending(brain)
|
||||
assert.Error(t, err, "pending must refuse a symlinked raw/")
|
||||
assert.Empty(t, notes)
|
||||
_, err = api.PromoteNote(brain, api.PromoteOptions{Filename: "x.md", Wing: "testing", Hall: "facts"})
|
||||
assert.Error(t, err, "promote must refuse a symlinked raw/")
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// errNotANote is returned for a raw/ entry that is not a regular file — in
|
||||
// particular a symlink, whose target may be anywhere (hyperguild#153).
|
||||
var errNotANote = errors.New("not a regular note file in raw/ (symlinks are refused)")
|
||||
|
||||
// errRawNotDir means brain/raw is not a real directory (e.g. a symlink), so
|
||||
// containment inside it would mean nothing (hyperguild#153 item 4).
|
||||
var errRawNotDir = errors.New("brain/raw is not a real directory")
|
||||
|
||||
// OpenRawRoot opens brain/raw as an os.Root, refusing a raw/ that is itself a
|
||||
// symlink. Reads through the root cannot leave it, even if an entry is
|
||||
// swapped for a symlink between a check and the read. It returns the root
|
||||
// and raw/'s path. A missing raw/ returns an error wrapping os.ErrNotExist.
|
||||
func OpenRawRoot(brainDir string) (*os.Root, string, error) {
|
||||
p := filepath.Join(brainDir, "raw")
|
||||
fi, err := os.Lstat(p)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() {
|
||||
return nil, "", errRawNotDir
|
||||
}
|
||||
r, err := os.OpenRoot(p)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("open raw root: %w", err)
|
||||
}
|
||||
return r, p, nil
|
||||
}
|
||||
|
||||
// readRawNote reads one raw/ entry by basename through the root, refusing
|
||||
// anything that is not a regular file.
|
||||
func readRawNote(r *os.Root, base string) ([]byte, os.FileInfo, error) {
|
||||
fi, err := r.Lstat(base)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !fi.Mode().IsRegular() {
|
||||
return nil, nil, errNotANote
|
||||
}
|
||||
f, err := r.Open(base)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return b, fi, nil
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package extract
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
@@ -41,3 +42,48 @@ func fileExt(path string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TextIn is Text for a file read through an os.Root (hyperguild#153): the
|
||||
// read cannot leave the root, so a path checked earlier cannot be swapped
|
||||
// for a symlink pointing elsewhere before it is read. Formats that need an
|
||||
// external tool (pdf, docmark) are copied to a temp file from the handle
|
||||
// opened through the root, then extracted from the copy.
|
||||
func TextIn(r *os.Root, rel string) (string, error) {
|
||||
fi, err := r.Lstat(rel)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stat %s: %w", rel, err)
|
||||
}
|
||||
if !fi.Mode().IsRegular() {
|
||||
return "", fmt.Errorf("%s: not a regular file", rel)
|
||||
}
|
||||
f, err := r.Open(rel)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("open %s: %w", rel, err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
ext := strings.ToLower(fileExt(rel))
|
||||
switch ext {
|
||||
case ".md", ".txt":
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", rel, err)
|
||||
}
|
||||
return string(b), nil
|
||||
case ".pdf", ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
||||
tmp, err := os.CreateTemp("", "ingest-*"+ext)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("temp copy: %w", err)
|
||||
}
|
||||
defer func() { _ = os.Remove(tmp.Name()) }()
|
||||
if _, err := io.Copy(tmp, f); err != nil {
|
||||
_ = tmp.Close()
|
||||
return "", fmt.Errorf("copy %s: %w", rel, err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return "", fmt.Errorf("copy %s: %w", rel, err)
|
||||
}
|
||||
return Text(tmp.Name())
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported file extension: %s", ext)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package extract
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// hyperguild#153 item 3: ingest checked a path, then extract.Text reopened it
|
||||
// by name, so a file swapped for a symlink in between was followed out of
|
||||
// raw/. TextIn reads through an os.Root, which refuses to leave the root.
|
||||
func TestTextInReadsInsideTheRootAndRefusesEscapes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(root, "ok.md"), []byte("inside"), 0o644))
|
||||
outside := filepath.Join(t.TempDir(), "secret.md")
|
||||
require.NoError(t, os.WriteFile(outside, []byte("SENTINEL-153-TEXTIN"), 0o644))
|
||||
require.NoError(t, os.Symlink(outside, filepath.Join(root, "leak.md")))
|
||||
|
||||
r, err := os.OpenRoot(root)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = r.Close() }()
|
||||
|
||||
got, err := TextIn(r, "ok.md")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "inside", got)
|
||||
|
||||
got, err = TextIn(r, "leak.md")
|
||||
assert.Error(t, err)
|
||||
assert.NotContains(t, got, "SENTINEL-153-TEXTIN")
|
||||
|
||||
_, err = TextIn(r, "../"+filepath.Base(outside))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/capture"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/graphsync"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/notes"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
||||
@@ -549,7 +548,7 @@ func (s *Server) brainIngest(ctx context.Context, args json.RawMessage) (json.Ra
|
||||
return nil, err
|
||||
}
|
||||
a.Path = resolved
|
||||
text, err := extract.Text(a.Path)
|
||||
text, err := api.ReadIngestFile(s.brainDir, a.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: %w", err)
|
||||
}
|
||||
|
||||
@@ -48,8 +48,25 @@ func Start(ctx context.Context, cfg Config) {
|
||||
func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
rawDir := filepath.Join(cfg.BrainDir, "raw")
|
||||
|
||||
// Every read goes through an os.Root on raw/ (hyperguild#153): raw/ is
|
||||
// writable via Syncthing, and a file swapped for a symlink between the walk
|
||||
// and the read must not lead out of it. A symlinked raw/ is refused.
|
||||
if fi, lerr := os.Lstat(rawDir); lerr != nil {
|
||||
if os.IsNotExist(lerr) {
|
||||
return nil
|
||||
}
|
||||
return []error{fmt.Errorf("stat raw dir: %w", lerr)}
|
||||
} else if fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() {
|
||||
return []error{fmt.Errorf("raw dir %s is not a real directory; refusing to ingest from it", rawDir)}
|
||||
}
|
||||
root, err := os.OpenRoot(rawDir)
|
||||
if err != nil {
|
||||
return []error{fmt.Errorf("open raw root: %w", err)}
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
var errs []error
|
||||
err := filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error {
|
||||
err = filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -86,6 +103,10 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
if strings.HasPrefix(d.Name(), "tunnel-candidates-") {
|
||||
return nil
|
||||
}
|
||||
// Only regular files are notes; a symlink's target may be anywhere.
|
||||
if !d.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip files that have already been processed or permanently failed.
|
||||
if _, err := os.Stat(path + ".processed"); err == nil {
|
||||
@@ -95,7 +116,7 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := processFile(ctx, cfg, path, date); err != nil {
|
||||
if err := processFile(ctx, cfg, root, rawDir, path, date); err != nil {
|
||||
errs = append(errs, fmt.Errorf("process %s: %w", filepath.Base(path), err))
|
||||
}
|
||||
return nil
|
||||
@@ -109,11 +130,15 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
// processFile reads a file, calls pipeline.Run, copies it to processed/ or failed/,
|
||||
// and writes a marker file next to the original so the watcher skips it next poll.
|
||||
// The original file is never deleted, keeping Syncthing-connected vaults (e.g. Obsidian) intact.
|
||||
func processFile(ctx context.Context, cfg Config, path, date string) error {
|
||||
func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, date string) error {
|
||||
filename := filepath.Base(path)
|
||||
source := deriveSource(filename)
|
||||
rel, err := filepath.Rel(rawDir, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("relative path: %w", err)
|
||||
}
|
||||
|
||||
content, err := extract.Text(path)
|
||||
content, err := extract.TextIn(root, rel)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extract text: %w", err)
|
||||
}
|
||||
@@ -126,7 +151,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
|
||||
return fmt.Errorf("mkdir failed dir: %w", mkErr)
|
||||
}
|
||||
dest := filepath.Join(failedDir, filename)
|
||||
if cpErr := copyFile(path, dest); cpErr != nil {
|
||||
if cpErr := copyFromRoot(root, rel, dest); cpErr != nil {
|
||||
return fmt.Errorf("copy to failed: %w", cpErr)
|
||||
}
|
||||
if mkErr := os.WriteFile(path+".failed", []byte(runErr.Error()), 0o644); mkErr != nil {
|
||||
@@ -154,7 +179,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
|
||||
base := strings.TrimSuffix(filename, ext)
|
||||
dest = filepath.Join(processedDir, base+"-"+time.Now().UTC().Format("150405")+ext)
|
||||
}
|
||||
if err := copyFile(path, dest); err != nil {
|
||||
if err := copyFromRoot(root, rel, dest); err != nil {
|
||||
return fmt.Errorf("copy to processed: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(path+".processed", []byte(date), 0o644); err != nil {
|
||||
@@ -165,26 +190,6 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies src to dst, creating dst if it doesn't exist.
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open src: %w", err)
|
||||
}
|
||||
defer in.Close() //nolint:errcheck
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create dst: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close() //nolint:errcheck
|
||||
return fmt.Errorf("copy: %w", err)
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
|
||||
// deriveSource turns a filename into a human-readable source name.
|
||||
// "shape-up-book.md" → "Shape Up Book"
|
||||
func deriveSource(filename string) string {
|
||||
@@ -221,3 +226,30 @@ func appendWatcherLog(brainDir, filename string, runErr error, date string) erro
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// copyFromRoot archives a raw/ file read through the root, refusing anything
|
||||
// that is not a regular file, so a swapped-in symlink cannot copy an outside
|
||||
// file into raw/processed/ or raw/failed/ (hyperguild#153).
|
||||
func copyFromRoot(root *os.Root, rel, dst string) error {
|
||||
fi, err := root.Lstat(rel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !fi.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s: not a regular file", rel)
|
||||
}
|
||||
in, err := root.Open(rel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -282,3 +283,32 @@ func TestProcessDir_SkipsTunnelCandidateFiles(t *testing.T) {
|
||||
_, err = os.Stat(filepath.Join(brainDir, "raw", "processed", date, "tunnel-candidates-2026-07-19.md"))
|
||||
assert.True(t, os.IsNotExist(err), "tunnel-candidates file should not be copied to processed/")
|
||||
}
|
||||
|
||||
// hyperguild#153: raw/ is writable via Syncthing, and the watcher ingests
|
||||
// whatever lands there with no API call at all. WalkDir yields symlinked
|
||||
// files, and extract.Text followed them, so a symlink to any pod-readable
|
||||
// file would be extracted into the brain. The watcher now ingests only
|
||||
// regular files, read through an os.Root on raw/.
|
||||
func TestProcessDir_DoesNotIngestSymlinks(t *testing.T) {
|
||||
brainDir := setupBrainDir(t)
|
||||
outside := filepath.Join(t.TempDir(), "secret.md")
|
||||
require.NoError(t, os.WriteFile(outside, []byte("SENTINEL-153-WATCHER"), 0o644))
|
||||
require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", "leak.md")))
|
||||
|
||||
var sawSentinel bool
|
||||
calls := 0
|
||||
cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{
|
||||
Complete: func(_ context.Context, _, user string) (string, error) {
|
||||
calls++
|
||||
if strings.Contains(user, "SENTINEL-153-WATCHER") {
|
||||
sawSentinel = true
|
||||
}
|
||||
b, _ := json.Marshal([]pipeline.RawPage{{Title: "X", Type: "source", Subtype: "article", Content: "## S\n\nx\n"}})
|
||||
return string(b), nil
|
||||
},
|
||||
Schema: "# Schema",
|
||||
}}
|
||||
_ = processDir(context.Background(), cfg, time.Now().UTC().Format("2006-01-02"))
|
||||
assert.False(t, sawSentinel, "the symlink target must never reach the pipeline")
|
||||
assert.Zero(t, calls, "a symlink is not a note")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user