feat(ingestion): symlink tripwire for the Syncthing-shared brain dir
The brain dir is a Syncthing sendreceive folder and Syncthing syncs symlinks; search, vectorstore, context and graph readers follow one out of the brain and would index or return what it points to (reproduced with search.Query). Decision (#157): a tripwire now, brain-wide os.Root readers next. - symlinkguard.Scan lists every symlink inside the brain dir (skipping .git; the brain dir itself may be a mount symlink). - At startup the server refuses to start while any exists, naming them. - symlinkguard.Watch rescans every BRAIN_SYMLINK_SCAN_INTERVAL seconds (default 300) and sends one ntfy alert per new set of symlinks. The live brain dir had 0 symlinks on 2026-10-03, checked before this change, so deploying it does not trip. Refs #157 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,7 @@ import (
|
|||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/startupretry"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/startupretry"
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/symlinkguard"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/watcher"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/watcher"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/webhook"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/webhook"
|
||||||
@@ -239,6 +240,10 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
brainDir := envOr("INGEST_BRAIN_DIR", "../brain")
|
brainDir := envOr("INGEST_BRAIN_DIR", "../brain")
|
||||||
|
if err := startupSymlinkCheck(brainDir); err != nil {
|
||||||
|
logger.Error("brain symlink tripwire", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
port := envOr("INGEST_PORT", "3300")
|
port := envOr("INGEST_PORT", "3300")
|
||||||
|
|
||||||
llmURL := envOr("INGEST_LLM_URL", "http://iguana:4000/v1")
|
llmURL := envOr("INGEST_LLM_URL", "http://iguana:4000/v1")
|
||||||
@@ -360,6 +365,8 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
// hyperguild#157: report a symlink that appears in the brain while running.
|
||||||
|
go symlinkguard.Watch(ctx, brainDir, time.Duration(envInt("BRAIN_SYMLINK_SCAN_INTERVAL", 300))*time.Second, buildNotifier())
|
||||||
|
|
||||||
// Built before the watcher, the graph backfill and session ingestion, and
|
// Built before the watcher, the graph backfill and session ingestion, and
|
||||||
// retried rather than fatal (#83). Two separate reasons:
|
// retried rather than fatal (#83). Two separate reasons:
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/symlinkguard"
|
||||||
|
)
|
||||||
|
|
||||||
|
// startupSymlinkCheck is the hyperguild#157 tripwire at boot: a symlink in the
|
||||||
|
// brain dir would let brain readers serve a file from outside the brain, so
|
||||||
|
// the server does not start until it is removed. A scan that cannot run (e.g.
|
||||||
|
// a missing brain dir on first boot) is not treated as an incident.
|
||||||
|
func startupSymlinkCheck(brainDir string) error {
|
||||||
|
found, err := symlinkguard.Scan(brainDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(found) > 0 {
|
||||||
|
return fmt.Errorf("refusing to start: %d symlink(s) in the brain dir %s (hyperguild#157); remove them: %s",
|
||||||
|
len(found), brainDir, strings.Join(found, ", "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hyperguild#157: the server refuses to start while a symlink exists in the
|
||||||
|
// brain dir (Mathias's chosen tripwire), naming it; a clean brain starts.
|
||||||
|
func TestStartupSymlinkCheckRefusesABrainWithASymlink(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "planted")))
|
||||||
|
err := startupSymlinkCheck(brain)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "planted")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartupSymlinkCheckPassesACleanBrain(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "n.md"), []byte("x"), 0o644))
|
||||||
|
assert.NoError(t, startupSymlinkCheck(brain))
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
// Package symlinkguard is the hyperguild#157 tripwire. The brain directory is
|
||||||
|
// a Syncthing sendreceive folder and Syncthing syncs symlinks; several brain
|
||||||
|
// readers (search, vectorstore, context, graph) follow a symlink out of the
|
||||||
|
// brain and would index or return whatever it points to. Until every reader
|
||||||
|
// reads through an os.Root, any symlink inside the brain is treated as an
|
||||||
|
// incident: the server refuses to start on one (main), and Watch reports one
|
||||||
|
// that appears while it runs.
|
||||||
|
package symlinkguard
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Notifier is the out-of-band alert sink (audit.Notifier satisfies it).
|
||||||
|
type Notifier interface {
|
||||||
|
Notify(ctx context.Context, msg string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scan returns the brain-relative paths of every symlink inside brainDir,
|
||||||
|
// skipping .git. brainDir itself may be a symlink (a volume mount); it is
|
||||||
|
// resolved first, and only entries inside it are checked. WalkDir never
|
||||||
|
// descends a symlinked directory, so each link is reported, not followed.
|
||||||
|
func Scan(brainDir string) ([]string, error) {
|
||||||
|
root, err := filepath.EvalSymlinks(brainDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve brain dir: %w", err)
|
||||||
|
}
|
||||||
|
var found []string
|
||||||
|
err = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() && d.Name() == ".git" {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
if d.Type()&os.ModeSymlink != 0 {
|
||||||
|
rel, _ := filepath.Rel(root, p)
|
||||||
|
found = append(found, filepath.ToSlash(rel))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("scan brain dir: %w", err)
|
||||||
|
}
|
||||||
|
sort.Strings(found)
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watch rescans every interval and notifies once per distinct set of
|
||||||
|
// symlinks found, until ctx ends. A scan error is logged, not fatal.
|
||||||
|
func Watch(ctx context.Context, brainDir string, interval time.Duration, n Notifier) {
|
||||||
|
t := time.NewTicker(interval)
|
||||||
|
defer t.Stop()
|
||||||
|
last := ""
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
}
|
||||||
|
found, err := Scan(brainDir)
|
||||||
|
if err != nil {
|
||||||
|
slog.Warn("symlink tripwire scan failed", "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := strings.Join(found, "\n")
|
||||||
|
if key == last {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
last = key
|
||||||
|
if len(found) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
msg := fmt.Sprintf("brain symlink tripwire (hyperguild#157): %d symlink(s) in the brain dir; "+
|
||||||
|
"readers may follow them out of the brain. Remove them: %s", len(found), strings.Join(found, ", "))
|
||||||
|
slog.Error("brain symlink tripwire", "count", len(found), "paths", found)
|
||||||
|
if n != nil {
|
||||||
|
if err := n.Notify(ctx, msg); err != nil {
|
||||||
|
slog.Warn("symlink tripwire notify failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package symlinkguard
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hyperguild#157: the brain dir is a Syncthing sendreceive folder and
|
||||||
|
// Syncthing syncs symlinks; search, vectorstore, context and graph readers
|
||||||
|
// follow them out of the brain. Until every reader goes through an os.Root,
|
||||||
|
// a symlink anywhere in the brain is a tripwire: the server refuses to start
|
||||||
|
// on one, and reports one that appears while it runs.
|
||||||
|
|
||||||
|
func TestScanFindsSymlinksAtAnyDepthAndSkipsGit(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki", "w", "facts"), 0o755))
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, ".git", "objects"), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "w", "facts", "ok.md"), []byte("x"), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(filepath.Join(outside, "f"), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
|
||||||
|
require.NoError(t, os.Symlink(outside, filepath.Join(brain, "wiki", "dirlink")))
|
||||||
|
require.NoError(t, os.Symlink(outside, filepath.Join(brain, ".git", "objects", "alt")))
|
||||||
|
|
||||||
|
found, err := Scan(brain)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.ElementsMatch(t, []string{"wiki/w/facts/leak.md", "wiki/dirlink"}, found)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScanOfACleanBrainFindsNothing(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "n.md"), []byte("x"), 0o644))
|
||||||
|
found, err := Scan(brain)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, found)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The brain dir itself being a symlink is fine (the volume mount may be one);
|
||||||
|
// only entries inside it are checked.
|
||||||
|
func TestScanFollowsTheRootButNotEntries(t *testing.T) {
|
||||||
|
real := t.TempDir()
|
||||||
|
link := filepath.Join(t.TempDir(), "brain")
|
||||||
|
require.NoError(t, os.Symlink(real, link))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(real, "n.md"), []byte("x"), 0o644))
|
||||||
|
found, err := Scan(link)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, found)
|
||||||
|
}
|
||||||
|
|
||||||
|
type recNotifier struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
msgs []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recNotifier) Notify(_ context.Context, m string) error {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.msgs = append(r.msgs, m)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recNotifier) count() int { r.mu.Lock(); defer r.mu.Unlock(); return len(r.msgs) }
|
||||||
|
|
||||||
|
// Watch reports a symlink that appears while running, once per distinct set.
|
||||||
|
func TestWatchNotifiesOncePerNewSymlinkSet(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
n := &recNotifier{}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 10*time.Millisecond, n)
|
||||||
|
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
assert.Zero(t, n.count(), "a clean brain raises nothing")
|
||||||
|
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "planted")))
|
||||||
|
require.Eventually(t, func() bool { return n.count() == 1 }, 2*time.Second, 10*time.Millisecond)
|
||||||
|
time.Sleep(60 * time.Millisecond)
|
||||||
|
assert.Equal(t, 1, n.count(), "the same symlink set is reported once, not every tick")
|
||||||
|
assert.Contains(t, n.msgs[0], "planted")
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user