Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4ba9d20b4e | ||
|
|
01de163452 | ||
|
|
504feb372f | ||
|
|
687afa0145 | ||
|
|
cad389f62f |
@@ -13,7 +13,6 @@ import (
|
|||||||
|
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/claim"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/claim"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
|
||||||
@@ -481,7 +480,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
|
|||||||
allWarnings = append(allWarnings, fmt.Sprintf("skipped %s: %v", path, rerr))
|
allWarnings = append(allWarnings, fmt.Sprintf("skipped %s: %v", path, rerr))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
content, readErr := extract.Text(path)
|
content, readErr := ReadIngestFile(h.brainDir, path)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
allWarnings = append(allWarnings, fmt.Sprintf("extract %s: %v", path, readErr))
|
allWarnings = append(allWarnings, fmt.Sprintf("extract %s: %v", path, readErr))
|
||||||
return nil
|
return nil
|
||||||
@@ -510,7 +509,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext))
|
writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
content, readErr := extract.Text(req.Path)
|
content, readErr := ReadIngestFile(h.brainDir, req.Path)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
writeError(w, http.StatusInternalServerError, fmt.Sprintf("extract text: %v", readErr))
|
writeError(w, http.StatusInternalServerError, fmt.Sprintf("extract text: %v", readErr))
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ package api
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
@@ -19,6 +22,11 @@ func ResolveIngestPath(brainDir, p string) (string, error) {
|
|||||||
if strings.TrimSpace(p) == "" {
|
if strings.TrimSpace(p) == "" {
|
||||||
return "", ErrOutsideIngestRoot
|
return "", ErrOutsideIngestRoot
|
||||||
}
|
}
|
||||||
|
// raw/ itself must be a real directory: if it were a symlink, containment
|
||||||
|
// inside it would mean nothing (hyperguild#153 item 4).
|
||||||
|
if fi, lerr := os.Lstat(filepath.Join(brainDir, "raw")); lerr != nil || fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() {
|
||||||
|
return "", ErrOutsideIngestRoot
|
||||||
|
}
|
||||||
root, err := filepath.EvalSymlinks(filepath.Join(brainDir, "raw"))
|
root, err := filepath.EvalSymlinks(filepath.Join(brainDir, "raw"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", ErrOutsideIngestRoot
|
return "", ErrOutsideIngestRoot
|
||||||
@@ -36,3 +44,24 @@ func ResolveIngestPath(brainDir, p string) (string, error) {
|
|||||||
}
|
}
|
||||||
return real, nil
|
return real, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReadIngestFile extracts text from an ingest path already confined by
|
||||||
|
// ResolveIngestPath, reading it through the raw/ root rather than reopening
|
||||||
|
// the path by name (hyperguild#153 item 3): if the file is swapped for a
|
||||||
|
// symlink after the check, the read still cannot leave raw/.
|
||||||
|
func ReadIngestFile(brainDir, resolved string) (string, error) {
|
||||||
|
root, rawPath, err := OpenRawRoot(brainDir)
|
||||||
|
if err != nil {
|
||||||
|
return "", ErrOutsideIngestRoot
|
||||||
|
}
|
||||||
|
defer func() { _ = root.Close() }()
|
||||||
|
realRoot, err := filepath.EvalSymlinks(rawPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", ErrOutsideIngestRoot
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(realRoot, resolved)
|
||||||
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||||
|
return "", ErrOutsideIngestRoot
|
||||||
|
}
|
||||||
|
return extract.TextIn(root, rel)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
@@ -28,25 +30,25 @@ var datePrefix = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}-`)
|
|||||||
// (natural review order). An absent raw/ dir yields an empty slice, not an
|
// (natural review order). An absent raw/ dir yields an empty slice, not an
|
||||||
// error. Only .md files are listed; tunnel-candidate files are skipped.
|
// error. Only .md files are listed; tunnel-candidate files are skipped.
|
||||||
func ListPending(brainDir string) ([]PendingNote, error) {
|
func ListPending(brainDir string) ([]PendingNote, error) {
|
||||||
dir := filepath.Join(brainDir, "raw")
|
root, _, err := OpenRawRoot(brainDir)
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
return []PendingNote{}, nil
|
return []PendingNote{}, nil
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("read raw dir: %w", err)
|
return nil, fmt.Errorf("read raw dir: %w", err)
|
||||||
}
|
}
|
||||||
|
defer func() { _ = root.Close() }()
|
||||||
|
entries, err := fs.ReadDir(root.FS(), ".")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read raw dir: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
out := make([]PendingNote, 0, len(entries))
|
out := make([]PendingNote, 0, len(entries))
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
|
if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
|
||||||
continue
|
continue // directories, and symlinks whose target may be anywhere (#153)
|
||||||
}
|
}
|
||||||
info, statErr := e.Info()
|
raw, info, readErr := readRawNote(root, e.Name())
|
||||||
if statErr != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
raw, readErr := os.ReadFile(filepath.Join(dir, e.Name()))
|
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -105,12 +107,9 @@ func PromoteNote(brainDir string, opts PromoteOptions) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
src := filepath.Join(brainDir, "raw", base)
|
src := filepath.Join(brainDir, "raw", base)
|
||||||
raw, err := os.ReadFile(src)
|
raw, err := readRawSource(brainDir, base)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
return "", err
|
||||||
return "", fmt.Errorf("pending note %q does not exist in raw/", base)
|
|
||||||
}
|
|
||||||
return "", fmt.Errorf("read source: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Collision: never silently overwrite an existing promoted note.
|
// Collision: never silently overwrite an existing promoted note.
|
||||||
@@ -180,12 +179,9 @@ func RejectNote(brainDir string, opts RejectOptions) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
src := filepath.Join(brainDir, "raw", base)
|
src := filepath.Join(brainDir, "raw", base)
|
||||||
raw, err := os.ReadFile(src)
|
raw, err := readRawSource(brainDir, base)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
return "", err
|
||||||
return "", fmt.Errorf("pending note %q does not exist in raw/", base)
|
|
||||||
}
|
|
||||||
return "", fmt.Errorf("read source: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
dest := filepath.Join(brainDir, "raw", "rejected", base)
|
dest := filepath.Join(brainDir, "raw", "rejected", base)
|
||||||
@@ -224,3 +220,24 @@ func excerpt(s string, n int) string {
|
|||||||
}
|
}
|
||||||
return strings.TrimSpace(string(r))
|
return strings.TrimSpace(string(r))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// readRawSource reads a pending note for promote/reject through the raw root,
|
||||||
|
// refusing symlinks and a symlinked raw/ (hyperguild#153).
|
||||||
|
func readRawSource(brainDir, base string) ([]byte, error) {
|
||||||
|
root, _, err := OpenRawRoot(brainDir)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, fmt.Errorf("pending note %q does not exist in raw/", base)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("read source: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = root.Close() }()
|
||||||
|
raw, _, err := readRawNote(root, base)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, fmt.Errorf("pending note %q does not exist in raw/", base)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("read source: %w", err)
|
||||||
|
}
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
package api_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hyperguild#153 / PR #156: behavioural race tests, so the control does not
|
||||||
|
// rest on lints that list forbidden APIs (round 3: an os.ReadFile or
|
||||||
|
// Root.ReadFile rewrite passed the lint and leaked 19k times in 4s). A
|
||||||
|
// swapper keeps renaming a regular note, a symlink to an outside file, or a
|
||||||
|
// FIFO into raw/r.md while the public readers loop. Any leaked byte fails;
|
||||||
|
// any hang fails.
|
||||||
|
|
||||||
|
const raceSentinel = "SENTINEL-153-RACE"
|
||||||
|
|
||||||
|
// startSwapper runs fn until the test ends, and waits for it to exit before
|
||||||
|
// t.TempDir's cleanup removes the directory it writes into (hyperguild#162:
|
||||||
|
// an unawaited swapper raced the cleanup, "directory not empty").
|
||||||
|
func startSwapper(t *testing.T, fn func(stop *atomic.Bool)) {
|
||||||
|
t.Helper()
|
||||||
|
var stop atomic.Bool
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
go func() { defer wg.Done(); fn(&stop) }()
|
||||||
|
t.Cleanup(func() { stop.Store(true); wg.Wait() })
|
||||||
|
}
|
||||||
|
|
||||||
|
func swap(raw, outside string, fifo bool, stop *atomic.Bool) {
|
||||||
|
tgt := filepath.Join(raw, "r.md")
|
||||||
|
for i := 0; !stop.Load(); i++ {
|
||||||
|
tmp := filepath.Join(raw, ".swap")
|
||||||
|
_ = os.Remove(tmp)
|
||||||
|
switch {
|
||||||
|
case i%2 == 0:
|
||||||
|
_ = os.WriteFile(tmp, []byte("# r\n\ninside"), 0o644)
|
||||||
|
case fifo:
|
||||||
|
_ = syscall.Mkfifo(tmp, 0o644)
|
||||||
|
default:
|
||||||
|
_ = os.Symlink(outside, tmp)
|
||||||
|
}
|
||||||
|
_ = os.Rename(tmp, tgt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func raceBrain(t *testing.T) (brain, raw, outside string) {
|
||||||
|
t.Helper()
|
||||||
|
brain = t.TempDir()
|
||||||
|
raw = filepath.Join(brain, "raw")
|
||||||
|
if err := os.MkdirAll(raw, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
outside = filepath.Join(t.TempDir(), "secret.md")
|
||||||
|
if err := os.WriteFile(outside, []byte("# s\n\n"+raceSentinel), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(raw, "r.md"), []byte("# r\n\ninside"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return brain, raw, outside
|
||||||
|
}
|
||||||
|
|
||||||
|
// readers runs every public raw/ reader once and returns all text they
|
||||||
|
// produced (excerpts, extracted text).
|
||||||
|
func readers(brain, raw string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
if ns, err := api.ListPending(brain); err == nil {
|
||||||
|
for _, n := range ns {
|
||||||
|
b.WriteString(n.Excerpt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s, err := api.ReadIngestFile(brain, filepath.Join(raw, "r.md")); err == nil {
|
||||||
|
b.WriteString(s)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRawReadersLeakNothingUnderASymlinkSwapRace(t *testing.T) {
|
||||||
|
brain, raw, outside := raceBrain(t)
|
||||||
|
startSwapper(t, func(stop *atomic.Bool) { swap(raw, outside, false, stop) })
|
||||||
|
leaks, iters := 0, 0
|
||||||
|
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); iters++ {
|
||||||
|
leaks += strings.Count(readers(brain, raw), raceSentinel)
|
||||||
|
}
|
||||||
|
if leaks > 0 {
|
||||||
|
t.Fatalf("outside file leaked %d times in %d iterations", leaks, iters)
|
||||||
|
}
|
||||||
|
t.Logf("%d iterations, 0 leaks", iters)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRawReadersDoNotHangUnderAFIFOSwapRace(t *testing.T) {
|
||||||
|
brain, raw, outside := raceBrain(t)
|
||||||
|
startSwapper(t, func(stop *atomic.Bool) { swap(raw, outside, true, stop) })
|
||||||
|
var iters atomic.Int64
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
|
||||||
|
_ = readers(brain, raw)
|
||||||
|
iters.Add(1)
|
||||||
|
}
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(6 * time.Second):
|
||||||
|
// Unblock the stuck open so the test process can exit.
|
||||||
|
if f, err := os.OpenFile(filepath.Join(raw, "r.md"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil {
|
||||||
|
_ = f.Close()
|
||||||
|
}
|
||||||
|
t.Fatalf("a raw/ reader hung on a swapped-in FIFO after %d iterations", iters.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #156 round 4: the pdf/docmark branch of TextIn copies the file through
|
||||||
|
// the root to a temp file; pointing the extractor back at the raw/ path by
|
||||||
|
// name stayed green because the race tests only used r.md (a stub docmark
|
||||||
|
// then received the outside file's bytes 34k times in 4s). This races a
|
||||||
|
// symlink into raw/r.docx and has a fake docmark count the sentinel in every
|
||||||
|
// document it is sent.
|
||||||
|
func TestDocmarkExtractionLeaksNothingUnderASymlinkSwapRace(t *testing.T) {
|
||||||
|
var seen atomic.Int64
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req struct {
|
||||||
|
Params struct {
|
||||||
|
Arguments struct {
|
||||||
|
ContentBase64 string `json:"content_base64"`
|
||||||
|
} `json:"arguments"`
|
||||||
|
} `json:"params"`
|
||||||
|
}
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||||
|
if b, err := base64.StdEncoding.DecodeString(req.Params.Arguments.ContentBase64); err == nil &&
|
||||||
|
strings.Contains(string(b), raceSentinel) {
|
||||||
|
seen.Add(1)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"converted"}]}}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
t.Setenv("DOCMARK_URL", srv.URL)
|
||||||
|
|
||||||
|
brain, raw, outside := raceBrain(t)
|
||||||
|
tgt := filepath.Join(raw, "r.docx")
|
||||||
|
if err := os.WriteFile(tgt, []byte("inside docx bytes"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
startSwapper(t, func(stop *atomic.Bool) {
|
||||||
|
for i := 0; !stop.Load(); i++ {
|
||||||
|
tmp := filepath.Join(raw, ".swapdocx")
|
||||||
|
_ = os.Remove(tmp)
|
||||||
|
if i%2 == 0 {
|
||||||
|
_ = os.WriteFile(tmp, []byte("inside docx bytes"), 0o644)
|
||||||
|
} else {
|
||||||
|
_ = os.Symlink(outside, tmp)
|
||||||
|
}
|
||||||
|
_ = os.Rename(tmp, tgt)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
iters := 0
|
||||||
|
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); iters++ {
|
||||||
|
_, _ = api.ReadIngestFile(brain, tgt)
|
||||||
|
}
|
||||||
|
if n := seen.Load(); n > 0 {
|
||||||
|
t.Fatalf("docmark received the outside file %d times in %d iterations", n, iters)
|
||||||
|
}
|
||||||
|
t.Logf("%d iterations, 0 leaks to docmark", iters)
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package api_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hyperguild#153: readers of brain/raw/ followed symlinks. /pending showed the
|
||||||
|
// target's contents and promote would copy it into wiki/ (and brain-sync to
|
||||||
|
// Gitea). raw/ is writable via Syncthing, so a symlink there is reachable
|
||||||
|
// input. Every raw/ reader now treats a symlink as not-a-note.
|
||||||
|
const sentinel153 = "SENTINEL-153-OUTSIDE"
|
||||||
|
|
||||||
|
func rawWithSymlink(t *testing.T) (brain string) {
|
||||||
|
t.Helper()
|
||||||
|
brain = t.TempDir()
|
||||||
|
raw := filepath.Join(brain, "raw")
|
||||||
|
require.NoError(t, os.MkdirAll(raw, 0o755))
|
||||||
|
outside := filepath.Join(t.TempDir(), "secret.md")
|
||||||
|
require.NoError(t, os.WriteFile(outside, []byte("---\ncreated_at: 2026-01-01T00:00:00Z\n---\n# s\n\n"+sentinel153), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(outside, filepath.Join(raw, "2026-01-01-leak.md")))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-01-02-real.md"), []byte("# real\n\nreal body"), 0o644))
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListPendingSkipsSymlinkedNotes(t *testing.T) {
|
||||||
|
notes, err := api.ListPending(rawWithSymlink(t))
|
||||||
|
require.NoError(t, err)
|
||||||
|
var names []string
|
||||||
|
for _, n := range notes {
|
||||||
|
names = append(names, n.Filename)
|
||||||
|
assert.NotContains(t, n.Excerpt, sentinel153)
|
||||||
|
}
|
||||||
|
assert.Equal(t, []string{"2026-01-02-real.md"}, names)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteNoteRefusesASymlinkedSource(t *testing.T) {
|
||||||
|
brain := rawWithSymlink(t)
|
||||||
|
_, err := api.PromoteNote(brain, api.PromoteOptions{Filename: "2026-01-01-leak.md", Wing: "testing", Hall: "facts"})
|
||||||
|
require.Error(t, err)
|
||||||
|
_, statErr := os.Stat(filepath.Join(brain, "wiki", "testing", "facts", "leak.md"))
|
||||||
|
assert.True(t, os.IsNotExist(statErr), "nothing may be promoted from a symlink")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRejectNoteRefusesASymlinkedSource(t *testing.T) {
|
||||||
|
brain := rawWithSymlink(t)
|
||||||
|
_, err := api.RejectNote(brain, api.RejectOptions{Filename: "2026-01-01-leak.md"})
|
||||||
|
require.Error(t, err)
|
||||||
|
b, _ := os.ReadFile(filepath.Join(brain, "raw", "rejected", "2026-01-01-leak.md"))
|
||||||
|
assert.False(t, strings.Contains(string(b), sentinel153))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Item 4: if raw/ itself is a symlink, containment inside it means nothing.
|
||||||
|
func TestResolvePendingPromoteRejectAndReadIngestRefuseASymlinkedRawRoot(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
elsewhere := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(elsewhere, "x.md"), []byte("# x\n\n"+sentinel153), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(elsewhere, filepath.Join(brain, "raw")))
|
||||||
|
|
||||||
|
_, err := api.ResolveIngestPath(brain, "x.md")
|
||||||
|
assert.Error(t, err, "ingest must refuse a symlinked raw/")
|
||||||
|
notes, err := api.ListPending(brain)
|
||||||
|
assert.Error(t, err, "pending must refuse a symlinked raw/")
|
||||||
|
assert.Empty(t, notes)
|
||||||
|
_, err = api.PromoteNote(brain, api.PromoteOptions{Filename: "x.md", Wing: "testing", Hall: "facts"})
|
||||||
|
assert.Error(t, err, "promote must refuse a symlinked raw/")
|
||||||
|
_, err = api.RejectNote(brain, api.RejectOptions{Filename: "x.md"})
|
||||||
|
assert.Error(t, err, "reject must refuse a symlinked raw/")
|
||||||
|
text, err := api.ReadIngestFile(brain, filepath.Join(elsewhere, "x.md"))
|
||||||
|
assert.Error(t, err, "ReadIngestFile must refuse a symlinked raw/")
|
||||||
|
assert.NotContains(t, text, sentinel153)
|
||||||
|
// The watcher's refusal is tested in internal/watcher.
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
||||||
|
)
|
||||||
|
|
||||||
|
// errNotANote is returned for a raw/ entry that is not a regular file — in
|
||||||
|
// particular a symlink, whose target may be anywhere (hyperguild#153).
|
||||||
|
var errNotANote = errors.New("not a regular note file in raw/ (symlinks are refused)")
|
||||||
|
|
||||||
|
// errRawNotDir means brain/raw is not a real directory (e.g. a symlink), so
|
||||||
|
// containment inside it would mean nothing (hyperguild#153 item 4).
|
||||||
|
var errRawNotDir = errors.New("brain/raw is not a real directory")
|
||||||
|
|
||||||
|
// OpenRawRoot opens brain/raw as an os.Root, refusing a raw/ that is itself a
|
||||||
|
// symlink. Reads through the root cannot leave it, even if an entry is
|
||||||
|
// swapped for a symlink between a check and the read. It returns the root
|
||||||
|
// and raw/'s path. A missing raw/ returns an error wrapping os.ErrNotExist.
|
||||||
|
func OpenRawRoot(brainDir string) (*os.Root, string, error) {
|
||||||
|
p := filepath.Join(brainDir, "raw")
|
||||||
|
fi, err := os.Lstat(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() {
|
||||||
|
return nil, "", errRawNotDir
|
||||||
|
}
|
||||||
|
r, err := os.OpenRoot(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("open raw root: %w", err)
|
||||||
|
}
|
||||||
|
return r, p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readRawNote reads one raw/ entry by basename through the root, refusing
|
||||||
|
// anything that is not a regular file.
|
||||||
|
//
|
||||||
|
// Accepted residual: a HARD link to an outside file is a regular file and is
|
||||||
|
// read. Syncthing syncs contents, not links, so a peer cannot create one;
|
||||||
|
// fs.protected_hardlinks=1 and the pod's secrets live on a different
|
||||||
|
// filesystem from the brain volume, so a local caller cannot link them in.
|
||||||
|
// Refusing Nlink > 1 is the next step if either assumption changes.
|
||||||
|
func readRawNote(r *os.Root, base string) ([]byte, os.FileInfo, error) {
|
||||||
|
fi, err := r.Lstat(base)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if !fi.Mode().IsRegular() {
|
||||||
|
return nil, nil, errNotANote
|
||||||
|
}
|
||||||
|
f, _, err := extract.OpenRegular(r, base)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
b, err := io.ReadAll(f)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return b, fi, nil
|
||||||
|
}
|
||||||
@@ -49,7 +49,19 @@ func (n *NtfyNotifier) Notify(ctx context.Context, msg string) error {
|
|||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
return fmt.Errorf("ntfy notify: status %d", resp.StatusCode)
|
return &RejectedError{Status: resp.StatusCode}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RejectedError is a non-2xx ntfy response.
|
||||||
|
type RejectedError struct{ Status int }
|
||||||
|
|
||||||
|
func (e *RejectedError) Error() string { return fmt.Sprintf("ntfy notify: status %d", e.Status) }
|
||||||
|
|
||||||
|
// Permanent reports whether resending the same message cannot succeed: a 4xx
|
||||||
|
// other than 429 (rate limited). Callers may drop such a message; anything
|
||||||
|
// else (5xx, 429, transport errors) is an outage and must be retried.
|
||||||
|
func (e *RejectedError) Permanent() bool {
|
||||||
|
return e.Status >= 400 && e.Status < 500 && e.Status != http.StatusTooManyRequests
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package audit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A non-2xx ntfy response is a *RejectedError, and only a client-side refusal
|
||||||
|
// other than rate limiting is permanent: callers may drop a message on a
|
||||||
|
// permanent rejection and must retry the rest (hyperguild#157, PR #160).
|
||||||
|
func TestNotifyClassifiesRejectionsAsPermanentOrNot(t *testing.T) {
|
||||||
|
for status, permanent := range map[int]bool{413: true, 400: true, 403: true, 429: false, 500: false, 503: false} {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) }))
|
||||||
|
err := NewNtfyNotifier(srv.URL, "").Notify(context.Background(), "m")
|
||||||
|
srv.Close()
|
||||||
|
var rej *RejectedError
|
||||||
|
require.True(t, errors.As(err, &rej), "status %d: %v", status, err)
|
||||||
|
assert.Equal(t, status, rej.Status)
|
||||||
|
assert.Equal(t, permanent, rej.Permanent(), "status %d", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package extract
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go/ast"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const extractPkg = "github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
||||||
|
|
||||||
|
// parseNonTest parses every non-test Go file under the module.
|
||||||
|
func parseNonTest(t *testing.T) map[string]*ast.File {
|
||||||
|
t.Helper()
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
out := map[string]*ast.File{}
|
||||||
|
err := filepath.WalkDir(filepath.Join("..", ".."), func(p string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() && (d.Name() == "vendor" || d.Name() == ".git") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
if d.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
f, perr := parser.ParseFile(fset, p, nil, 0)
|
||||||
|
if perr != nil {
|
||||||
|
return perr
|
||||||
|
}
|
||||||
|
out[filepath.ToSlash(p)] = f
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #156 review: reading a caller-chosen path by name reopens it after the
|
||||||
|
// containment check (hyperguild#153 item 3). Outside this package, nothing
|
||||||
|
// may reference extract.Text — called OR taken as a value — under whatever
|
||||||
|
// name the package is imported (round 2: an alias and a func value slipped
|
||||||
|
// past a plain text search).
|
||||||
|
func TestNoPackageReferencesExtractTextOutsideExtract(t *testing.T) {
|
||||||
|
for path, f := range parseNonTest(t) {
|
||||||
|
if strings.Contains(path, "internal/extract/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, imp := range f.Imports {
|
||||||
|
if p, _ := strconv.Unquote(imp.Path.Value); p == extractPkg {
|
||||||
|
name := "extract"
|
||||||
|
if imp.Name != nil {
|
||||||
|
name = imp.Name.Name
|
||||||
|
}
|
||||||
|
if name == "." {
|
||||||
|
t.Errorf("%s dot-imports extract; import it by name so its use stays checkable", path)
|
||||||
|
}
|
||||||
|
names[name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(names) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ast.Inspect(f, func(n ast.Node) bool {
|
||||||
|
if sel, ok := n.(*ast.SelectorExpr); ok {
|
||||||
|
if id, ok := sel.X.(*ast.Ident); ok && names[id.Name] && sel.Sel.Name == "Text" {
|
||||||
|
t.Errorf("%s references %s.Text; use extract.TextIn through an os.Root", path, id.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #156 review round 2: the FIFO fix (open non-blocking, check the handle)
|
||||||
|
// lives in OpenRegular, and reverting any caller to a plain Root.Open stayed
|
||||||
|
// green. In the three raw/ reader FILES parsed below (and only those: a
|
||||||
|
// wrapper in another file is not seen), no Open/OpenFile/ReadFile selector
|
||||||
|
// may appear outside OpenRegular, called or taken as a method value. This is
|
||||||
|
// a second layer; the behavioural race tests in internal/api and
|
||||||
|
// internal/watcher are what catch a rewrite through any API (round 3).
|
||||||
|
// osOpenAllowed names the functions in guarded files that may call os.Open*
|
||||||
|
// on a path that is not a raw/ read.
|
||||||
|
var osOpenAllowed = map[string]bool{
|
||||||
|
"internal/watcher/watcher.go:appendWatcherLog": true, // ingest log append, not a raw/ read
|
||||||
|
// Text reads by path; outside this package only TextIn's temp copy reaches
|
||||||
|
// it (TestNoPackageReferencesExtractTextOutsideExtract enforces that).
|
||||||
|
"internal/extract/extract.go:Text": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRawReaderFilesUseNoOpenOrReadFileOutsideOpenRegular(t *testing.T) {
|
||||||
|
guarded := []string{"internal/api/rawroot.go", "internal/extract/extract.go", "internal/watcher/watcher.go"}
|
||||||
|
files := parseNonTest(t)
|
||||||
|
for _, g := range guarded {
|
||||||
|
var f *ast.File
|
||||||
|
for p, af := range files {
|
||||||
|
if strings.HasSuffix(p, g) {
|
||||||
|
f = af
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if f == nil {
|
||||||
|
t.Fatalf("guarded file %s not found", g)
|
||||||
|
}
|
||||||
|
for _, decl := range f.Decls {
|
||||||
|
fn, ok := decl.(*ast.FuncDecl)
|
||||||
|
if !ok || fn.Name.Name == "OpenRegular" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ast.Inspect(fn, func(n ast.Node) bool {
|
||||||
|
sel, ok := n.(*ast.SelectorExpr)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if sel.Sel.Name == "Open" || sel.Sel.Name == "OpenFile" || sel.Sel.Name == "ReadFile" {
|
||||||
|
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "os" && osOpenAllowed[g+":"+fn.Name.Name] {
|
||||||
|
return true // a named non-raw write (the ingest log append)
|
||||||
|
}
|
||||||
|
t.Errorf("%s: %s calls .%s; raw/ reads must go through OpenRegular", g, fn.Name.Name, sel.Sel.Name)
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,8 +3,10 @@ package extract
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Text reads the file at path and returns its plain-text content.
|
// Text reads the file at path and returns its plain-text content.
|
||||||
@@ -41,3 +43,70 @@ func fileExt(path string) string {
|
|||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TextIn is Text for a file read through an os.Root (hyperguild#153): the
|
||||||
|
// read cannot leave the root, so a path checked earlier cannot be swapped
|
||||||
|
// for a symlink pointing elsewhere before it is read. Formats that need an
|
||||||
|
// external tool (pdf, docmark) are copied to a temp file from the handle
|
||||||
|
// opened through the root, then extracted from the copy.
|
||||||
|
func TextIn(r *os.Root, rel string) (string, error) {
|
||||||
|
fi, err := r.Lstat(rel)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("stat %s: %w", rel, err)
|
||||||
|
}
|
||||||
|
if !fi.Mode().IsRegular() {
|
||||||
|
return "", fmt.Errorf("%s: not a regular file", rel)
|
||||||
|
}
|
||||||
|
f, _, err := OpenRegular(r, rel)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("open %s: %w", rel, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
ext := strings.ToLower(fileExt(rel))
|
||||||
|
switch ext {
|
||||||
|
case ".md", ".txt":
|
||||||
|
b, err := io.ReadAll(f)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("read %s: %w", rel, err)
|
||||||
|
}
|
||||||
|
return string(b), nil
|
||||||
|
case ".pdf", ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
||||||
|
tmp, err := os.CreateTemp("", "ingest-*"+ext)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("temp copy: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = os.Remove(tmp.Name()) }()
|
||||||
|
if _, err := io.Copy(tmp, f); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return "", fmt.Errorf("copy %s: %w", rel, err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return "", fmt.Errorf("copy %s: %w", rel, err)
|
||||||
|
}
|
||||||
|
return Text(tmp.Name())
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unsupported file extension: %s", ext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRegular opens rel inside r and returns it only if the OPENED handle is
|
||||||
|
// a regular file. The open is non-blocking, so a FIFO swapped in after an
|
||||||
|
// Lstat (PR #156 review: it hung open(2) forever) is refused promptly rather
|
||||||
|
// than stalling the reader. Callers keep their Lstat check, which refuses
|
||||||
|
// symlinks outright; this closes the race between that check and the open.
|
||||||
|
func OpenRegular(r *os.Root, rel string) (*os.File, os.FileInfo, error) {
|
||||||
|
f, err := r.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
fi, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if !fi.Mode().IsRegular() {
|
||||||
|
_ = f.Close()
|
||||||
|
return nil, nil, fmt.Errorf("%s: not a regular file", rel)
|
||||||
|
}
|
||||||
|
return f, fi, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package extract
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hyperguild#153 item 3: ingest checked a path, then extract.Text reopened it
|
||||||
|
// by name, so a file swapped for a symlink in between was followed out of
|
||||||
|
// raw/. TextIn reads through an os.Root, which refuses to leave the root.
|
||||||
|
func TestTextInReadsInsideTheRootAndRefusesEscapes(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(root, "ok.md"), []byte("inside"), 0o644))
|
||||||
|
outside := filepath.Join(t.TempDir(), "secret.md")
|
||||||
|
require.NoError(t, os.WriteFile(outside, []byte("SENTINEL-153-TEXTIN"), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(outside, filepath.Join(root, "leak.md")))
|
||||||
|
|
||||||
|
r, err := os.OpenRoot(root)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = r.Close() }()
|
||||||
|
|
||||||
|
got, err := TextIn(r, "ok.md")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "inside", got)
|
||||||
|
|
||||||
|
got, err = TextIn(r, "leak.md")
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.NotContains(t, got, "SENTINEL-153-TEXTIN")
|
||||||
|
|
||||||
|
_, err = TextIn(r, "../"+filepath.Base(outside))
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #156 review: a regular file swapped for a FIFO between Lstat and Open
|
||||||
|
// blocked open(2) forever. OpenRegular opens non-blocking and checks the type
|
||||||
|
// on the opened handle, so a FIFO is refused promptly instead of hanging.
|
||||||
|
func TestOpenRegularRefusesAFIFOWithoutBlocking(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "pipe.md"), 0o644))
|
||||||
|
r, err := os.OpenRoot(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = r.Close() }()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
f, _, err := OpenRegular(r, "pipe.md")
|
||||||
|
if f != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
}
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
assert.Error(t, err)
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("OpenRegular blocked on a FIFO")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/capture"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/capture"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/graphsync"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/graphsync"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/notes"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/notes"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
||||||
@@ -549,7 +548,7 @@ func (s *Server) brainIngest(ctx context.Context, args json.RawMessage) (json.Ra
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
a.Path = resolved
|
a.Path = resolved
|
||||||
text, err := extract.Text(a.Path)
|
text, err := api.ReadIngestFile(s.brainDir, a.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("extract: %w", err)
|
return nil, fmt.Errorf("extract: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ package symlinkguard
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -21,7 +22,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// DefaultInterval is used when Watch is given a non-positive interval.
|
// DefaultInterval is used when Watch is given a non-positive interval.
|
||||||
const DefaultInterval = 5 * time.Minute
|
const DefaultInterval = 30 * time.Second
|
||||||
|
|
||||||
|
// Alert bounds (PR #160 review): the planter chooses the paths, so neither an
|
||||||
|
// alert's size nor the queue may grow with what was planted.
|
||||||
|
const (
|
||||||
|
maxPathsPerAlert = 10 // further paths are counted, not listed
|
||||||
|
maxPending = 5 // queued alerts; the rest become one summary
|
||||||
|
)
|
||||||
|
|
||||||
// Notifier is the out-of-band alert sink (audit.Notifier satisfies it).
|
// Notifier is the out-of-band alert sink (audit.Notifier satisfies it).
|
||||||
type Notifier interface {
|
type Notifier interface {
|
||||||
@@ -41,13 +49,7 @@ func (r Result) Clean() bool { return len(r.Symlinks) == 0 && len(r.Unreadable)
|
|||||||
// String describes the findings for a log line or alert. Paths are quoted, so
|
// String describes the findings for a log line or alert. Paths are quoted, so
|
||||||
// a name containing a newline cannot blur two findings together.
|
// a name containing a newline cannot blur two findings together.
|
||||||
func (r Result) String() string {
|
func (r Result) String() string {
|
||||||
q := func(xs []string) string {
|
q := r.quoted
|
||||||
out := make([]string, len(xs))
|
|
||||||
for i, x := range xs {
|
|
||||||
out[i] = strconv.Quote(x)
|
|
||||||
}
|
|
||||||
return strings.Join(out, ", ")
|
|
||||||
}
|
|
||||||
var parts []string
|
var parts []string
|
||||||
if len(r.Symlinks) > 0 {
|
if len(r.Symlinks) > 0 {
|
||||||
parts = append(parts, fmt.Sprintf("%d symlink(s): %s", len(r.Symlinks), q(r.Symlinks)))
|
parts = append(parts, fmt.Sprintf("%d symlink(s): %s", len(r.Symlinks), q(r.Symlinks)))
|
||||||
@@ -58,6 +60,24 @@ func (r Result) String() string {
|
|||||||
return strings.Join(parts, "; ")
|
return strings.Join(parts, "; ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// quoted joins at most maxPathsPerAlert paths, each strconv-quoted, and
|
||||||
|
// counts the rest.
|
||||||
|
func (Result) quoted(xs []string) string {
|
||||||
|
shown := xs
|
||||||
|
if len(shown) > maxPathsPerAlert {
|
||||||
|
shown = shown[:maxPathsPerAlert]
|
||||||
|
}
|
||||||
|
out := make([]string, len(shown))
|
||||||
|
for i, x := range shown {
|
||||||
|
out[i] = strconv.Quote(x)
|
||||||
|
}
|
||||||
|
s := strings.Join(out, ", ")
|
||||||
|
if extra := len(xs) - len(shown); extra > 0 {
|
||||||
|
s += fmt.Sprintf(" and %d more", extra)
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
// Scan walks the brain dir and reports every symlink in it and every
|
// Scan walks the brain dir and reports every symlink in it and every
|
||||||
// directory it could not read, continuing past both. Only the ROOT .git is
|
// directory it could not read, continuing past both. Only the ROOT .git is
|
||||||
// skipped (it is brain-sync's, and no reader walks it); a nested .git is
|
// skipped (it is brain-sync's, and no reader walks it); a nested .git is
|
||||||
@@ -106,15 +126,20 @@ func Scan(brainDir string) (Result, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch rescans every interval (DefaultInterval if non-positive) until ctx
|
// Watch rescans every interval (DefaultInterval if non-positive) until ctx
|
||||||
// ends, and notifies once per distinct non-clean result. A result counts as
|
// ends. Every symlink found is removed (the link, never its target:
|
||||||
// reported only after Notify succeeds, so an alert lost to a notifier outage
|
// hyperguild#157 decision), so one is readable for at most one interval.
|
||||||
// is retried on the next tick.
|
// Each removal is an alert; a persistent finding (a link it could not remove,
|
||||||
|
// an unreadable dir, a scan error) alerts once per distinct state. Alerts
|
||||||
|
// queue until Notify succeeds (see alertQueue): an outage delays them, and
|
||||||
|
// any that do not fit the bounded queue, or are undeliverable, are counted in
|
||||||
|
// a summary that is never dropped, so every incident is delivered or counted.
|
||||||
func Watch(ctx context.Context, brainDir string, interval time.Duration, n Notifier) {
|
func Watch(ctx context.Context, brainDir string, interval time.Duration, n Notifier) {
|
||||||
if interval <= 0 {
|
if interval <= 0 {
|
||||||
interval = DefaultInterval
|
interval = DefaultInterval
|
||||||
}
|
}
|
||||||
t := time.NewTicker(interval)
|
t := time.NewTicker(interval)
|
||||||
defer t.Stop()
|
defer t.Stop()
|
||||||
|
q := &alertQueue{}
|
||||||
reported := ""
|
reported := ""
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
@@ -123,31 +148,117 @@ func Watch(ctx context.Context, brainDir string, interval time.Duration, n Notif
|
|||||||
case <-t.C:
|
case <-t.C:
|
||||||
}
|
}
|
||||||
r, err := Scan(brainDir)
|
r, err := Scan(brainDir)
|
||||||
var key, msg string
|
key := ""
|
||||||
switch {
|
if err != nil {
|
||||||
case err != nil:
|
key = "cannot scan the brain dir: " + err.Error()
|
||||||
key = "error: " + err.Error()
|
} else {
|
||||||
msg = "brain symlink tripwire (hyperguild#157): cannot scan the brain dir: " + err.Error()
|
removed, failed := removeSymlinks(brainDir, r.Symlinks)
|
||||||
case r.Clean():
|
if len(removed) > 0 {
|
||||||
reported = ""
|
slog.Error("brain symlink tripwire removed symlinks", "count", len(removed))
|
||||||
continue
|
q.push(fmt.Sprintf("brain symlink tripwire (hyperguild#157): removed %d symlink(s) from the brain dir: %s",
|
||||||
default:
|
len(removed), Result{}.quoted(removed)))
|
||||||
key = r.String()
|
}
|
||||||
msg = "brain symlink tripwire (hyperguild#157): " + key +
|
key = Result{Symlinks: failed, Unreadable: r.Unreadable}.String()
|
||||||
". Readers may follow symlinks out of the brain; remove them."
|
|
||||||
}
|
}
|
||||||
if key == reported {
|
if key != reported {
|
||||||
continue
|
|
||||||
}
|
|
||||||
slog.Error("brain symlink tripwire", "finding", key)
|
|
||||||
if n == nil {
|
|
||||||
reported = key
|
reported = key
|
||||||
continue
|
if key != "" {
|
||||||
|
slog.Error("brain symlink tripwire", "finding", key)
|
||||||
|
q.push("brain symlink tripwire (hyperguild#157): " + key +
|
||||||
|
". Readers may follow symlinks out of the brain; fix them.")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := n.Notify(ctx, msg); err != nil {
|
q.flush(ctx, n)
|
||||||
slog.Warn("symlink tripwire notify failed; will retry", "err", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
reported = key
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// alertQueue holds alerts until Notify accepts them. It is bounded (at most
|
||||||
|
// maxPending queued) and loses nothing from the record: every overflow and
|
||||||
|
// every dropped message increments suppressed, sent as a summary that is
|
||||||
|
// retried every tick and never dropped.
|
||||||
|
//
|
||||||
|
// A message is dropped only when the notifier rejects it permanently (an
|
||||||
|
// error with Permanent() == true, e.g. ntfy 413). Every other failure is an
|
||||||
|
// outage, and nothing is dropped however long it lasts (PR #160 round 2:
|
||||||
|
// dropping after N raw failures lost the only incident of a long outage).
|
||||||
|
// Each flush tries every queued message, so one undeliverable message cannot
|
||||||
|
// block the others.
|
||||||
|
type alertQueue struct {
|
||||||
|
items []string
|
||||||
|
suppressed int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *alertQueue) push(msg string) {
|
||||||
|
if len(q.items) >= maxPending {
|
||||||
|
q.suppressed++
|
||||||
|
return
|
||||||
|
}
|
||||||
|
q.items = append(q.items, msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *alertQueue) flush(ctx context.Context, n Notifier) {
|
||||||
|
if n == nil {
|
||||||
|
q.items, q.suppressed = nil, 0
|
||||||
|
return
|
||||||
|
}
|
||||||
|
kept := q.items[:0]
|
||||||
|
for _, msg := range q.items {
|
||||||
|
err := n.Notify(ctx, msg)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
case permanent(err):
|
||||||
|
slog.Error("symlink tripwire alert rejected permanently; counted in the summary", "err", err)
|
||||||
|
q.suppressed++
|
||||||
|
default:
|
||||||
|
slog.Warn("symlink tripwire notify failed; will retry", "err", err)
|
||||||
|
kept = append(kept, msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
q.items = kept
|
||||||
|
if q.suppressed > 0 {
|
||||||
|
msg := fmt.Sprintf("brain symlink tripwire (hyperguild#157): %d further alert(s) were suppressed or undeliverable; check the ingestion logs", q.suppressed)
|
||||||
|
if err := n.Notify(ctx, msg); err == nil {
|
||||||
|
q.suppressed = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// permanent reports whether err says the message itself can never be sent.
|
||||||
|
func permanent(err error) bool {
|
||||||
|
var p interface{ Permanent() bool }
|
||||||
|
return errors.As(err, &p) && p.Permanent()
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeSymlinks unlinks each brain-relative symlink through an os.Root on
|
||||||
|
// the brain dir, after re-checking it is still a symlink. Removing a symlink
|
||||||
|
// never touches its target. It returns what was removed and what was not.
|
||||||
|
func removeSymlinks(brainDir string, rels []string) (removed, failed []string) {
|
||||||
|
if len(rels) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
resolved, err := filepath.EvalSymlinks(brainDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, rels
|
||||||
|
}
|
||||||
|
root, err := os.OpenRoot(resolved)
|
||||||
|
if err != nil {
|
||||||
|
return nil, rels
|
||||||
|
}
|
||||||
|
defer func() { _ = root.Close() }()
|
||||||
|
for _, rel := range rels {
|
||||||
|
fi, err := root.Lstat(rel)
|
||||||
|
if err != nil || fi.Mode()&os.ModeSymlink == 0 {
|
||||||
|
failed = append(failed, rel)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// root.Remove, not os.RemoveAll: if the link is swapped for a
|
||||||
|
// non-empty directory after the Lstat, Remove refuses rather than
|
||||||
|
// deleting it recursively, and it cannot act outside the root.
|
||||||
|
if err := root.Remove(rel); err != nil {
|
||||||
|
failed = append(failed, rel)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
removed = append(removed, rel)
|
||||||
|
}
|
||||||
|
return removed, failed
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,9 +3,13 @@ package symlinkguard
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -182,8 +186,313 @@ func TestWatchReportsASetAgainAfterItWasCleared(t *testing.T) {
|
|||||||
go Watch(ctx, brain, 10*time.Millisecond, n)
|
go Watch(ctx, brain, 10*time.Millisecond, n)
|
||||||
require.NoError(t, os.Symlink(t.TempDir(), link))
|
require.NoError(t, os.Symlink(t.TempDir(), link))
|
||||||
require.Eventually(t, func() bool { return n.count() == 1 }, 2*time.Second, 10*time.Millisecond)
|
require.Eventually(t, func() bool { return n.count() == 1 }, 2*time.Second, 10*time.Millisecond)
|
||||||
require.NoError(t, os.Remove(link))
|
// The tripwire removed it; planting it again is a new event.
|
||||||
time.Sleep(60 * time.Millisecond)
|
require.Eventually(t, func() bool { _, err := os.Lstat(link); return os.IsNotExist(err) }, 2*time.Second, 10*time.Millisecond)
|
||||||
require.NoError(t, os.Symlink(t.TempDir(), link))
|
require.NoError(t, os.Symlink(t.TempDir(), link))
|
||||||
require.Eventually(t, func() bool { return n.count() == 2 }, 2*time.Second, 10*time.Millisecond)
|
require.Eventually(t, func() bool { return n.count() == 2 }, 2*time.Second, 10*time.Millisecond)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hyperguild#157 decision: the runtime tripwire removes a symlink it finds
|
||||||
|
// (the link, never its target), so it is readable for at most one interval
|
||||||
|
// instead of until a human acts, and the alert says what was removed.
|
||||||
|
func TestWatchRemovesAnOutsideFileLinkButNotItsTargetOrRegularFiles(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
outside := filepath.Join(t.TempDir(), "target.md")
|
||||||
|
require.NoError(t, os.WriteFile(outside, []byte("keep me"), 0o644))
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki"), 0o755))
|
||||||
|
link := filepath.Join(brain, "wiki", "leak.md")
|
||||||
|
require.NoError(t, os.Symlink(outside, link))
|
||||||
|
regular := filepath.Join(brain, "wiki", "note.md")
|
||||||
|
require.NoError(t, os.WriteFile(regular, []byte("note"), 0o644))
|
||||||
|
|
||||||
|
n := &recNotifier{}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 10*time.Millisecond, n)
|
||||||
|
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
_, err := os.Lstat(link)
|
||||||
|
return os.IsNotExist(err)
|
||||||
|
}, 2*time.Second, 10*time.Millisecond, "the symlink is removed")
|
||||||
|
require.Eventually(t, func() bool { return n.count() >= 1 }, 2*time.Second, 10*time.Millisecond)
|
||||||
|
b, err := os.ReadFile(outside)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "keep me", string(b), "the target is untouched")
|
||||||
|
b, err = os.ReadFile(regular)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "note", string(b), "regular files are untouched")
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
assert.Contains(t, n.msgs[0], "removed")
|
||||||
|
assert.Contains(t, n.msgs[0], "wiki/leak.md")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultIntervalIsThirtySeconds(t *testing.T) {
|
||||||
|
assert.Equal(t, 30*time.Second, DefaultInterval)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #160 review: the Lstat re-check and the os.Root confinement in
|
||||||
|
// removeSymlinks had no test. Anything that is not a symlink inside the brain
|
||||||
|
// is reported failed and nothing is deleted.
|
||||||
|
func TestRemoveSymlinksRefusesNonSymlinksEscapesAndPathsThroughLinks(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki", "real"), 0o755))
|
||||||
|
regular := filepath.Join(brain, "wiki", "note.md")
|
||||||
|
require.NoError(t, os.WriteFile(regular, []byte("note"), 0o644))
|
||||||
|
inLinked := filepath.Join(brain, "wiki", "real", "x.md")
|
||||||
|
require.NoError(t, os.WriteFile(inLinked, []byte("x"), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(filepath.Join(brain, "wiki", "real"), filepath.Join(brain, "wiki", "linkdir")))
|
||||||
|
outside := filepath.Join(filepath.Dir(brain), "outside-157.md")
|
||||||
|
require.NoError(t, os.WriteFile(outside, []byte("o"), 0o644))
|
||||||
|
t.Cleanup(func() { _ = os.Remove(outside) })
|
||||||
|
|
||||||
|
removed, failed := removeSymlinks(brain, []string{
|
||||||
|
"wiki/note.md", "wiki/linkdir/x.md", "../outside-157.md", "wiki/missing.md",
|
||||||
|
})
|
||||||
|
assert.Empty(t, removed)
|
||||||
|
assert.ElementsMatch(t, []string{"wiki/note.md", "wiki/linkdir/x.md", "../outside-157.md", "wiki/missing.md"}, failed)
|
||||||
|
for _, p := range []string{regular, inLinked, outside} {
|
||||||
|
_, err := os.Stat(p)
|
||||||
|
assert.NoError(t, err, "%s must not be deleted", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dir targets and in-brain targets survive too, not only an outside file.
|
||||||
|
func TestWatchRemovesDirAndInBrainLinksButNotTheirTargets(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
outsideDir := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(outsideDir, "keep.md"), []byte("keep"), 0o644))
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki"), 0o755))
|
||||||
|
inBrain := filepath.Join(brain, "wiki", "target.md")
|
||||||
|
require.NoError(t, os.WriteFile(inBrain, []byte("t"), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(outsideDir, filepath.Join(brain, "wiki", "dirlink")))
|
||||||
|
require.NoError(t, os.Symlink(inBrain, filepath.Join(brain, "wiki", "inlink.md")))
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 10*time.Millisecond, nil)
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
r, err := Scan(brain)
|
||||||
|
return err == nil && len(r.Symlinks) == 0
|
||||||
|
}, 2*time.Second, 10*time.Millisecond)
|
||||||
|
for _, p := range []string{filepath.Join(outsideDir, "keep.md"), inBrain} {
|
||||||
|
_, err := os.Stat(p)
|
||||||
|
assert.NoError(t, err, "%s must survive", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #160 review: the planter chooses the paths. An alert names at most
|
||||||
|
// maxPathsPerAlert of them and counts the rest, so its size stays bounded.
|
||||||
|
func TestWatchAlertNamesBoundedPathsAndCountsTheRest(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, strings.Repeat("n", 100)+strconv.Itoa(i))))
|
||||||
|
}
|
||||||
|
n := &recNotifier{}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 10*time.Millisecond, n)
|
||||||
|
require.Eventually(t, func() bool { return n.count() >= 1 }, 2*time.Second, 10*time.Millisecond)
|
||||||
|
n.mu.Lock()
|
||||||
|
msg := n.msgs[0]
|
||||||
|
n.mu.Unlock()
|
||||||
|
assert.Contains(t, msg, "removed 50 symlink(s)")
|
||||||
|
assert.Contains(t, msg, "and 40 more")
|
||||||
|
assert.Less(t, len(msg), 2500, "alert size is bounded regardless of how many links were planted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// rejectNotifier refuses every message containing a marker, forever.
|
||||||
|
type rejectNotifier struct {
|
||||||
|
recNotifier
|
||||||
|
reject string
|
||||||
|
}
|
||||||
|
|
||||||
|
// permanentErr is a rejection the notifier says can never succeed (like ntfy 413).
|
||||||
|
type permanentErr struct{}
|
||||||
|
|
||||||
|
func (permanentErr) Error() string { return "413 payload too large" }
|
||||||
|
func (permanentErr) Permanent() bool { return true }
|
||||||
|
|
||||||
|
func (r *rejectNotifier) Notify(ctx context.Context, m string) error {
|
||||||
|
if strings.Contains(m, r.reject) {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.msgs = append(r.msgs, "REJECTED")
|
||||||
|
r.mu.Unlock()
|
||||||
|
return permanentErr{}
|
||||||
|
}
|
||||||
|
return r.recNotifier.Notify(ctx, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #160 review: one message the server will never accept blocked every
|
||||||
|
// later alert forever (head-of-line). It is dropped after maxAttempts.
|
||||||
|
func TestWatchDropsAMessageThatNeverSendsSoLaterAlertsFlow(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
n := &rejectNotifier{reject: "POISON"}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 10*time.Millisecond, n)
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "POISON")))
|
||||||
|
time.Sleep(30 * time.Millisecond)
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "later")))
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
for _, m := range n.msgs {
|
||||||
|
if strings.Contains(m, "later") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}, 3*time.Second, 10*time.Millisecond, "a later alert is delivered despite a poison message")
|
||||||
|
}
|
||||||
|
|
||||||
|
// toggleNotifier fails while down is set.
|
||||||
|
type toggleNotifier struct {
|
||||||
|
recNotifier
|
||||||
|
down atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *toggleNotifier) Notify(ctx context.Context, m string) error {
|
||||||
|
if r.down.Load() {
|
||||||
|
return errors.New("ntfy down")
|
||||||
|
}
|
||||||
|
return r.recNotifier.Notify(ctx, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #160 review: an outage queued one message per removal, unbounded. The
|
||||||
|
// queue holds at most maxPending messages; the rest become one summary.
|
||||||
|
func TestWatchQueueIsBoundedDuringAnOutage(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
n := &toggleNotifier{}
|
||||||
|
n.down.Store(true)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 5*time.Millisecond, n)
|
||||||
|
for i := 0; i < 30; i++ {
|
||||||
|
link := filepath.Join(brain, "p"+strconv.Itoa(i))
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), link))
|
||||||
|
require.Eventually(t, func() bool { _, err := os.Lstat(link); return os.IsNotExist(err) }, 2*time.Second, 2*time.Millisecond)
|
||||||
|
}
|
||||||
|
n.down.Store(false)
|
||||||
|
require.Eventually(t, func() bool { return n.count() >= 1 }, 2*time.Second, 5*time.Millisecond)
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
assert.LessOrEqual(t, n.count(), maxPending+1, "queued alerts plus at most one suppression summary")
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
found := false
|
||||||
|
for _, m := range n.msgs {
|
||||||
|
if strings.Contains(m, "suppressed") {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.True(t, found, "suppressed alerts are counted in a summary")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Within the bound, every alert queued during an outage is delivered after
|
||||||
|
// it, each naming its own removal (PR #160 review: truncating the queue after
|
||||||
|
// the first send stayed green).
|
||||||
|
func TestWatchDeliversEveryQueuedAlertAfterAShortOutage(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
n := &toggleNotifier{}
|
||||||
|
n.down.Store(true)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 5*time.Millisecond, n)
|
||||||
|
for _, name := range []string{"qa", "qb", "qc"} {
|
||||||
|
link := filepath.Join(brain, name)
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), link))
|
||||||
|
require.Eventually(t, func() bool { _, err := os.Lstat(link); return os.IsNotExist(err) }, 2*time.Second, 2*time.Millisecond)
|
||||||
|
}
|
||||||
|
n.down.Store(false)
|
||||||
|
require.Eventually(t, func() bool { return n.count() == 3 }, 2*time.Second, 5*time.Millisecond)
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
for i, name := range []string{"qa", "qb", "qc"} {
|
||||||
|
assert.Contains(t, n.msgs[i], strconv.Quote(name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #160 round 2: dropping after N failures could not tell a poison message
|
||||||
|
// from an outage, so a 15-minute outage silently lost the only incident. A
|
||||||
|
// message is only dropped when it keeps failing while others send; through
|
||||||
|
// an outage nothing is dropped, however long.
|
||||||
|
func TestWatchLosesNoAlertThroughALongOutage(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
n := &toggleNotifier{}
|
||||||
|
n.down.Store(true)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 2*time.Millisecond, n)
|
||||||
|
link := filepath.Join(brain, "only-incident")
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), link))
|
||||||
|
require.Eventually(t, func() bool { _, err := os.Lstat(link); return os.IsNotExist(err) }, 2*time.Second, 2*time.Millisecond)
|
||||||
|
time.Sleep(200 * time.Millisecond) // ~100 ticks of outage, far past any attempt limit
|
||||||
|
n.down.Store(false)
|
||||||
|
require.Eventually(t, func() bool { return n.count() >= 1 }, 2*time.Second, 5*time.Millisecond)
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
assert.Contains(t, n.msgs[0], strconv.Quote("only-incident"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every removal is either delivered or counted in the suppression summary:
|
||||||
|
// none disappears from the record (round 2: 74 of 300 were in neither).
|
||||||
|
func TestWatchAccountsForEveryRemovalDuringAnOutage(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
n := &toggleNotifier{}
|
||||||
|
n.down.Store(true)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 2*time.Millisecond, n)
|
||||||
|
const plants = 25
|
||||||
|
for i := 0; i < plants; i++ {
|
||||||
|
link := filepath.Join(brain, "acct"+strconv.Itoa(i))
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), link))
|
||||||
|
require.Eventually(t, func() bool { _, err := os.Lstat(link); return os.IsNotExist(err) }, 2*time.Second, 1*time.Millisecond)
|
||||||
|
}
|
||||||
|
n.down.Store(false)
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
for _, m := range n.msgs {
|
||||||
|
if strings.Contains(m, "suppressed") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}, 2*time.Second, 5*time.Millisecond)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
delivered, counted := 0, 0
|
||||||
|
for _, m := range n.msgs {
|
||||||
|
if strings.Contains(m, "removed 1 symlink(s)") {
|
||||||
|
delivered++
|
||||||
|
}
|
||||||
|
if i := strings.Index(m, "): "); i >= 0 && strings.Contains(m, "suppressed") {
|
||||||
|
_, _ = fmt.Sscanf(m[i+3:], "%d", &counted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.Equal(t, plants, delivered+counted, "delivered %d + counted %d must equal %d removals", delivered, counted, plants)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A poison message is still dropped (others send while it fails), and the
|
||||||
|
// drop is counted in the summary.
|
||||||
|
func TestWatchDropsAPoisonMessageAndCountsIt(t *testing.T) {
|
||||||
|
brain := t.TempDir()
|
||||||
|
n := &rejectNotifier{reject: "POISON"}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go Watch(ctx, brain, 5*time.Millisecond, n)
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "POISON")))
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "fine")))
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
for _, m := range n.msgs {
|
||||||
|
if strings.Contains(m, "1 further alert(s)") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}, 3*time.Second, 5*time.Millisecond, "the dropped poison alert is counted in a summary")
|
||||||
|
}
|
||||||
|
|||||||
@@ -48,8 +48,25 @@ func Start(ctx context.Context, cfg Config) {
|
|||||||
func processDir(ctx context.Context, cfg Config, date string) []error {
|
func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||||
rawDir := filepath.Join(cfg.BrainDir, "raw")
|
rawDir := filepath.Join(cfg.BrainDir, "raw")
|
||||||
|
|
||||||
|
// Every read goes through an os.Root on raw/ (hyperguild#153): raw/ is
|
||||||
|
// writable via Syncthing, and a file swapped for a symlink between the walk
|
||||||
|
// and the read must not lead out of it. A symlinked raw/ is refused.
|
||||||
|
if fi, lerr := os.Lstat(rawDir); lerr != nil {
|
||||||
|
if os.IsNotExist(lerr) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []error{fmt.Errorf("stat raw dir: %w", lerr)}
|
||||||
|
} else if fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir() {
|
||||||
|
return []error{fmt.Errorf("raw dir %s is not a real directory; refusing to ingest from it", rawDir)}
|
||||||
|
}
|
||||||
|
root, err := os.OpenRoot(rawDir)
|
||||||
|
if err != nil {
|
||||||
|
return []error{fmt.Errorf("open raw root: %w", err)}
|
||||||
|
}
|
||||||
|
defer func() { _ = root.Close() }()
|
||||||
|
|
||||||
var errs []error
|
var errs []error
|
||||||
err := filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error {
|
err = filepath.WalkDir(rawDir, func(path string, d os.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -86,6 +103,10 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
|||||||
if strings.HasPrefix(d.Name(), "tunnel-candidates-") {
|
if strings.HasPrefix(d.Name(), "tunnel-candidates-") {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
// Only regular files are notes; a symlink's target may be anywhere.
|
||||||
|
if !d.Type().IsRegular() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Skip files that have already been processed or permanently failed.
|
// Skip files that have already been processed or permanently failed.
|
||||||
if _, err := os.Stat(path + ".processed"); err == nil {
|
if _, err := os.Stat(path + ".processed"); err == nil {
|
||||||
@@ -95,7 +116,7 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := processFile(ctx, cfg, path, date); err != nil {
|
if err := processFile(ctx, cfg, root, rawDir, path, date); err != nil {
|
||||||
errs = append(errs, fmt.Errorf("process %s: %w", filepath.Base(path), err))
|
errs = append(errs, fmt.Errorf("process %s: %w", filepath.Base(path), err))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -109,11 +130,15 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
|||||||
// processFile reads a file, calls pipeline.Run, copies it to processed/ or failed/,
|
// processFile reads a file, calls pipeline.Run, copies it to processed/ or failed/,
|
||||||
// and writes a marker file next to the original so the watcher skips it next poll.
|
// and writes a marker file next to the original so the watcher skips it next poll.
|
||||||
// The original file is never deleted, keeping Syncthing-connected vaults (e.g. Obsidian) intact.
|
// The original file is never deleted, keeping Syncthing-connected vaults (e.g. Obsidian) intact.
|
||||||
func processFile(ctx context.Context, cfg Config, path, date string) error {
|
func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, date string) error {
|
||||||
filename := filepath.Base(path)
|
filename := filepath.Base(path)
|
||||||
source := deriveSource(filename)
|
source := deriveSource(filename)
|
||||||
|
rel, err := filepath.Rel(rawDir, path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("relative path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
content, err := extract.Text(path)
|
content, err := extract.TextIn(root, rel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("extract text: %w", err)
|
return fmt.Errorf("extract text: %w", err)
|
||||||
}
|
}
|
||||||
@@ -126,7 +151,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
|
|||||||
return fmt.Errorf("mkdir failed dir: %w", mkErr)
|
return fmt.Errorf("mkdir failed dir: %w", mkErr)
|
||||||
}
|
}
|
||||||
dest := filepath.Join(failedDir, filename)
|
dest := filepath.Join(failedDir, filename)
|
||||||
if cpErr := copyFile(path, dest); cpErr != nil {
|
if cpErr := copyFromRoot(root, rel, dest); cpErr != nil {
|
||||||
return fmt.Errorf("copy to failed: %w", cpErr)
|
return fmt.Errorf("copy to failed: %w", cpErr)
|
||||||
}
|
}
|
||||||
if mkErr := os.WriteFile(path+".failed", []byte(runErr.Error()), 0o644); mkErr != nil {
|
if mkErr := os.WriteFile(path+".failed", []byte(runErr.Error()), 0o644); mkErr != nil {
|
||||||
@@ -154,7 +179,7 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
|
|||||||
base := strings.TrimSuffix(filename, ext)
|
base := strings.TrimSuffix(filename, ext)
|
||||||
dest = filepath.Join(processedDir, base+"-"+time.Now().UTC().Format("150405")+ext)
|
dest = filepath.Join(processedDir, base+"-"+time.Now().UTC().Format("150405")+ext)
|
||||||
}
|
}
|
||||||
if err := copyFile(path, dest); err != nil {
|
if err := copyFromRoot(root, rel, dest); err != nil {
|
||||||
return fmt.Errorf("copy to processed: %w", err)
|
return fmt.Errorf("copy to processed: %w", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(path+".processed", []byte(date), 0o644); err != nil {
|
if err := os.WriteFile(path+".processed", []byte(date), 0o644); err != nil {
|
||||||
@@ -165,26 +190,6 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// copyFile copies src to dst, creating dst if it doesn't exist.
|
|
||||||
func copyFile(src, dst string) error {
|
|
||||||
in, err := os.Open(src)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("open src: %w", err)
|
|
||||||
}
|
|
||||||
defer in.Close() //nolint:errcheck
|
|
||||||
|
|
||||||
out, err := os.Create(dst)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create dst: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := io.Copy(out, in); err != nil {
|
|
||||||
out.Close() //nolint:errcheck
|
|
||||||
return fmt.Errorf("copy: %w", err)
|
|
||||||
}
|
|
||||||
return out.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// deriveSource turns a filename into a human-readable source name.
|
// deriveSource turns a filename into a human-readable source name.
|
||||||
// "shape-up-book.md" → "Shape Up Book"
|
// "shape-up-book.md" → "Shape Up Book"
|
||||||
func deriveSource(filename string) string {
|
func deriveSource(filename string) string {
|
||||||
@@ -221,3 +226,30 @@ func appendWatcherLog(brainDir, filename string, runErr error, date string) erro
|
|||||||
}
|
}
|
||||||
return f.Close()
|
return f.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// copyFromRoot archives a raw/ file read through the root, refusing anything
|
||||||
|
// that is not a regular file, so a swapped-in symlink cannot copy an outside
|
||||||
|
// file into raw/processed/ or raw/failed/ (hyperguild#153).
|
||||||
|
func copyFromRoot(root *os.Root, rel, dst string) error {
|
||||||
|
fi, err := root.Lstat(rel)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !fi.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("%s: not a regular file", rel)
|
||||||
|
}
|
||||||
|
in, _, err := extract.OpenRegular(root, rel)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = in.Close() }()
|
||||||
|
out, err := os.Create(dst)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(out, in); err != nil {
|
||||||
|
_ = out.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return out.Close()
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -282,3 +286,107 @@ func TestProcessDir_SkipsTunnelCandidateFiles(t *testing.T) {
|
|||||||
_, err = os.Stat(filepath.Join(brainDir, "raw", "processed", date, "tunnel-candidates-2026-07-19.md"))
|
_, err = os.Stat(filepath.Join(brainDir, "raw", "processed", date, "tunnel-candidates-2026-07-19.md"))
|
||||||
assert.True(t, os.IsNotExist(err), "tunnel-candidates file should not be copied to processed/")
|
assert.True(t, os.IsNotExist(err), "tunnel-candidates file should not be copied to processed/")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hyperguild#153: raw/ is writable via Syncthing, and the watcher ingests
|
||||||
|
// whatever lands there with no API call at all. WalkDir yields symlinked
|
||||||
|
// files, and extract.Text followed them, so a symlink to any pod-readable
|
||||||
|
// file would be extracted into the brain. The watcher now ingests only
|
||||||
|
// regular files, read through an os.Root on raw/.
|
||||||
|
func TestProcessDir_IngestsTheRealNoteAndSkipsTheSymlinkBesideIt(t *testing.T) {
|
||||||
|
brainDir := setupBrainDir(t)
|
||||||
|
outside := filepath.Join(t.TempDir(), "secret.md")
|
||||||
|
require.NoError(t, os.WriteFile(outside, []byte("SENTINEL-153-WATCHER"), 0o644))
|
||||||
|
// "a-" sorts before "real": a walk that stopped at the first symlink
|
||||||
|
// (filepath.SkipAll — one synced link halting all ingestion) would fail
|
||||||
|
// the call count below (PR #156 review).
|
||||||
|
require.NoError(t, os.Symlink(outside, filepath.Join(brainDir, "raw", "a-leak.md")))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(brainDir, "raw", "real.md"), []byte("real note"), 0o644))
|
||||||
|
|
||||||
|
var sawSentinel bool
|
||||||
|
calls := 0
|
||||||
|
cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{
|
||||||
|
Complete: func(_ context.Context, _, user string) (string, error) {
|
||||||
|
calls++
|
||||||
|
if strings.Contains(user, "SENTINEL-153-WATCHER") {
|
||||||
|
sawSentinel = true
|
||||||
|
}
|
||||||
|
b, _ := json.Marshal([]pipeline.RawPage{{Title: "X", Type: "source", Subtype: "article", Content: "## S\n\nx\n"}})
|
||||||
|
return string(b), nil
|
||||||
|
},
|
||||||
|
Schema: "# Schema",
|
||||||
|
}}
|
||||||
|
errs := processDir(context.Background(), cfg, time.Now().UTC().Format("2006-01-02"))
|
||||||
|
assert.Empty(t, errs)
|
||||||
|
assert.False(t, sawSentinel, "the symlink target must never reach the pipeline")
|
||||||
|
assert.Equal(t, 1, calls, "exactly the real note is ingested; the symlink is not a note")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A symlinked raw/ is refused outright: os.OpenRoot would otherwise follow
|
||||||
|
// it and confine reads to wherever it points.
|
||||||
|
func TestProcessDir_RefusesASymlinkedRawDir(t *testing.T) {
|
||||||
|
brainDir := t.TempDir()
|
||||||
|
elsewhere := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(elsewhere, "x.md"), []byte("SENTINEL-153-RAWLINK"), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(elsewhere, filepath.Join(brainDir, "raw")))
|
||||||
|
calls := 0
|
||||||
|
cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{
|
||||||
|
Complete: func(context.Context, string, string) (string, error) { calls++; return "[]", nil },
|
||||||
|
Schema: "# Schema",
|
||||||
|
}}
|
||||||
|
errs := processDir(context.Background(), cfg, time.Now().UTC().Format("2006-01-02"))
|
||||||
|
assert.NotEmpty(t, errs, "a symlinked raw/ must be reported, not silently ingested")
|
||||||
|
assert.Zero(t, calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PR #156: a FIFO swapped into raw/ between the watcher's checks and its
|
||||||
|
// read or archive copy must not hang the watcher, which runs unattended.
|
||||||
|
func TestProcessDir_DoesNotHangUnderAFIFOSwapRace(t *testing.T) {
|
||||||
|
brainDir := setupBrainDir(t)
|
||||||
|
raw := filepath.Join(brainDir, "raw")
|
||||||
|
var stop atomic.Bool
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
// Awaited before t.TempDir's cleanup (hyperguild#162).
|
||||||
|
t.Cleanup(func() { stop.Store(true); wg.Wait() })
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for i := 0; !stop.Load(); i++ {
|
||||||
|
tmp := filepath.Join(raw, ".swap")
|
||||||
|
_ = os.Remove(tmp)
|
||||||
|
if i%2 == 0 {
|
||||||
|
_ = os.WriteFile(tmp, []byte("x"), 0o644)
|
||||||
|
} else {
|
||||||
|
_ = syscall.Mkfifo(tmp, 0o644)
|
||||||
|
}
|
||||||
|
_ = os.Rename(tmp, filepath.Join(raw, "r.md"))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
cfg := Config{BrainDir: brainDir, Interval: time.Hour, Pipeline: pipeline.Config{
|
||||||
|
Complete: func(context.Context, string, string) (string, error) {
|
||||||
|
b, _ := json.Marshal([]pipeline.RawPage{{Title: "R", Type: "source", Subtype: "article", Content: "## S\n\nx\n"}})
|
||||||
|
return string(b), nil
|
||||||
|
},
|
||||||
|
Schema: "# Schema",
|
||||||
|
}}
|
||||||
|
var iters atomic.Int64
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
|
||||||
|
_ = os.Remove(filepath.Join(raw, "r.md.processed"))
|
||||||
|
_ = os.Remove(filepath.Join(raw, "r.md.failed"))
|
||||||
|
_ = processDir(context.Background(), cfg, "2026-10-03")
|
||||||
|
iters.Add(1)
|
||||||
|
}
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
stop.Store(true)
|
||||||
|
case <-time.After(8 * time.Second):
|
||||||
|
stop.Store(true)
|
||||||
|
if f, err := os.OpenFile(filepath.Join(raw, "r.md"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil {
|
||||||
|
_ = f.Close()
|
||||||
|
}
|
||||||
|
t.Fatalf("the watcher hung on a swapped-in FIFO after %d iterations", iters.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user