mathiasandClaude Opus 5.5
1a2a93d2f4
Review round 1 on PR #195: - A refused marker or archive write left no marker, so the note was reprocessed every interval (archive copies piling up; an LLM call per round on the failure path). The marker check now uses Lstat (any entry at the marker path counts, so a dangling marker link skips the note instead of looping), and a note whose archive copy or marker write is refused is remembered (path, size, mtime) and not retried until it changes. - cleanRel no longer refuses a backslash: it is a legal filename byte on Linux, not a traversal, and a raw note named with one was never archived or marked. - Tests now reach every converted writer: the log.md append (real raw/failed so the log is reached, symlinked log.md), a dangling marker link, the refused-retry loop with a positive control, a backslash name, a buildAndWrite swap race, and the claude-session archive with in-brain and outside symlinked archive dirs. The stale lint allowlist entry for appendWatcherLog is removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
218 lines
11 KiB
Go
218 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
|
)
|
|
|
|
// hyperguild#157 slice 6: the pipeline read the source page, schema.md, the
|
|
// pages it revises and the pages it adds source refs to with os.ReadFile, so
|
|
// a symlinked file's content reached the LLM prompt and the written pages,
|
|
// and a FIFO hung the ingest.
|
|
|
|
func mkdirs(t *testing.T, brain string, dirs ...string) {
|
|
t.Helper()
|
|
for _, d := range dirs {
|
|
require.NoError(t, os.MkdirAll(filepath.Join(brain, filepath.FromSlash(d)), 0o755))
|
|
}
|
|
}
|
|
|
|
func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources")
|
|
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644))
|
|
assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control")
|
|
assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control")
|
|
|
|
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md")))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel)
|
|
assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel)
|
|
assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel)
|
|
|
|
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
read := func() string { return readSourcePage(brain, "r") }
|
|
brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read)
|
|
brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read)
|
|
brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
|
|
|
|
schema := filepath.Join(brain, "schema.md")
|
|
rs := func() string { return loadSchema(brain) }
|
|
brainfstest.AssertNoLeakFrom(t, schema, "zqinside schema", inBrain, 1500*time.Millisecond, rs)
|
|
brainfstest.AssertNoHang(t, schema, "zqinside schema", 1500*time.Millisecond, 6*time.Second, func() { _ = rs() })
|
|
}
|
|
|
|
func TestReviseOrCreateRefusesASymlinkedExistingPage(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/concepts")
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
page := wiki.Page{Path: "wiki/concepts/c.md", Content: "---\ntitle: C\n---\nnew\n"}
|
|
out, _, err := reviseOrCreate(brain, page)
|
|
assert.Error(t, err)
|
|
assert.NotContains(t, out, brainfstest.Sentinel)
|
|
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("---\ntitle: D\n---\nzqold\n"), 0o644))
|
|
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/d.md", Content: "---\ntitle: D\n---\nnew\n"})
|
|
require.NoError(t, err)
|
|
assert.Contains(t, out, "zqold", "positive control: the existing page is read")
|
|
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/new.md", Content: "fresh"})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "fresh", out, "positive control: a missing page is created")
|
|
}
|
|
|
|
func TestInjectSourceRefsDoesNotReadASymlinkedPage(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/concepts")
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("zqinside d"), 0o644))
|
|
inv := map[wiki.PageType][]wiki.Entry{wiki.PageTypeConcept: {
|
|
{Slug: "c", Type: wiki.PageTypeConcept}, {Slug: "d", Type: wiki.PageTypeConcept},
|
|
}}
|
|
src := wiki.Page{Path: "wiki/sources/src.md", Content: "---\ntitle: Src\n---\nsee [[c|C]] and [[d|D]]\n"}
|
|
out := injectSourceRefs([]wiki.Page{src}, inv, brain)
|
|
got := fmt.Sprintf("%+v", out)
|
|
assert.Contains(t, got, "zqinside d", "positive control")
|
|
assert.NotContains(t, got, brainfstest.Sentinel)
|
|
}
|
|
|
|
func TestBackfillRefsSkipsSymlinkedSourcesAndPages(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "real.md"), []byte("---\ntitle: R\n---\nbody\n"), 0o644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("---\ntitle: zqinside\n---\n[[real|R]]\n"), 0o644))
|
|
// A source elsewhere in the brain whose title is the sentinel and which
|
|
// links the real concept with the [[slug|title]] form the pipeline follows.
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "src.md"),
|
|
[]byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "src.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
// A concept page linked by the regular source that is itself a symlink.
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok2.md"), []byte("---\ntitle: Two\n---\n[[lnk|L]]\n"), 0o644))
|
|
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "page.md"))
|
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "page.md"), filepath.Join(brain, "wiki", "concepts", "lnk.md")))
|
|
|
|
_, err := BackfillRefs(context.Background(), brain)
|
|
require.NoError(t, err)
|
|
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "concepts", "real.md"))
|
|
assert.Contains(t, string(b), "zqinside", "positive control: the regular source's ref is added")
|
|
assert.NotContains(t, string(b), brainfstest.Sentinel, "a symlinked source's title was written as a ref")
|
|
raw, _ := os.ReadFile(filepath.Join(brain, "raw", "page.md"))
|
|
assert.NotContains(t, string(raw), "[[ok2", "the symlinked concept page's target was rewritten")
|
|
}
|
|
|
|
// A source swapped between a regular file and a symlink to an in-brain source
|
|
// whose title is the sentinel: the read refuses the swapped-in link, so the
|
|
// sentinel never lands in the concept page as a ref.
|
|
func TestBackfillRefsDoesNotWriteARefFromASourceSwappedInBehindASymlink(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
realPage := filepath.Join(brain, "wiki", "concepts", "real.md")
|
|
target := filepath.Join(brain, "raw", "src.md")
|
|
require.NoError(t, os.WriteFile(target, []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
inside := "---\ntitle: zqinside\n---\n[[real|R]]\n"
|
|
read := func() string {
|
|
_ = os.WriteFile(realPage, []byte("---\ntitle: R\n---\nbody\n"), 0o644)
|
|
_, _ = BackfillRefs(context.Background(), brain)
|
|
b, _ := os.ReadFile(realPage)
|
|
return string(b)
|
|
}
|
|
require.NoError(t, os.WriteFile(src, []byte(inside), 0o644))
|
|
require.Contains(t, read(), "zqinside", "positive control")
|
|
brainfstest.AssertNoLeakFrom(t, src, inside, target, 1500*time.Millisecond, read)
|
|
}
|
|
|
|
// A concept page that is regular when the inventory lists it, then swapped
|
|
// for a symlink to an in-brain file before BackfillRefs reads it: only the
|
|
// page read's own check refuses it (a static symlink never gets this far,
|
|
// because LoadInventory does not list it).
|
|
func TestBackfillRefsDoesNotCopyAPageSwappedInBehindASymlink(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "s.md"), []byte("---\ntitle: S\n---\n[[c|C]]\n"), 0o644))
|
|
page := filepath.Join(brain, "wiki", "concepts", "c.md")
|
|
target := filepath.Join(brain, "raw", "secret.md")
|
|
inside := "---\ntitle: C\n---\nzqinside\n"
|
|
read := func() string {
|
|
// The write path still follows a swapped-in link (#158) and can
|
|
// overwrite the target, so restore it each round.
|
|
_ = os.WriteFile(target, []byte("---\ntitle: X\n---\n"+brainfstest.Sentinel+"\n"), 0o644)
|
|
_, _ = BackfillRefs(context.Background(), brain)
|
|
f, err := os.OpenFile(page, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
b, _ := io.ReadAll(f)
|
|
return string(b)
|
|
}
|
|
require.NoError(t, os.WriteFile(page, []byte(inside), 0o644))
|
|
require.Contains(t, read(), "[[s|S]]", "positive control: the regular page gets the source ref")
|
|
brainfstest.AssertNoLeakFrom(t, page, inside, target, 3*time.Second, read)
|
|
}
|
|
|
|
// hyperguild#158: BackfillRefs read a regular concept page, a symlink could
|
|
// be swapped in, and os.WriteFile then followed it and overwrote the link's
|
|
// target. The target is not restored between rounds here, so any
|
|
// write-through shows.
|
|
func TestBackfillRefsNeverWritesThroughAPageSwappedForASymlink(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "s.md"), []byte("---\ntitle: S\n---\n[[c|C]]\n"), 0o644))
|
|
page := filepath.Join(brain, "wiki", "concepts", "c.md")
|
|
target := filepath.Join(brain, "raw", "secret.md")
|
|
require.NoError(t, os.WriteFile(target, []byte("untouched"), 0o644))
|
|
inside := "---\ntitle: C\n---\nbody\n"
|
|
require.NoError(t, os.WriteFile(page, []byte(inside), 0o644))
|
|
brainfstest.StartSwap(t, page, inside, target, false)
|
|
ok := 0
|
|
for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); {
|
|
if _, err := BackfillRefs(context.Background(), brain); err == nil {
|
|
ok++
|
|
}
|
|
}
|
|
got, _ := os.ReadFile(target)
|
|
assert.Equal(t, "untouched", string(got), "BackfillRefs wrote through a swapped-in symlink")
|
|
assert.Positive(t, ok, "positive control: some backfills ran")
|
|
}
|
|
|
|
// hyperguild#158: buildAndWrite wrote each page with os.WriteFile by path, so
|
|
// a page read as regular and swapped for a symlink before the write was
|
|
// written through. The target is not restored between rounds, so any
|
|
// write-through shows.
|
|
func TestBuildAndWriteNeverWritesThroughAPageSwappedForASymlink(t *testing.T) {
|
|
brain := t.TempDir()
|
|
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "wiki/entities", "raw")
|
|
page := filepath.Join(brain, "wiki", "concepts", "zq-concept.md")
|
|
target := filepath.Join(brain, "raw", "secret.md")
|
|
require.NoError(t, os.WriteFile(target, []byte("untouched"), 0o644))
|
|
inside := "---\ntitle: Zq Concept\n---\nbody\n"
|
|
require.NoError(t, os.WriteFile(page, []byte(inside), 0o644))
|
|
pages := []RawPage{{Title: "Zq Concept", Type: "concept", Content: "## About\n\nmore\n"}}
|
|
brainfstest.StartSwap(t, page, inside, target, false)
|
|
ok := 0
|
|
for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); {
|
|
if _, err := RunRaw(context.Background(), brain, "zq source", pages, false); err == nil {
|
|
ok++
|
|
}
|
|
}
|
|
got, _ := os.ReadFile(target)
|
|
assert.Equal(t, "untouched", string(got), "buildAndWrite wrote through a swapped-in symlink")
|
|
assert.Positive(t, ok, "positive control: some runs wrote the page")
|
|
}
|