From e5f8947f3b460b36546479cfbd4b6a946aa9d85b Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 20 Oct 2026 20:24:03 +0200 Subject: [PATCH 1/3] feat(legacymove): clear the retired wiki/sources layout with an archive and a rollback (#209) Videos and articles move to the default wing (wing/hall stamped, body and links byte-preserved); auto-tunnel review logs are deleted; notes, projects and untyped pages stay. Plan is read-only; Apply archives every page it touches to archive/wing-migration-/ (outside raw/) before changing anything, creates destinations O_EXCL and removes sources only after; Rollback restores byte for byte. concepts/ and entities/ are the shared layer and are untouched. Dry run against the live brain: 16 move, 24 delete, 15 left. Mutation-checked: junk guard, type filter, slug collision, no-overwrite, archive-before-change, action gate. Co-Authored-By: Claude Sonnet 5.5 --- ingestion/cmd/legacymove/main.go | 59 ++++ ingestion/internal/legacymove/legacymove.go | 254 ++++++++++++++++++ .../internal/legacymove/legacymove_test.go | 173 ++++++++++++ 3 files changed, 486 insertions(+) create mode 100644 ingestion/cmd/legacymove/main.go create mode 100644 ingestion/internal/legacymove/legacymove.go create mode 100644 ingestion/internal/legacymove/legacymove_test.go diff --git a/ingestion/cmd/legacymove/main.go b/ingestion/cmd/legacymove/main.go new file mode 100644 index 0000000..619f9c9 --- /dev/null +++ b/ingestion/cmd/legacymove/main.go @@ -0,0 +1,59 @@ +// Command legacymove clears wiki/sources/ of the retired layout (hyperguild +// #209). Default is a dry run that prints the plan; -apply performs it after +// archiving every page it touches; -rollback restores from an archive. +package main + +import ( + "flag" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/mathiasbq/hyperguild/ingestion/internal/legacymove" +) + +func main() { + brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory") + wing := flag.String("wing", "web-clip", "wing that receives video and article sources") + apply := flag.Bool("apply", false, "perform the plan (default: dry run)") + rollback := flag.String("rollback", "", "restore from this archive dir (brain-relative, e.g. archive/wing-migration-2026-10-20)") + flag.Parse() + + if err := run(*brain, *wing, *apply, *rollback); err != nil { + slog.Error("legacymove", "err", err) + os.Exit(1) + } +} + +func run(brain, wing string, apply bool, rollback string) error { + if rollback != "" { + if err := legacymove.Rollback(brain, rollback); err != nil { + return err + } + fmt.Println("rolled back from", rollback) + return nil + } + items, err := legacymove.Plan(brain, wing) + if err != nil { + return err + } + counts := map[legacymove.Action]int{} + for _, it := range items { + counts[it.Action]++ + } + fmt.Printf("# legacy source plan\n\n%d pages: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip]) + fmt.Println("| action | from | to | why |") + fmt.Println("|---|---|---|---|") + for _, it := range items { + fmt.Printf("| %s | %s | %s | %s |\n", it.Action, it.From, it.To, strings.ReplaceAll(it.Reason, "|", "/")) + } + if !apply { + fmt.Println("\n_dry run: nothing was changed_") + return nil + } + res, err := legacymove.Apply(brain, items, time.Now()) + fmt.Printf("\nmoved %d, deleted %d; archive: %s\n", res.Moved, res.Deleted, res.ArchiveDir) + return err +} diff --git a/ingestion/internal/legacymove/legacymove.go b/ingestion/internal/legacymove/legacymove.go new file mode 100644 index 0000000..89c43fa --- /dev/null +++ b/ingestion/internal/legacymove/legacymove.go @@ -0,0 +1,254 @@ +// Package legacymove clears wiki/sources/ of the retired layout (hyperguild +// #209). It does the one thing that is safe to do mechanically: video and +// article source pages move to the default wing, and the auto-tunnel review +// logs that were once ingested as "sources" are deleted. Notes, projects and +// untyped pages stay where they are, and wiki/concepts and wiki/entities are +// not touched at all (they are the shared layer). +// +// Plan only reads. Apply copies every page it touches into +// archive/wing-migration-/ first, so Rollback can restore the brain +// byte for byte. The archive is outside raw/, so the watcher never ingests it. +package legacymove + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "path" + "sort" + "strings" + "time" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brain" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" + "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" +) + +// Action is what the plan does with one page. +type Action string + +const ( + Move Action = "move" + Delete Action = "delete" + Skip Action = "skip" +) + +// Item is one planned operation. +type Item struct { + From string + To string // Move only + Slug string + Action Action + Wing string // Move only + Reason string +} + +const legacyDir = "wiki/sources" + +// Plan reads the retired source dir and decides. It writes nothing. +func Plan(brainDir, wing string) ([]Item, error) { + if wing == "" || brain.Sanitise(wing) != wing { + return nil, fmt.Errorf("wing %q is not a plain wing name", wing) + } + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + names, err := bfs.ReadDirRegular(legacyDir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", legacyDir, err) + } + var items []Item + for _, name := range names { + if !strings.HasSuffix(name, ".md") || name == "_index.md" { + continue + } + from := legacyDir + "/" + name + slug := strings.TrimSuffix(name, ".md") + it := Item{From: from, Slug: slug} + if strings.HasPrefix(name, "tunnel-candidates-") { + it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source" + items = append(items, it) + continue + } + content, err := bfs.ReadRegular(from) + if err != nil { + it.Action, it.Reason = Skip, fmt.Sprintf("unreadable: %v", err) + items = append(items, it) + continue + } + switch typ := frontmatterValue(string(content), "type"); typ { + case "video", "article": + to := "wiki/" + wing + "/sources/" + name + it.Action, it.To, it.Wing = Move, to, wing + existing, ferr := wiki.FindSlugs(brainDir, slug) + if ferr != nil { + it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("cannot check slug: %v", ferr) + } + for _, p := range existing { + if p != from { + it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("slug %q already used by %s", slug, p) + break + } + } + default: + it.Action, it.Reason = Skip, fmt.Sprintf("type %q: left in place", typ) + } + items = append(items, it) + } + sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) + return items, nil +} + +// Result reports what Apply did and where the archive is (brain-relative). +type Result struct { + Moved, Deleted int + ArchiveDir string +} + +type manifest struct { + Moves []moveRec `json:"moves"` + Deletes []string `json:"deletes"` +} + +type moveRec struct { + From string `json:"from"` + To string `json:"to"` +} + +// Apply performs the Move and Delete items, one at a time, stopping at the +// first failure. Each original is archived before anything changes; a move +// writes its destination with O_EXCL before the source is removed, so a failure +// never loses a page. The manifest is rewritten after every step so a partial +// run can still be rolled back. +func Apply(brainDir string, items []Item, now time.Time) (Result, error) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return Result{}, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + + res := Result{ArchiveDir: "archive/wing-migration-" + now.UTC().Format("2006-01-02")} + var man manifest + save := func() error { + b, err := json.MarshalIndent(man, "", " ") + if err != nil { + return err + } + return bfs.PutRegular(res.ArchiveDir+"/manifest.json", b) + } + for _, it := range items { + if it.Action != Move && it.Action != Delete { + continue + } + content, err := bfs.ReadRegular(it.From) + if err != nil { + return res, fmt.Errorf("read %s: %w", it.From, err) + } + if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+path.Base(it.From), content); err != nil { + return res, fmt.Errorf("archive %s: %w", it.From, err) + } + switch it.Action { + case Move: + if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing))); err != nil { + return res, fmt.Errorf("write %s: %w", it.To, err) + } + man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To}) + res.Moved++ + case Delete: + man.Deletes = append(man.Deletes, it.From) + res.Deleted++ + } + if err := save(); err != nil { + return res, fmt.Errorf("write manifest: %w", err) + } + if err := bfs.Remove(it.From); err != nil { + return res, fmt.Errorf("handled %s but removing it failed: %w", it.From, err) + } + } + for w := range wingsOf(items) { + _ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved + } + return res, nil +} + +func wingsOf(items []Item) map[string]bool { + out := map[string]bool{} + for _, it := range items { + if it.Action == Move { + out[it.Wing] = true + } + } + return out +} + +// Rollback restores every page named in the archive's manifest from its +// archived original and removes the moved copies. It refuses to overwrite a +// page that already exists at the original location. +func Rollback(brainDir, archiveDir string) error { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + raw, err := bfs.ReadRegular(archiveDir + "/manifest.json") + if err != nil { + return fmt.Errorf("read manifest: %w", err) + } + var man manifest + if err := json.Unmarshal(raw, &man); err != nil { + return fmt.Errorf("parse manifest: %w", err) + } + restore := func(from string) error { + orig, err := bfs.ReadRegular(archiveDir + "/originals/" + path.Base(from)) + if err != nil { + return fmt.Errorf("read archived %s: %w", from, err) + } + if err := bfs.CreateRegular(from, orig); err != nil { + return fmt.Errorf("restore %s: %w", from, err) + } + return nil + } + for _, m := range man.Moves { + if err := restore(m.From); err != nil { + return err + } + if err := bfs.Remove(m.To); err != nil { + return fmt.Errorf("remove moved copy %s: %w", m.To, err) + } + } + for _, d := range man.Deletes { + if err := restore(d); err != nil { + return err + } + } + return nil +} + +// stamp inserts wing/hall after the opening fence, leaving every other byte +// alone. A page without frontmatter gets a block. +func stamp(content, wing string) string { + extra := "wing: " + wing + "\nhall: sources\n" + if strings.HasPrefix(content, "---\n") { + return "---\n" + extra + content[4:] + } + return "---\n" + extra + "---\n" + content +} + +func frontmatterValue(content, key string) string { + if !strings.HasPrefix(content, "---\n") { + return "" + } + block, _, _ := strings.Cut(content[4:], "\n---") + for _, line := range strings.Split(block, "\n") { + if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key { + return strings.Trim(strings.TrimSpace(v), `"'`) + } + } + return "" +} diff --git a/ingestion/internal/legacymove/legacymove_test.go b/ingestion/internal/legacymove/legacymove_test.go new file mode 100644 index 0000000..43b7f7e --- /dev/null +++ b/ingestion/internal/legacymove/legacymove_test.go @@ -0,0 +1,173 @@ +package legacymove_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/legacymove" +) + +func page(t *testing.T, dir, rel, body string) { + t.Helper() + full := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(t, os.WriteFile(full, []byte(body), 0o644)) +} + +func src(typ string) string { + return "---\ntitle: T\ntype: " + typ + "\n---\n\n## Summary\n\nBody with [[a-link|A]] kept as is.\n" +} + +func byFrom(items []legacymove.Item) map[string]legacymove.Item { + m := map[string]legacymove.Item{} + for _, it := range items { + m[it.From] = it + } + return m +} + +func TestPlan_VideosAndArticlesMoveToTheDefaultWing(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + page(t, dir, "wiki/sources/an-article.md", src("article")) + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, legacymove.Move, m["wiki/sources/a-talk.md"].Action) + assert.Equal(t, "wiki/web-clip/sources/a-talk.md", m["wiki/sources/a-talk.md"].To) + assert.Equal(t, legacymove.Move, m["wiki/sources/an-article.md"].Action) +} + +func TestPlan_NotesProjectsAndUntypedPagesStayWhereTheyAre(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-note.md", src("note")) + page(t, dir, "wiki/sources/a-project.md", src("project")) + page(t, dir, "wiki/sources/untyped.md", "---\ntitle: T\n---\nbody") + page(t, dir, "wiki/sources/_index.md", "index") + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + m := byFrom(items) + for _, f := range []string{"a-note", "a-project", "untyped"} { + assert.Equal(t, legacymove.Skip, m["wiki/sources/"+f+".md"].Action, f) + } + _, listed := m["wiki/sources/_index.md"] + assert.False(t, listed, "the index is not a page") +} + +func TestPlan_AutoTunnelLogsAreDeleted(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("video")) + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + assert.Equal(t, legacymove.Delete, byFrom(items)["wiki/sources/tunnel-candidates-2026-06-14.md"].Action, + "never moved, whatever the type says") +} + +func TestPlan_ASlugAlreadyUsedElsewhereIsSkippedNotOverwritten(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + page(t, dir, "wiki/entities/a-talk.md", "an entity with the same slug") + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + it := byFrom(items)["wiki/sources/a-talk.md"] + assert.Equal(t, legacymove.Skip, it.Action) + assert.Contains(t, it.Reason, "already used") +} + +func TestPlan_ADefaultWingThatIsNotAPlainNameIsRefused(t *testing.T) { + _, err := legacymove.Plan(t.TempDir(), "../x") + assert.Error(t, err) +} + +func TestApply_MovesStampsAndArchivesEverythingItTouches(t *testing.T) { + dir := t.TempDir() + talk := src("video") + junk := "tunnel log" + page(t, dir, "wiki/sources/a-talk.md", talk) + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", junk) + page(t, dir, "wiki/sources/a-note.md", src("note")) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + + res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + assert.Equal(t, 1, res.Moved) + assert.Equal(t, 1, res.Deleted) + + moved, err := os.ReadFile(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md")) + require.NoError(t, err) + assert.Contains(t, string(moved), "wing: web-clip") + assert.Contains(t, string(moved), "hall: sources") + assert.Contains(t, string(moved), "Body with [[a-link|A]] kept as is.", "body and links are byte-preserved") + + for _, gone := range []string{"wiki/sources/a-talk.md", "wiki/sources/tunnel-candidates-2026-06-14.md"} { + _, statErr := os.Stat(filepath.Join(dir, gone)) + assert.ErrorIs(t, statErr, os.ErrNotExist, gone) + } + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-note.md")) + assert.NoError(t, statErr, "a skipped page is left exactly where it is") + + again, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + for _, it := range again { + assert.Equal(t, legacymove.Skip, it.Action, "nothing is left to do: %s", it.From) + } +} + +func TestRollback_RestoresEveryTouchedPageByteForByte(t *testing.T) { + dir := t.TempDir() + talk := src("video") + junk := "tunnel log" + page(t, dir, "wiki/sources/a-talk.md", talk) + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", junk) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + + require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir)) + + got, err := os.ReadFile(filepath.Join(dir, "wiki/sources/a-talk.md")) + require.NoError(t, err) + assert.Equal(t, talk, string(got)) + got, err = os.ReadFile(filepath.Join(dir, "wiki/sources/tunnel-candidates-2026-06-14.md")) + require.NoError(t, err) + assert.Equal(t, junk, string(got)) + _, statErr := os.Stat(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md")) + assert.ErrorIs(t, statErr, os.ErrNotExist, "the moved copy is removed") +} + +func TestApply_NeverOverwritesAndKeepsTheSourceIfTheDestinationAppearsLate(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + page(t, dir, "wiki/web-clip/sources/a-talk.md", "appeared after the plan") + + _, err = legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.Error(t, err) + got, _ := os.ReadFile(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md")) + assert.Equal(t, "appeared after the plan", string(got)) + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-talk.md")) + assert.NoError(t, statErr, "the source survives") +} + +func TestApply_TheArchiveIsOutsideRawSoTheWatcherNeverIngestsIt(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", "x") + items, _ := legacymove.Plan(dir, "web-clip") + res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + assert.False(t, strings.Contains(res.ArchiveDir, "/raw"), res.ArchiveDir) + assert.True(t, strings.HasPrefix(res.ArchiveDir, "archive/"), res.ArchiveDir) +} -- 2.54.0 From 2b16f2cc2f8f46553b1492472149c42054257660 Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 20 Oct 2026 21:02:41 +0200 Subject: [PATCH 2/3] feat(legacymove): also clear raw/ leftovers from before inbox zero (#209) PlanRaw: an original that was archived but never removed (it has a .processed marker AND a byte-identical copy under raw/processed/) is deleted with its marker; a marker without a note is deleted; an unmarked note, a note without an identical archive copy, and AutoTunnel's tunnel-candidates-* review queue are never touched. The archive key flattens the whole path so raw/x.md and wiki/sources/x.md cannot overwrite each other. Live dry run: 84 entries, 16 move, 53 delete (24 junk + 29 raw leftovers), 15 left. Mutation-checked. Co-Authored-By: Claude Sonnet 5.5 --- ingestion/cmd/legacymove/main.go | 18 +++- ingestion/internal/legacymove/legacymove.go | 97 +++++++++++++++++- .../internal/legacymove/legacymove_test.go | 98 +++++++++++++++++++ 3 files changed, 205 insertions(+), 8 deletions(-) diff --git a/ingestion/cmd/legacymove/main.go b/ingestion/cmd/legacymove/main.go index 619f9c9..e3ac10d 100644 --- a/ingestion/cmd/legacymove/main.go +++ b/ingestion/cmd/legacymove/main.go @@ -1,6 +1,9 @@ -// Command legacymove clears wiki/sources/ of the retired layout (hyperguild -// #209). Default is a dry run that prints the plan; -apply performs it after -// archiving every page it touches; -rollback restores from an archive. +// Command legacymove clears the leftovers of the retired layout (hyperguild +// #209): wiki/sources/ (videos and articles move to the default wing, the +// auto-tunnel review logs are deleted) and raw/ (originals archived before inbox +// zero, their markers, orphan markers). Default is a dry run that prints the +// plan; -apply performs it after archiving every page it touches; -rollback +// restores from an archive. package main import ( @@ -35,15 +38,20 @@ func run(brain, wing string, apply bool, rollback string) error { fmt.Println("rolled back from", rollback) return nil } - items, err := legacymove.Plan(brain, wing) + sources, err := legacymove.Plan(brain, wing) if err != nil { return err } + rawLeft, err := legacymove.PlanRaw(brain) + if err != nil { + return err + } + items := append(sources, rawLeft...) counts := map[legacymove.Action]int{} for _, it := range items { counts[it.Action]++ } - fmt.Printf("# legacy source plan\n\n%d pages: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip]) + fmt.Printf("# legacy plan\n\n%d entries: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip]) fmt.Println("| action | from | to | why |") fmt.Println("|---|---|---|---|") for _, it := range items { diff --git a/ingestion/internal/legacymove/legacymove.go b/ingestion/internal/legacymove/legacymove.go index 89c43fa..f3a064e 100644 --- a/ingestion/internal/legacymove/legacymove.go +++ b/ingestion/internal/legacymove/legacymove.go @@ -11,11 +11,11 @@ package legacymove import ( + "crypto/sha256" "encoding/json" "errors" "fmt" "io/fs" - "path" "sort" "strings" "time" @@ -150,7 +150,7 @@ func Apply(brainDir string, items []Item, now time.Time) (Result, error) { if err != nil { return res, fmt.Errorf("read %s: %w", it.From, err) } - if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+path.Base(it.From), content); err != nil { + if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil { return res, fmt.Errorf("archive %s: %w", it.From, err) } switch it.Action { @@ -205,7 +205,7 @@ func Rollback(brainDir, archiveDir string) error { return fmt.Errorf("parse manifest: %w", err) } restore := func(from string) error { - orig, err := bfs.ReadRegular(archiveDir + "/originals/" + path.Base(from)) + orig, err := bfs.ReadRegular(archiveDir + "/originals/" + archiveKey(from)) if err != nil { return fmt.Errorf("read archived %s: %w", from, err) } @@ -230,6 +230,11 @@ func Rollback(brainDir, archiveDir string) error { return nil } +// archiveKey flattens a brain path into one file name, so two pages with the +// same basename in different places (raw/x.md and wiki/sources/x.md) never +// overwrite each other in the archive. +func archiveKey(p string) string { return strings.ReplaceAll(p, "/", "__") } + // stamp inserts wing/hall after the opening fence, leaving every other byte // alone. A page without frontmatter gets a block. func stamp(content, wing string) string { @@ -252,3 +257,89 @@ func frontmatterValue(content, key string) string { } return "" } + +const rawDir = "raw" + +// PlanRaw finds what the watcher left in raw/ before inbox zero: originals that +// were archived but never removed, with their .processed markers, and markers +// whose note is gone. An original is deleted only when a byte-identical copy +// exists under raw/processed/ (so nothing is lost), and only when it has a +// marker (an unmarked note is still in the inbox). tunnel-candidates-* are +// AutoTunnel's own review queue and are never touched. +func PlanRaw(brainDir string) ([]Item, error) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + names, err := bfs.ReadDirRegular(rawDir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", rawDir, err) + } + present := map[string]bool{} + for _, n := range names { + present[n] = true + } + archived, err := archivedCopies(bfs) + if err != nil { + return nil, err + } + var items []Item + for _, n := range names { + if strings.HasPrefix(n, "tunnel-candidates-") { + continue + } + switch { + case strings.HasSuffix(n, ".processed"): + note := strings.TrimSuffix(n, ".processed") + if !present[note] { + items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "marker without a note"}) + } + case present[n+".processed"]: + content, rerr := bfs.ReadRegular(rawDir + "/" + n) + if rerr != nil { + items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", rerr)}) + continue + } + if !archived[sha256.Sum256(content)] { + items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "marked processed but no identical copy under raw/processed/; kept"}) + continue + } + items = append(items, + Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "archived copy exists under raw/processed/"}, + Item{From: rawDir + "/" + n + ".processed", Slug: n + ".processed", Action: Delete, Reason: "marker of an archived note"}) + } + } + sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) + return items, nil +} + +// archivedCopies returns the SHA-256 of every file under raw/processed/, as a +// set, so "is there an identical copy" is one lookup. +func archivedCopies(bfs *brainfs.Brain) (map[[32]byte]bool, error) { + out := map[[32]byte]bool{} + days, err := bfs.ReadDirDirs(rawDir + "/processed") + if errors.Is(err, fs.ErrNotExist) { + return out, nil + } + if err != nil { + return nil, fmt.Errorf("list %s/processed: %w", rawDir, err) + } + for _, d := range days { + files, err := bfs.ReadDirRegular(rawDir + "/processed/" + d) + if err != nil { + continue + } + for _, f := range files { + b, err := bfs.ReadRegular(rawDir + "/processed/" + d + "/" + f) + if err != nil { + continue + } + out[sha256.Sum256(b)] = true + } + } + return out, nil +} diff --git a/ingestion/internal/legacymove/legacymove_test.go b/ingestion/internal/legacymove/legacymove_test.go index 43b7f7e..10e8e40 100644 --- a/ingestion/internal/legacymove/legacymove_test.go +++ b/ingestion/internal/legacymove/legacymove_test.go @@ -171,3 +171,101 @@ func TestApply_TheArchiveIsOutsideRawSoTheWatcherNeverIngestsIt(t *testing.T) { assert.False(t, strings.Contains(res.ArchiveDir, "/raw"), res.ArchiveDir) assert.True(t, strings.HasPrefix(res.ArchiveDir, "archive/"), res.ArchiveDir) } + +// --- raw/ leftovers from before inbox zero --- + +func rawFile(t *testing.T, dir, rel, body string) { page(t, dir, rel, body) } + +func TestPlanRaw_AnArchivedOriginalAndItsMarkerAreDeleted(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/clip.md", "the clip text") + rawFile(t, dir, "raw/clip.md.processed", "2026-10-19") + rawFile(t, dir, "raw/processed/2026-10-19/clip.md", "the clip text") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, legacymove.Delete, m["raw/clip.md"].Action) + assert.Equal(t, legacymove.Delete, m["raw/clip.md.processed"].Action) +} + +func TestPlanRaw_AnOriginalWhoseArchiveCopyDiffersIsLeftAlone(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/clip.md", "the clip text, edited since") + rawFile(t, dir, "raw/clip.md.processed", "2026-10-19") + rawFile(t, dir, "raw/processed/2026-10-19/clip.md", "the clip text") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, legacymove.Skip, m["raw/clip.md"].Action, "no identical archive copy: not ours to delete") + assert.Contains(t, m["raw/clip.md"].Reason, "no identical copy") + _, hasMarker := m["raw/clip.md.processed"] + assert.False(t, hasMarker, "the marker of a note we keep is kept") +} + +func TestPlanRaw_ACopyUnderATimestampedNameCounts(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/clip.md", "same text") + rawFile(t, dir, "raw/clip.md.processed", "d") + rawFile(t, dir, "raw/processed/2026-10-19/clip-180200.md", "same text") + rawFile(t, dir, "raw/processed/2026-10-19/clip-2.md", "other text") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + assert.Equal(t, legacymove.Delete, byFrom(items)["raw/clip.md"].Action) +} + +func TestPlanRaw_AnOrphanMarkerIsDeleted(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/gone.md.processed", "2026-06-16") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + assert.Equal(t, legacymove.Delete, byFrom(items)["raw/gone.md.processed"].Action) +} + +func TestPlanRaw_TheAutoTunnelReviewQueueAndANoteWithoutAMarkerAreNotTouched(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/tunnel-candidates-2026-06-14.md", "AutoTunnel's own queue") + rawFile(t, dir, "raw/tunnel-candidates-2026-06-14.md.processed", "x") + rawFile(t, dir, "raw/fresh.md", "an unprocessed clip") + rawFile(t, dir, "raw/processed/2026-10-19/fresh.md", "an unprocessed clip") // same bytes, but no marker: still in the inbox + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + m := byFrom(items) + for _, f := range []string{"raw/tunnel-candidates-2026-06-14.md", "raw/tunnel-candidates-2026-06-14.md.processed", "raw/fresh.md"} { + _, listed := m[f] + assert.False(t, listed, "%s is not a leftover", f) + } +} + +func TestApplyAndRollback_HandleRawLeftoversAndNeverCollideWithSourcesOfTheSameName(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/same.md", "raw text") + rawFile(t, dir, "raw/same.md.processed", "d") + rawFile(t, dir, "raw/processed/2026-10-19/same.md", "raw text") + page(t, dir, "wiki/sources/same.md", src("video")) // a wiki source with the same basename + rawItems, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + srcItems, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + + res, err := legacymove.Apply(dir, append(rawItems, srcItems...), time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + for _, gone := range []string{"raw/same.md", "raw/same.md.processed", "wiki/sources/same.md"} { + _, statErr := os.Stat(filepath.Join(dir, gone)) + assert.ErrorIs(t, statErr, os.ErrNotExist, gone) + } + _, statErr := os.Stat(filepath.Join(dir, "raw/processed/2026-10-19/same.md")) + assert.NoError(t, statErr, "the archive copy in raw/processed is never touched") + + require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir)) + got, err := os.ReadFile(filepath.Join(dir, "raw/same.md")) + require.NoError(t, err) + assert.Equal(t, "raw text", string(got), "the raw note and the wiki source of one name did not overwrite each other in the archive") + got, err = os.ReadFile(filepath.Join(dir, "wiki/sources/same.md")) + require.NoError(t, err) + assert.Equal(t, src("video"), string(got)) +} -- 2.54.0 From b66b45b8fdd9bb8ce8e6d2602c613187bbbd6b22 Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 20 Oct 2026 21:42:44 +0200 Subject: [PATCH 3/3] fix(legacymove): second run never reuses an archive; Apply re-checks the planned bytes; rollback checks first and is repeatable (review of #213) From the independent review: - the archive dir is unique per run (-2, -3 ...): a second Apply the same day replaced the first run's archived original and rewrote its manifest - every Move/Delete carries the SHA-256 it was planned on; Apply skips and reports (Result.Changed) a page whose bytes changed since planning, so a note dropped again after PlanRaw is not deleted unprocessed - Rollback verifies all targets before changing anything, recognises pages already restored (so it can be run again), and refuses to delete a moved copy that was edited since the move - names whose flattened archive key exceeds NAME_MAX are skipped at planning time instead of wedging the run - a page that already has wing/hall keys, or whose type sits in a nested/block scalar line or an unterminated block, is not moved Each has a test that fails without it (mutation-checked). Residual: a swap between Apply's read and its remove; the archived copy is still the bytes read. Co-Authored-By: Claude Sonnet 5.5 --- ingestion/internal/legacymove/legacymove.go | 225 +++++++++++++++--- .../internal/legacymove/legacymove_test.go | 174 ++++++++++++++ 2 files changed, 370 insertions(+), 29 deletions(-) diff --git a/ingestion/internal/legacymove/legacymove.go b/ingestion/internal/legacymove/legacymove.go index f3a064e..8aeecb9 100644 --- a/ingestion/internal/legacymove/legacymove.go +++ b/ingestion/internal/legacymove/legacymove.go @@ -11,11 +11,14 @@ package legacymove import ( + "bytes" "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "io/fs" + "path" "sort" "strings" "time" @@ -42,8 +45,17 @@ type Item struct { Action Action Wing string // Move only Reason string + // SHA is the SHA-256 (hex) of the bytes the plan was decided on. Apply + // re-reads the page and does nothing if the bytes differ: a page changed + // between planning and applying is not the page that was judged safe. + SHA string } +// maxArchiveName is NAME_MAX. The archive key (the whole path, flattened) is +// longer than the page's own name, so a long name must be refused at planning +// time rather than wedge Apply halfway. +const maxArchiveName = 255 + const legacyDir = "wiki/sources" // Plan reads the retired source dir and decides. It writes nothing. @@ -71,8 +83,8 @@ func Plan(brainDir, wing string) ([]Item, error) { from := legacyDir + "/" + name slug := strings.TrimSuffix(name, ".md") it := Item{From: from, Slug: slug} - if strings.HasPrefix(name, "tunnel-candidates-") { - it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source" + if len(archiveKey(from)) > maxArchiveName { + it.Action, it.Reason = Skip, "name too long to archive; left in place" items = append(items, it) continue } @@ -82,6 +94,17 @@ func Plan(brainDir, wing string) ([]Item, error) { items = append(items, it) continue } + it.SHA = sum(content) + if strings.HasPrefix(name, "tunnel-candidates-") { + it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source" + items = append(items, it) + continue + } + if has := topLevelKeys(string(content)); has["wing"] || has["hall"] { + it.Action, it.Reason = Skip, "already has wing/hall in its frontmatter; left in place" + items = append(items, it) + continue + } switch typ := frontmatterValue(string(content), "type"); typ { case "video", "article": to := "wiki/" + wing + "/sources/" + name @@ -109,6 +132,9 @@ func Plan(brainDir, wing string) ([]Item, error) { type Result struct { Moved, Deleted int ArchiveDir string + // Changed lists planned pages that were NOT touched because their bytes no + // longer match what the plan was decided on. + Changed []string } type manifest struct { @@ -119,13 +145,18 @@ type manifest struct { type moveRec struct { From string `json:"from"` To string `json:"to"` + Wing string `json:"wing"` } // Apply performs the Move and Delete items, one at a time, stopping at the -// first failure. Each original is archived before anything changes; a move -// writes its destination with O_EXCL before the source is removed, so a failure -// never loses a page. The manifest is rewritten after every step so a partial -// run can still be rolled back. +// first failure. For each page it re-reads the bytes and compares them with the +// plan's hash: a page that changed since planning is left alone and reported in +// Result.Changed. Each original is archived (O_EXCL, into an archive directory +// no earlier run has used) before anything changes; a move writes its +// destination O_EXCL before the source is removed, so a failure never loses a +// page. The manifest is rewritten after every step so a partial run can still +// be rolled back. A swap between the read and the remove is a residual window +// that only a locking watcher could close. func Apply(brainDir string, items []Item, now time.Time) (Result, error) { bfs, err := brainfs.Open(brainDir) if err != nil { @@ -133,7 +164,7 @@ func Apply(brainDir string, items []Item, now time.Time) (Result, error) { } defer func() { _ = bfs.Close() }() - res := Result{ArchiveDir: "archive/wing-migration-" + now.UTC().Format("2006-01-02")} + res := Result{ArchiveDir: freshArchiveDir(bfs, "archive/wing-migration-"+now.UTC().Format("2006-01-02"))} var man manifest save := func() error { b, err := json.MarshalIndent(man, "", " ") @@ -150,7 +181,11 @@ func Apply(brainDir string, items []Item, now time.Time) (Result, error) { if err != nil { return res, fmt.Errorf("read %s: %w", it.From, err) } - if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil { + if it.SHA == "" || sum(content) != it.SHA { + res.Changed = append(res.Changed, it.From) + continue + } + if err := bfs.CreateRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil { return res, fmt.Errorf("archive %s: %w", it.From, err) } switch it.Action { @@ -158,7 +193,7 @@ func Apply(brainDir string, items []Item, now time.Time) (Result, error) { if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing))); err != nil { return res, fmt.Errorf("write %s: %w", it.To, err) } - man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To}) + man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To, Wing: it.Wing}) res.Moved++ case Delete: man.Deletes = append(man.Deletes, it.From) @@ -177,6 +212,23 @@ func Apply(brainDir string, items []Item, now time.Time) (Result, error) { return res, nil } +// freshArchiveDir returns base, or base-2, base-3, ... : the first that does not +// exist, so a second run never reuses (and never overwrites) an earlier archive. +func freshArchiveDir(bfs *brainfs.Brain, base string) string { + dir := base + for n := 2; ; n++ { + if _, err := bfs.Lstat(dir); err != nil { + return dir + } + dir = fmt.Sprintf("%s-%d", base, n) + } +} + +func sum(b []byte) string { + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + func wingsOf(items []Item) map[string]bool { out := map[string]bool{} for _, it := range items { @@ -188,8 +240,11 @@ func wingsOf(items []Item) map[string]bool { } // Rollback restores every page named in the archive's manifest from its -// archived original and removes the moved copies. It refuses to overwrite a -// page that already exists at the original location. +// archived original and removes the moved copies. It checks everything first +// and changes nothing if anything is in the way: a restore target that exists +// with different bytes, or a moved copy that was edited since the move (which +// is never deleted). Pages already restored are recognised by their bytes, so a +// rollback can be run again. mtime and mode of the restored pages are new. func Rollback(brainDir, archiveDir string) error { bfs, err := brainfs.Open(brainDir) if err != nil { @@ -204,28 +259,77 @@ func Rollback(brainDir, archiveDir string) error { if err := json.Unmarshal(raw, &man); err != nil { return fmt.Errorf("parse manifest: %w", err) } - restore := func(from string) error { + original := func(from string) ([]byte, error) { orig, err := bfs.ReadRegular(archiveDir + "/originals/" + archiveKey(from)) if err != nil { - return fmt.Errorf("read archived %s: %w", from, err) + return nil, fmt.Errorf("read archived %s: %w", from, err) } - if err := bfs.CreateRegular(from, orig); err != nil { - return fmt.Errorf("restore %s: %w", from, err) + return orig, nil + } + + type step struct { + from string + orig []byte + restore bool // From is missing and must be written + remove string // moved copy to delete, if any + } + var steps []step + var problems []string + plan := func(from string, orig []byte, moved string, expectMoved []byte) { + st := step{from: from, orig: orig} + cur, err := bfs.ReadRegular(from) + switch { + case err == nil && bytes.Equal(cur, orig): + // already restored + case err == nil: + problems = append(problems, fmt.Sprintf("%s exists with different content", from)) + case errors.Is(err, fs.ErrNotExist): + st.restore = true + default: + problems = append(problems, fmt.Sprintf("%s: %v", from, err)) } - return nil + if moved != "" { + cur, err := bfs.ReadRegular(moved) + switch { + case err == nil && bytes.Equal(cur, expectMoved): + st.remove = moved + case err == nil: + problems = append(problems, fmt.Sprintf("%s was edited since the move; not deleting it", moved)) + case errors.Is(err, fs.ErrNotExist): + default: + problems = append(problems, fmt.Sprintf("%s: %v", moved, err)) + } + } + steps = append(steps, st) } for _, m := range man.Moves { - if err := restore(m.From); err != nil { + orig, err := original(m.From) + if err != nil { return err } - if err := bfs.Remove(m.To); err != nil { - return fmt.Errorf("remove moved copy %s: %w", m.To, err) - } + plan(m.From, orig, m.To, []byte(stamp(string(orig), m.Wing))) } for _, d := range man.Deletes { - if err := restore(d); err != nil { + orig, err := original(d) + if err != nil { return err } + plan(d, orig, "", nil) + } + if len(problems) > 0 { + return fmt.Errorf("rollback refused, nothing changed: %s", strings.Join(problems, "; ")) + } + for _, st := range steps { + if st.restore { + if err := bfs.CreateRegular(st.from, st.orig); err != nil { + return fmt.Errorf("restore %s: %w", st.from, err) + } + } + if st.remove != "" { + if err := bfs.Remove(st.remove); err != nil { + return fmt.Errorf("remove moved copy %s: %w", st.remove, err) + } + } } return nil } @@ -245,13 +349,53 @@ func stamp(content, wing string) string { return "---\n" + extra + "---\n" + content } -func frontmatterValue(content, key string) string { +// frontmatterBlock returns the text between the opening fence and a real +// closing fence (a line that is exactly "---"), or "" when there is none. +func frontmatterBlock(content string) (string, bool) { if !strings.HasPrefix(content, "---\n") { + return "", false + } + rest := content[4:] + if i := strings.Index(rest, "\n---\n"); i >= 0 { + return rest[:i], true + } + if strings.HasSuffix(rest, "\n---") { + return rest[:len(rest)-4], true + } + return "", false +} + +// topLevelKeys is the set of unindented `key:` names in the frontmatter block. +func topLevelKeys(content string) map[string]bool { + out := map[string]bool{} + block, ok := frontmatterBlock(content) + if !ok { + return out + } + for _, line := range strings.Split(block, "\n") { + if line == "" || line[0] == ' ' || line[0] == '\t' { + continue + } + if k, _, found := strings.Cut(line, ":"); found { + out[strings.TrimSpace(k)] = true + } + } + return out +} + +// frontmatterValue reads a top-level key from a CLOSED frontmatter block. A +// nested or block-scalar line that merely looks like `type: video` does not +// count, and neither does an unterminated block. +func frontmatterValue(content, key string) string { + block, ok := frontmatterBlock(content) + if !ok { return "" } - block, _, _ := strings.Cut(content[4:], "\n---") for _, line := range strings.Split(block, "\n") { - if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key { + if line == "" || line[0] == ' ' || line[0] == '\t' { + continue + } + if k, v, found := strings.Cut(line, ":"); found && strings.TrimSpace(k) == key { return strings.Trim(strings.TrimSpace(v), `"'`) } } @@ -265,7 +409,8 @@ const rawDir = "raw" // whose note is gone. An original is deleted only when a byte-identical copy // exists under raw/processed/ (so nothing is lost), and only when it has a // marker (an unmarked note is still in the inbox). tunnel-candidates-* are -// AutoTunnel's own review queue and are never touched. +// AutoTunnel's own review queue and are never touched. Every Delete carries the +// SHA-256 it was decided on, which Apply re-checks. func PlanRaw(brainDir string) ([]Item, error) { bfs, err := brainfs.Open(brainDir) if err != nil { @@ -287,6 +432,15 @@ func PlanRaw(brainDir string) ([]Item, error) { if err != nil { return nil, err } + del := func(name, why string) Item { + it := Item{From: rawDir + "/" + name, Slug: name, Action: Delete, Reason: why} + b, err := bfs.ReadRegular(it.From) + if err != nil { + return Item{From: it.From, Slug: name, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", err)} + } + it.SHA = sum(b) + return it + } var items []Item for _, n := range names { if strings.HasPrefix(n, "tunnel-candidates-") { @@ -296,7 +450,7 @@ func PlanRaw(brainDir string) ([]Item, error) { case strings.HasSuffix(n, ".processed"): note := strings.TrimSuffix(n, ".processed") if !present[note] { - items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "marker without a note"}) + items = append(items, tooLongOr(rawDir+"/"+n, func() Item { return del(n, "marker without a note") })) } case present[n+".processed"]: content, rerr := bfs.ReadRegular(rawDir + "/" + n) @@ -308,15 +462,28 @@ func PlanRaw(brainDir string) ([]Item, error) { items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "marked processed but no identical copy under raw/processed/; kept"}) continue } - items = append(items, - Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "archived copy exists under raw/processed/"}, - Item{From: rawDir + "/" + n + ".processed", Slug: n + ".processed", Action: Delete, Reason: "marker of an archived note"}) + // Both or neither: the archive keys of the note and its (longer) + // marker must fit, or a run would delete the note and wedge on the marker. + if len(archiveKey(rawDir+"/"+n+".processed")) > maxArchiveName { + items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "name too long to archive; kept"}) + continue + } + items = append(items, del(n, "archived copy exists under raw/processed/"), del(n+".processed", "marker of an archived note")) } } sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) return items, nil } +// tooLongOr returns a Skip when the page's archive key cannot fit in a file +// name, else whatever mk builds. +func tooLongOr(from string, mk func() Item) Item { + if len(archiveKey(from)) > maxArchiveName { + return Item{From: from, Slug: path.Base(from), Action: Skip, Reason: "name too long to archive; kept"} + } + return mk() +} + // archivedCopies returns the SHA-256 of every file under raw/processed/, as a // set, so "is there an identical copy" is one lookup. func archivedCopies(bfs *brainfs.Brain) (map[[32]byte]bool, error) { diff --git a/ingestion/internal/legacymove/legacymove_test.go b/ingestion/internal/legacymove/legacymove_test.go index 10e8e40..bf63cba 100644 --- a/ingestion/internal/legacymove/legacymove_test.go +++ b/ingestion/internal/legacymove/legacymove_test.go @@ -269,3 +269,177 @@ func TestApplyAndRollback_HandleRawLeftoversAndNeverCollideWithSourcesOfTheSameN require.NoError(t, err) assert.Equal(t, src("video"), string(got)) } + +// --- review of #213 --- + +var day = time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC) + +// F1: a second Apply on the same day must not touch the first run's archive. +func TestApply_ASecondRunTheSameDayGetsItsOwnArchiveAndKeepsTheFirst(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + page(t, dir, "wiki/sources/tunnel-candidates-1.md", "FIRST tunnel bytes") + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + first, err := legacymove.Apply(dir, items, day) + require.NoError(t, err) + + page(t, dir, "wiki/sources/tunnel-candidates-1.md", "SECOND tunnel bytes") // same name reappears + items2, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + second, err := legacymove.Apply(dir, items2, day) + require.NoError(t, err) + + assert.NotEqual(t, first.ArchiveDir, second.ArchiveDir) + got, err := os.ReadFile(filepath.Join(dir, first.ArchiveDir, "originals", "wiki__sources__tunnel-candidates-1.md")) + require.NoError(t, err) + assert.Equal(t, "FIRST tunnel bytes", string(got), "the first run's archived original is intact") + + require.NoError(t, legacymove.Rollback(dir, first.ArchiveDir)) + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-talk.md")) + assert.NoError(t, statErr, "the first run's move is still rolled back by its own manifest") +} + +// F2: Plan decided on the bytes it saw. Apply must not delete different bytes. +func TestApply_ARawNoteThatChangedSincePlanningIsNotDeleted(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/a.md", "bytes that have an archive copy") + rawFile(t, dir, "raw/a.md.processed", "d") + rawFile(t, dir, "raw/processed/2026-10-19/a.md", "bytes that have an archive copy") + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + + rawFile(t, dir, "raw/a.md", "NEW UNPROCESSED BYTES") // dropped again before Apply runs + res, err := legacymove.Apply(dir, items, day) + require.NoError(t, err) + + got, err := os.ReadFile(filepath.Join(dir, "raw/a.md")) + require.NoError(t, err, "the changed note is still in the inbox") + assert.Equal(t, "NEW UNPROCESSED BYTES", string(got)) + assert.Contains(t, strings.Join(res.Changed, ","), "raw/a.md") +} + +func TestApply_AWikiSourceThatChangedSincePlanningIsNotMoved(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + page(t, dir, "wiki/sources/a-talk.md", src("video")+"\nan edit made after the plan\n") + + res, err := legacymove.Apply(dir, items, day) + require.NoError(t, err) + assert.Zero(t, res.Moved) + got, _ := os.ReadFile(filepath.Join(dir, "wiki/sources/a-talk.md")) + assert.Contains(t, string(got), "an edit made after the plan") + assert.NotEmpty(t, res.Changed) +} + +// F3: rollback checks everything first, changes nothing on a conflict, can be +// repeated, and never throws away an edit. +func TestRollback_RefusesBeforeChangingAnythingWhenATargetExists(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + page(t, dir, "wiki/sources/b-talk.md", src("video")) + items, _ := legacymove.Plan(dir, "web-clip") + res, err := legacymove.Apply(dir, items, day) + require.NoError(t, err) + + page(t, dir, "wiki/sources/b-talk.md", "someone put a different b here") + err = legacymove.Rollback(dir, res.ArchiveDir) + require.Error(t, err) + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-talk.md")) + assert.ErrorIs(t, statErr, os.ErrNotExist, "nothing was restored: a was not touched either") + _, statErr = os.Stat(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md")) + assert.NoError(t, statErr, "and the moved copies are still there") +} + +func TestRollback_CanBeRunTwice(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + items, _ := legacymove.Plan(dir, "web-clip") + res, err := legacymove.Apply(dir, items, day) + require.NoError(t, err) + require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir)) + require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir), "a repeat finds everything already restored") + got, _ := os.ReadFile(filepath.Join(dir, "wiki/sources/a-talk.md")) + assert.Equal(t, src("video"), string(got)) +} + +func TestRollback_DoesNotDeleteAMovedPageThatWasEditedSinceTheMove(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + items, _ := legacymove.Plan(dir, "web-clip") + res, err := legacymove.Apply(dir, items, day) + require.NoError(t, err) + moved := filepath.Join(dir, "wiki/web-clip/sources/a-talk.md") + b, _ := os.ReadFile(moved) + require.NoError(t, os.WriteFile(moved, append(b, []byte("\nmy edit\n")...), 0o644)) + + err = legacymove.Rollback(dir, res.ArchiveDir) + require.Error(t, err) + got, readErr := os.ReadFile(moved) + require.NoError(t, readErr) + assert.Contains(t, string(got), "my edit", "the edit survives") + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-talk.md")) + assert.ErrorIs(t, statErr, os.ErrNotExist, "nothing was changed") +} + +// F4: a name whose archive key cannot fit must be skipped at planning time. +func TestPlanRaw_ANameTooLongToArchiveIsSkippedNotAbortedOn(t *testing.T) { + dir := t.TempDir() + long := strings.Repeat("a", 240) + ".md" + rawFile(t, dir, "raw/"+long, "text") + rawFile(t, dir, "raw/"+long+".processed", "d") + rawFile(t, dir, "raw/processed/2026-10-19/"+long, "text") + rawFile(t, dir, "raw/ok.md", "fine") + rawFile(t, dir, "raw/ok.md.processed", "d") + rawFile(t, dir, "raw/processed/2026-10-19/ok.md", "fine") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, legacymove.Skip, m["raw/"+long].Action) + assert.Contains(t, m["raw/"+long].Reason, "too long") + assert.Equal(t, legacymove.Delete, m["raw/ok.md"].Action) + + _, err = legacymove.Apply(dir, items, day) + require.NoError(t, err, "the long name does not wedge the run") + _, statErr := os.Stat(filepath.Join(dir, "raw/"+long)) + assert.NoError(t, statErr, "and it is left where it is") +} + +// F6: do not stamp a page that already carries a wing or hall; only a top-level +// type inside a closed frontmatter block decides. +func TestPlan_AMovedPageIsNeverGivenDuplicateWingKeys(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/has-wing.md", "---\ntitle: T\ntype: video\nwing: old\nhall: facts\n---\nbody") + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + it := byFrom(items)["wiki/sources/has-wing.md"] + assert.Equal(t, legacymove.Skip, it.Action) + assert.Contains(t, it.Reason, "already has") +} + +func TestPlan_OnlyATopLevelTypeInAClosedBlockClassifiesAPage(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/nested.md", "---\ntitle: T\nmeta:\n type: video\n---\nbody") + page(t, dir, "wiki/sources/blockscalar.md", "---\ntitle: T\ndesc: |\n type: article\n---\nbody") + page(t, dir, "wiki/sources/unterminated.md", "---\ntitle: T\ntype: video\nbody with no closing fence") + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + m := byFrom(items) + for _, f := range []string{"nested", "blockscalar", "unterminated"} { + assert.Equal(t, legacymove.Skip, m["wiki/sources/"+f+".md"].Action, f) + } +} + +func TestPlan_AWikiSourceWithATooLongNameIsSkipped(t *testing.T) { + dir := t.TempDir() + long := strings.Repeat("a", 240) + ".md" + page(t, dir, "wiki/sources/"+long, src("video")) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + it := byFrom(items)["wiki/sources/"+long] + assert.Equal(t, legacymove.Skip, it.Action) + assert.Contains(t, it.Reason, "too long") +} -- 2.54.0