diff --git a/ingestion/cmd/legacymove/main.go b/ingestion/cmd/legacymove/main.go index 619f9c9..e3ac10d 100644 --- a/ingestion/cmd/legacymove/main.go +++ b/ingestion/cmd/legacymove/main.go @@ -1,6 +1,9 @@ -// Command legacymove clears wiki/sources/ of the retired layout (hyperguild -// #209). Default is a dry run that prints the plan; -apply performs it after -// archiving every page it touches; -rollback restores from an archive. +// Command legacymove clears the leftovers of the retired layout (hyperguild +// #209): wiki/sources/ (videos and articles move to the default wing, the +// auto-tunnel review logs are deleted) and raw/ (originals archived before inbox +// zero, their markers, orphan markers). Default is a dry run that prints the +// plan; -apply performs it after archiving every page it touches; -rollback +// restores from an archive. package main import ( @@ -35,15 +38,20 @@ func run(brain, wing string, apply bool, rollback string) error { fmt.Println("rolled back from", rollback) return nil } - items, err := legacymove.Plan(brain, wing) + sources, err := legacymove.Plan(brain, wing) if err != nil { return err } + rawLeft, err := legacymove.PlanRaw(brain) + if err != nil { + return err + } + items := append(sources, rawLeft...) counts := map[legacymove.Action]int{} for _, it := range items { counts[it.Action]++ } - fmt.Printf("# legacy source plan\n\n%d pages: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip]) + fmt.Printf("# legacy plan\n\n%d entries: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip]) fmt.Println("| action | from | to | why |") fmt.Println("|---|---|---|---|") for _, it := range items { diff --git a/ingestion/internal/legacymove/legacymove.go b/ingestion/internal/legacymove/legacymove.go index 89c43fa..f3a064e 100644 --- a/ingestion/internal/legacymove/legacymove.go +++ b/ingestion/internal/legacymove/legacymove.go @@ -11,11 +11,11 @@ package legacymove import ( + "crypto/sha256" "encoding/json" "errors" "fmt" "io/fs" - "path" "sort" "strings" "time" @@ -150,7 +150,7 @@ func Apply(brainDir string, items []Item, now time.Time) (Result, error) { if err != nil { return res, fmt.Errorf("read %s: %w", it.From, err) } - if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+path.Base(it.From), content); err != nil { + if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil { return res, fmt.Errorf("archive %s: %w", it.From, err) } switch it.Action { @@ -205,7 +205,7 @@ func Rollback(brainDir, archiveDir string) error { return fmt.Errorf("parse manifest: %w", err) } restore := func(from string) error { - orig, err := bfs.ReadRegular(archiveDir + "/originals/" + path.Base(from)) + orig, err := bfs.ReadRegular(archiveDir + "/originals/" + archiveKey(from)) if err != nil { return fmt.Errorf("read archived %s: %w", from, err) } @@ -230,6 +230,11 @@ func Rollback(brainDir, archiveDir string) error { return nil } +// archiveKey flattens a brain path into one file name, so two pages with the +// same basename in different places (raw/x.md and wiki/sources/x.md) never +// overwrite each other in the archive. +func archiveKey(p string) string { return strings.ReplaceAll(p, "/", "__") } + // stamp inserts wing/hall after the opening fence, leaving every other byte // alone. A page without frontmatter gets a block. func stamp(content, wing string) string { @@ -252,3 +257,89 @@ func frontmatterValue(content, key string) string { } return "" } + +const rawDir = "raw" + +// PlanRaw finds what the watcher left in raw/ before inbox zero: originals that +// were archived but never removed, with their .processed markers, and markers +// whose note is gone. An original is deleted only when a byte-identical copy +// exists under raw/processed/ (so nothing is lost), and only when it has a +// marker (an unmarked note is still in the inbox). tunnel-candidates-* are +// AutoTunnel's own review queue and are never touched. +func PlanRaw(brainDir string) ([]Item, error) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + names, err := bfs.ReadDirRegular(rawDir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", rawDir, err) + } + present := map[string]bool{} + for _, n := range names { + present[n] = true + } + archived, err := archivedCopies(bfs) + if err != nil { + return nil, err + } + var items []Item + for _, n := range names { + if strings.HasPrefix(n, "tunnel-candidates-") { + continue + } + switch { + case strings.HasSuffix(n, ".processed"): + note := strings.TrimSuffix(n, ".processed") + if !present[note] { + items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "marker without a note"}) + } + case present[n+".processed"]: + content, rerr := bfs.ReadRegular(rawDir + "/" + n) + if rerr != nil { + items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", rerr)}) + continue + } + if !archived[sha256.Sum256(content)] { + items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "marked processed but no identical copy under raw/processed/; kept"}) + continue + } + items = append(items, + Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "archived copy exists under raw/processed/"}, + Item{From: rawDir + "/" + n + ".processed", Slug: n + ".processed", Action: Delete, Reason: "marker of an archived note"}) + } + } + sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) + return items, nil +} + +// archivedCopies returns the SHA-256 of every file under raw/processed/, as a +// set, so "is there an identical copy" is one lookup. +func archivedCopies(bfs *brainfs.Brain) (map[[32]byte]bool, error) { + out := map[[32]byte]bool{} + days, err := bfs.ReadDirDirs(rawDir + "/processed") + if errors.Is(err, fs.ErrNotExist) { + return out, nil + } + if err != nil { + return nil, fmt.Errorf("list %s/processed: %w", rawDir, err) + } + for _, d := range days { + files, err := bfs.ReadDirRegular(rawDir + "/processed/" + d) + if err != nil { + continue + } + for _, f := range files { + b, err := bfs.ReadRegular(rawDir + "/processed/" + d + "/" + f) + if err != nil { + continue + } + out[sha256.Sum256(b)] = true + } + } + return out, nil +} diff --git a/ingestion/internal/legacymove/legacymove_test.go b/ingestion/internal/legacymove/legacymove_test.go index 43b7f7e..10e8e40 100644 --- a/ingestion/internal/legacymove/legacymove_test.go +++ b/ingestion/internal/legacymove/legacymove_test.go @@ -171,3 +171,101 @@ func TestApply_TheArchiveIsOutsideRawSoTheWatcherNeverIngestsIt(t *testing.T) { assert.False(t, strings.Contains(res.ArchiveDir, "/raw"), res.ArchiveDir) assert.True(t, strings.HasPrefix(res.ArchiveDir, "archive/"), res.ArchiveDir) } + +// --- raw/ leftovers from before inbox zero --- + +func rawFile(t *testing.T, dir, rel, body string) { page(t, dir, rel, body) } + +func TestPlanRaw_AnArchivedOriginalAndItsMarkerAreDeleted(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/clip.md", "the clip text") + rawFile(t, dir, "raw/clip.md.processed", "2026-10-19") + rawFile(t, dir, "raw/processed/2026-10-19/clip.md", "the clip text") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, legacymove.Delete, m["raw/clip.md"].Action) + assert.Equal(t, legacymove.Delete, m["raw/clip.md.processed"].Action) +} + +func TestPlanRaw_AnOriginalWhoseArchiveCopyDiffersIsLeftAlone(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/clip.md", "the clip text, edited since") + rawFile(t, dir, "raw/clip.md.processed", "2026-10-19") + rawFile(t, dir, "raw/processed/2026-10-19/clip.md", "the clip text") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, legacymove.Skip, m["raw/clip.md"].Action, "no identical archive copy: not ours to delete") + assert.Contains(t, m["raw/clip.md"].Reason, "no identical copy") + _, hasMarker := m["raw/clip.md.processed"] + assert.False(t, hasMarker, "the marker of a note we keep is kept") +} + +func TestPlanRaw_ACopyUnderATimestampedNameCounts(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/clip.md", "same text") + rawFile(t, dir, "raw/clip.md.processed", "d") + rawFile(t, dir, "raw/processed/2026-10-19/clip-180200.md", "same text") + rawFile(t, dir, "raw/processed/2026-10-19/clip-2.md", "other text") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + assert.Equal(t, legacymove.Delete, byFrom(items)["raw/clip.md"].Action) +} + +func TestPlanRaw_AnOrphanMarkerIsDeleted(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/gone.md.processed", "2026-06-16") + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + assert.Equal(t, legacymove.Delete, byFrom(items)["raw/gone.md.processed"].Action) +} + +func TestPlanRaw_TheAutoTunnelReviewQueueAndANoteWithoutAMarkerAreNotTouched(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/tunnel-candidates-2026-06-14.md", "AutoTunnel's own queue") + rawFile(t, dir, "raw/tunnel-candidates-2026-06-14.md.processed", "x") + rawFile(t, dir, "raw/fresh.md", "an unprocessed clip") + rawFile(t, dir, "raw/processed/2026-10-19/fresh.md", "an unprocessed clip") // same bytes, but no marker: still in the inbox + + items, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + m := byFrom(items) + for _, f := range []string{"raw/tunnel-candidates-2026-06-14.md", "raw/tunnel-candidates-2026-06-14.md.processed", "raw/fresh.md"} { + _, listed := m[f] + assert.False(t, listed, "%s is not a leftover", f) + } +} + +func TestApplyAndRollback_HandleRawLeftoversAndNeverCollideWithSourcesOfTheSameName(t *testing.T) { + dir := t.TempDir() + rawFile(t, dir, "raw/same.md", "raw text") + rawFile(t, dir, "raw/same.md.processed", "d") + rawFile(t, dir, "raw/processed/2026-10-19/same.md", "raw text") + page(t, dir, "wiki/sources/same.md", src("video")) // a wiki source with the same basename + rawItems, err := legacymove.PlanRaw(dir) + require.NoError(t, err) + srcItems, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + + res, err := legacymove.Apply(dir, append(rawItems, srcItems...), time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + for _, gone := range []string{"raw/same.md", "raw/same.md.processed", "wiki/sources/same.md"} { + _, statErr := os.Stat(filepath.Join(dir, gone)) + assert.ErrorIs(t, statErr, os.ErrNotExist, gone) + } + _, statErr := os.Stat(filepath.Join(dir, "raw/processed/2026-10-19/same.md")) + assert.NoError(t, statErr, "the archive copy in raw/processed is never touched") + + require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir)) + got, err := os.ReadFile(filepath.Join(dir, "raw/same.md")) + require.NoError(t, err) + assert.Equal(t, "raw text", string(got), "the raw note and the wiki source of one name did not overwrite each other in the archive") + got, err = os.ReadFile(filepath.Join(dir, "wiki/sources/same.md")) + require.NoError(t, err) + assert.Equal(t, src("video"), string(got)) +}