3 changed files with 205 additions and 8 deletions
@@ -1,6 +1,9 @@
-// Command legacymove clears wiki/sources/ of the retired layout (hyperguild
-// #209). Default is a dry run that prints the plan; -apply performs it after
-// archiving every page it touches; -rollback restores from an archive.
+// Command legacymove clears the leftovers of the retired layout (hyperguild
+// #209): wiki/sources/ (videos and articles move to the default wing, the
+// auto-tunnel review logs are deleted) and raw/ (originals archived before inbox
+// zero, their markers, orphan markers). Default is a dry run that prints the
+// plan; -apply performs it after archiving every page it touches; -rollback
+// restores from an archive.
 package main

 import (
@@ -35,15 +38,20 @@ func run(brain, wing string, apply bool, rollback string) error {
 		fmt.Println("rolled back from", rollback)
 		return nil
 	}
-	items, err := legacymove.Plan(brain, wing)
+	sources, err := legacymove.Plan(brain, wing)
 	if err != nil {
 		return err
 	}
+	rawLeft, err := legacymove.PlanRaw(brain)
+	if err != nil {
+		return err
+	}
+	items := append(sources, rawLeft...)
 	counts := map[legacymove.Action]int{}
 	for _, it := range items {
 		counts[it.Action]++
 	}
-	fmt.Printf("# legacy source plan\n\n%d pages: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip])
+	fmt.Printf("# legacy plan\n\n%d entries: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip])
 	fmt.Println("| action | from | to | why |")
 	fmt.Println("|---|---|---|---|")
 	for _, it := range items {
@@ -11,11 +11,11 @@
 package legacymove

 import (
+	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
-	"path"
 	"sort"
 	"strings"
 	"time"
@@ -150,7 +150,7 @@ func Apply(brainDir string, items []Item, now time.Time) (Result, error) {
 		if err != nil {
 			return res, fmt.Errorf("read %s: %w", it.From, err)
 		}
-		if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+path.Base(it.From), content); err != nil {
+		if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil {
 			return res, fmt.Errorf("archive %s: %w", it.From, err)
 		}
 		switch it.Action {
@@ -205,7 +205,7 @@ func Rollback(brainDir, archiveDir string) error {
 		return fmt.Errorf("parse manifest: %w", err)
 	}
 	restore := func(from string) error {
-		orig, err := bfs.ReadRegular(archiveDir + "/originals/" + path.Base(from))
+		orig, err := bfs.ReadRegular(archiveDir + "/originals/" + archiveKey(from))
 		if err != nil {
 			return fmt.Errorf("read archived %s: %w", from, err)
 		}
@@ -230,6 +230,11 @@ func Rollback(brainDir, archiveDir string) error {
 	return nil
 }

+// archiveKey flattens a brain path into one file name, so two pages with the
+// same basename in different places (raw/x.md and wiki/sources/x.md) never
+// overwrite each other in the archive.
+func archiveKey(p string) string { return strings.ReplaceAll(p, "/", "__") }
+
 // stamp inserts wing/hall after the opening fence, leaving every other byte
 // alone. A page without frontmatter gets a block.
 func stamp(content, wing string) string {
@@ -252,3 +257,89 @@ func frontmatterValue(content, key string) string {
 	}
 	return ""
 }
+
+const rawDir = "raw"
+
+// PlanRaw finds what the watcher left in raw/ before inbox zero: originals that
+// were archived but never removed, with their .processed markers, and markers
+// whose note is gone. An original is deleted only when a byte-identical copy
+// exists under raw/processed/ (so nothing is lost), and only when it has a
+// marker (an unmarked note is still in the inbox). tunnel-candidates-* are
+// AutoTunnel's own review queue and are never touched.
+func PlanRaw(brainDir string) ([]Item, error) {
+	bfs, err := brainfs.Open(brainDir)
+	if err != nil {
+		return nil, fmt.Errorf("open brain: %w", err)
+	}
+	defer func() { _ = bfs.Close() }()
+	names, err := bfs.ReadDirRegular(rawDir)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", rawDir, err)
+	}
+	present := map[string]bool{}
+	for _, n := range names {
+		present[n] = true
+	}
+	archived, err := archivedCopies(bfs)
+	if err != nil {
+		return nil, err
+	}
+	var items []Item
+	for _, n := range names {
+		if strings.HasPrefix(n, "tunnel-candidates-") {
+			continue
+		}
+		switch {
+		case strings.HasSuffix(n, ".processed"):
+			note := strings.TrimSuffix(n, ".processed")
+			if !present[note] {
+				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "marker without a note"})
+			}
+		case present[n+".processed"]:
+			content, rerr := bfs.ReadRegular(rawDir + "/" + n)
+			if rerr != nil {
+				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", rerr)})
+				continue
+			}
+			if !archived[sha256.Sum256(content)] {
+				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "marked processed but no identical copy under raw/processed/; kept"})
+				continue
+			}
+			items = append(items,
+				Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "archived copy exists under raw/processed/"},
+				Item{From: rawDir + "/" + n + ".processed", Slug: n + ".processed", Action: Delete, Reason: "marker of an archived note"})
+		}
+	}
+	sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
+	return items, nil
+}
+
+// archivedCopies returns the SHA-256 of every file under raw/processed/, as a
+// set, so "is there an identical copy" is one lookup.
+func archivedCopies(bfs *brainfs.Brain) (map[[32]byte]bool, error) {
+	out := map[[32]byte]bool{}
+	days, err := bfs.ReadDirDirs(rawDir + "/processed")
+	if errors.Is(err, fs.ErrNotExist) {
+		return out, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("list %s/processed: %w", rawDir, err)
+	}
+	for _, d := range days {
+		files, err := bfs.ReadDirRegular(rawDir + "/processed/" + d)
+		if err != nil {
+			continue
+		}
+		for _, f := range files {
+			b, err := bfs.ReadRegular(rawDir + "/processed/" + d + "/" + f)
+			if err != nil {
+				continue
+			}
+			out[sha256.Sum256(b)] = true
+		}
+	}
+	return out, nil
+}
@@ -171,3 +171,101 @@ func TestApply_TheArchiveIsOutsideRawSoTheWatcherNeverIngestsIt(t *testing.T) {
 	assert.False(t, strings.Contains(res.ArchiveDir, "/raw"), res.ArchiveDir)
 	assert.True(t, strings.HasPrefix(res.ArchiveDir, "archive/"), res.ArchiveDir)
 }
+
+// --- raw/ leftovers from before inbox zero ---
+
+func rawFile(t *testing.T, dir, rel, body string) { page(t, dir, rel, body) }
+
+func TestPlanRaw_AnArchivedOriginalAndItsMarkerAreDeleted(t *testing.T) {
+	dir := t.TempDir()
+	rawFile(t, dir, "raw/clip.md", "the clip text")
+	rawFile(t, dir, "raw/clip.md.processed", "2026-10-19")
+	rawFile(t, dir, "raw/processed/2026-10-19/clip.md", "the clip text")
+
+	items, err := legacymove.PlanRaw(dir)
+	require.NoError(t, err)
+	m := byFrom(items)
+	assert.Equal(t, legacymove.Delete, m["raw/clip.md"].Action)
+	assert.Equal(t, legacymove.Delete, m["raw/clip.md.processed"].Action)
+}
+
+func TestPlanRaw_AnOriginalWhoseArchiveCopyDiffersIsLeftAlone(t *testing.T) {
+	dir := t.TempDir()
+	rawFile(t, dir, "raw/clip.md", "the clip text, edited since")
+	rawFile(t, dir, "raw/clip.md.processed", "2026-10-19")
+	rawFile(t, dir, "raw/processed/2026-10-19/clip.md", "the clip text")
+
+	items, err := legacymove.PlanRaw(dir)
+	require.NoError(t, err)
+	m := byFrom(items)
+	assert.Equal(t, legacymove.Skip, m["raw/clip.md"].Action, "no identical archive copy: not ours to delete")
+	assert.Contains(t, m["raw/clip.md"].Reason, "no identical copy")
+	_, hasMarker := m["raw/clip.md.processed"]
+	assert.False(t, hasMarker, "the marker of a note we keep is kept")
+}
+
+func TestPlanRaw_ACopyUnderATimestampedNameCounts(t *testing.T) {
+	dir := t.TempDir()
+	rawFile(t, dir, "raw/clip.md", "same text")
+	rawFile(t, dir, "raw/clip.md.processed", "d")
+	rawFile(t, dir, "raw/processed/2026-10-19/clip-180200.md", "same text")
+	rawFile(t, dir, "raw/processed/2026-10-19/clip-2.md", "other text")
+
+	items, err := legacymove.PlanRaw(dir)
+	require.NoError(t, err)
+	assert.Equal(t, legacymove.Delete, byFrom(items)["raw/clip.md"].Action)
+}
+
+func TestPlanRaw_AnOrphanMarkerIsDeleted(t *testing.T) {
+	dir := t.TempDir()
+	rawFile(t, dir, "raw/gone.md.processed", "2026-06-16")
+
+	items, err := legacymove.PlanRaw(dir)
+	require.NoError(t, err)
+	assert.Equal(t, legacymove.Delete, byFrom(items)["raw/gone.md.processed"].Action)
+}
+
+func TestPlanRaw_TheAutoTunnelReviewQueueAndANoteWithoutAMarkerAreNotTouched(t *testing.T) {
+	dir := t.TempDir()
+	rawFile(t, dir, "raw/tunnel-candidates-2026-06-14.md", "AutoTunnel's own queue")
+	rawFile(t, dir, "raw/tunnel-candidates-2026-06-14.md.processed", "x")
+	rawFile(t, dir, "raw/fresh.md", "an unprocessed clip")
+	rawFile(t, dir, "raw/processed/2026-10-19/fresh.md", "an unprocessed clip") // same bytes, but no marker: still in the inbox
+
+	items, err := legacymove.PlanRaw(dir)
+	require.NoError(t, err)
+	m := byFrom(items)
+	for _, f := range []string{"raw/tunnel-candidates-2026-06-14.md", "raw/tunnel-candidates-2026-06-14.md.processed", "raw/fresh.md"} {
+		_, listed := m[f]
+		assert.False(t, listed, "%s is not a leftover", f)
+	}
+}
+
+func TestApplyAndRollback_HandleRawLeftoversAndNeverCollideWithSourcesOfTheSameName(t *testing.T) {
+	dir := t.TempDir()
+	rawFile(t, dir, "raw/same.md", "raw text")
+	rawFile(t, dir, "raw/same.md.processed", "d")
+	rawFile(t, dir, "raw/processed/2026-10-19/same.md", "raw text")
+	page(t, dir, "wiki/sources/same.md", src("video")) // a wiki source with the same basename
+	rawItems, err := legacymove.PlanRaw(dir)
+	require.NoError(t, err)
+	srcItems, err := legacymove.Plan(dir, "web-clip")
+	require.NoError(t, err)
+
+	res, err := legacymove.Apply(dir, append(rawItems, srcItems...), time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC))
+	require.NoError(t, err)
+	for _, gone := range []string{"raw/same.md", "raw/same.md.processed", "wiki/sources/same.md"} {
+		_, statErr := os.Stat(filepath.Join(dir, gone))
+		assert.ErrorIs(t, statErr, os.ErrNotExist, gone)
+	}
+	_, statErr := os.Stat(filepath.Join(dir, "raw/processed/2026-10-19/same.md"))
+	assert.NoError(t, statErr, "the archive copy in raw/processed is never touched")
+
+	require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir))
+	got, err := os.ReadFile(filepath.Join(dir, "raw/same.md"))
+	require.NoError(t, err)
+	assert.Equal(t, "raw text", string(got), "the raw note and the wiki source of one name did not overwrite each other in the archive")
+	got, err = os.ReadFile(filepath.Join(dir, "wiki/sources/same.md"))
+	require.NoError(t, err)
+	assert.Equal(t, src("video"), string(got))
+}