diff --git a/ingestion/cmd/legacymove/main.go b/ingestion/cmd/legacymove/main.go new file mode 100644 index 0000000..619f9c9 --- /dev/null +++ b/ingestion/cmd/legacymove/main.go @@ -0,0 +1,59 @@ +// Command legacymove clears wiki/sources/ of the retired layout (hyperguild +// #209). Default is a dry run that prints the plan; -apply performs it after +// archiving every page it touches; -rollback restores from an archive. +package main + +import ( + "flag" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/mathiasbq/hyperguild/ingestion/internal/legacymove" +) + +func main() { + brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory") + wing := flag.String("wing", "web-clip", "wing that receives video and article sources") + apply := flag.Bool("apply", false, "perform the plan (default: dry run)") + rollback := flag.String("rollback", "", "restore from this archive dir (brain-relative, e.g. archive/wing-migration-2026-10-20)") + flag.Parse() + + if err := run(*brain, *wing, *apply, *rollback); err != nil { + slog.Error("legacymove", "err", err) + os.Exit(1) + } +} + +func run(brain, wing string, apply bool, rollback string) error { + if rollback != "" { + if err := legacymove.Rollback(brain, rollback); err != nil { + return err + } + fmt.Println("rolled back from", rollback) + return nil + } + items, err := legacymove.Plan(brain, wing) + if err != nil { + return err + } + counts := map[legacymove.Action]int{} + for _, it := range items { + counts[it.Action]++ + } + fmt.Printf("# legacy source plan\n\n%d pages: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip]) + fmt.Println("| action | from | to | why |") + fmt.Println("|---|---|---|---|") + for _, it := range items { + fmt.Printf("| %s | %s | %s | %s |\n", it.Action, it.From, it.To, strings.ReplaceAll(it.Reason, "|", "/")) + } + if !apply { + fmt.Println("\n_dry run: nothing was changed_") + return nil + } + res, err := legacymove.Apply(brain, items, time.Now()) + fmt.Printf("\nmoved %d, deleted %d; archive: %s\n", res.Moved, res.Deleted, res.ArchiveDir) + return err +} diff --git a/ingestion/internal/legacymove/legacymove.go b/ingestion/internal/legacymove/legacymove.go new file mode 100644 index 0000000..89c43fa --- /dev/null +++ b/ingestion/internal/legacymove/legacymove.go @@ -0,0 +1,254 @@ +// Package legacymove clears wiki/sources/ of the retired layout (hyperguild +// #209). It does the one thing that is safe to do mechanically: video and +// article source pages move to the default wing, and the auto-tunnel review +// logs that were once ingested as "sources" are deleted. Notes, projects and +// untyped pages stay where they are, and wiki/concepts and wiki/entities are +// not touched at all (they are the shared layer). +// +// Plan only reads. Apply copies every page it touches into +// archive/wing-migration-/ first, so Rollback can restore the brain +// byte for byte. The archive is outside raw/, so the watcher never ingests it. +package legacymove + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "path" + "sort" + "strings" + "time" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brain" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" + "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" +) + +// Action is what the plan does with one page. +type Action string + +const ( + Move Action = "move" + Delete Action = "delete" + Skip Action = "skip" +) + +// Item is one planned operation. +type Item struct { + From string + To string // Move only + Slug string + Action Action + Wing string // Move only + Reason string +} + +const legacyDir = "wiki/sources" + +// Plan reads the retired source dir and decides. It writes nothing. +func Plan(brainDir, wing string) ([]Item, error) { + if wing == "" || brain.Sanitise(wing) != wing { + return nil, fmt.Errorf("wing %q is not a plain wing name", wing) + } + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + names, err := bfs.ReadDirRegular(legacyDir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", legacyDir, err) + } + var items []Item + for _, name := range names { + if !strings.HasSuffix(name, ".md") || name == "_index.md" { + continue + } + from := legacyDir + "/" + name + slug := strings.TrimSuffix(name, ".md") + it := Item{From: from, Slug: slug} + if strings.HasPrefix(name, "tunnel-candidates-") { + it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source" + items = append(items, it) + continue + } + content, err := bfs.ReadRegular(from) + if err != nil { + it.Action, it.Reason = Skip, fmt.Sprintf("unreadable: %v", err) + items = append(items, it) + continue + } + switch typ := frontmatterValue(string(content), "type"); typ { + case "video", "article": + to := "wiki/" + wing + "/sources/" + name + it.Action, it.To, it.Wing = Move, to, wing + existing, ferr := wiki.FindSlugs(brainDir, slug) + if ferr != nil { + it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("cannot check slug: %v", ferr) + } + for _, p := range existing { + if p != from { + it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("slug %q already used by %s", slug, p) + break + } + } + default: + it.Action, it.Reason = Skip, fmt.Sprintf("type %q: left in place", typ) + } + items = append(items, it) + } + sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) + return items, nil +} + +// Result reports what Apply did and where the archive is (brain-relative). +type Result struct { + Moved, Deleted int + ArchiveDir string +} + +type manifest struct { + Moves []moveRec `json:"moves"` + Deletes []string `json:"deletes"` +} + +type moveRec struct { + From string `json:"from"` + To string `json:"to"` +} + +// Apply performs the Move and Delete items, one at a time, stopping at the +// first failure. Each original is archived before anything changes; a move +// writes its destination with O_EXCL before the source is removed, so a failure +// never loses a page. The manifest is rewritten after every step so a partial +// run can still be rolled back. +func Apply(brainDir string, items []Item, now time.Time) (Result, error) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return Result{}, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + + res := Result{ArchiveDir: "archive/wing-migration-" + now.UTC().Format("2006-01-02")} + var man manifest + save := func() error { + b, err := json.MarshalIndent(man, "", " ") + if err != nil { + return err + } + return bfs.PutRegular(res.ArchiveDir+"/manifest.json", b) + } + for _, it := range items { + if it.Action != Move && it.Action != Delete { + continue + } + content, err := bfs.ReadRegular(it.From) + if err != nil { + return res, fmt.Errorf("read %s: %w", it.From, err) + } + if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+path.Base(it.From), content); err != nil { + return res, fmt.Errorf("archive %s: %w", it.From, err) + } + switch it.Action { + case Move: + if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing))); err != nil { + return res, fmt.Errorf("write %s: %w", it.To, err) + } + man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To}) + res.Moved++ + case Delete: + man.Deletes = append(man.Deletes, it.From) + res.Deleted++ + } + if err := save(); err != nil { + return res, fmt.Errorf("write manifest: %w", err) + } + if err := bfs.Remove(it.From); err != nil { + return res, fmt.Errorf("handled %s but removing it failed: %w", it.From, err) + } + } + for w := range wingsOf(items) { + _ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved + } + return res, nil +} + +func wingsOf(items []Item) map[string]bool { + out := map[string]bool{} + for _, it := range items { + if it.Action == Move { + out[it.Wing] = true + } + } + return out +} + +// Rollback restores every page named in the archive's manifest from its +// archived original and removes the moved copies. It refuses to overwrite a +// page that already exists at the original location. +func Rollback(brainDir, archiveDir string) error { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + raw, err := bfs.ReadRegular(archiveDir + "/manifest.json") + if err != nil { + return fmt.Errorf("read manifest: %w", err) + } + var man manifest + if err := json.Unmarshal(raw, &man); err != nil { + return fmt.Errorf("parse manifest: %w", err) + } + restore := func(from string) error { + orig, err := bfs.ReadRegular(archiveDir + "/originals/" + path.Base(from)) + if err != nil { + return fmt.Errorf("read archived %s: %w", from, err) + } + if err := bfs.CreateRegular(from, orig); err != nil { + return fmt.Errorf("restore %s: %w", from, err) + } + return nil + } + for _, m := range man.Moves { + if err := restore(m.From); err != nil { + return err + } + if err := bfs.Remove(m.To); err != nil { + return fmt.Errorf("remove moved copy %s: %w", m.To, err) + } + } + for _, d := range man.Deletes { + if err := restore(d); err != nil { + return err + } + } + return nil +} + +// stamp inserts wing/hall after the opening fence, leaving every other byte +// alone. A page without frontmatter gets a block. +func stamp(content, wing string) string { + extra := "wing: " + wing + "\nhall: sources\n" + if strings.HasPrefix(content, "---\n") { + return "---\n" + extra + content[4:] + } + return "---\n" + extra + "---\n" + content +} + +func frontmatterValue(content, key string) string { + if !strings.HasPrefix(content, "---\n") { + return "" + } + block, _, _ := strings.Cut(content[4:], "\n---") + for _, line := range strings.Split(block, "\n") { + if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key { + return strings.Trim(strings.TrimSpace(v), `"'`) + } + } + return "" +} diff --git a/ingestion/internal/legacymove/legacymove_test.go b/ingestion/internal/legacymove/legacymove_test.go new file mode 100644 index 0000000..43b7f7e --- /dev/null +++ b/ingestion/internal/legacymove/legacymove_test.go @@ -0,0 +1,173 @@ +package legacymove_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/legacymove" +) + +func page(t *testing.T, dir, rel, body string) { + t.Helper() + full := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(t, os.WriteFile(full, []byte(body), 0o644)) +} + +func src(typ string) string { + return "---\ntitle: T\ntype: " + typ + "\n---\n\n## Summary\n\nBody with [[a-link|A]] kept as is.\n" +} + +func byFrom(items []legacymove.Item) map[string]legacymove.Item { + m := map[string]legacymove.Item{} + for _, it := range items { + m[it.From] = it + } + return m +} + +func TestPlan_VideosAndArticlesMoveToTheDefaultWing(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + page(t, dir, "wiki/sources/an-article.md", src("article")) + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, legacymove.Move, m["wiki/sources/a-talk.md"].Action) + assert.Equal(t, "wiki/web-clip/sources/a-talk.md", m["wiki/sources/a-talk.md"].To) + assert.Equal(t, legacymove.Move, m["wiki/sources/an-article.md"].Action) +} + +func TestPlan_NotesProjectsAndUntypedPagesStayWhereTheyAre(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-note.md", src("note")) + page(t, dir, "wiki/sources/a-project.md", src("project")) + page(t, dir, "wiki/sources/untyped.md", "---\ntitle: T\n---\nbody") + page(t, dir, "wiki/sources/_index.md", "index") + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + m := byFrom(items) + for _, f := range []string{"a-note", "a-project", "untyped"} { + assert.Equal(t, legacymove.Skip, m["wiki/sources/"+f+".md"].Action, f) + } + _, listed := m["wiki/sources/_index.md"] + assert.False(t, listed, "the index is not a page") +} + +func TestPlan_AutoTunnelLogsAreDeleted(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("video")) + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + assert.Equal(t, legacymove.Delete, byFrom(items)["wiki/sources/tunnel-candidates-2026-06-14.md"].Action, + "never moved, whatever the type says") +} + +func TestPlan_ASlugAlreadyUsedElsewhereIsSkippedNotOverwritten(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + page(t, dir, "wiki/entities/a-talk.md", "an entity with the same slug") + + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + it := byFrom(items)["wiki/sources/a-talk.md"] + assert.Equal(t, legacymove.Skip, it.Action) + assert.Contains(t, it.Reason, "already used") +} + +func TestPlan_ADefaultWingThatIsNotAPlainNameIsRefused(t *testing.T) { + _, err := legacymove.Plan(t.TempDir(), "../x") + assert.Error(t, err) +} + +func TestApply_MovesStampsAndArchivesEverythingItTouches(t *testing.T) { + dir := t.TempDir() + talk := src("video") + junk := "tunnel log" + page(t, dir, "wiki/sources/a-talk.md", talk) + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", junk) + page(t, dir, "wiki/sources/a-note.md", src("note")) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + + res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + assert.Equal(t, 1, res.Moved) + assert.Equal(t, 1, res.Deleted) + + moved, err := os.ReadFile(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md")) + require.NoError(t, err) + assert.Contains(t, string(moved), "wing: web-clip") + assert.Contains(t, string(moved), "hall: sources") + assert.Contains(t, string(moved), "Body with [[a-link|A]] kept as is.", "body and links are byte-preserved") + + for _, gone := range []string{"wiki/sources/a-talk.md", "wiki/sources/tunnel-candidates-2026-06-14.md"} { + _, statErr := os.Stat(filepath.Join(dir, gone)) + assert.ErrorIs(t, statErr, os.ErrNotExist, gone) + } + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-note.md")) + assert.NoError(t, statErr, "a skipped page is left exactly where it is") + + again, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + for _, it := range again { + assert.Equal(t, legacymove.Skip, it.Action, "nothing is left to do: %s", it.From) + } +} + +func TestRollback_RestoresEveryTouchedPageByteForByte(t *testing.T) { + dir := t.TempDir() + talk := src("video") + junk := "tunnel log" + page(t, dir, "wiki/sources/a-talk.md", talk) + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", junk) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + + require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir)) + + got, err := os.ReadFile(filepath.Join(dir, "wiki/sources/a-talk.md")) + require.NoError(t, err) + assert.Equal(t, talk, string(got)) + got, err = os.ReadFile(filepath.Join(dir, "wiki/sources/tunnel-candidates-2026-06-14.md")) + require.NoError(t, err) + assert.Equal(t, junk, string(got)) + _, statErr := os.Stat(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md")) + assert.ErrorIs(t, statErr, os.ErrNotExist, "the moved copy is removed") +} + +func TestApply_NeverOverwritesAndKeepsTheSourceIfTheDestinationAppearsLate(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a-talk.md", src("video")) + items, err := legacymove.Plan(dir, "web-clip") + require.NoError(t, err) + page(t, dir, "wiki/web-clip/sources/a-talk.md", "appeared after the plan") + + _, err = legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.Error(t, err) + got, _ := os.ReadFile(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md")) + assert.Equal(t, "appeared after the plan", string(got)) + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-talk.md")) + assert.NoError(t, statErr, "the source survives") +} + +func TestApply_TheArchiveIsOutsideRawSoTheWatcherNeverIngestsIt(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", "x") + items, _ := legacymove.Plan(dir, "web-clip") + res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + assert.False(t, strings.Contains(res.ArchiveDir, "/raw"), res.ArchiveDir) + assert.True(t, strings.HasPrefix(res.ArchiveDir, "archive/"), res.ArchiveDir) +}