feat(legacymove): clear the retired wiki/sources layout with an archive and a rollback (#209)
Videos and articles move to the default wing (wing/hall stamped, body and links byte-preserved); auto-tunnel review logs are deleted; notes, projects and untyped pages stay. Plan is read-only; Apply archives every page it touches to archive/wing-migration-<date>/ (outside raw/) before changing anything, creates destinations O_EXCL and removes sources only after; Rollback restores byte for byte. concepts/ and entities/ are the shared layer and are untouched. Dry run against the live brain: 16 move, 24 delete, 15 left. Mutation-checked: junk guard, type filter, slug collision, no-overwrite, archive-before-change, action gate. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,59 @@
|
||||
// Command legacymove clears wiki/sources/ of the retired layout (hyperguild
|
||||
// #209). Default is a dry run that prints the plan; -apply performs it after
|
||||
// archiving every page it touches; -rollback restores from an archive.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/legacymove"
|
||||
)
|
||||
|
||||
func main() {
|
||||
brain := flag.String("brain", "/var/lib/supervisor/brain", "brain directory")
|
||||
wing := flag.String("wing", "web-clip", "wing that receives video and article sources")
|
||||
apply := flag.Bool("apply", false, "perform the plan (default: dry run)")
|
||||
rollback := flag.String("rollback", "", "restore from this archive dir (brain-relative, e.g. archive/wing-migration-2026-10-20)")
|
||||
flag.Parse()
|
||||
|
||||
if err := run(*brain, *wing, *apply, *rollback); err != nil {
|
||||
slog.Error("legacymove", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(brain, wing string, apply bool, rollback string) error {
|
||||
if rollback != "" {
|
||||
if err := legacymove.Rollback(brain, rollback); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("rolled back from", rollback)
|
||||
return nil
|
||||
}
|
||||
items, err := legacymove.Plan(brain, wing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
counts := map[legacymove.Action]int{}
|
||||
for _, it := range items {
|
||||
counts[it.Action]++
|
||||
}
|
||||
fmt.Printf("# legacy source plan\n\n%d pages: %d move, %d delete, %d left\n\n", len(items), counts[legacymove.Move], counts[legacymove.Delete], counts[legacymove.Skip])
|
||||
fmt.Println("| action | from | to | why |")
|
||||
fmt.Println("|---|---|---|---|")
|
||||
for _, it := range items {
|
||||
fmt.Printf("| %s | %s | %s | %s |\n", it.Action, it.From, it.To, strings.ReplaceAll(it.Reason, "|", "/"))
|
||||
}
|
||||
if !apply {
|
||||
fmt.Println("\n_dry run: nothing was changed_")
|
||||
return nil
|
||||
}
|
||||
res, err := legacymove.Apply(brain, items, time.Now())
|
||||
fmt.Printf("\nmoved %d, deleted %d; archive: %s\n", res.Moved, res.Deleted, res.ArchiveDir)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
// Package legacymove clears wiki/sources/ of the retired layout (hyperguild
|
||||
// #209). It does the one thing that is safe to do mechanically: video and
|
||||
// article source pages move to the default wing, and the auto-tunnel review
|
||||
// logs that were once ingested as "sources" are deleted. Notes, projects and
|
||||
// untyped pages stay where they are, and wiki/concepts and wiki/entities are
|
||||
// not touched at all (they are the shared layer).
|
||||
//
|
||||
// Plan only reads. Apply copies every page it touches into
|
||||
// archive/wing-migration-<date>/ first, so Rollback can restore the brain
|
||||
// byte for byte. The archive is outside raw/, so the watcher never ingests it.
|
||||
package legacymove
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
||||
)
|
||||
|
||||
// Action is what the plan does with one page.
|
||||
type Action string
|
||||
|
||||
const (
|
||||
Move Action = "move"
|
||||
Delete Action = "delete"
|
||||
Skip Action = "skip"
|
||||
)
|
||||
|
||||
// Item is one planned operation.
|
||||
type Item struct {
|
||||
From string
|
||||
To string // Move only
|
||||
Slug string
|
||||
Action Action
|
||||
Wing string // Move only
|
||||
Reason string
|
||||
}
|
||||
|
||||
const legacyDir = "wiki/sources"
|
||||
|
||||
// Plan reads the retired source dir and decides. It writes nothing.
|
||||
func Plan(brainDir, wing string) ([]Item, error) {
|
||||
if wing == "" || brain.Sanitise(wing) != wing {
|
||||
return nil, fmt.Errorf("wing %q is not a plain wing name", wing)
|
||||
}
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open brain: %w", err)
|
||||
}
|
||||
defer func() { _ = bfs.Close() }()
|
||||
names, err := bfs.ReadDirRegular(legacyDir)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", legacyDir, err)
|
||||
}
|
||||
var items []Item
|
||||
for _, name := range names {
|
||||
if !strings.HasSuffix(name, ".md") || name == "_index.md" {
|
||||
continue
|
||||
}
|
||||
from := legacyDir + "/" + name
|
||||
slug := strings.TrimSuffix(name, ".md")
|
||||
it := Item{From: from, Slug: slug}
|
||||
if strings.HasPrefix(name, "tunnel-candidates-") {
|
||||
it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source"
|
||||
items = append(items, it)
|
||||
continue
|
||||
}
|
||||
content, err := bfs.ReadRegular(from)
|
||||
if err != nil {
|
||||
it.Action, it.Reason = Skip, fmt.Sprintf("unreadable: %v", err)
|
||||
items = append(items, it)
|
||||
continue
|
||||
}
|
||||
switch typ := frontmatterValue(string(content), "type"); typ {
|
||||
case "video", "article":
|
||||
to := "wiki/" + wing + "/sources/" + name
|
||||
it.Action, it.To, it.Wing = Move, to, wing
|
||||
existing, ferr := wiki.FindSlugs(brainDir, slug)
|
||||
if ferr != nil {
|
||||
it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("cannot check slug: %v", ferr)
|
||||
}
|
||||
for _, p := range existing {
|
||||
if p != from {
|
||||
it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("slug %q already used by %s", slug, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
default:
|
||||
it.Action, it.Reason = Skip, fmt.Sprintf("type %q: left in place", typ)
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Result reports what Apply did and where the archive is (brain-relative).
|
||||
type Result struct {
|
||||
Moved, Deleted int
|
||||
ArchiveDir string
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Moves []moveRec `json:"moves"`
|
||||
Deletes []string `json:"deletes"`
|
||||
}
|
||||
|
||||
type moveRec struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
}
|
||||
|
||||
// Apply performs the Move and Delete items, one at a time, stopping at the
|
||||
// first failure. Each original is archived before anything changes; a move
|
||||
// writes its destination with O_EXCL before the source is removed, so a failure
|
||||
// never loses a page. The manifest is rewritten after every step so a partial
|
||||
// run can still be rolled back.
|
||||
func Apply(brainDir string, items []Item, now time.Time) (Result, error) {
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("open brain: %w", err)
|
||||
}
|
||||
defer func() { _ = bfs.Close() }()
|
||||
|
||||
res := Result{ArchiveDir: "archive/wing-migration-" + now.UTC().Format("2006-01-02")}
|
||||
var man manifest
|
||||
save := func() error {
|
||||
b, err := json.MarshalIndent(man, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return bfs.PutRegular(res.ArchiveDir+"/manifest.json", b)
|
||||
}
|
||||
for _, it := range items {
|
||||
if it.Action != Move && it.Action != Delete {
|
||||
continue
|
||||
}
|
||||
content, err := bfs.ReadRegular(it.From)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("read %s: %w", it.From, err)
|
||||
}
|
||||
if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+path.Base(it.From), content); err != nil {
|
||||
return res, fmt.Errorf("archive %s: %w", it.From, err)
|
||||
}
|
||||
switch it.Action {
|
||||
case Move:
|
||||
if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing))); err != nil {
|
||||
return res, fmt.Errorf("write %s: %w", it.To, err)
|
||||
}
|
||||
man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To})
|
||||
res.Moved++
|
||||
case Delete:
|
||||
man.Deletes = append(man.Deletes, it.From)
|
||||
res.Deleted++
|
||||
}
|
||||
if err := save(); err != nil {
|
||||
return res, fmt.Errorf("write manifest: %w", err)
|
||||
}
|
||||
if err := bfs.Remove(it.From); err != nil {
|
||||
return res, fmt.Errorf("handled %s but removing it failed: %w", it.From, err)
|
||||
}
|
||||
}
|
||||
for w := range wingsOf(items) {
|
||||
_ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func wingsOf(items []Item) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, it := range items {
|
||||
if it.Action == Move {
|
||||
out[it.Wing] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Rollback restores every page named in the archive's manifest from its
|
||||
// archived original and removes the moved copies. It refuses to overwrite a
|
||||
// page that already exists at the original location.
|
||||
func Rollback(brainDir, archiveDir string) error {
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open brain: %w", err)
|
||||
}
|
||||
defer func() { _ = bfs.Close() }()
|
||||
raw, err := bfs.ReadRegular(archiveDir + "/manifest.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
var man manifest
|
||||
if err := json.Unmarshal(raw, &man); err != nil {
|
||||
return fmt.Errorf("parse manifest: %w", err)
|
||||
}
|
||||
restore := func(from string) error {
|
||||
orig, err := bfs.ReadRegular(archiveDir + "/originals/" + path.Base(from))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read archived %s: %w", from, err)
|
||||
}
|
||||
if err := bfs.CreateRegular(from, orig); err != nil {
|
||||
return fmt.Errorf("restore %s: %w", from, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, m := range man.Moves {
|
||||
if err := restore(m.From); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bfs.Remove(m.To); err != nil {
|
||||
return fmt.Errorf("remove moved copy %s: %w", m.To, err)
|
||||
}
|
||||
}
|
||||
for _, d := range man.Deletes {
|
||||
if err := restore(d); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stamp inserts wing/hall after the opening fence, leaving every other byte
|
||||
// alone. A page without frontmatter gets a block.
|
||||
func stamp(content, wing string) string {
|
||||
extra := "wing: " + wing + "\nhall: sources\n"
|
||||
if strings.HasPrefix(content, "---\n") {
|
||||
return "---\n" + extra + content[4:]
|
||||
}
|
||||
return "---\n" + extra + "---\n" + content
|
||||
}
|
||||
|
||||
func frontmatterValue(content, key string) string {
|
||||
if !strings.HasPrefix(content, "---\n") {
|
||||
return ""
|
||||
}
|
||||
block, _, _ := strings.Cut(content[4:], "\n---")
|
||||
for _, line := range strings.Split(block, "\n") {
|
||||
if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key {
|
||||
return strings.Trim(strings.TrimSpace(v), `"'`)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package legacymove_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/legacymove"
|
||||
)
|
||||
|
||||
func page(t *testing.T, dir, rel, body string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, rel)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
||||
require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
|
||||
}
|
||||
|
||||
func src(typ string) string {
|
||||
return "---\ntitle: T\ntype: " + typ + "\n---\n\n## Summary\n\nBody with [[a-link|A]] kept as is.\n"
|
||||
}
|
||||
|
||||
func byFrom(items []legacymove.Item) map[string]legacymove.Item {
|
||||
m := map[string]legacymove.Item{}
|
||||
for _, it := range items {
|
||||
m[it.From] = it
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestPlan_VideosAndArticlesMoveToTheDefaultWing(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
page(t, dir, "wiki/sources/a-talk.md", src("video"))
|
||||
page(t, dir, "wiki/sources/an-article.md", src("article"))
|
||||
|
||||
items, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
m := byFrom(items)
|
||||
assert.Equal(t, legacymove.Move, m["wiki/sources/a-talk.md"].Action)
|
||||
assert.Equal(t, "wiki/web-clip/sources/a-talk.md", m["wiki/sources/a-talk.md"].To)
|
||||
assert.Equal(t, legacymove.Move, m["wiki/sources/an-article.md"].Action)
|
||||
}
|
||||
|
||||
func TestPlan_NotesProjectsAndUntypedPagesStayWhereTheyAre(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
page(t, dir, "wiki/sources/a-note.md", src("note"))
|
||||
page(t, dir, "wiki/sources/a-project.md", src("project"))
|
||||
page(t, dir, "wiki/sources/untyped.md", "---\ntitle: T\n---\nbody")
|
||||
page(t, dir, "wiki/sources/_index.md", "index")
|
||||
|
||||
items, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
m := byFrom(items)
|
||||
for _, f := range []string{"a-note", "a-project", "untyped"} {
|
||||
assert.Equal(t, legacymove.Skip, m["wiki/sources/"+f+".md"].Action, f)
|
||||
}
|
||||
_, listed := m["wiki/sources/_index.md"]
|
||||
assert.False(t, listed, "the index is not a page")
|
||||
}
|
||||
|
||||
func TestPlan_AutoTunnelLogsAreDeleted(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("video"))
|
||||
|
||||
items, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, legacymove.Delete, byFrom(items)["wiki/sources/tunnel-candidates-2026-06-14.md"].Action,
|
||||
"never moved, whatever the type says")
|
||||
}
|
||||
|
||||
func TestPlan_ASlugAlreadyUsedElsewhereIsSkippedNotOverwritten(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
page(t, dir, "wiki/sources/a-talk.md", src("video"))
|
||||
page(t, dir, "wiki/entities/a-talk.md", "an entity with the same slug")
|
||||
|
||||
items, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
it := byFrom(items)["wiki/sources/a-talk.md"]
|
||||
assert.Equal(t, legacymove.Skip, it.Action)
|
||||
assert.Contains(t, it.Reason, "already used")
|
||||
}
|
||||
|
||||
func TestPlan_ADefaultWingThatIsNotAPlainNameIsRefused(t *testing.T) {
|
||||
_, err := legacymove.Plan(t.TempDir(), "../x")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestApply_MovesStampsAndArchivesEverythingItTouches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
talk := src("video")
|
||||
junk := "tunnel log"
|
||||
page(t, dir, "wiki/sources/a-talk.md", talk)
|
||||
page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", junk)
|
||||
page(t, dir, "wiki/sources/a-note.md", src("note"))
|
||||
items, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
|
||||
res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, res.Moved)
|
||||
assert.Equal(t, 1, res.Deleted)
|
||||
|
||||
moved, err := os.ReadFile(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md"))
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(moved), "wing: web-clip")
|
||||
assert.Contains(t, string(moved), "hall: sources")
|
||||
assert.Contains(t, string(moved), "Body with [[a-link|A]] kept as is.", "body and links are byte-preserved")
|
||||
|
||||
for _, gone := range []string{"wiki/sources/a-talk.md", "wiki/sources/tunnel-candidates-2026-06-14.md"} {
|
||||
_, statErr := os.Stat(filepath.Join(dir, gone))
|
||||
assert.ErrorIs(t, statErr, os.ErrNotExist, gone)
|
||||
}
|
||||
_, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-note.md"))
|
||||
assert.NoError(t, statErr, "a skipped page is left exactly where it is")
|
||||
|
||||
again, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
for _, it := range again {
|
||||
assert.Equal(t, legacymove.Skip, it.Action, "nothing is left to do: %s", it.From)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollback_RestoresEveryTouchedPageByteForByte(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
talk := src("video")
|
||||
junk := "tunnel log"
|
||||
page(t, dir, "wiki/sources/a-talk.md", talk)
|
||||
page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", junk)
|
||||
items, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, legacymove.Rollback(dir, res.ArchiveDir))
|
||||
|
||||
got, err := os.ReadFile(filepath.Join(dir, "wiki/sources/a-talk.md"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, talk, string(got))
|
||||
got, err = os.ReadFile(filepath.Join(dir, "wiki/sources/tunnel-candidates-2026-06-14.md"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, junk, string(got))
|
||||
_, statErr := os.Stat(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md"))
|
||||
assert.ErrorIs(t, statErr, os.ErrNotExist, "the moved copy is removed")
|
||||
}
|
||||
|
||||
func TestApply_NeverOverwritesAndKeepsTheSourceIfTheDestinationAppearsLate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
page(t, dir, "wiki/sources/a-talk.md", src("video"))
|
||||
items, err := legacymove.Plan(dir, "web-clip")
|
||||
require.NoError(t, err)
|
||||
page(t, dir, "wiki/web-clip/sources/a-talk.md", "appeared after the plan")
|
||||
|
||||
_, err = legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC))
|
||||
require.Error(t, err)
|
||||
got, _ := os.ReadFile(filepath.Join(dir, "wiki/web-clip/sources/a-talk.md"))
|
||||
assert.Equal(t, "appeared after the plan", string(got))
|
||||
_, statErr := os.Stat(filepath.Join(dir, "wiki/sources/a-talk.md"))
|
||||
assert.NoError(t, statErr, "the source survives")
|
||||
}
|
||||
|
||||
func TestApply_TheArchiveIsOutsideRawSoTheWatcherNeverIngestsIt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", "x")
|
||||
items, _ := legacymove.Plan(dir, "web-clip")
|
||||
res, err := legacymove.Apply(dir, items, time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC))
|
||||
require.NoError(t, err)
|
||||
assert.False(t, strings.Contains(res.ArchiveDir, "/raw"), res.ArchiveDir)
|
||||
assert.True(t, strings.HasPrefix(res.ArchiveDir, "archive/"), res.ArchiveDir)
|
||||
}
|
||||
Reference in New Issue
Block a user