// Package legacymove clears wiki/sources/ of the retired layout (hyperguild // #209). It does the one thing that is safe to do mechanically: video and // article source pages move to the default wing, and the auto-tunnel review // logs that were once ingested as "sources" are deleted. Notes, projects and // untyped pages stay where they are, and wiki/concepts and wiki/entities are // not touched at all (they are the shared layer). // // Plan only reads. Apply copies every page it touches into // archive/wing-migration-/ first, so Rollback can restore the brain // byte for byte. The archive is outside raw/, so the watcher never ingests it. package legacymove import ( "crypto/sha256" "encoding/json" "errors" "fmt" "io/fs" "sort" "strings" "time" "github.com/mathiasbq/hyperguild/ingestion/internal/brain" "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" ) // Action is what the plan does with one page. type Action string const ( Move Action = "move" Delete Action = "delete" Skip Action = "skip" ) // Item is one planned operation. type Item struct { From string To string // Move only Slug string Action Action Wing string // Move only Reason string } const legacyDir = "wiki/sources" // Plan reads the retired source dir and decides. It writes nothing. func Plan(brainDir, wing string) ([]Item, error) { if wing == "" || brain.Sanitise(wing) != wing { return nil, fmt.Errorf("wing %q is not a plain wing name", wing) } bfs, err := brainfs.Open(brainDir) if err != nil { return nil, fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() names, err := bfs.ReadDirRegular(legacyDir) if errors.Is(err, fs.ErrNotExist) { return nil, nil } if err != nil { return nil, fmt.Errorf("read %s: %w", legacyDir, err) } var items []Item for _, name := range names { if !strings.HasSuffix(name, ".md") || name == "_index.md" { continue } from := legacyDir + "/" + name slug := strings.TrimSuffix(name, ".md") it := Item{From: from, Slug: slug} if strings.HasPrefix(name, "tunnel-candidates-") { it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source" items = append(items, it) continue } content, err := bfs.ReadRegular(from) if err != nil { it.Action, it.Reason = Skip, fmt.Sprintf("unreadable: %v", err) items = append(items, it) continue } switch typ := frontmatterValue(string(content), "type"); typ { case "video", "article": to := "wiki/" + wing + "/sources/" + name it.Action, it.To, it.Wing = Move, to, wing existing, ferr := wiki.FindSlugs(brainDir, slug) if ferr != nil { it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("cannot check slug: %v", ferr) } for _, p := range existing { if p != from { it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("slug %q already used by %s", slug, p) break } } default: it.Action, it.Reason = Skip, fmt.Sprintf("type %q: left in place", typ) } items = append(items, it) } sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) return items, nil } // Result reports what Apply did and where the archive is (brain-relative). type Result struct { Moved, Deleted int ArchiveDir string } type manifest struct { Moves []moveRec `json:"moves"` Deletes []string `json:"deletes"` } type moveRec struct { From string `json:"from"` To string `json:"to"` } // Apply performs the Move and Delete items, one at a time, stopping at the // first failure. Each original is archived before anything changes; a move // writes its destination with O_EXCL before the source is removed, so a failure // never loses a page. The manifest is rewritten after every step so a partial // run can still be rolled back. func Apply(brainDir string, items []Item, now time.Time) (Result, error) { bfs, err := brainfs.Open(brainDir) if err != nil { return Result{}, fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() res := Result{ArchiveDir: "archive/wing-migration-" + now.UTC().Format("2006-01-02")} var man manifest save := func() error { b, err := json.MarshalIndent(man, "", " ") if err != nil { return err } return bfs.PutRegular(res.ArchiveDir+"/manifest.json", b) } for _, it := range items { if it.Action != Move && it.Action != Delete { continue } content, err := bfs.ReadRegular(it.From) if err != nil { return res, fmt.Errorf("read %s: %w", it.From, err) } if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil { return res, fmt.Errorf("archive %s: %w", it.From, err) } switch it.Action { case Move: if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing))); err != nil { return res, fmt.Errorf("write %s: %w", it.To, err) } man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To}) res.Moved++ case Delete: man.Deletes = append(man.Deletes, it.From) res.Deleted++ } if err := save(); err != nil { return res, fmt.Errorf("write manifest: %w", err) } if err := bfs.Remove(it.From); err != nil { return res, fmt.Errorf("handled %s but removing it failed: %w", it.From, err) } } for w := range wingsOf(items) { _ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved } return res, nil } func wingsOf(items []Item) map[string]bool { out := map[string]bool{} for _, it := range items { if it.Action == Move { out[it.Wing] = true } } return out } // Rollback restores every page named in the archive's manifest from its // archived original and removes the moved copies. It refuses to overwrite a // page that already exists at the original location. func Rollback(brainDir, archiveDir string) error { bfs, err := brainfs.Open(brainDir) if err != nil { return fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() raw, err := bfs.ReadRegular(archiveDir + "/manifest.json") if err != nil { return fmt.Errorf("read manifest: %w", err) } var man manifest if err := json.Unmarshal(raw, &man); err != nil { return fmt.Errorf("parse manifest: %w", err) } restore := func(from string) error { orig, err := bfs.ReadRegular(archiveDir + "/originals/" + archiveKey(from)) if err != nil { return fmt.Errorf("read archived %s: %w", from, err) } if err := bfs.CreateRegular(from, orig); err != nil { return fmt.Errorf("restore %s: %w", from, err) } return nil } for _, m := range man.Moves { if err := restore(m.From); err != nil { return err } if err := bfs.Remove(m.To); err != nil { return fmt.Errorf("remove moved copy %s: %w", m.To, err) } } for _, d := range man.Deletes { if err := restore(d); err != nil { return err } } return nil } // archiveKey flattens a brain path into one file name, so two pages with the // same basename in different places (raw/x.md and wiki/sources/x.md) never // overwrite each other in the archive. func archiveKey(p string) string { return strings.ReplaceAll(p, "/", "__") } // stamp inserts wing/hall after the opening fence, leaving every other byte // alone. A page without frontmatter gets a block. func stamp(content, wing string) string { extra := "wing: " + wing + "\nhall: sources\n" if strings.HasPrefix(content, "---\n") { return "---\n" + extra + content[4:] } return "---\n" + extra + "---\n" + content } func frontmatterValue(content, key string) string { if !strings.HasPrefix(content, "---\n") { return "" } block, _, _ := strings.Cut(content[4:], "\n---") for _, line := range strings.Split(block, "\n") { if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key { return strings.Trim(strings.TrimSpace(v), `"'`) } } return "" } const rawDir = "raw" // PlanRaw finds what the watcher left in raw/ before inbox zero: originals that // were archived but never removed, with their .processed markers, and markers // whose note is gone. An original is deleted only when a byte-identical copy // exists under raw/processed/ (so nothing is lost), and only when it has a // marker (an unmarked note is still in the inbox). tunnel-candidates-* are // AutoTunnel's own review queue and are never touched. func PlanRaw(brainDir string) ([]Item, error) { bfs, err := brainfs.Open(brainDir) if err != nil { return nil, fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() names, err := bfs.ReadDirRegular(rawDir) if errors.Is(err, fs.ErrNotExist) { return nil, nil } if err != nil { return nil, fmt.Errorf("read %s: %w", rawDir, err) } present := map[string]bool{} for _, n := range names { present[n] = true } archived, err := archivedCopies(bfs) if err != nil { return nil, err } var items []Item for _, n := range names { if strings.HasPrefix(n, "tunnel-candidates-") { continue } switch { case strings.HasSuffix(n, ".processed"): note := strings.TrimSuffix(n, ".processed") if !present[note] { items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "marker without a note"}) } case present[n+".processed"]: content, rerr := bfs.ReadRegular(rawDir + "/" + n) if rerr != nil { items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", rerr)}) continue } if !archived[sha256.Sum256(content)] { items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "marked processed but no identical copy under raw/processed/; kept"}) continue } items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "archived copy exists under raw/processed/"}, Item{From: rawDir + "/" + n + ".processed", Slug: n + ".processed", Action: Delete, Reason: "marker of an archived note"}) } } sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) return items, nil } // archivedCopies returns the SHA-256 of every file under raw/processed/, as a // set, so "is there an identical copy" is one lookup. func archivedCopies(bfs *brainfs.Brain) (map[[32]byte]bool, error) { out := map[[32]byte]bool{} days, err := bfs.ReadDirDirs(rawDir + "/processed") if errors.Is(err, fs.ErrNotExist) { return out, nil } if err != nil { return nil, fmt.Errorf("list %s/processed: %w", rawDir, err) } for _, d := range days { files, err := bfs.ReadDirRegular(rawDir + "/processed/" + d) if err != nil { continue } for _, f := range files { b, err := bfs.ReadRegular(rawDir + "/processed/" + d + "/" + f) if err != nil { continue } out[sha256.Sum256(b)] = true } } return out, nil }