hyperguild/ingestion/internal/legacymove/legacymove.go

// Package legacymove clears wiki/sources/ of the retired layout (hyperguild
// #209). It does the one thing that is safe to do mechanically: video and
// article source pages move to the default wing, and the auto-tunnel review
// logs that were once ingested as "sources" are deleted. Notes, projects and
// untyped pages stay where they are, and wiki/concepts and wiki/entities are
// not touched at all (they are the shared layer).
//
// Plan only reads. Apply copies every page it touches into
// archive/wing-migration-<date>/ first, so Rollback can restore the brain
// byte for byte. The archive is outside raw/, so the watcher never ingests it.
package legacymove

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"path"
	"sort"
	"strings"
	"time"

	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
	"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
)

// Action is what the plan does with one page.
type Action string

const (
	Move   Action = "move"
	Delete Action = "delete"
	Skip   Action = "skip"
)

// Item is one planned operation.
type Item struct {
	From   string
	To     string // Move only
	Slug   string
	Action Action
	Wing   string // Move only
	Reason string
	// SHA is the SHA-256 (hex) of the bytes the plan was decided on. Apply
	// re-reads the page and does nothing if the bytes differ: a page changed
	// between planning and applying is not the page that was judged safe.
	SHA string
}

// maxArchiveName is NAME_MAX. The archive key (the whole path, flattened) is
// longer than the page's own name, so a long name must be refused at planning
// time rather than wedge Apply halfway.
const maxArchiveName = 255

const legacyDir = "wiki/sources"

// Plan reads the retired source dir and decides. It writes nothing.
func Plan(brainDir, wing string) ([]Item, error) {
	if wing == "" || brain.Sanitise(wing) != wing {
		return nil, fmt.Errorf("wing %q is not a plain wing name", wing)
	}
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return nil, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()
	names, err := bfs.ReadDirRegular(legacyDir)
	if errors.Is(err, fs.ErrNotExist) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", legacyDir, err)
	}
	var items []Item
	for _, name := range names {
		if !strings.HasSuffix(name, ".md") || name == "_index.md" {
			continue
		}
		from := legacyDir + "/" + name
		slug := strings.TrimSuffix(name, ".md")
		it := Item{From: from, Slug: slug}
		if len(archiveKey(from)) > maxArchiveName {
			it.Action, it.Reason = Skip, "name too long to archive; left in place"
			items = append(items, it)
			continue
		}
		content, err := bfs.ReadRegular(from)
		if err != nil {
			it.Action, it.Reason = Skip, fmt.Sprintf("unreadable: %v", err)
			items = append(items, it)
			continue
		}
		it.SHA = sum(content)
		if strings.HasPrefix(name, "tunnel-candidates-") {
			it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source"
			items = append(items, it)
			continue
		}
		if has := topLevelKeys(string(content)); has["wing"] || has["hall"] {
			it.Action, it.Reason = Skip, "already has wing/hall in its frontmatter; left in place"
			items = append(items, it)
			continue
		}
		switch typ := frontmatterValue(string(content), "type"); typ {
		case "video", "article":
			to := "wiki/" + wing + "/sources/" + name
			it.Action, it.To, it.Wing = Move, to, wing
			existing, ferr := wiki.FindSlugs(brainDir, slug)
			if ferr != nil {
				it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("cannot check slug: %v", ferr)
			}
			for _, p := range existing {
				if p != from {
					it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("slug %q already used by %s", slug, p)
					break
				}
			}
		default:
			it.Action, it.Reason = Skip, fmt.Sprintf("type %q: left in place", typ)
		}
		items = append(items, it)
	}
	sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
	return items, nil
}

// Result reports what Apply did and where the archive is (brain-relative).
type Result struct {
	Moved, Deleted int
	ArchiveDir     string
	// Changed lists planned pages that were NOT touched because their bytes no
	// longer match what the plan was decided on.
	Changed []string
}

type manifest struct {
	Moves   []moveRec `json:"moves"`
	Deletes []string  `json:"deletes"`
}

type moveRec struct {
	From string `json:"from"`
	To   string `json:"to"`
	Wing string `json:"wing"`
}

// Apply performs the Move and Delete items, one at a time, stopping at the
// first failure. For each page it re-reads the bytes and compares them with the
// plan's hash: a page that changed since planning is left alone and reported in
// Result.Changed. Each original is archived (O_EXCL, into an archive directory
// no earlier run has used) before anything changes; a move writes its
// destination O_EXCL before the source is removed, so a failure never loses a
// page. The manifest is rewritten after every step so a partial run can still
// be rolled back. A swap between the read and the remove is a residual window
// that only a locking watcher could close.
func Apply(brainDir string, items []Item, now time.Time) (Result, error) {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return Result{}, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()

	res := Result{ArchiveDir: freshArchiveDir(bfs, "archive/wing-migration-"+now.UTC().Format("2006-01-02"))}
	var man manifest
	save := func() error {
		b, err := json.MarshalIndent(man, "", "  ")
		if err != nil {
			return err
		}
		return bfs.PutRegular(res.ArchiveDir+"/manifest.json", b)
	}
	for _, it := range items {
		if it.Action != Move && it.Action != Delete {
			continue
		}
		content, err := bfs.ReadRegular(it.From)
		if err != nil {
			return res, fmt.Errorf("read %s: %w", it.From, err)
		}
		if it.SHA == "" || sum(content) != it.SHA {
			res.Changed = append(res.Changed, it.From)
			continue
		}
		if err := bfs.CreateRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil {
			return res, fmt.Errorf("archive %s: %w", it.From, err)
		}
		switch it.Action {
		case Move:
			if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing))); err != nil {
				return res, fmt.Errorf("write %s: %w", it.To, err)
			}
			man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To, Wing: it.Wing})
			res.Moved++
		case Delete:
			man.Deletes = append(man.Deletes, it.From)
			res.Deleted++
		}
		if err := save(); err != nil {
			return res, fmt.Errorf("write manifest: %w", err)
		}
		if err := bfs.Remove(it.From); err != nil {
			return res, fmt.Errorf("handled %s but removing it failed: %w", it.From, err)
		}
	}
	for w := range wingsOf(items) {
		_ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved
	}
	return res, nil
}

// freshArchiveDir returns base, or base-2, base-3, ... : the first that does not
// exist, so a second run never reuses (and never overwrites) an earlier archive.
func freshArchiveDir(bfs *brainfs.Brain, base string) string {
	dir := base
	for n := 2; ; n++ {
		if _, err := bfs.Lstat(dir); err != nil {
			return dir
		}
		dir = fmt.Sprintf("%s-%d", base, n)
	}
}

func sum(b []byte) string {
	h := sha256.Sum256(b)
	return hex.EncodeToString(h[:])
}

func wingsOf(items []Item) map[string]bool {
	out := map[string]bool{}
	for _, it := range items {
		if it.Action == Move {
			out[it.Wing] = true
		}
	}
	return out
}

// Rollback restores every page named in the archive's manifest from its
// archived original and removes the moved copies. It checks everything first
// and changes nothing if anything is in the way: a restore target that exists
// with different bytes, or a moved copy that was edited since the move (which
// is never deleted). Pages already restored are recognised by their bytes, so a
// rollback can be run again. mtime and mode of the restored pages are new.
func Rollback(brainDir, archiveDir string) error {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()
	raw, err := bfs.ReadRegular(archiveDir + "/manifest.json")
	if err != nil {
		return fmt.Errorf("read manifest: %w", err)
	}
	var man manifest
	if err := json.Unmarshal(raw, &man); err != nil {
		return fmt.Errorf("parse manifest: %w", err)
	}
	original := func(from string) ([]byte, error) {
		orig, err := bfs.ReadRegular(archiveDir + "/originals/" + archiveKey(from))
		if err != nil {
			return nil, fmt.Errorf("read archived %s: %w", from, err)
		}
		return orig, nil
	}

	type step struct {
		from    string
		orig    []byte
		restore bool   // From is missing and must be written
		remove  string // moved copy to delete, if any
	}
	var steps []step
	var problems []string
	plan := func(from string, orig []byte, moved string, expectMoved []byte) {
		st := step{from: from, orig: orig}
		cur, err := bfs.ReadRegular(from)
		switch {
		case err == nil && bytes.Equal(cur, orig):
			// already restored
		case err == nil:
			problems = append(problems, fmt.Sprintf("%s exists with different content", from))
		case errors.Is(err, fs.ErrNotExist):
			st.restore = true
		default:
			problems = append(problems, fmt.Sprintf("%s: %v", from, err))
		}
		if moved != "" {
			cur, err := bfs.ReadRegular(moved)
			switch {
			case err == nil && bytes.Equal(cur, expectMoved):
				st.remove = moved
			case err == nil:
				problems = append(problems, fmt.Sprintf("%s was edited since the move; not deleting it", moved))
			case errors.Is(err, fs.ErrNotExist):
			default:
				problems = append(problems, fmt.Sprintf("%s: %v", moved, err))
			}
		}
		steps = append(steps, st)
	}
	for _, m := range man.Moves {
		orig, err := original(m.From)
		if err != nil {
			return err
		}
		plan(m.From, orig, m.To, []byte(stamp(string(orig), m.Wing)))
	}
	for _, d := range man.Deletes {
		orig, err := original(d)
		if err != nil {
			return err
		}
		plan(d, orig, "", nil)
	}
	if len(problems) > 0 {
		return fmt.Errorf("rollback refused, nothing changed: %s", strings.Join(problems, "; "))
	}
	for _, st := range steps {
		if st.restore {
			if err := bfs.CreateRegular(st.from, st.orig); err != nil {
				return fmt.Errorf("restore %s: %w", st.from, err)
			}
		}
		if st.remove != "" {
			if err := bfs.Remove(st.remove); err != nil {
				return fmt.Errorf("remove moved copy %s: %w", st.remove, err)
			}
		}
	}
	return nil
}

// archiveKey flattens a brain path into one file name, so two pages with the
// same basename in different places (raw/x.md and wiki/sources/x.md) never
// overwrite each other in the archive.
func archiveKey(p string) string { return strings.ReplaceAll(p, "/", "__") }

// stamp inserts wing/hall after the opening fence, leaving every other byte
// alone. A page without frontmatter gets a block.
func stamp(content, wing string) string {
	extra := "wing: " + wing + "\nhall: sources\n"
	if strings.HasPrefix(content, "---\n") {
		return "---\n" + extra + content[4:]
	}
	return "---\n" + extra + "---\n" + content
}

// frontmatterBlock returns the text between the opening fence and a real
// closing fence (a line that is exactly "---"), or "" when there is none.
func frontmatterBlock(content string) (string, bool) {
	if !strings.HasPrefix(content, "---\n") {
		return "", false
	}
	rest := content[4:]
	if i := strings.Index(rest, "\n---\n"); i >= 0 {
		return rest[:i], true
	}
	if strings.HasSuffix(rest, "\n---") {
		return rest[:len(rest)-4], true
	}
	return "", false
}

// topLevelKeys is the set of unindented `key:` names in the frontmatter block.
func topLevelKeys(content string) map[string]bool {
	out := map[string]bool{}
	block, ok := frontmatterBlock(content)
	if !ok {
		return out
	}
	for _, line := range strings.Split(block, "\n") {
		if line == "" || line[0] == ' ' || line[0] == '\t' {
			continue
		}
		if k, _, found := strings.Cut(line, ":"); found {
			out[strings.TrimSpace(k)] = true
		}
	}
	return out
}

// frontmatterValue reads a top-level key from a CLOSED frontmatter block. A
// nested or block-scalar line that merely looks like `type: video` does not
// count, and neither does an unterminated block.
func frontmatterValue(content, key string) string {
	block, ok := frontmatterBlock(content)
	if !ok {
		return ""
	}
	for _, line := range strings.Split(block, "\n") {
		if line == "" || line[0] == ' ' || line[0] == '\t' {
			continue
		}
		if k, v, found := strings.Cut(line, ":"); found && strings.TrimSpace(k) == key {
			return strings.Trim(strings.TrimSpace(v), `"'`)
		}
	}
	return ""
}

const rawDir = "raw"

// PlanRaw finds what the watcher left in raw/ before inbox zero: originals that
// were archived but never removed, with their .processed markers, and markers
// whose note is gone. An original is deleted only when a byte-identical copy
// exists under raw/processed/ (so nothing is lost), and only when it has a
// marker (an unmarked note is still in the inbox). tunnel-candidates-* are
// AutoTunnel's own review queue and are never touched. Every Delete carries the
// SHA-256 it was decided on, which Apply re-checks.
func PlanRaw(brainDir string) ([]Item, error) {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return nil, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()
	names, err := bfs.ReadDirRegular(rawDir)
	if errors.Is(err, fs.ErrNotExist) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", rawDir, err)
	}
	present := map[string]bool{}
	for _, n := range names {
		present[n] = true
	}
	archived, err := archivedCopies(bfs)
	if err != nil {
		return nil, err
	}
	del := func(name, why string) Item {
		it := Item{From: rawDir + "/" + name, Slug: name, Action: Delete, Reason: why}
		b, err := bfs.ReadRegular(it.From)
		if err != nil {
			return Item{From: it.From, Slug: name, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", err)}
		}
		it.SHA = sum(b)
		return it
	}
	var items []Item
	for _, n := range names {
		if strings.HasPrefix(n, "tunnel-candidates-") {
			continue
		}
		switch {
		case strings.HasSuffix(n, ".processed"):
			note := strings.TrimSuffix(n, ".processed")
			if !present[note] {
				items = append(items, tooLongOr(rawDir+"/"+n, func() Item { return del(n, "marker without a note") }))
			}
		case present[n+".processed"]:
			content, rerr := bfs.ReadRegular(rawDir + "/" + n)
			if rerr != nil {
				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", rerr)})
				continue
			}
			if !archived[sha256.Sum256(content)] {
				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "marked processed but no identical copy under raw/processed/; kept"})
				continue
			}
			// Both or neither: the archive keys of the note and its (longer)
			// marker must fit, or a run would delete the note and wedge on the marker.
			if len(archiveKey(rawDir+"/"+n+".processed")) > maxArchiveName {
				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "name too long to archive; kept"})
				continue
			}
			items = append(items, del(n, "archived copy exists under raw/processed/"), del(n+".processed", "marker of an archived note"))
		}
	}
	sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
	return items, nil
}

// tooLongOr returns a Skip when the page's archive key cannot fit in a file
// name, else whatever mk builds.
func tooLongOr(from string, mk func() Item) Item {
	if len(archiveKey(from)) > maxArchiveName {
		return Item{From: from, Slug: path.Base(from), Action: Skip, Reason: "name too long to archive; kept"}
	}
	return mk()
}

// archivedCopies returns the SHA-256 of every file under raw/processed/, as a
// set, so "is there an identical copy" is one lookup.
func archivedCopies(bfs *brainfs.Brain) (map[[32]byte]bool, error) {
	out := map[[32]byte]bool{}
	days, err := bfs.ReadDirDirs(rawDir + "/processed")
	if errors.Is(err, fs.ErrNotExist) {
		return out, nil
	}
	if err != nil {
		return nil, fmt.Errorf("list %s/processed: %w", rawDir, err)
	}
	for _, d := range days {
		files, err := bfs.ReadDirRegular(rawDir + "/processed/" + d)
		if err != nil {
			continue
		}
		for _, f := range files {
			b, err := bfs.ReadRegular(rawDir + "/processed/" + d + "/" + f)
			if err != nil {
				continue
			}
			out[sha256.Sum256(b)] = true
		}
	}
	return out, nil
}