hyperguild/ingestion/internal/legacymove/legacymove.go

// Package legacymove clears wiki/sources/ of the retired layout (hyperguild
// #209). It does the one thing that is safe to do mechanically: video and
// article source pages move to the default wing, and the auto-tunnel review
// logs that were once ingested as "sources" are deleted. Notes, projects and
// untyped pages stay where they are, and wiki/concepts and wiki/entities are
// not touched at all (they are the shared layer).
//
// Plan only reads. Apply copies every page it touches into
// archive/wing-migration-<date>/ first, so Rollback can restore the brain
// byte for byte. The archive is outside raw/, so the watcher never ingests it.
package legacymove

import (
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"sort"
	"strings"
	"time"

	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
	"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
)

// Action is what the plan does with one page.
type Action string

const (
	Move   Action = "move"
	Delete Action = "delete"
	Skip   Action = "skip"
)

// Item is one planned operation.
type Item struct {
	From   string
	To     string // Move only
	Slug   string
	Action Action
	Wing   string // Move only
	Reason string
}

const legacyDir = "wiki/sources"

// Plan reads the retired source dir and decides. It writes nothing.
func Plan(brainDir, wing string) ([]Item, error) {
	if wing == "" || brain.Sanitise(wing) != wing {
		return nil, fmt.Errorf("wing %q is not a plain wing name", wing)
	}
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return nil, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()
	names, err := bfs.ReadDirRegular(legacyDir)
	if errors.Is(err, fs.ErrNotExist) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", legacyDir, err)
	}
	var items []Item
	for _, name := range names {
		if !strings.HasSuffix(name, ".md") || name == "_index.md" {
			continue
		}
		from := legacyDir + "/" + name
		slug := strings.TrimSuffix(name, ".md")
		it := Item{From: from, Slug: slug}
		if strings.HasPrefix(name, "tunnel-candidates-") {
			it.Action, it.Reason = Delete, "auto-tunnel review log ingested as a source; not a source"
			items = append(items, it)
			continue
		}
		content, err := bfs.ReadRegular(from)
		if err != nil {
			it.Action, it.Reason = Skip, fmt.Sprintf("unreadable: %v", err)
			items = append(items, it)
			continue
		}
		switch typ := frontmatterValue(string(content), "type"); typ {
		case "video", "article":
			to := "wiki/" + wing + "/sources/" + name
			it.Action, it.To, it.Wing = Move, to, wing
			existing, ferr := wiki.FindSlugs(brainDir, slug)
			if ferr != nil {
				it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("cannot check slug: %v", ferr)
			}
			for _, p := range existing {
				if p != from {
					it.Action, it.To, it.Wing, it.Reason = Skip, "", "", fmt.Sprintf("slug %q already used by %s", slug, p)
					break
				}
			}
		default:
			it.Action, it.Reason = Skip, fmt.Sprintf("type %q: left in place", typ)
		}
		items = append(items, it)
	}
	sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
	return items, nil
}

// Result reports what Apply did and where the archive is (brain-relative).
type Result struct {
	Moved, Deleted int
	ArchiveDir     string
}

type manifest struct {
	Moves   []moveRec `json:"moves"`
	Deletes []string  `json:"deletes"`
}

type moveRec struct {
	From string `json:"from"`
	To   string `json:"to"`
}

// Apply performs the Move and Delete items, one at a time, stopping at the
// first failure. Each original is archived before anything changes; a move
// writes its destination with O_EXCL before the source is removed, so a failure
// never loses a page. The manifest is rewritten after every step so a partial
// run can still be rolled back.
func Apply(brainDir string, items []Item, now time.Time) (Result, error) {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return Result{}, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()

	res := Result{ArchiveDir: "archive/wing-migration-" + now.UTC().Format("2006-01-02")}
	var man manifest
	save := func() error {
		b, err := json.MarshalIndent(man, "", "  ")
		if err != nil {
			return err
		}
		return bfs.PutRegular(res.ArchiveDir+"/manifest.json", b)
	}
	for _, it := range items {
		if it.Action != Move && it.Action != Delete {
			continue
		}
		content, err := bfs.ReadRegular(it.From)
		if err != nil {
			return res, fmt.Errorf("read %s: %w", it.From, err)
		}
		if err := bfs.PutRegular(res.ArchiveDir+"/originals/"+archiveKey(it.From), content); err != nil {
			return res, fmt.Errorf("archive %s: %w", it.From, err)
		}
		switch it.Action {
		case Move:
			if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing))); err != nil {
				return res, fmt.Errorf("write %s: %w", it.To, err)
			}
			man.Moves = append(man.Moves, moveRec{From: it.From, To: it.To})
			res.Moved++
		case Delete:
			man.Deletes = append(man.Deletes, it.From)
			res.Deleted++
		}
		if err := save(); err != nil {
			return res, fmt.Errorf("write manifest: %w", err)
		}
		if err := bfs.Remove(it.From); err != nil {
			return res, fmt.Errorf("handled %s but removing it failed: %w", it.From, err)
		}
	}
	for w := range wingsOf(items) {
		_ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved
	}
	return res, nil
}

func wingsOf(items []Item) map[string]bool {
	out := map[string]bool{}
	for _, it := range items {
		if it.Action == Move {
			out[it.Wing] = true
		}
	}
	return out
}

// Rollback restores every page named in the archive's manifest from its
// archived original and removes the moved copies. It refuses to overwrite a
// page that already exists at the original location.
func Rollback(brainDir, archiveDir string) error {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()
	raw, err := bfs.ReadRegular(archiveDir + "/manifest.json")
	if err != nil {
		return fmt.Errorf("read manifest: %w", err)
	}
	var man manifest
	if err := json.Unmarshal(raw, &man); err != nil {
		return fmt.Errorf("parse manifest: %w", err)
	}
	restore := func(from string) error {
		orig, err := bfs.ReadRegular(archiveDir + "/originals/" + archiveKey(from))
		if err != nil {
			return fmt.Errorf("read archived %s: %w", from, err)
		}
		if err := bfs.CreateRegular(from, orig); err != nil {
			return fmt.Errorf("restore %s: %w", from, err)
		}
		return nil
	}
	for _, m := range man.Moves {
		if err := restore(m.From); err != nil {
			return err
		}
		if err := bfs.Remove(m.To); err != nil {
			return fmt.Errorf("remove moved copy %s: %w", m.To, err)
		}
	}
	for _, d := range man.Deletes {
		if err := restore(d); err != nil {
			return err
		}
	}
	return nil
}

// archiveKey flattens a brain path into one file name, so two pages with the
// same basename in different places (raw/x.md and wiki/sources/x.md) never
// overwrite each other in the archive.
func archiveKey(p string) string { return strings.ReplaceAll(p, "/", "__") }

// stamp inserts wing/hall after the opening fence, leaving every other byte
// alone. A page without frontmatter gets a block.
func stamp(content, wing string) string {
	extra := "wing: " + wing + "\nhall: sources\n"
	if strings.HasPrefix(content, "---\n") {
		return "---\n" + extra + content[4:]
	}
	return "---\n" + extra + "---\n" + content
}

func frontmatterValue(content, key string) string {
	if !strings.HasPrefix(content, "---\n") {
		return ""
	}
	block, _, _ := strings.Cut(content[4:], "\n---")
	for _, line := range strings.Split(block, "\n") {
		if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key {
			return strings.Trim(strings.TrimSpace(v), `"'`)
		}
	}
	return ""
}

const rawDir = "raw"

// PlanRaw finds what the watcher left in raw/ before inbox zero: originals that
// were archived but never removed, with their .processed markers, and markers
// whose note is gone. An original is deleted only when a byte-identical copy
// exists under raw/processed/ (so nothing is lost), and only when it has a
// marker (an unmarked note is still in the inbox). tunnel-candidates-* are
// AutoTunnel's own review queue and are never touched.
func PlanRaw(brainDir string) ([]Item, error) {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return nil, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()
	names, err := bfs.ReadDirRegular(rawDir)
	if errors.Is(err, fs.ErrNotExist) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", rawDir, err)
	}
	present := map[string]bool{}
	for _, n := range names {
		present[n] = true
	}
	archived, err := archivedCopies(bfs)
	if err != nil {
		return nil, err
	}
	var items []Item
	for _, n := range names {
		if strings.HasPrefix(n, "tunnel-candidates-") {
			continue
		}
		switch {
		case strings.HasSuffix(n, ".processed"):
			note := strings.TrimSuffix(n, ".processed")
			if !present[note] {
				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "marker without a note"})
			}
		case present[n+".processed"]:
			content, rerr := bfs.ReadRegular(rawDir + "/" + n)
			if rerr != nil {
				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: fmt.Sprintf("unreadable: %v", rerr)})
				continue
			}
			if !archived[sha256.Sum256(content)] {
				items = append(items, Item{From: rawDir + "/" + n, Slug: n, Action: Skip, Reason: "marked processed but no identical copy under raw/processed/; kept"})
				continue
			}
			items = append(items,
				Item{From: rawDir + "/" + n, Slug: n, Action: Delete, Reason: "archived copy exists under raw/processed/"},
				Item{From: rawDir + "/" + n + ".processed", Slug: n + ".processed", Action: Delete, Reason: "marker of an archived note"})
		}
	}
	sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
	return items, nil
}

// archivedCopies returns the SHA-256 of every file under raw/processed/, as a
// set, so "is there an identical copy" is one lookup.
func archivedCopies(bfs *brainfs.Brain) (map[[32]byte]bool, error) {
	out := map[[32]byte]bool{}
	days, err := bfs.ReadDirDirs(rawDir + "/processed")
	if errors.Is(err, fs.ErrNotExist) {
		return out, nil
	}
	if err != nil {
		return nil, fmt.Errorf("list %s/processed: %w", rawDir, err)
	}
	for _, d := range days {
		files, err := bfs.ReadDirRegular(rawDir + "/processed/" + d)
		if err != nil {
			continue
		}
		for _, f := range files {
			b, err := bfs.ReadRegular(rawDir + "/processed/" + d + "/" + f)
			if err != nil {
				continue
			}
			out[sha256.Sum256(b)] = true
		}
	}
	return out, nil
}