hyperguild/ingestion/internal/migrate/migrate.go

// Package migrate moves pages out of the retired wiki/{sources,concepts,
// entities}/ layout into wiki/<wing>/{sources,facts}/ (hyperguild#205).
//
// It is a two-step tool on purpose. BuildPlan only reads and decides; Apply
// performs exactly the items the plan marked Auto and nothing else. Sources are
// placed by the wing Resolver. Concepts and entities have no wing of their own:
// they inherit the strict-majority wing of the sources that cite them, which is
// deterministic and needs no model. Whatever cannot be placed safely is an
// Exception, left where it is for a human.
package migrate

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"path"
	"regexp"
	"sort"
	"strings"

	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
	"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
)

// Status is whether the plan moves a page by itself.
type Status string

const (
	Auto      Status = "auto"
	Exception Status = "exception"
)

// Item is one planned move.
type Item struct {
	From       string // brain-relative, e.g. wiki/entities/jev.md
	To         string // brain-relative; empty for an Exception without a proposal
	Slug       string
	Kind       string // source | concept | entity
	Wing       string // decided wing, or the proposal for an Exception
	Status     Status
	Method     string // resolver | inherited | junk | blocked
	Confidence float64
	Reason     string
}

// ResolverFor returns the Resolver to use for one page. It takes the page's
// own path so a scorer can exclude the page from its evidence.
type ResolverFor func(excludePath string) *wing.Resolver

var (
	linkSlugRE = regexp.MustCompile(`\[\[([^|\]]+)`)
	legacy     = []struct{ dir, kind string }{
		{"wiki/sources", "source"},
		{"wiki/concepts", "concept"},
		{"wiki/entities", "entity"},
	}
)

// BuildPlan reads the legacy layout and decides, without writing anything.
func BuildPlan(ctx context.Context, brainDir string, rf ResolverFor) ([]Item, error) {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return nil, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()

	taken, err := slugsInWings(bfs)
	if err != nil {
		return nil, err
	}

	var items []Item
	decided := map[string]Item{} // source slug -> its decision
	for _, l := range legacy {
		names, err := bfs.ReadDirRegular(l.dir)
		if errors.Is(err, fs.ErrNotExist) {
			continue
		}
		if err != nil {
			return nil, fmt.Errorf("read %s: %w", l.dir, err)
		}
		for _, name := range names {
			if !strings.HasSuffix(name, ".md") || name == "_index.md" {
				continue
			}
			rel := l.dir + "/" + name
			slug := strings.TrimSuffix(name, ".md")
			it := Item{From: rel, Slug: slug, Kind: l.kind}
			if strings.HasPrefix(name, "tunnel-candidates-") {
				it.Status, it.Method, it.Reason = Exception, "junk", "auto-tunnel review log ingested as a source; not a source"
				items = append(items, it)
				continue
			}
			content, err := bfs.ReadRegular(rel)
			if err != nil {
				it.Status, it.Method, it.Reason = Exception, "blocked", fmt.Sprintf("unreadable: %v", err)
				items = append(items, it)
				continue
			}
			if l.kind == "source" {
				d := rf(rel).Resolve(ctx, wing.Note{Text: string(content), FrontmatterWing: frontmatterValue(string(content), "wing")})
				it.Method, it.Wing, it.Confidence, it.Reason = "resolver", d.Wing, d.Confidence, d.Reason
				if d.Source == wing.Exception {
					it.Status = Exception
				} else {
					it.Status, it.To = Auto, "wiki/"+d.Wing+"/sources/"+name
				}
				decided[slug] = it
			}
			items = append(items, it)
		}
	}

	// Second pass: concepts and entities inherit from their sources.
	for i, it := range items {
		if it.Kind == "source" || it.Method == "junk" || it.Method == "blocked" {
			continue
		}
		content, err := bfs.ReadRegular(it.From)
		if err != nil {
			continue
		}
		it.Method = "inherited"
		w, reason := inheritedWing(string(content), decided)
		if w == "" {
			it.Status, it.Reason = Exception, reason
		} else {
			it.Status, it.Wing, it.To = Auto, w, "wiki/"+w+"/facts/"+it.Slug+".md"
			it.Reason = fmt.Sprintf("strict majority of its sources are in %q", w)
		}
		items[i] = it
	}

	blockCollisions(bfs, items, taken)
	sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
	return items, nil
}

// inheritedWing returns the wing held by a strict majority of the legacy
// sources cited in the page's "## Sources" section, or "" with the reason.
func inheritedWing(content string, decided map[string]Item) (string, string) {
	section := sourcesSection(content)
	if section == "" {
		return "", "page has no Sources section to inherit a wing from"
	}
	total := 0
	votes := map[string]int{}
	for _, m := range linkSlugRE.FindAllStringSubmatch(section, -1) {
		d, ok := decided[strings.TrimSpace(m[1])]
		if !ok {
			continue // not a legacy source (already migrated, or not a source)
		}
		total++
		if d.Status == Auto {
			votes[d.Wing]++
		}
	}
	if total == 0 {
		return "", "none of its cited sources are in the legacy layout"
	}
	for w, n := range votes {
		if n*2 > total {
			return w, ""
		}
	}
	return "", fmt.Sprintf("its %d cited sources disagree on a wing or were not placed", total)
}

// blockCollisions turns an Auto item into an Exception when its destination
// exists, its slug is already used in some wing (graph identity is the slug
// alone), or two planned items would land on the same path.
func blockCollisions(bfs *brainfs.Brain, items []Item, taken map[string]string) {
	planned := map[string]int{}
	for _, it := range items {
		if it.Status == Auto {
			planned[it.To]++
		}
	}
	for i, it := range items {
		if it.Status != Auto {
			continue
		}
		switch {
		case planned[it.To] > 1:
			items[i].Status, items[i].Reason = Exception, fmt.Sprintf("another legacy page would land on %s", it.To)
		case exists(bfs, it.To):
			items[i].Status, items[i].Reason = Exception, fmt.Sprintf("%s already exists", it.To)
		default:
			if other, ok := taken[it.Slug]; ok {
				items[i].Status, items[i].Reason = Exception, fmt.Sprintf("slug %q is already used by %s", it.Slug, other)
			}
		}
	}
}

func exists(bfs *brainfs.Brain, rel string) bool {
	_, err := bfs.Lstat(rel)
	return err == nil
}

// slugsInWings maps every slug under wiki/<wing>/<hall>/ to its path.
func slugsInWings(bfs *brainfs.Brain) (map[string]string, error) {
	out := map[string]string{}
	wings, err := bfs.ReadDirDirs("wiki")
	if errors.Is(err, fs.ErrNotExist) {
		return out, nil
	}
	if err != nil {
		return nil, fmt.Errorf("list wiki: %w", err)
	}
	for _, w := range wings {
		if w == "sources" || w == "concepts" || w == "entities" {
			continue
		}
		halls, err := bfs.ReadDirDirs("wiki/" + w)
		if err != nil {
			continue
		}
		for _, h := range halls {
			names, err := bfs.ReadDirRegular("wiki/" + w + "/" + h)
			if err != nil {
				continue
			}
			for _, n := range names {
				if strings.HasSuffix(n, ".md") && n != "_index.md" {
					out[strings.TrimSuffix(n, ".md")] = "wiki/" + w + "/" + h + "/" + n
				}
			}
		}
	}
	return out, nil
}

// Apply performs the Auto items, one at a time, and stops at the first
// failure. Each move writes the destination with O_EXCL, then removes the
// source, so a failure never loses a page. Exceptions are not touched.
func Apply(brainDir string, items []Item) (int, error) {
	bfs, err := brainfs.Open(brainDir)
	if err != nil {
		return 0, fmt.Errorf("open brain: %w", err)
	}
	defer func() { _ = bfs.Close() }()

	n := 0
	touched := map[string]bool{}
	for _, it := range items {
		if it.Status != Auto {
			continue
		}
		content, err := bfs.ReadRegular(it.From)
		if err != nil {
			return n, fmt.Errorf("read %s: %w", it.From, err)
		}
		hall := path.Base(path.Dir(it.To))
		if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing, hall, it.Kind))); err != nil {
			return n, fmt.Errorf("write %s: %w", it.To, err)
		}
		if err := bfs.Remove(it.From); err != nil {
			return n, fmt.Errorf("moved to %s but removing %s failed: %w", it.To, it.From, err)
		}
		touched[it.Wing] = true
		n++
	}
	for w := range touched {
		_ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved
	}
	return n, nil
}

// stamp inserts wing/hall (and kind for facts) after the opening fence, leaving
// every other byte of the page alone. A page without frontmatter gets a block.
func stamp(content, w, hall, kind string) string {
	extra := "wing: " + w + "\nhall: " + hall + "\n"
	if hall == "facts" {
		extra += "kind: " + kind + "\n"
	}
	if strings.HasPrefix(content, "---\n") {
		return "---\n" + extra + content[4:]
	}
	return "---\n" + extra + "---\n" + content
}

// sourcesSection returns the text under a "## Sources" heading up to the next
// "## " heading.
func sourcesSection(content string) string {
	_, rest, ok := strings.Cut(content, "\n## Sources")
	if !ok {
		return ""
	}
	if i := strings.Index(rest, "\n## "); i >= 0 {
		rest = rest[:i]
	}
	return rest
}

func frontmatterValue(content, key string) string {
	if !strings.HasPrefix(content, "---\n") {
		return ""
	}
	block, _, _ := strings.Cut(content[4:], "\n---")
	for _, line := range strings.Split(block, "\n") {
		if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key {
			return strings.Trim(strings.TrimSpace(v), `"'`)
		}
	}
	return ""
}