ingestion/internal/migrate/migrate.go

@@ -0,0 +1,309 @@
+// Package migrate moves pages out of the retired wiki/{sources,concepts,
+// entities}/ layout into wiki/<wing>/{sources,facts}/ (hyperguild#205).
+//
+// It is a two-step tool on purpose. BuildPlan only reads and decides; Apply
+// performs exactly the items the plan marked Auto and nothing else. Sources are
+// placed by the wing Resolver. Concepts and entities have no wing of their own:
+// they inherit the strict-majority wing of the sources that cite them, which is
+// deterministic and needs no model. Whatever cannot be placed safely is an
+// Exception, left where it is for a human.
+package migrate
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"path"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
+)
+
+// Status is whether the plan moves a page by itself.
+type Status string
+
+const (
+	Auto      Status = "auto"
+	Exception Status = "exception"
+)
+
+// Item is one planned move.
+type Item struct {
+	From       string // brain-relative, e.g. wiki/entities/jev.md
+	To         string // brain-relative; empty for an Exception without a proposal
+	Slug       string
+	Kind       string // source | concept | entity
+	Wing       string // decided wing, or the proposal for an Exception
+	Status     Status
+	Method     string // resolver | inherited | junk | blocked
+	Confidence float64
+	Reason     string
+}
+
+// ResolverFor returns the Resolver to use for one page. It takes the page's
+// own path so a scorer can exclude the page from its evidence.
+type ResolverFor func(excludePath string) *wing.Resolver
+
+var (
+	linkSlugRE = regexp.MustCompile(`\[\[([^|\]]+)`)
+	legacy     = []struct{ dir, kind string }{
+		{"wiki/sources", "source"},
+		{"wiki/concepts", "concept"},
+		{"wiki/entities", "entity"},
+	}
+)
+
+// BuildPlan reads the legacy layout and decides, without writing anything.
+func BuildPlan(ctx context.Context, brainDir string, rf ResolverFor) ([]Item, error) {
+	bfs, err := brainfs.Open(brainDir)
+	if err != nil {
+		return nil, fmt.Errorf("open brain: %w", err)
+	}
+	defer func() { _ = bfs.Close() }()
+
+	taken, err := slugsInWings(bfs)
+	if err != nil {
+		return nil, err
+	}
+
+	var items []Item
+	decided := map[string]Item{} // source slug -> its decision
+	for _, l := range legacy {
+		names, err := bfs.ReadDirRegular(l.dir)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read %s: %w", l.dir, err)
+		}
+		for _, name := range names {
+			if !strings.HasSuffix(name, ".md") || name == "_index.md" {
+				continue
+			}
+			rel := l.dir + "/" + name
+			slug := strings.TrimSuffix(name, ".md")
+			it := Item{From: rel, Slug: slug, Kind: l.kind}
+			if strings.HasPrefix(name, "tunnel-candidates-") {
+				it.Status, it.Method, it.Reason = Exception, "junk", "auto-tunnel review log ingested as a source; not a source"
+				items = append(items, it)
+				continue
+			}
+			content, err := bfs.ReadRegular(rel)
+			if err != nil {
+				it.Status, it.Method, it.Reason = Exception, "blocked", fmt.Sprintf("unreadable: %v", err)
+				items = append(items, it)
+				continue
+			}
+			if l.kind == "source" {
+				d := rf(rel).Resolve(ctx, wing.Note{Text: string(content), FrontmatterWing: frontmatterValue(string(content), "wing")})
+				it.Method, it.Wing, it.Confidence, it.Reason = "resolver", d.Wing, d.Confidence, d.Reason
+				if d.Source == wing.Exception {
+					it.Status = Exception
+				} else {
+					it.Status, it.To = Auto, "wiki/"+d.Wing+"/sources/"+name
+				}
+				decided[slug] = it
+			}
+			items = append(items, it)
+		}
+	}
+
+	// Second pass: concepts and entities inherit from their sources.
+	for i, it := range items {
+		if it.Kind == "source" || it.Method == "junk" || it.Method == "blocked" {
+			continue
+		}
+		content, err := bfs.ReadRegular(it.From)
+		if err != nil {
+			continue
+		}
+		it.Method = "inherited"
+		w, reason := inheritedWing(string(content), decided)
+		if w == "" {
+			it.Status, it.Reason = Exception, reason
+		} else {
+			it.Status, it.Wing, it.To = Auto, w, "wiki/"+w+"/facts/"+it.Slug+".md"
+			it.Reason = fmt.Sprintf("strict majority of its sources are in %q", w)
+		}
+		items[i] = it
+	}
+
+	blockCollisions(bfs, items, taken)
+	sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From })
+	return items, nil
+}
+
+// inheritedWing returns the wing held by a strict majority of the legacy
+// sources cited in the page's "## Sources" section, or "" with the reason.
+func inheritedWing(content string, decided map[string]Item) (string, string) {
+	section := sourcesSection(content)
+	if section == "" {
+		return "", "page has no Sources section to inherit a wing from"
+	}
+	total := 0
+	votes := map[string]int{}
+	for _, m := range linkSlugRE.FindAllStringSubmatch(section, -1) {
+		d, ok := decided[strings.TrimSpace(m[1])]
+		if !ok {
+			continue // not a legacy source (already migrated, or not a source)
+		}
+		total++
+		if d.Status == Auto {
+			votes[d.Wing]++
+		}
+	}
+	if total == 0 {
+		return "", "none of its cited sources are in the legacy layout"
+	}
+	for w, n := range votes {
+		if n*2 > total {
+			return w, ""
+		}
+	}
+	return "", fmt.Sprintf("its %d cited sources disagree on a wing or were not placed", total)
+}
+
+// blockCollisions turns an Auto item into an Exception when its destination
+// exists, its slug is already used in some wing (graph identity is the slug
+// alone), or two planned items would land on the same path.
+func blockCollisions(bfs *brainfs.Brain, items []Item, taken map[string]string) {
+	planned := map[string]int{}
+	for _, it := range items {
+		if it.Status == Auto {
+			planned[it.To]++
+		}
+	}
+	for i, it := range items {
+		if it.Status != Auto {
+			continue
+		}
+		switch {
+		case planned[it.To] > 1:
+			items[i].Status, items[i].Reason = Exception, fmt.Sprintf("another legacy page would land on %s", it.To)
+		case exists(bfs, it.To):
+			items[i].Status, items[i].Reason = Exception, fmt.Sprintf("%s already exists", it.To)
+		default:
+			if other, ok := taken[it.Slug]; ok {
+				items[i].Status, items[i].Reason = Exception, fmt.Sprintf("slug %q is already used by %s", it.Slug, other)
+			}
+		}
+	}
+}
+
+func exists(bfs *brainfs.Brain, rel string) bool {
+	_, err := bfs.Lstat(rel)
+	return err == nil
+}
+
+// slugsInWings maps every slug under wiki/<wing>/<hall>/ to its path.
+func slugsInWings(bfs *brainfs.Brain) (map[string]string, error) {
+	out := map[string]string{}
+	wings, err := bfs.ReadDirDirs("wiki")
+	if errors.Is(err, fs.ErrNotExist) {
+		return out, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("list wiki: %w", err)
+	}
+	for _, w := range wings {
+		if w == "sources" || w == "concepts" || w == "entities" {
+			continue
+		}
+		halls, err := bfs.ReadDirDirs("wiki/" + w)
+		if err != nil {
+			continue
+		}
+		for _, h := range halls {
+			names, err := bfs.ReadDirRegular("wiki/" + w + "/" + h)
+			if err != nil {
+				continue
+			}
+			for _, n := range names {
+				if strings.HasSuffix(n, ".md") && n != "_index.md" {
+					out[strings.TrimSuffix(n, ".md")] = "wiki/" + w + "/" + h + "/" + n
+				}
+			}
+		}
+	}
+	return out, nil
+}
+
+// Apply performs the Auto items, one at a time, and stops at the first
+// failure. Each move writes the destination with O_EXCL, then removes the
+// source, so a failure never loses a page. Exceptions are not touched.
+func Apply(brainDir string, items []Item) (int, error) {
+	bfs, err := brainfs.Open(brainDir)
+	if err != nil {
+		return 0, fmt.Errorf("open brain: %w", err)
+	}
+	defer func() { _ = bfs.Close() }()
+
+	n := 0
+	touched := map[string]bool{}
+	for _, it := range items {
+		if it.Status != Auto {
+			continue
+		}
+		content, err := bfs.ReadRegular(it.From)
+		if err != nil {
+			return n, fmt.Errorf("read %s: %w", it.From, err)
+		}
+		hall := path.Base(path.Dir(it.To))
+		if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing, hall, it.Kind))); err != nil {
+			return n, fmt.Errorf("write %s: %w", it.To, err)
+		}
+		if err := bfs.Remove(it.From); err != nil {
+			return n, fmt.Errorf("moved to %s but removing %s failed: %w", it.To, it.From, err)
+		}
+		touched[it.Wing] = true
+		n++
+	}
+	for w := range touched {
+		_ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved
+	}
+	return n, nil
+}
+
+// stamp inserts wing/hall (and kind for facts) after the opening fence, leaving
+// every other byte of the page alone. A page without frontmatter gets a block.
+func stamp(content, w, hall, kind string) string {
+	extra := "wing: " + w + "\nhall: " + hall + "\n"
+	if hall == "facts" {
+		extra += "kind: " + kind + "\n"
+	}
+	if strings.HasPrefix(content, "---\n") {
+		return "---\n" + extra + content[4:]
+	}
+	return "---\n" + extra + "---\n" + content
+}
+
+// sourcesSection returns the text under a "## Sources" heading up to the next
+// "## " heading.
+func sourcesSection(content string) string {
+	_, rest, ok := strings.Cut(content, "\n## Sources")
+	if !ok {
+		return ""
+	}
+	if i := strings.Index(rest, "\n## "); i >= 0 {
+		rest = rest[:i]
+	}
+	return rest
+}
+
+func frontmatterValue(content, key string) string {
+	if !strings.HasPrefix(content, "---\n") {
+		return ""
+	}
+	block, _, _ := strings.Cut(content[4:], "\n---")
+	for _, line := range strings.Split(block, "\n") {
+		if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key {
+			return strings.Trim(strings.TrimSpace(v), `"'`)
+		}
+	}
+	return ""
+}

ingestion/internal/migrate/migrate_test.go

@@ -0,0 +1,243 @@
+package migrate_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/migrate"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
+)
+
+// keywordClassifier picks a wing by a marker word in the text, and the scorer
+// agrees by the same marker, so a page "belongs" to the wing whose marker it holds.
+type keywordClassifier struct{}
+
+func (keywordClassifier) Classify(_ context.Context, text string, _ []string) (wing.Suggestion, error) {
+	for _, w := range []string{"alpha", "beta"} {
+		if strings.Contains(text, strings.ToUpper(w)) {
+			return wing.Suggestion{Wing: w, Confidence: 0.95, Reason: "marker"}, nil
+		}
+	}
+	return wing.Suggestion{}, nil
+}
+
+type keywordScorer struct{}
+
+func (keywordScorer) Score(text string) map[string]float64 {
+	out := map[string]float64{}
+	for _, w := range []string{"alpha", "beta"} {
+		if strings.Contains(text, strings.ToUpper(w)) {
+			out[w] = 1
+		}
+	}
+	return out
+}
+
+func resolverFor(string) *wing.Resolver {
+	return &wing.Resolver{
+		Known:      []string{"alpha", "beta", "client"},
+		Eligible:   func(w string) bool { return w != "client" },
+		Classifier: keywordClassifier{},
+		Scorer:     keywordScorer{},
+		Threshold:  0.8,
+	}
+}
+
+func page(t *testing.T, dir, rel, body string) {
+	t.Helper()
+	full := filepath.Join(dir, rel)
+	require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
+	require.NoError(t, os.WriteFile(full, []byte(body), 0o644))
+}
+
+func src(title, marker string) string {
+	return "---\ntitle: " + title + "\ntype: video\n---\n\n## Summary\n\n" + marker + " content.\n"
+}
+
+func entity(title string, sources ...string) string {
+	var b strings.Builder
+	b.WriteString("---\ntitle: " + title + "\ntype: tool\n---\n\n## Definition\n\nA thing.\n\n## Sources\n\n")
+	for _, s := range sources {
+		b.WriteString("- [[" + s + "|" + s + "]]\n")
+	}
+	return b.String()
+}
+
+func byFrom(items []migrate.Item) map[string]migrate.Item {
+	m := map[string]migrate.Item{}
+	for _, it := range items {
+		m[it.From] = it
+	}
+	return m
+}
+
+func TestPlan_AResolvedSourceGoesToItsWingsSourcesHall(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	it := byFrom(items)["wiki/sources/talk-a.md"]
+	assert.Equal(t, migrate.Auto, it.Status)
+	assert.Equal(t, "wiki/alpha/sources/talk-a.md", it.To)
+}
+
+func TestPlan_AutoTunnelLogsAreJunkNotSources(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("t", "ALPHA"))
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	it := byFrom(items)["wiki/sources/tunnel-candidates-2026-06-14.md"]
+	assert.Equal(t, migrate.Exception, it.Status, "never auto-moved, whatever the classifier says")
+	assert.Equal(t, "junk", it.Method)
+}
+
+func TestPlan_AnEntityFollowsTheMajorityWingOfItsSources(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/a1.md", src("A1", "ALPHA"))
+	page(t, dir, "wiki/sources/a2.md", src("A2", "ALPHA"))
+	page(t, dir, "wiki/sources/b1.md", src("B1", "BETA"))
+	page(t, dir, "wiki/entities/jev.md", entity("Jev", "a1", "a2", "b1"))
+	page(t, dir, "wiki/concepts/factory.md", "---\ntitle: Factory\n---\n\n## Definition\n\nx\n\n## Sources\n\n- [[a1|A1]]\n")
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	m := byFrom(items)
+	assert.Equal(t, "wiki/alpha/facts/jev.md", m["wiki/entities/jev.md"].To)
+	assert.Equal(t, "entity", m["wiki/entities/jev.md"].Kind)
+	assert.Equal(t, "wiki/alpha/facts/factory.md", m["wiki/concepts/factory.md"].To)
+	assert.Equal(t, "concept", m["wiki/concepts/factory.md"].Kind)
+}
+
+func TestPlan_ATieBetweenWingsIsAnException(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/a1.md", src("A1", "ALPHA"))
+	page(t, dir, "wiki/sources/b1.md", src("B1", "BETA"))
+	page(t, dir, "wiki/entities/jev.md", entity("Jev", "a1", "b1"))
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	it := byFrom(items)["wiki/entities/jev.md"]
+	assert.Equal(t, migrate.Exception, it.Status)
+	assert.Contains(t, it.Reason, "disagree")
+}
+
+func TestPlan_AnEntityWhoseSourcesWereNotPlacedIsAnException(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/mystery.md", src("Mystery", "nothing recognisable"))
+	page(t, dir, "wiki/entities/orphan.md", entity("Orphan", "mystery"))
+	page(t, dir, "wiki/entities/nosources.md", "---\ntitle: No Sources\n---\n\n## Definition\n\nx\n")
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	m := byFrom(items)
+	assert.Equal(t, migrate.Exception, m["wiki/entities/orphan.md"].Status)
+	assert.Equal(t, migrate.Exception, m["wiki/entities/nosources.md"].Status)
+}
+
+func TestPlan_ADestinationThatAlreadyExistsIsAnException(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
+	page(t, dir, "wiki/alpha/sources/talk-a.md", "someone else's page")
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	it := byFrom(items)["wiki/sources/talk-a.md"]
+	assert.Equal(t, migrate.Exception, it.Status)
+	assert.Contains(t, it.Reason, "already exists", "the occupied destination is named as the reason, not a generic slug clash")
+}
+
+func TestPlan_ASlugAlreadyUsedInAnotherWingIsAnException(t *testing.T) {
+	// Graph identity is the slug alone; two pages with one slug would merge.
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
+	page(t, dir, "wiki/beta/facts/talk-a.md", "a beta page with the same slug")
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	it := byFrom(items)["wiki/sources/talk-a.md"]
+	assert.Equal(t, migrate.Exception, it.Status)
+	assert.Contains(t, it.Reason, "slug")
+}
+
+func TestApply_MovesOnlyAutoItemsAndStampsFrontmatterWithoutTouchingTheBody(t *testing.T) {
+	dir := t.TempDir()
+	body := src("Talk A", "ALPHA") + "\nSee [[jev|Jev]] and [[unknown-slug]].\n"
+	page(t, dir, "wiki/sources/talk-a.md", body)
+	page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("t", "ALPHA"))
+	page(t, dir, "wiki/entities/jev.md", entity("Jev", "talk-a"))
+
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	n, err := migrate.Apply(dir, items)
+	require.NoError(t, err)
+	assert.Equal(t, 2, n)
+
+	moved, err := os.ReadFile(filepath.Join(dir, "wiki/alpha/sources/talk-a.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(moved), "wing: alpha")
+	assert.Contains(t, string(moved), "hall: sources")
+	assert.Contains(t, string(moved), "See [[jev|Jev]] and [[unknown-slug]].", "links and body are byte-preserved")
+
+	ent, err := os.ReadFile(filepath.Join(dir, "wiki/alpha/facts/jev.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(ent), "kind: entity")
+
+	_, err = os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md"))
+	assert.ErrorIs(t, err, os.ErrNotExist, "the original is removed after the copy is written")
+	_, err = os.Stat(filepath.Join(dir, "wiki/sources/tunnel-candidates-2026-06-14.md"))
+	assert.NoError(t, err, "an exception is left exactly where it is")
+}
+
+func TestApply_NeverOverwritesAndKeepsTheSourceWhenTheDestinationAppearsLate(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	page(t, dir, "wiki/alpha/sources/talk-a.md", "appeared after the plan")
+
+	n, err := migrate.Apply(dir, items)
+	require.Error(t, err)
+	assert.Zero(t, n)
+	got, _ := os.ReadFile(filepath.Join(dir, "wiki/alpha/sources/talk-a.md"))
+	assert.Equal(t, "appeared after the plan", string(got))
+	_, statErr := os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md"))
+	assert.NoError(t, statErr, "the source survives a failed write")
+}
+
+func TestPlan_IsEmptyOnceEverythingIsMigrated(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	_, err = migrate.Apply(dir, items)
+	require.NoError(t, err)
+
+	again, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	assert.Empty(t, again)
+}
+
+// An exception that blocked on a collision still carries a destination; Apply
+// must treat Status, not the presence of a path, as the decision.
+func TestApply_LeavesACollisionExceptionAloneEvenThoughItHasADestination(t *testing.T) {
+	dir := t.TempDir()
+	page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA"))
+	page(t, dir, "wiki/alpha/sources/talk-a.md", "occupied")
+	items, err := migrate.BuildPlan(context.Background(), dir, resolverFor)
+	require.NoError(t, err)
+	require.Equal(t, migrate.Exception, byFrom(items)["wiki/sources/talk-a.md"].Status)
+
+	n, err := migrate.Apply(dir, items)
+	require.NoError(t, err)
+	assert.Zero(t, n)
+	_, statErr := os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md"))
+	assert.NoError(t, statErr)
+}