From dd4a825f0a57ad7da31db422efd9752e2ed5ca6b Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 20 Oct 2026 06:14:52 +0200 Subject: [PATCH] feat(migrate): plan and apply the move out of the retired wiki layout (#205 S5) Sources are placed by the wing Resolver; concepts/entities inherit the strict majority wing of the sources that cite them. Collisions, junk (auto-tunnel logs) and anything unplaceable are Exceptions left in place. Apply writes the destination O_EXCL then removes the source, only for Auto items. Mutation-checked (junk guard, majority, slug identity, status gate, no-overwrite, destination-exists). Co-Authored-By: Claude Sonnet 5.5 --- ingestion/internal/migrate/migrate.go | 309 +++++++++++++++++++++ ingestion/internal/migrate/migrate_test.go | 243 ++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 ingestion/internal/migrate/migrate.go create mode 100644 ingestion/internal/migrate/migrate_test.go diff --git a/ingestion/internal/migrate/migrate.go b/ingestion/internal/migrate/migrate.go new file mode 100644 index 0000000..d92001b --- /dev/null +++ b/ingestion/internal/migrate/migrate.go @@ -0,0 +1,309 @@ +// Package migrate moves pages out of the retired wiki/{sources,concepts, +// entities}/ layout into wiki//{sources,facts}/ (hyperguild#205). +// +// It is a two-step tool on purpose. BuildPlan only reads and decides; Apply +// performs exactly the items the plan marked Auto and nothing else. Sources are +// placed by the wing Resolver. Concepts and entities have no wing of their own: +// they inherit the strict-majority wing of the sources that cite them, which is +// deterministic and needs no model. Whatever cannot be placed safely is an +// Exception, left where it is for a human. +package migrate + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path" + "regexp" + "sort" + "strings" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brain" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" +) + +// Status is whether the plan moves a page by itself. +type Status string + +const ( + Auto Status = "auto" + Exception Status = "exception" +) + +// Item is one planned move. +type Item struct { + From string // brain-relative, e.g. wiki/entities/jev.md + To string // brain-relative; empty for an Exception without a proposal + Slug string + Kind string // source | concept | entity + Wing string // decided wing, or the proposal for an Exception + Status Status + Method string // resolver | inherited | junk | blocked + Confidence float64 + Reason string +} + +// ResolverFor returns the Resolver to use for one page. It takes the page's +// own path so a scorer can exclude the page from its evidence. +type ResolverFor func(excludePath string) *wing.Resolver + +var ( + linkSlugRE = regexp.MustCompile(`\[\[([^|\]]+)`) + legacy = []struct{ dir, kind string }{ + {"wiki/sources", "source"}, + {"wiki/concepts", "concept"}, + {"wiki/entities", "entity"}, + } +) + +// BuildPlan reads the legacy layout and decides, without writing anything. +func BuildPlan(ctx context.Context, brainDir string, rf ResolverFor) ([]Item, error) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return nil, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + + taken, err := slugsInWings(bfs) + if err != nil { + return nil, err + } + + var items []Item + decided := map[string]Item{} // source slug -> its decision + for _, l := range legacy { + names, err := bfs.ReadDirRegular(l.dir) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", l.dir, err) + } + for _, name := range names { + if !strings.HasSuffix(name, ".md") || name == "_index.md" { + continue + } + rel := l.dir + "/" + name + slug := strings.TrimSuffix(name, ".md") + it := Item{From: rel, Slug: slug, Kind: l.kind} + if strings.HasPrefix(name, "tunnel-candidates-") { + it.Status, it.Method, it.Reason = Exception, "junk", "auto-tunnel review log ingested as a source; not a source" + items = append(items, it) + continue + } + content, err := bfs.ReadRegular(rel) + if err != nil { + it.Status, it.Method, it.Reason = Exception, "blocked", fmt.Sprintf("unreadable: %v", err) + items = append(items, it) + continue + } + if l.kind == "source" { + d := rf(rel).Resolve(ctx, wing.Note{Text: string(content), FrontmatterWing: frontmatterValue(string(content), "wing")}) + it.Method, it.Wing, it.Confidence, it.Reason = "resolver", d.Wing, d.Confidence, d.Reason + if d.Source == wing.Exception { + it.Status = Exception + } else { + it.Status, it.To = Auto, "wiki/"+d.Wing+"/sources/"+name + } + decided[slug] = it + } + items = append(items, it) + } + } + + // Second pass: concepts and entities inherit from their sources. + for i, it := range items { + if it.Kind == "source" || it.Method == "junk" || it.Method == "blocked" { + continue + } + content, err := bfs.ReadRegular(it.From) + if err != nil { + continue + } + it.Method = "inherited" + w, reason := inheritedWing(string(content), decided) + if w == "" { + it.Status, it.Reason = Exception, reason + } else { + it.Status, it.Wing, it.To = Auto, w, "wiki/"+w+"/facts/"+it.Slug+".md" + it.Reason = fmt.Sprintf("strict majority of its sources are in %q", w) + } + items[i] = it + } + + blockCollisions(bfs, items, taken) + sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) + return items, nil +} + +// inheritedWing returns the wing held by a strict majority of the legacy +// sources cited in the page's "## Sources" section, or "" with the reason. +func inheritedWing(content string, decided map[string]Item) (string, string) { + section := sourcesSection(content) + if section == "" { + return "", "page has no Sources section to inherit a wing from" + } + total := 0 + votes := map[string]int{} + for _, m := range linkSlugRE.FindAllStringSubmatch(section, -1) { + d, ok := decided[strings.TrimSpace(m[1])] + if !ok { + continue // not a legacy source (already migrated, or not a source) + } + total++ + if d.Status == Auto { + votes[d.Wing]++ + } + } + if total == 0 { + return "", "none of its cited sources are in the legacy layout" + } + for w, n := range votes { + if n*2 > total { + return w, "" + } + } + return "", fmt.Sprintf("its %d cited sources disagree on a wing or were not placed", total) +} + +// blockCollisions turns an Auto item into an Exception when its destination +// exists, its slug is already used in some wing (graph identity is the slug +// alone), or two planned items would land on the same path. +func blockCollisions(bfs *brainfs.Brain, items []Item, taken map[string]string) { + planned := map[string]int{} + for _, it := range items { + if it.Status == Auto { + planned[it.To]++ + } + } + for i, it := range items { + if it.Status != Auto { + continue + } + switch { + case planned[it.To] > 1: + items[i].Status, items[i].Reason = Exception, fmt.Sprintf("another legacy page would land on %s", it.To) + case exists(bfs, it.To): + items[i].Status, items[i].Reason = Exception, fmt.Sprintf("%s already exists", it.To) + default: + if other, ok := taken[it.Slug]; ok { + items[i].Status, items[i].Reason = Exception, fmt.Sprintf("slug %q is already used by %s", it.Slug, other) + } + } + } +} + +func exists(bfs *brainfs.Brain, rel string) bool { + _, err := bfs.Lstat(rel) + return err == nil +} + +// slugsInWings maps every slug under wiki/// to its path. +func slugsInWings(bfs *brainfs.Brain) (map[string]string, error) { + out := map[string]string{} + wings, err := bfs.ReadDirDirs("wiki") + if errors.Is(err, fs.ErrNotExist) { + return out, nil + } + if err != nil { + return nil, fmt.Errorf("list wiki: %w", err) + } + for _, w := range wings { + if w == "sources" || w == "concepts" || w == "entities" { + continue + } + halls, err := bfs.ReadDirDirs("wiki/" + w) + if err != nil { + continue + } + for _, h := range halls { + names, err := bfs.ReadDirRegular("wiki/" + w + "/" + h) + if err != nil { + continue + } + for _, n := range names { + if strings.HasSuffix(n, ".md") && n != "_index.md" { + out[strings.TrimSuffix(n, ".md")] = "wiki/" + w + "/" + h + "/" + n + } + } + } + } + return out, nil +} + +// Apply performs the Auto items, one at a time, and stops at the first +// failure. Each move writes the destination with O_EXCL, then removes the +// source, so a failure never loses a page. Exceptions are not touched. +func Apply(brainDir string, items []Item) (int, error) { + bfs, err := brainfs.Open(brainDir) + if err != nil { + return 0, fmt.Errorf("open brain: %w", err) + } + defer func() { _ = bfs.Close() }() + + n := 0 + touched := map[string]bool{} + for _, it := range items { + if it.Status != Auto { + continue + } + content, err := bfs.ReadRegular(it.From) + if err != nil { + return n, fmt.Errorf("read %s: %w", it.From, err) + } + hall := path.Base(path.Dir(it.To)) + if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing, hall, it.Kind))); err != nil { + return n, fmt.Errorf("write %s: %w", it.To, err) + } + if err := bfs.Remove(it.From); err != nil { + return n, fmt.Errorf("moved to %s but removing %s failed: %w", it.To, it.From, err) + } + touched[it.Wing] = true + n++ + } + for w := range touched { + _ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved + } + return n, nil +} + +// stamp inserts wing/hall (and kind for facts) after the opening fence, leaving +// every other byte of the page alone. A page without frontmatter gets a block. +func stamp(content, w, hall, kind string) string { + extra := "wing: " + w + "\nhall: " + hall + "\n" + if hall == "facts" { + extra += "kind: " + kind + "\n" + } + if strings.HasPrefix(content, "---\n") { + return "---\n" + extra + content[4:] + } + return "---\n" + extra + "---\n" + content +} + +// sourcesSection returns the text under a "## Sources" heading up to the next +// "## " heading. +func sourcesSection(content string) string { + _, rest, ok := strings.Cut(content, "\n## Sources") + if !ok { + return "" + } + if i := strings.Index(rest, "\n## "); i >= 0 { + rest = rest[:i] + } + return rest +} + +func frontmatterValue(content, key string) string { + if !strings.HasPrefix(content, "---\n") { + return "" + } + block, _, _ := strings.Cut(content[4:], "\n---") + for _, line := range strings.Split(block, "\n") { + if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key { + return strings.Trim(strings.TrimSpace(v), `"'`) + } + } + return "" +} diff --git a/ingestion/internal/migrate/migrate_test.go b/ingestion/internal/migrate/migrate_test.go new file mode 100644 index 0000000..6dbc1d8 --- /dev/null +++ b/ingestion/internal/migrate/migrate_test.go @@ -0,0 +1,243 @@ +package migrate_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/migrate" + "github.com/mathiasbq/hyperguild/ingestion/internal/wing" +) + +// keywordClassifier picks a wing by a marker word in the text, and the scorer +// agrees by the same marker, so a page "belongs" to the wing whose marker it holds. +type keywordClassifier struct{} + +func (keywordClassifier) Classify(_ context.Context, text string, _ []string) (wing.Suggestion, error) { + for _, w := range []string{"alpha", "beta"} { + if strings.Contains(text, strings.ToUpper(w)) { + return wing.Suggestion{Wing: w, Confidence: 0.95, Reason: "marker"}, nil + } + } + return wing.Suggestion{}, nil +} + +type keywordScorer struct{} + +func (keywordScorer) Score(text string) map[string]float64 { + out := map[string]float64{} + for _, w := range []string{"alpha", "beta"} { + if strings.Contains(text, strings.ToUpper(w)) { + out[w] = 1 + } + } + return out +} + +func resolverFor(string) *wing.Resolver { + return &wing.Resolver{ + Known: []string{"alpha", "beta", "client"}, + Eligible: func(w string) bool { return w != "client" }, + Classifier: keywordClassifier{}, + Scorer: keywordScorer{}, + Threshold: 0.8, + } +} + +func page(t *testing.T, dir, rel, body string) { + t.Helper() + full := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(t, os.WriteFile(full, []byte(body), 0o644)) +} + +func src(title, marker string) string { + return "---\ntitle: " + title + "\ntype: video\n---\n\n## Summary\n\n" + marker + " content.\n" +} + +func entity(title string, sources ...string) string { + var b strings.Builder + b.WriteString("---\ntitle: " + title + "\ntype: tool\n---\n\n## Definition\n\nA thing.\n\n## Sources\n\n") + for _, s := range sources { + b.WriteString("- [[" + s + "|" + s + "]]\n") + } + return b.String() +} + +func byFrom(items []migrate.Item) map[string]migrate.Item { + m := map[string]migrate.Item{} + for _, it := range items { + m[it.From] = it + } + return m +} + +func TestPlan_AResolvedSourceGoesToItsWingsSourcesHall(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA")) + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + it := byFrom(items)["wiki/sources/talk-a.md"] + assert.Equal(t, migrate.Auto, it.Status) + assert.Equal(t, "wiki/alpha/sources/talk-a.md", it.To) +} + +func TestPlan_AutoTunnelLogsAreJunkNotSources(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("t", "ALPHA")) + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + it := byFrom(items)["wiki/sources/tunnel-candidates-2026-06-14.md"] + assert.Equal(t, migrate.Exception, it.Status, "never auto-moved, whatever the classifier says") + assert.Equal(t, "junk", it.Method) +} + +func TestPlan_AnEntityFollowsTheMajorityWingOfItsSources(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a1.md", src("A1", "ALPHA")) + page(t, dir, "wiki/sources/a2.md", src("A2", "ALPHA")) + page(t, dir, "wiki/sources/b1.md", src("B1", "BETA")) + page(t, dir, "wiki/entities/jev.md", entity("Jev", "a1", "a2", "b1")) + page(t, dir, "wiki/concepts/factory.md", "---\ntitle: Factory\n---\n\n## Definition\n\nx\n\n## Sources\n\n- [[a1|A1]]\n") + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, "wiki/alpha/facts/jev.md", m["wiki/entities/jev.md"].To) + assert.Equal(t, "entity", m["wiki/entities/jev.md"].Kind) + assert.Equal(t, "wiki/alpha/facts/factory.md", m["wiki/concepts/factory.md"].To) + assert.Equal(t, "concept", m["wiki/concepts/factory.md"].Kind) +} + +func TestPlan_ATieBetweenWingsIsAnException(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/a1.md", src("A1", "ALPHA")) + page(t, dir, "wiki/sources/b1.md", src("B1", "BETA")) + page(t, dir, "wiki/entities/jev.md", entity("Jev", "a1", "b1")) + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + it := byFrom(items)["wiki/entities/jev.md"] + assert.Equal(t, migrate.Exception, it.Status) + assert.Contains(t, it.Reason, "disagree") +} + +func TestPlan_AnEntityWhoseSourcesWereNotPlacedIsAnException(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/mystery.md", src("Mystery", "nothing recognisable")) + page(t, dir, "wiki/entities/orphan.md", entity("Orphan", "mystery")) + page(t, dir, "wiki/entities/nosources.md", "---\ntitle: No Sources\n---\n\n## Definition\n\nx\n") + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + m := byFrom(items) + assert.Equal(t, migrate.Exception, m["wiki/entities/orphan.md"].Status) + assert.Equal(t, migrate.Exception, m["wiki/entities/nosources.md"].Status) +} + +func TestPlan_ADestinationThatAlreadyExistsIsAnException(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA")) + page(t, dir, "wiki/alpha/sources/talk-a.md", "someone else's page") + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + it := byFrom(items)["wiki/sources/talk-a.md"] + assert.Equal(t, migrate.Exception, it.Status) + assert.Contains(t, it.Reason, "already exists", "the occupied destination is named as the reason, not a generic slug clash") +} + +func TestPlan_ASlugAlreadyUsedInAnotherWingIsAnException(t *testing.T) { + // Graph identity is the slug alone; two pages with one slug would merge. + dir := t.TempDir() + page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA")) + page(t, dir, "wiki/beta/facts/talk-a.md", "a beta page with the same slug") + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + it := byFrom(items)["wiki/sources/talk-a.md"] + assert.Equal(t, migrate.Exception, it.Status) + assert.Contains(t, it.Reason, "slug") +} + +func TestApply_MovesOnlyAutoItemsAndStampsFrontmatterWithoutTouchingTheBody(t *testing.T) { + dir := t.TempDir() + body := src("Talk A", "ALPHA") + "\nSee [[jev|Jev]] and [[unknown-slug]].\n" + page(t, dir, "wiki/sources/talk-a.md", body) + page(t, dir, "wiki/sources/tunnel-candidates-2026-06-14.md", src("t", "ALPHA")) + page(t, dir, "wiki/entities/jev.md", entity("Jev", "talk-a")) + + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + n, err := migrate.Apply(dir, items) + require.NoError(t, err) + assert.Equal(t, 2, n) + + moved, err := os.ReadFile(filepath.Join(dir, "wiki/alpha/sources/talk-a.md")) + require.NoError(t, err) + assert.Contains(t, string(moved), "wing: alpha") + assert.Contains(t, string(moved), "hall: sources") + assert.Contains(t, string(moved), "See [[jev|Jev]] and [[unknown-slug]].", "links and body are byte-preserved") + + ent, err := os.ReadFile(filepath.Join(dir, "wiki/alpha/facts/jev.md")) + require.NoError(t, err) + assert.Contains(t, string(ent), "kind: entity") + + _, err = os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md")) + assert.ErrorIs(t, err, os.ErrNotExist, "the original is removed after the copy is written") + _, err = os.Stat(filepath.Join(dir, "wiki/sources/tunnel-candidates-2026-06-14.md")) + assert.NoError(t, err, "an exception is left exactly where it is") +} + +func TestApply_NeverOverwritesAndKeepsTheSourceWhenTheDestinationAppearsLate(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA")) + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + page(t, dir, "wiki/alpha/sources/talk-a.md", "appeared after the plan") + + n, err := migrate.Apply(dir, items) + require.Error(t, err) + assert.Zero(t, n) + got, _ := os.ReadFile(filepath.Join(dir, "wiki/alpha/sources/talk-a.md")) + assert.Equal(t, "appeared after the plan", string(got)) + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md")) + assert.NoError(t, statErr, "the source survives a failed write") +} + +func TestPlan_IsEmptyOnceEverythingIsMigrated(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA")) + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + _, err = migrate.Apply(dir, items) + require.NoError(t, err) + + again, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + assert.Empty(t, again) +} + +// An exception that blocked on a collision still carries a destination; Apply +// must treat Status, not the presence of a path, as the decision. +func TestApply_LeavesACollisionExceptionAloneEvenThoughItHasADestination(t *testing.T) { + dir := t.TempDir() + page(t, dir, "wiki/sources/talk-a.md", src("Talk A", "ALPHA")) + page(t, dir, "wiki/alpha/sources/talk-a.md", "occupied") + items, err := migrate.BuildPlan(context.Background(), dir, resolverFor) + require.NoError(t, err) + require.Equal(t, migrate.Exception, byFrom(items)["wiki/sources/talk-a.md"].Status) + + n, err := migrate.Apply(dir, items) + require.NoError(t, err) + assert.Zero(t, n) + _, statErr := os.Stat(filepath.Join(dir, "wiki/sources/talk-a.md")) + assert.NoError(t, statErr) +}