// Package migrate moves pages out of the retired wiki/{sources,concepts, // entities}/ layout into wiki//{sources,facts}/ (hyperguild#205). // // It is a two-step tool on purpose. BuildPlan only reads and decides; Apply // performs exactly the items the plan marked Auto and nothing else. Sources are // placed by the wing Resolver. Concepts and entities have no wing of their own: // they inherit the strict-majority wing of the sources that cite them, which is // deterministic and needs no model. Whatever cannot be placed safely is an // Exception, left where it is for a human. package migrate import ( "context" "errors" "fmt" "io/fs" "path" "regexp" "sort" "strings" "github.com/mathiasbq/hyperguild/ingestion/internal/brain" "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "github.com/mathiasbq/hyperguild/ingestion/internal/wing" ) // Status is whether the plan moves a page by itself. type Status string const ( Auto Status = "auto" Exception Status = "exception" ) // Item is one planned move. type Item struct { From string // brain-relative, e.g. wiki/entities/jev.md To string // brain-relative; empty for an Exception without a proposal Slug string Kind string // source | concept | entity Wing string // decided wing, or the proposal for an Exception Status Status Method string // resolver | inherited | junk | blocked Confidence float64 Reason string } // ResolverFor returns the Resolver to use for one page. It takes the page's // own path so a scorer can exclude the page from its evidence. type ResolverFor func(excludePath string) *wing.Resolver var ( linkSlugRE = regexp.MustCompile(`\[\[([^|\]]+)`) legacy = []struct{ dir, kind string }{ {"wiki/sources", "source"}, {"wiki/concepts", "concept"}, {"wiki/entities", "entity"}, } ) // BuildPlan reads the legacy layout and decides, without writing anything. func BuildPlan(ctx context.Context, brainDir string, rf ResolverFor) ([]Item, error) { bfs, err := brainfs.Open(brainDir) if err != nil { return nil, fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() taken, err := slugsInWings(bfs) if err != nil { return nil, err } var items []Item decided := map[string]Item{} // source slug -> its decision for _, l := range legacy { names, err := bfs.ReadDirRegular(l.dir) if errors.Is(err, fs.ErrNotExist) { continue } if err != nil { return nil, fmt.Errorf("read %s: %w", l.dir, err) } for _, name := range names { if !strings.HasSuffix(name, ".md") || name == "_index.md" { continue } rel := l.dir + "/" + name slug := strings.TrimSuffix(name, ".md") it := Item{From: rel, Slug: slug, Kind: l.kind} if strings.HasPrefix(name, "tunnel-candidates-") { it.Status, it.Method, it.Reason = Exception, "junk", "auto-tunnel review log ingested as a source; not a source" items = append(items, it) continue } content, err := bfs.ReadRegular(rel) if err != nil { it.Status, it.Method, it.Reason = Exception, "blocked", fmt.Sprintf("unreadable: %v", err) items = append(items, it) continue } if l.kind == "source" { d := rf(rel).Resolve(ctx, wing.Note{Text: string(content), FrontmatterWing: frontmatterValue(string(content), "wing")}) it.Method, it.Wing, it.Confidence, it.Reason = "resolver", d.Wing, d.Confidence, d.Reason if d.Source == wing.Exception { it.Status = Exception } else { it.Status, it.To = Auto, "wiki/"+d.Wing+"/sources/"+name } decided[slug] = it } items = append(items, it) } } // Second pass: concepts and entities inherit from their sources. for i, it := range items { if it.Kind == "source" || it.Method == "junk" || it.Method == "blocked" { continue } content, err := bfs.ReadRegular(it.From) if err != nil { continue } it.Method = "inherited" w, reason := inheritedWing(string(content), decided) if w == "" { it.Status, it.Reason = Exception, reason } else { it.Status, it.Wing, it.To = Auto, w, "wiki/"+w+"/facts/"+it.Slug+".md" it.Reason = fmt.Sprintf("strict majority of its sources are in %q", w) } items[i] = it } blockCollisions(bfs, items, taken) sort.SliceStable(items, func(i, j int) bool { return items[i].From < items[j].From }) return items, nil } // inheritedWing returns the wing held by a strict majority of the legacy // sources cited in the page's "## Sources" section, or "" with the reason. func inheritedWing(content string, decided map[string]Item) (string, string) { section := sourcesSection(content) if section == "" { return "", "page has no Sources section to inherit a wing from" } total := 0 votes := map[string]int{} for _, m := range linkSlugRE.FindAllStringSubmatch(section, -1) { d, ok := decided[strings.TrimSpace(m[1])] if !ok { continue // not a legacy source (already migrated, or not a source) } total++ if d.Status == Auto { votes[d.Wing]++ } } if total == 0 { return "", "none of its cited sources are in the legacy layout" } for w, n := range votes { if n*2 > total { return w, "" } } return "", fmt.Sprintf("its %d cited sources disagree on a wing or were not placed", total) } // blockCollisions turns an Auto item into an Exception when its destination // exists, its slug is already used in some wing (graph identity is the slug // alone), or two planned items would land on the same path. func blockCollisions(bfs *brainfs.Brain, items []Item, taken map[string]string) { planned := map[string]int{} for _, it := range items { if it.Status == Auto { planned[it.To]++ } } for i, it := range items { if it.Status != Auto { continue } switch { case planned[it.To] > 1: items[i].Status, items[i].Reason = Exception, fmt.Sprintf("another legacy page would land on %s", it.To) case exists(bfs, it.To): items[i].Status, items[i].Reason = Exception, fmt.Sprintf("%s already exists", it.To) default: if other, ok := taken[it.Slug]; ok { items[i].Status, items[i].Reason = Exception, fmt.Sprintf("slug %q is already used by %s", it.Slug, other) } } } } func exists(bfs *brainfs.Brain, rel string) bool { _, err := bfs.Lstat(rel) return err == nil } // slugsInWings maps every slug under wiki/// to its path. func slugsInWings(bfs *brainfs.Brain) (map[string]string, error) { out := map[string]string{} wings, err := bfs.ReadDirDirs("wiki") if errors.Is(err, fs.ErrNotExist) { return out, nil } if err != nil { return nil, fmt.Errorf("list wiki: %w", err) } for _, w := range wings { if w == "sources" || w == "concepts" || w == "entities" { continue } halls, err := bfs.ReadDirDirs("wiki/" + w) if err != nil { continue } for _, h := range halls { names, err := bfs.ReadDirRegular("wiki/" + w + "/" + h) if err != nil { continue } for _, n := range names { if strings.HasSuffix(n, ".md") && n != "_index.md" { out[strings.TrimSuffix(n, ".md")] = "wiki/" + w + "/" + h + "/" + n } } } } return out, nil } // Apply performs the Auto items, one at a time, and stops at the first // failure. Each move writes the destination with O_EXCL, then removes the // source, so a failure never loses a page. Exceptions are not touched. func Apply(brainDir string, items []Item) (int, error) { bfs, err := brainfs.Open(brainDir) if err != nil { return 0, fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() n := 0 touched := map[string]bool{} for _, it := range items { if it.Status != Auto { continue } content, err := bfs.ReadRegular(it.From) if err != nil { return n, fmt.Errorf("read %s: %w", it.From, err) } hall := path.Base(path.Dir(it.To)) if err := bfs.CreateRegular(it.To, []byte(stamp(string(content), it.Wing, hall, it.Kind))); err != nil { return n, fmt.Errorf("write %s: %w", it.To, err) } if err := bfs.Remove(it.From); err != nil { return n, fmt.Errorf("moved to %s but removing %s failed: %w", it.To, it.From, err) } touched[it.Wing] = true n++ } for w := range touched { _ = brain.BuildWingIndex(brainDir, w) // best-effort upkeep; the pages are already moved } return n, nil } // stamp inserts wing/hall (and kind for facts) after the opening fence, leaving // every other byte of the page alone. A page without frontmatter gets a block. func stamp(content, w, hall, kind string) string { extra := "wing: " + w + "\nhall: " + hall + "\n" if hall == "facts" { extra += "kind: " + kind + "\n" } if strings.HasPrefix(content, "---\n") { return "---\n" + extra + content[4:] } return "---\n" + extra + "---\n" + content } // sourcesSection returns the text under a "## Sources" heading up to the next // "## " heading. func sourcesSection(content string) string { _, rest, ok := strings.Cut(content, "\n## Sources") if !ok { return "" } if i := strings.Index(rest, "\n## "); i >= 0 { rest = rest[:i] } return rest } func frontmatterValue(content, key string) string { if !strings.HasPrefix(content, "---\n") { return "" } block, _, _ := strings.Cut(content[4:], "\n---") for _, line := range strings.Split(block, "\n") { if k, v, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(k) == key { return strings.Trim(strings.TrimSpace(v), `"'`) } } return "" }