mathiasandClaude Opus 5.5
aee5f66398
The ingest pipeline read wiki/sources/<slug>.md and schema.md (both go into the LLM prompt), the pages it revises, and the pages it adds source refs to, with os.ReadFile; BackfillRefs listed wiki/sources with os.ReadDir. A symlinked file's content reached the prompt and the written pages (1.6M leaks in the race), and a FIFO hung the ingest. classification.Load (the I1 sovereignty levels) read classification.yaml the same way and hung startup on a FIFO. - pipeline: readSourcePage, loadSchema, reviseOrCreate, injectSourceRefs read through readBrainFile (brainfs ReadRegular); reviseOrCreate takes the brain dir and reads page.Path, a missing page is still created and a non-regular one is an error. BackfillRefs lists with ReadDirRegular and reads with ReadRegular. - classification.Load: read through brainfs; a symlinked or special classification.yaml is a hard error (as a malformed one is); an absent file or brain dir still yields the default config. Tests use in-brain targets and the [[slug|title]] link form the pipeline actually follows (bare [[slug]] links are ignored by it, which made a first draft of the backfill test vacuous). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
114 lines
3.1 KiB
Go
114 lines
3.1 KiB
Go
// ingestion/internal/pipeline/refs.go
|
|
package pipeline
|
|
|
|
import (
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
|
)
|
|
|
|
var wikilinkRE = regexp.MustCompile(`\[\[([^|\]]+)\|`)
|
|
|
|
// injectSourceRefs finds the source page in the proposed batch, extracts its
|
|
// wikilinks, and injects a back-reference into every linked concept or entity page.
|
|
// Pages that exist on disk but are not in the current batch are loaded and
|
|
// appended so they will be updated on write.
|
|
func injectSourceRefs(pages []wiki.Page, inventory map[wiki.PageType][]wiki.Entry, brainDir string) []wiki.Page {
|
|
sourceSlug, sourceTitle, found := findSourcePage(pages)
|
|
if !found {
|
|
return pages
|
|
}
|
|
|
|
var sourceContent string
|
|
for _, p := range pages {
|
|
if strings.HasPrefix(p.Path, "wiki/sources/") &&
|
|
strings.TrimSuffix(filepath.Base(p.Path), ".md") == sourceSlug {
|
|
sourceContent = p.Content
|
|
break
|
|
}
|
|
}
|
|
|
|
linkedSlugs := extractWikilinks(sourceContent)
|
|
sourceRef := "- [[" + sourceSlug + "|" + sourceTitle + "]]"
|
|
|
|
bySlug := make(map[string]int, len(pages))
|
|
for i, p := range pages {
|
|
if !strings.HasPrefix(p.Path, "wiki/sources/") {
|
|
bySlug[strings.TrimSuffix(filepath.Base(p.Path), ".md")] = i
|
|
}
|
|
}
|
|
|
|
for slug := range linkedSlugs {
|
|
if slug == sourceSlug {
|
|
continue
|
|
}
|
|
if idx, ok := bySlug[slug]; ok {
|
|
pages[idx] = addSourceRef(pages[idx], sourceRef)
|
|
continue
|
|
}
|
|
pt, ok := findInInventory(slug, inventory)
|
|
if !ok {
|
|
continue
|
|
}
|
|
b, err := readBrainFile(brainDir, "wiki/"+string(pt)+"/"+slug+".md")
|
|
if err != nil {
|
|
continue
|
|
}
|
|
page := wiki.Page{
|
|
Path: "wiki/" + string(pt) + "/" + slug + ".md",
|
|
Content: string(b),
|
|
}
|
|
pages = append(pages, addSourceRef(page, sourceRef))
|
|
}
|
|
|
|
return pages
|
|
}
|
|
|
|
// addSourceRef injects sourceRef into the ## Sources bullet section of page
|
|
// using wiki.Merge, which deduplicates bullets automatically.
|
|
func addSourceRef(page wiki.Page, sourceRef string) wiki.Page {
|
|
patch := wiki.Page{
|
|
Path: page.Path,
|
|
Content: "\n## Sources\n\n" + sourceRef + "\n",
|
|
}
|
|
return wiki.Merge(page, patch)
|
|
}
|
|
|
|
// extractWikilinks returns the set of slugs referenced as [[slug|...]] in content.
|
|
func extractWikilinks(content string) map[string]bool {
|
|
slugs := make(map[string]bool)
|
|
for _, m := range wikilinkRE.FindAllStringSubmatch(content, -1) {
|
|
slugs[m[1]] = true
|
|
}
|
|
return slugs
|
|
}
|
|
|
|
// findSourcePage returns the slug and title of the first wiki/sources/ page in pages.
|
|
func findSourcePage(pages []wiki.Page) (slug, title string, found bool) {
|
|
for _, p := range pages {
|
|
if strings.HasPrefix(p.Path, "wiki/sources/") {
|
|
slug = strings.TrimSuffix(filepath.Base(p.Path), ".md")
|
|
title = extractTitle(p.Content)
|
|
if title == "" {
|
|
title = slug
|
|
}
|
|
return slug, title, true
|
|
}
|
|
}
|
|
return "", "", false
|
|
}
|
|
|
|
// findInInventory returns the PageType for a slug if it appears in the inventory.
|
|
func findInInventory(slug string, inventory map[wiki.PageType][]wiki.Entry) (wiki.PageType, bool) {
|
|
for pt, entries := range inventory {
|
|
for _, e := range entries {
|
|
if e.Slug == slug {
|
|
return pt, true
|
|
}
|
|
}
|
|
}
|
|
return "", false
|
|
}
|