fix(pipeline): same-named sources in two wings, entity/source slug clashes, model-chosen types (review of #211)
- a source whose slug is taken by anything but this wing's own page gets a wing-qualified, stable slug (talk-homelab) instead of failing the ingest - an extracted entity/concept whose slug belongs to a different page is skipped with a warning, so one page can no longer lock a source out of re-ingest - in a wing only source/concept/entity are valid page types; the model (and so the clip) can no longer choose wiki/<type>/ directories Mutation-checked; stable-slug choice pinned directly because title resolution masks it at the Run level. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -52,20 +53,6 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
|
||||
var inventory map[wiki.PageType][]wiki.Entry
|
||||
var err error
|
||||
if cfg.Wing != "" {
|
||||
// A source slug is a page's graph identity: refuse to take over a slug
|
||||
// that lives anywhere but this wing's own sources/ — including when
|
||||
// this wing's own page of that name also exists.
|
||||
slug := wiki.Slug(source)
|
||||
own := "wiki/" + cfg.Wing + "/sources/" + slug + ".md"
|
||||
existing, ferr := wiki.FindSlugs(brainDir, slug)
|
||||
if ferr != nil {
|
||||
return Result{}, fmt.Errorf("check source slug: %w", ferr)
|
||||
}
|
||||
for _, p := range existing {
|
||||
if p != own {
|
||||
return Result{}, fmt.Errorf("source slug %q is already used by %s", slug, p)
|
||||
}
|
||||
}
|
||||
inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing)
|
||||
} else {
|
||||
inventory, err = wiki.LoadInventory(brainDir)
|
||||
@@ -80,6 +67,12 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
|
||||
}
|
||||
|
||||
sourceSlug := wiki.Slug(source)
|
||||
if cfg.Wing != "" {
|
||||
var rerr error
|
||||
if sourceSlug, rerr = resolveSourceSlug(brainDir, cfg.Wing, sourceSlug); rerr != nil {
|
||||
return Result{}, rerr
|
||||
}
|
||||
}
|
||||
date := time.Now().UTC().Format("2006-01-02")
|
||||
|
||||
// Drift check BEFORE any extraction: an unchanged source costs a full LLM
|
||||
@@ -151,6 +144,15 @@ func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceS
|
||||
warnings = append(warnings, linkWarnings...)
|
||||
withRefs := injectSourceRefs(canonicalized, inventory, brainDir)
|
||||
merged := mergeAll(withRefs)
|
||||
if wing != "" {
|
||||
var clashWarnings []string
|
||||
var cerr error
|
||||
merged, clashWarnings, cerr = dropSlugClashes(brainDir, merged)
|
||||
if cerr != nil {
|
||||
return Result{}, cerr
|
||||
}
|
||||
warnings = append(warnings, clashWarnings...)
|
||||
}
|
||||
|
||||
var written []string
|
||||
for _, page := range merged {
|
||||
@@ -333,3 +335,68 @@ func loadSchema(brainDir string) string {
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
// resolveSourceSlug returns the slug this wing's source page uses. A slug is a
|
||||
// page's graph identity and must be unique brain-wide, but two wings dropping a
|
||||
// file of the same name (notes.md, readme.md) is ordinary, so when the plain
|
||||
// slug is taken by anything but this wing's own source page the wing name is
|
||||
// appended (talk-homelab, then talk-homelab-2, ...). The choice is stable: a
|
||||
// re-ingest finds the page it made, because that page is "its own".
|
||||
func resolveSourceSlug(brainDir, wing, base string) (string, error) {
|
||||
for n := 1; n <= 100; n++ {
|
||||
cand := base
|
||||
switch n {
|
||||
case 1:
|
||||
case 2:
|
||||
cand = base + "-" + wing
|
||||
default:
|
||||
cand = fmt.Sprintf("%s-%s-%d", base, wing, n-1)
|
||||
}
|
||||
existing, err := wiki.FindSlugs(brainDir, cand)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("check source slug: %w", err)
|
||||
}
|
||||
own := "wiki/" + wing + "/sources/" + cand + ".md"
|
||||
free := true
|
||||
for _, p := range existing {
|
||||
if p != own {
|
||||
free = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if free {
|
||||
return cand, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no free source slug for %q in wing %q", base, wing)
|
||||
}
|
||||
|
||||
// dropSlugClashes removes, from the pages about to be written in a wing, any
|
||||
// page whose slug already belongs to a different page elsewhere in the brain.
|
||||
// The page's own path is not a clash (a shared entity merging into itself).
|
||||
// A skipped page is a warning, not a failure: the ingest as a whole is fine,
|
||||
// and one extracted page must not be able to lock a source out of re-ingest.
|
||||
func dropSlugClashes(brainDir string, pages []wiki.Page) ([]wiki.Page, []string, error) {
|
||||
var kept []wiki.Page
|
||||
var warnings []string
|
||||
for _, p := range pages {
|
||||
slug := strings.TrimSuffix(filepath.Base(p.Path), ".md")
|
||||
existing, err := wiki.FindSlugs(brainDir, slug)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("check slug %q: %w", slug, err)
|
||||
}
|
||||
clash := ""
|
||||
for _, e := range existing {
|
||||
if e != p.Path {
|
||||
clash = e
|
||||
break
|
||||
}
|
||||
}
|
||||
if clash != "" {
|
||||
warnings = append(warnings, fmt.Sprintf("skipped %s: slug %q is already used by %s", p.Path, slug, clash))
|
||||
continue
|
||||
}
|
||||
kept = append(kept, p)
|
||||
}
|
||||
return kept, warnings, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user