feat(pipeline): per-wing sources, shared concepts/entities, one slug one page (#208)

With Config.Wing a source goes to wiki/<wing>/sources/ (wing/hall frontmatter);
concepts and entities stay in the shared wiki/concepts and wiki/entities, so an
entity cited from two wings is ONE page with both sources (the per-wing facts/
layout duplicated it). A source slug already used anywhere but its own wing's
sources/ is refused (slug is the graph identity). wiki.FindSlug added.
Mutation-checked: collision guard, shared inventory, per-wing source path.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

ingestion/internal/pipeline/build.go

@@ -17,9 +17,11 @@ func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []str
 	return BuildPagesIn("", rawPages, sourceSlug, date)
 }
 
-// BuildPagesIn is BuildPages for a wing: sources go to wiki/<wing>/sources/,
-// concepts and entities to wiki/<wing>/facts/ with `kind:` recording which.
-// An empty wing is the retired wiki/{sources,concepts,entities}/ layout.
+// BuildPagesIn is BuildPages for a wing: a source goes to wiki/<wing>/sources/
+// with wing/hall frontmatter. Concepts and entities stay in the shared
+// wiki/concepts/ and wiki/entities/, since their slug is the graph's global
+// identity and they are cited from many wings. An empty wing is the retired
+// layout for sources too.
 func BuildPagesIn(wing string, rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) {
 	out := make([]wiki.Page, 0, len(rawPages))
 	var warnings []string
@@ -66,20 +68,13 @@ func buildPage(rp RawPage, sourceSlug, date, wing string) wiki.Page {
 		dir = "wiki/" + rp.Type
 	}
 
-	hall := ""
-	if wing != "" {
-		switch rp.Type {
-		case "source":
-			hall = "sources"
-		default:
-			hall = "facts"
-		}
-		dir = "wiki/" + wing + "/" + hall
+	if wing != "" && rp.Type == "source" {
+		dir = "wiki/" + wing + "/sources"
 	}
 	path := dir + "/" + slug + ".md"
 	fm := buildFrontmatter(rp, date)
-	if wing != "" {
-		fm = addWingFrontmatter(fm, wing, hall, rp.Type)
+	if wing != "" && rp.Type == "source" {
+		fm = addWingFrontmatter(fm, wing, "sources")
 	}
 
 	return wiki.Page{
@@ -142,14 +137,10 @@ func buildFrontmatter(rp RawPage, date string) string {
 	return sb.String()
 }
 
-// addWingFrontmatter inserts wing/hall (and kind for facts pages) right after
-// the opening fence of an already-built frontmatter block.
-func addWingFrontmatter(fm, wing, hall, rpType string) string {
-	extra := "wing: " + wing + "\nhall: " + hall + "\n"
-	if hall == "facts" {
-		extra += "kind: " + rpType + "\n"
-	}
-	return strings.Replace(fm, "---\n", "---\n"+extra, 1)
+// addWingFrontmatter inserts wing/hall right after the opening fence of an
+// already-built frontmatter block.
+func addWingFrontmatter(fm, wing, hall string) string {
+	return strings.Replace(fm, "---\n", "---\nwing: "+wing+"\nhall: "+hall+"\n", 1)
 }
 
 func yamlScalar(s string) string {

ingestion/internal/pipeline/pipeline.go

@@ -52,6 +52,13 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
 	var inventory map[wiki.PageType][]wiki.Entry
 	var err error
 	if cfg.Wing != "" {
+		// A source slug is a page's graph identity: refuse to take over a slug
+		// that lives anywhere but this wing's own sources/.
+		if existing, found, ferr := wiki.FindSlug(brainDir, wiki.Slug(source)); ferr != nil {
+			return Result{}, fmt.Errorf("check source slug: %w", ferr)
+		} else if own := "wiki/" + cfg.Wing + "/sources/" + wiki.Slug(source) + ".md"; found && existing != own {
+			return Result{}, fmt.Errorf("source slug %q is already used by %s", wiki.Slug(source), existing)
+		}
 		inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing)
 	} else {
 		inventory, err = wiki.LoadInventory(brainDir)

ingestion/internal/pipeline/resolve.go

@@ -56,17 +56,12 @@ func normalizeTitle(s string) string {
 }
 
 // pageTypeOf is the page's type: from its directory in the retired layout, or
-// from `kind:` for a wing-layout facts page (concepts and entities share facts/).
+// "sources" for wiki/<wing>/sources/<slug>.md. Concepts and entities are always
+// in their shared directories.
 func pageTypeOf(p wiki.Page) wiki.PageType {
 	parts := strings.Split(filepath.ToSlash(p.Path), "/")
-	if len(parts) >= 4 && parts[0] == "wiki" {
-		switch parts[2] {
-		case "sources":
-			return wiki.PageTypeSource
-		case "facts":
-			pt, _ := wiki.KindFromContent(p.Content)
-			return pt
-		}
+	if len(parts) == 4 && parts[0] == "wiki" && parts[2] == "sources" {
+		return wiki.PageTypeSource
 	}
 	return pageTypeFromPath(p.Path)
 }

ingestion/internal/pipeline/wing_layout_test.go

@@ -20,12 +20,14 @@ func wingCfg(wing string, pages ...RawPage) Config {
 	}
 }
 
-func sourceAndEntity() []RawPage {
+func sourceEntityConcept() []RawPage {
 	return []RawPage{
 		{Title: "Dark Factory Talk", Type: "source", Subtype: "video", Domain: "ai",
-			Content: "## Summary\n\nA talk.\n\n## Entities Mentioned\n\n[[Jev]]\n"},
+			Content: "## Summary\n\nA talk.\n\n## Entities Mentioned\n\n[[Jev]]\n\n## Concepts Introduced or Reinforced\n\n[[Dark Factory]]\n"},
 		{Title: "Jev", Type: "entity", Subtype: "tool", Domain: "ai",
 			Content: "## Definition\n\nA decision model.\n\n## Sources\n"},
+		{Title: "Dark Factory", Type: "concept", Domain: "ai",
+			Content: "## Definition\n\nA fully automated build line.\n\n## Sources\n"},
 	}
 }
 
@@ -41,73 +43,91 @@ func read(t *testing.T, dir, rel string) string {
 	return string(b)
 }
 
-func TestRun_WithAWingWritesIntoThatWingsHalls(t *testing.T) {
+// With a wing, the SOURCE belongs to the wing; concepts and entities are the
+// shared cross-wing layer (wiki/concepts, wiki/entities) because their slug is
+// the graph's global identity and they are cited from many wings.
+func TestRun_WithAWingTheSourceIsPerWingAndConceptsEntitiesAreShared(t *testing.T) {
 	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body of the talk", "dark-factory-talk", false)
 	require.NoError(t, err)
 
-	src := read(t, dir, "wiki/radar/sources/dark-factory-talk.md")
-	assert.Contains(t, src, "wing: radar")
+	src := read(t, dir, "wiki/web-clip/sources/dark-factory-talk.md")
+	assert.Contains(t, src, "wing: web-clip")
 	assert.Contains(t, src, "hall: sources")
-
-	ent := read(t, dir, "wiki/radar/facts/jev.md")
-	assert.Contains(t, ent, "wing: radar")
-	assert.Contains(t, ent, "hall: facts")
-	assert.Contains(t, ent, "kind: entity")
+	assert.True(t, exists(dir, "wiki/entities/jev.md"))
+	assert.True(t, exists(dir, "wiki/concepts/dark-factory.md"))
+	assert.NotContains(t, read(t, dir, "wiki/entities/jev.md"), "wing:", "a shared page belongs to no single wing")
+	assert.False(t, exists(dir, "wiki/web-clip/facts"), "no per-wing copy of a shared page")
+	assert.False(t, exists(dir, "wiki/sources"), "the retired source layout is not used")
 }
 
-func TestRun_WithAWingDoesNotTouchTheRetiredLayout(t *testing.T) {
+func TestRun_WithAWingTheSharedPageLinksBackToTheSource(t *testing.T) {
 	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "dark-factory-talk", false)
 	require.NoError(t, err)
-	for _, retired := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} {
-		assert.False(t, exists(dir, retired), "%s is the retired layout and must not be written", retired)
-	}
+	assert.Contains(t, read(t, dir, "wiki/entities/jev.md"), "[[dark-factory-talk|Dark Factory Talk]]")
 }
 
-func TestRun_WithAWingTheSourceLinksBackFromTheEntity(t *testing.T) {
+func TestRun_TheSameEntityCitedFromTwoWingsIsOnePageWithBothSources(t *testing.T) {
 	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
-	require.NoError(t, err)
-	assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "[[dark-factory-talk|Dark Factory Talk]]")
-}
-
-func TestRun_WithAWingASecondSourceReusesTheSameEntityPage(t *testing.T) {
-	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "first body", "talk-one", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "first body", "talk-one", false)
 	require.NoError(t, err)
+	// A wing must already exist to be named; create it the way a note would.
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/homelab/facts"), 0o755))
 
 	again := []RawPage{
 		{Title: "Second Talk", Type: "source", Subtype: "video", Content: "## Summary\n\nMore.\n\n## Entities Mentioned\n\n[[The Jev]]\n"},
 		{Title: "The Jev", Type: "entity", Subtype: "tool", Content: "## Definition\n\nStill a decision model.\n\n## Sources\n"},
 	}
-	_, err = Run(context.Background(), wingCfg("radar", again...), dir, "second body", "talk-two", false)
+	_, err = Run(context.Background(), wingCfg("homelab", again...), dir, "second body", "talk-two", false)
 	require.NoError(t, err)
 
-	files, err := filepath.Glob(filepath.Join(dir, "wiki/radar/facts/*.md"))
+	files, err := filepath.Glob(filepath.Join(dir, "wiki/entities/*.md"))
 	require.NoError(t, err)
-	assert.Len(t, files, 1, "an entity matched by title (leading article ignored) is merged, not duplicated")
-	assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "talk-two")
+	assert.Len(t, files, 1, "one entity page, matched by title with the leading article ignored")
+	jev := read(t, dir, "wiki/entities/jev.md")
+	assert.Contains(t, jev, "talk-one")
+	assert.Contains(t, jev, "talk-two")
+	assert.False(t, exists(dir, "wiki/homelab/facts/jev.md"), "no second copy in the second wing")
 }
 
-func TestRun_WithAWingNeverMergesIntoAnotherWingsPage(t *testing.T) {
+// A source slug is the graph's identity too: a second wing must not take over
+// the slug of a source that lives elsewhere.
+func TestRun_ASourceSlugAlreadyUsedInAnotherWingIsRefused(t *testing.T) {
 	dir := t.TempDir()
-	other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n"
-	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755))
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644))
-
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body", "dark-factory-talk", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "talk", false)
 	require.NoError(t, err)
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/homelab/facts"), 0o755))
 
-	assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "another wing's page is never read into or rewritten")
-	assert.True(t, exists(dir, "wiki/radar/facts/jev.md"))
+	_, err = Run(context.Background(), wingCfg("homelab", sourceEntityConcept()...), dir, "different body", "talk", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "already used")
+	assert.False(t, exists(dir, "wiki/homelab/sources/talk.md"))
+}
+
+func TestRun_ASourceSlugAlreadyInTheRetiredLayoutIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/sources"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/sources/talk.md"), []byte("---\ntitle: Talk\n---\nold"), 0o644))
+
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "talk", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "already used")
+}
+
+func TestRun_ReIngestingTheSameSourceIntoItsOwnWingIsNotACollision(t *testing.T) {
+	dir := t.TempDir()
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body one", "talk", false)
+	require.NoError(t, err)
+	_, err = Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body two, edited", "talk", false)
+	require.NoError(t, err)
 }
 
 func TestRun_WithAWingRejectsANameThatIsNotAPlainWing(t *testing.T) {
 	for _, bad := range []string{"../escape", "a/b", "Radar", "has space", "."} {
 		t.Run(bad, func(t *testing.T) {
 			dir := t.TempDir()
-			_, err := Run(context.Background(), wingCfg(bad, sourceAndEntity()...), dir, "body", "talk", false)
+			_, err := Run(context.Background(), wingCfg(bad, sourceEntityConcept()...), dir, "body", "talk", false)
 			require.Error(t, err)
 			assert.Len(t, listAll(t, dir), 1, "nothing is written for a rejected wing (only the temp dir itself exists)")
 		})
@@ -120,19 +140,3 @@ func listAll(t *testing.T, dir string) []string {
 	_ = filepath.WalkDir(dir, func(p string, _ os.DirEntry, _ error) error { out = append(out, p); return nil })
 	return out
 }
-
-// The dangerous path is the back-reference: a source that links [[Jev]] without
-// proposing a Jev page makes the pipeline go and edit an existing Jev page. That
-// lookup must be confined to the target wing.
-func TestRun_WithAWingASourceLinkingToAnotherWingsEntityDoesNotEditIt(t *testing.T) {
-	dir := t.TempDir()
-	other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n\n## Sources\n"
-	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755))
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644))
-
-	onlySource := sourceAndEntity()[:1]
-	_, err := Run(context.Background(), wingCfg("radar", onlySource...), dir, "body", "dark-factory-talk", false)
-	require.NoError(t, err)
-
-	assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "no back-reference may be written into another wing")
-}

ingestion/internal/wiki/findslug_test.go

@@ -0,0 +1,52 @@
+package wiki
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func put(t *testing.T, dir, rel string) {
+	t.Helper()
+	full := filepath.Join(dir, rel)
+	require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
+	require.NoError(t, os.WriteFile(full, []byte("---\ntitle: x\n---\n"), 0o644))
+}
+
+func TestFindSlug(t *testing.T) {
+	dir := t.TempDir()
+	put(t, dir, "wiki/sources/legacy-src.md")
+	put(t, dir, "wiki/concepts/a-concept.md")
+	put(t, dir, "wiki/entities/an-entity.md")
+	put(t, dir, "wiki/homelab/failures/in-a-hall.md")
+	put(t, dir, "wiki/web-clip/sources/a-clip.md")
+
+	tests := []struct {
+		slug string
+		want string
+	}{
+		{"legacy-src", "wiki/sources/legacy-src.md"},
+		{"a-concept", "wiki/concepts/a-concept.md"},
+		{"an-entity", "wiki/entities/an-entity.md"},
+		{"in-a-hall", "wiki/homelab/failures/in-a-hall.md"},
+		{"a-clip", "wiki/web-clip/sources/a-clip.md"},
+		{"nothing-here", ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.slug, func(t *testing.T) {
+			got, found, err := FindSlug(dir, tt.slug)
+			require.NoError(t, err)
+			assert.Equal(t, tt.want != "", found)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestFindSlug_NoWikiDirIsNotAFinding(t *testing.T) {
+	_, found, err := FindSlug(t.TempDir(), "x")
+	require.NoError(t, err)
+	assert.False(t, found)
+}

ingestion/internal/wiki/inventory.go

@@ -97,78 +97,97 @@ func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string
 	return
 }
 
-// LoadWingInventory returns the pages of ONE wing, grouped by type. Sources
-// come from wiki/<wing>/sources/; concepts and entities live together in
-// wiki/<wing>/facts/ and are told apart by their `kind:` frontmatter. A facts
-// page without a kind (a hand-written note) is left out, so extraction can
-// never merge into something it did not create. Other wings are never read.
+// LoadWingInventory returns what an ingest into one wing can match against:
+// that wing's sources (wiki/<wing>/sources/) and the SHARED concepts and
+// entities (wiki/concepts/, wiki/entities/), which belong to no single wing
+// because their slug is the graph's global identity. Other wings' sources are
+// not read.
 func LoadWingInventory(brainDir, wing string) (map[PageType][]Entry, error) {
-	result := map[PageType][]Entry{PageTypeConcept: {}, PageTypeEntity: {}, PageTypeSource: {}}
+	result, err := LoadInventory(brainDir)
+	if err != nil {
+		return nil, err
+	}
+	result[PageTypeSource] = []Entry{} // the retired source layout is not this wing's
 	bfs, err := brainfs.Open(brainDir)
 	if err != nil {
 		return nil, fmt.Errorf("open brain: %w", err)
 	}
 	defer func() { _ = bfs.Close() }()
-	for _, hall := range []string{"sources", "facts"} {
-		dir := "wiki/" + wing + "/" + hall
-		names, err := bfs.ReadDirRegular(dir)
-		if errors.Is(err, fs.ErrNotExist) {
+	dir := "wiki/" + wing + "/sources"
+	names, err := bfs.ReadDirRegular(dir)
+	if errors.Is(err, fs.ErrNotExist) {
+		return result, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read dir %s: %w", dir, err)
+	}
+	for _, name := range names {
+		if !strings.HasSuffix(name, ".md") {
 			continue
 		}
-		if err != nil {
-			return nil, fmt.Errorf("read dir %s: %w", dir, err)
-		}
-		for _, name := range names {
-			if !strings.HasSuffix(name, ".md") {
-				continue
-			}
-			rel := dir + "/" + name
-			slug := strings.TrimSuffix(name, ".md")
-			pt := PageTypeSource
-			if hall == "facts" {
-				var ok bool
-				if pt, ok = kindOf(bfs, rel); !ok {
-					continue
-				}
-			}
-			title, aliases := readFrontmatter(bfs, rel, slug)
-			result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt, Path: rel})
-		}
+		rel := dir + "/" + name
+		slug := strings.TrimSuffix(name, ".md")
+		title, aliases := readFrontmatter(bfs, rel, slug)
+		result[PageTypeSource] = append(result[PageTypeSource], Entry{Slug: slug, Title: title, Aliases: aliases, Type: PageTypeSource, Path: rel})
 	}
 	return result, nil
 }
 
-// kindOf reads `kind:` from a facts page's frontmatter.
-func kindOf(bfs *brainfs.Brain, rel string) (PageType, bool) {
-	content, err := bfs.ReadRegular(rel)
+// FindSlug returns the brain-relative path of an existing page whose basename is
+// slug, searching the retired dirs (wiki/{sources,concepts,entities}/) and every
+// wing hall (wiki/<wing>/<hall>/). The slug is a page's graph identity, so two
+// pages must not share one.
+func FindSlug(brainDir, slug string) (string, bool, error) {
+	bfs, err := brainfs.Open(brainDir)
 	if err != nil {
-		return "", false
+		return "", false, fmt.Errorf("open brain: %w", err)
 	}
-	return KindFromContent(string(content))
-}
-
-// KindFromContent maps frontmatter `kind: concept|entity` to a PageType.
-func KindFromContent(content string) (PageType, bool) {
-	inFM := false
-	for _, line := range strings.Split(content, "\n") {
-		if strings.TrimSpace(line) == "---" {
-			if inFM {
-				break
+	defer func() { _ = bfs.Close() }()
+	file := slug + ".md"
+	has := func(dir string) (bool, error) {
+		names, err := bfs.ReadDirRegular(dir)
+		if errors.Is(err, fs.ErrNotExist) {
+			return false, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		for _, n := range names {
+			if n == file {
+				return true, nil
 			}
-			inFM = true
+		}
+		return false, nil
+	}
+	for _, d := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} {
+		if ok, err := has(d); err != nil {
+			return "", false, err
+		} else if ok {
+			return d + "/" + file, true, nil
+		}
+	}
+	wings, err := bfs.ReadDirDirs("wiki")
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	for _, w := range wings {
+		if w == "sources" || w == "concepts" || w == "entities" {
 			continue
 		}
-		if !inFM {
+		halls, err := bfs.ReadDirDirs("wiki/" + w)
+		if err != nil {
 			continue
 		}
-		if key, val, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "kind" {
-			switch strings.Trim(strings.TrimSpace(val), `"'`) {
-			case "concept":
-				return PageTypeConcept, true
-			case "entity":
-				return PageTypeEntity, true
+		for _, h := range halls {
+			if ok, err := has("wiki/" + w + "/" + h); err != nil {
+				return "", false, err
+			} else if ok {
+				return "wiki/" + w + "/" + h + "/" + file, true, nil
 			}
 		}
 	}
-	return "", false
+	return "", false, nil
 }