feat(pipeline): per-wing sources, shared concepts/entities, one slug one page (#208)

With Config.Wing a source goes to wiki/<wing>/sources/ (wing/hall frontmatter);
concepts and entities stay in the shared wiki/concepts and wiki/entities, so an
entity cited from two wings is ONE page with both sources (the per-wing facts/
layout duplicated it). A source slug already used anywhere but its own wing's
sources/ is refused (slug is the graph identity). wiki.FindSlug added.
Mutation-checked: collision guard, shared inventory, per-wing source path.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

ingestion/internal/pipeline/build.go

@@ -17,9 +17,11 @@ func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []str
 	return BuildPagesIn("", rawPages, sourceSlug, date)
 }
 
-// BuildPagesIn is BuildPages for a wing: sources go to wiki/<wing>/sources/,
-// concepts and entities to wiki/<wing>/facts/ with `kind:` recording which.
-// An empty wing is the retired wiki/{sources,concepts,entities}/ layout.
+// BuildPagesIn is BuildPages for a wing: a source goes to wiki/<wing>/sources/
+// with wing/hall frontmatter. Concepts and entities stay in the shared
+// wiki/concepts/ and wiki/entities/, since their slug is the graph's global
+// identity and they are cited from many wings. An empty wing is the retired
+// layout for sources too.
 func BuildPagesIn(wing string, rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) {
 	out := make([]wiki.Page, 0, len(rawPages))
 	var warnings []string
@@ -66,20 +68,13 @@ func buildPage(rp RawPage, sourceSlug, date, wing string) wiki.Page {
 		dir = "wiki/" + rp.Type
 	}
 
-	hall := ""
-	if wing != "" {
-		switch rp.Type {
-		case "source":
-			hall = "sources"
-		default:
-			hall = "facts"
-		}
-		dir = "wiki/" + wing + "/" + hall
+	if wing != "" && rp.Type == "source" {
+		dir = "wiki/" + wing + "/sources"
 	}
 	path := dir + "/" + slug + ".md"
 	fm := buildFrontmatter(rp, date)
-	if wing != "" {
-		fm = addWingFrontmatter(fm, wing, hall, rp.Type)
+	if wing != "" && rp.Type == "source" {
+		fm = addWingFrontmatter(fm, wing, "sources")
 	}
 
 	return wiki.Page{
@@ -142,14 +137,10 @@ func buildFrontmatter(rp RawPage, date string) string {
 	return sb.String()
 }
 
-// addWingFrontmatter inserts wing/hall (and kind for facts pages) right after
-// the opening fence of an already-built frontmatter block.
-func addWingFrontmatter(fm, wing, hall, rpType string) string {
-	extra := "wing: " + wing + "\nhall: " + hall + "\n"
-	if hall == "facts" {
-		extra += "kind: " + rpType + "\n"
-	}
-	return strings.Replace(fm, "---\n", "---\n"+extra, 1)
+// addWingFrontmatter inserts wing/hall right after the opening fence of an
+// already-built frontmatter block.
+func addWingFrontmatter(fm, wing, hall string) string {
+	return strings.Replace(fm, "---\n", "---\nwing: "+wing+"\nhall: "+hall+"\n", 1)
 }
 
 func yamlScalar(s string) string {

ingestion/internal/pipeline/pipeline.go

@@ -52,6 +52,13 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
 	var inventory map[wiki.PageType][]wiki.Entry
 	var err error
 	if cfg.Wing != "" {
+		// A source slug is a page's graph identity: refuse to take over a slug
+		// that lives anywhere but this wing's own sources/.
+		if existing, found, ferr := wiki.FindSlug(brainDir, wiki.Slug(source)); ferr != nil {
+			return Result{}, fmt.Errorf("check source slug: %w", ferr)
+		} else if own := "wiki/" + cfg.Wing + "/sources/" + wiki.Slug(source) + ".md"; found && existing != own {
+			return Result{}, fmt.Errorf("source slug %q is already used by %s", wiki.Slug(source), existing)
+		}
 		inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing)
 	} else {
 		inventory, err = wiki.LoadInventory(brainDir)

ingestion/internal/pipeline/resolve.go

@@ -56,17 +56,12 @@ func normalizeTitle(s string) string {
 }
 
 // pageTypeOf is the page's type: from its directory in the retired layout, or
-// from `kind:` for a wing-layout facts page (concepts and entities share facts/).
+// "sources" for wiki/<wing>/sources/<slug>.md. Concepts and entities are always
+// in their shared directories.
 func pageTypeOf(p wiki.Page) wiki.PageType {
 	parts := strings.Split(filepath.ToSlash(p.Path), "/")
-	if len(parts) >= 4 && parts[0] == "wiki" {
-		switch parts[2] {
-		case "sources":
-			return wiki.PageTypeSource
-		case "facts":
-			pt, _ := wiki.KindFromContent(p.Content)
-			return pt
-		}
+	if len(parts) == 4 && parts[0] == "wiki" && parts[2] == "sources" {
+		return wiki.PageTypeSource
 	}
 	return pageTypeFromPath(p.Path)
 }

ingestion/internal/pipeline/wing_layout_test.go

@@ -20,12 +20,14 @@ func wingCfg(wing string, pages ...RawPage) Config {
 	}
 }
 
-func sourceAndEntity() []RawPage {
+func sourceEntityConcept() []RawPage {
 	return []RawPage{
 		{Title: "Dark Factory Talk", Type: "source", Subtype: "video", Domain: "ai",
-			Content: "## Summary\n\nA talk.\n\n## Entities Mentioned\n\n[[Jev]]\n"},
+			Content: "## Summary\n\nA talk.\n\n## Entities Mentioned\n\n[[Jev]]\n\n## Concepts Introduced or Reinforced\n\n[[Dark Factory]]\n"},
 		{Title: "Jev", Type: "entity", Subtype: "tool", Domain: "ai",
 			Content: "## Definition\n\nA decision model.\n\n## Sources\n"},
+		{Title: "Dark Factory", Type: "concept", Domain: "ai",
+			Content: "## Definition\n\nA fully automated build line.\n\n## Sources\n"},
 	}
 }
 
@@ -41,73 +43,91 @@ func read(t *testing.T, dir, rel string) string {
 	return string(b)
 }
 
-func TestRun_WithAWingWritesIntoThatWingsHalls(t *testing.T) {
+// With a wing, the SOURCE belongs to the wing; concepts and entities are the
+// shared cross-wing layer (wiki/concepts, wiki/entities) because their slug is
+// the graph's global identity and they are cited from many wings.
+func TestRun_WithAWingTheSourceIsPerWingAndConceptsEntitiesAreShared(t *testing.T) {
 	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body of the talk", "dark-factory-talk", false)
 	require.NoError(t, err)
 
-	src := read(t, dir, "wiki/radar/sources/dark-factory-talk.md")
-	assert.Contains(t, src, "wing: radar")
+	src := read(t, dir, "wiki/web-clip/sources/dark-factory-talk.md")
+	assert.Contains(t, src, "wing: web-clip")
 	assert.Contains(t, src, "hall: sources")
-
-	ent := read(t, dir, "wiki/radar/facts/jev.md")
-	assert.Contains(t, ent, "wing: radar")
-	assert.Contains(t, ent, "hall: facts")
-	assert.Contains(t, ent, "kind: entity")
+	assert.True(t, exists(dir, "wiki/entities/jev.md"))
+	assert.True(t, exists(dir, "wiki/concepts/dark-factory.md"))
+	assert.NotContains(t, read(t, dir, "wiki/entities/jev.md"), "wing:", "a shared page belongs to no single wing")
+	assert.False(t, exists(dir, "wiki/web-clip/facts"), "no per-wing copy of a shared page")
+	assert.False(t, exists(dir, "wiki/sources"), "the retired source layout is not used")
 }
 
-func TestRun_WithAWingDoesNotTouchTheRetiredLayout(t *testing.T) {
+func TestRun_WithAWingTheSharedPageLinksBackToTheSource(t *testing.T) {
 	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "dark-factory-talk", false)
 	require.NoError(t, err)
-	for _, retired := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} {
-		assert.False(t, exists(dir, retired), "%s is the retired layout and must not be written", retired)
-	}
+	assert.Contains(t, read(t, dir, "wiki/entities/jev.md"), "[[dark-factory-talk|Dark Factory Talk]]")
 }
 
-func TestRun_WithAWingTheSourceLinksBackFromTheEntity(t *testing.T) {
+func TestRun_TheSameEntityCitedFromTwoWingsIsOnePageWithBothSources(t *testing.T) {
 	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
-	require.NoError(t, err)
-	assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "[[dark-factory-talk|Dark Factory Talk]]")
-}
-
-func TestRun_WithAWingASecondSourceReusesTheSameEntityPage(t *testing.T) {
-	dir := t.TempDir()
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "first body", "talk-one", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "first body", "talk-one", false)
 	require.NoError(t, err)
+	// A wing must already exist to be named; create it the way a note would.
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/homelab/facts"), 0o755))
 
 	again := []RawPage{
 		{Title: "Second Talk", Type: "source", Subtype: "video", Content: "## Summary\n\nMore.\n\n## Entities Mentioned\n\n[[The Jev]]\n"},
 		{Title: "The Jev", Type: "entity", Subtype: "tool", Content: "## Definition\n\nStill a decision model.\n\n## Sources\n"},
 	}
-	_, err = Run(context.Background(), wingCfg("radar", again...), dir, "second body", "talk-two", false)
+	_, err = Run(context.Background(), wingCfg("homelab", again...), dir, "second body", "talk-two", false)
 	require.NoError(t, err)
 
-	files, err := filepath.Glob(filepath.Join(dir, "wiki/radar/facts/*.md"))
+	files, err := filepath.Glob(filepath.Join(dir, "wiki/entities/*.md"))
 	require.NoError(t, err)
-	assert.Len(t, files, 1, "an entity matched by title (leading article ignored) is merged, not duplicated")
-	assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "talk-two")
+	assert.Len(t, files, 1, "one entity page, matched by title with the leading article ignored")
+	jev := read(t, dir, "wiki/entities/jev.md")
+	assert.Contains(t, jev, "talk-one")
+	assert.Contains(t, jev, "talk-two")
+	assert.False(t, exists(dir, "wiki/homelab/facts/jev.md"), "no second copy in the second wing")
 }
 
-func TestRun_WithAWingNeverMergesIntoAnotherWingsPage(t *testing.T) {
+// A source slug is the graph's identity too: a second wing must not take over
+// the slug of a source that lives elsewhere.
+func TestRun_ASourceSlugAlreadyUsedInAnotherWingIsRefused(t *testing.T) {
 	dir := t.TempDir()
-	other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n"
-	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755))
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644))
-
-	_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body", "dark-factory-talk", false)
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "talk", false)
 	require.NoError(t, err)
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/homelab/facts"), 0o755))
 
-	assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "another wing's page is never read into or rewritten")
-	assert.True(t, exists(dir, "wiki/radar/facts/jev.md"))
+	_, err = Run(context.Background(), wingCfg("homelab", sourceEntityConcept()...), dir, "different body", "talk", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "already used")
+	assert.False(t, exists(dir, "wiki/homelab/sources/talk.md"))
+}
+
+func TestRun_ASourceSlugAlreadyInTheRetiredLayoutIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/sources"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/sources/talk.md"), []byte("---\ntitle: Talk\n---\nold"), 0o644))
+
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "talk", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "already used")
+}
+
+func TestRun_ReIngestingTheSameSourceIntoItsOwnWingIsNotACollision(t *testing.T) {
+	dir := t.TempDir()
+	_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body one", "talk", false)
+	require.NoError(t, err)
+	_, err = Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body two, edited", "talk", false)
+	require.NoError(t, err)
 }
 
 func TestRun_WithAWingRejectsANameThatIsNotAPlainWing(t *testing.T) {
 	for _, bad := range []string{"../escape", "a/b", "Radar", "has space", "."} {
 		t.Run(bad, func(t *testing.T) {
 			dir := t.TempDir()
-			_, err := Run(context.Background(), wingCfg(bad, sourceAndEntity()...), dir, "body", "talk", false)
+			_, err := Run(context.Background(), wingCfg(bad, sourceEntityConcept()...), dir, "body", "talk", false)
 			require.Error(t, err)
 			assert.Len(t, listAll(t, dir), 1, "nothing is written for a rejected wing (only the temp dir itself exists)")
 		})
@@ -120,19 +140,3 @@ func listAll(t *testing.T, dir string) []string {
 	_ = filepath.WalkDir(dir, func(p string, _ os.DirEntry, _ error) error { out = append(out, p); return nil })
 	return out
 }
-
-// The dangerous path is the back-reference: a source that links [[Jev]] without
-// proposing a Jev page makes the pipeline go and edit an existing Jev page. That
-// lookup must be confined to the target wing.
-func TestRun_WithAWingASourceLinkingToAnotherWingsEntityDoesNotEditIt(t *testing.T) {
-	dir := t.TempDir()
-	other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n\n## Sources\n"
-	require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755))
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644))
-
-	onlySource := sourceAndEntity()[:1]
-	_, err := Run(context.Background(), wingCfg("radar", onlySource...), dir, "body", "dark-factory-talk", false)
-	require.NoError(t, err)
-
-	assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "no back-reference may be written into another wing")
-}