diff --git a/ingestion/internal/pipeline/build.go b/ingestion/internal/pipeline/build.go index c45fdac..5479db5 100644 --- a/ingestion/internal/pipeline/build.go +++ b/ingestion/internal/pipeline/build.go @@ -17,9 +17,11 @@ func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []str return BuildPagesIn("", rawPages, sourceSlug, date) } -// BuildPagesIn is BuildPages for a wing: sources go to wiki//sources/, -// concepts and entities to wiki//facts/ with `kind:` recording which. -// An empty wing is the retired wiki/{sources,concepts,entities}/ layout. +// BuildPagesIn is BuildPages for a wing: a source goes to wiki//sources/ +// with wing/hall frontmatter. Concepts and entities stay in the shared +// wiki/concepts/ and wiki/entities/, since their slug is the graph's global +// identity and they are cited from many wings. An empty wing is the retired +// layout for sources too. func BuildPagesIn(wing string, rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) { out := make([]wiki.Page, 0, len(rawPages)) var warnings []string @@ -66,20 +68,13 @@ func buildPage(rp RawPage, sourceSlug, date, wing string) wiki.Page { dir = "wiki/" + rp.Type } - hall := "" - if wing != "" { - switch rp.Type { - case "source": - hall = "sources" - default: - hall = "facts" - } - dir = "wiki/" + wing + "/" + hall + if wing != "" && rp.Type == "source" { + dir = "wiki/" + wing + "/sources" } path := dir + "/" + slug + ".md" fm := buildFrontmatter(rp, date) - if wing != "" { - fm = addWingFrontmatter(fm, wing, hall, rp.Type) + if wing != "" && rp.Type == "source" { + fm = addWingFrontmatter(fm, wing, "sources") } return wiki.Page{ @@ -142,14 +137,10 @@ func buildFrontmatter(rp RawPage, date string) string { return sb.String() } -// addWingFrontmatter inserts wing/hall (and kind for facts pages) right after -// the opening fence of an already-built frontmatter block. -func addWingFrontmatter(fm, wing, hall, rpType string) string { - extra := "wing: " + wing + "\nhall: " + hall + "\n" - if hall == "facts" { - extra += "kind: " + rpType + "\n" - } - return strings.Replace(fm, "---\n", "---\n"+extra, 1) +// addWingFrontmatter inserts wing/hall right after the opening fence of an +// already-built frontmatter block. +func addWingFrontmatter(fm, wing, hall string) string { + return strings.Replace(fm, "---\n", "---\nwing: "+wing+"\nhall: "+hall+"\n", 1) } func yamlScalar(s string) string { diff --git a/ingestion/internal/pipeline/pipeline.go b/ingestion/internal/pipeline/pipeline.go index 9269f3a..cd4ec03 100644 --- a/ingestion/internal/pipeline/pipeline.go +++ b/ingestion/internal/pipeline/pipeline.go @@ -52,6 +52,13 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR var inventory map[wiki.PageType][]wiki.Entry var err error if cfg.Wing != "" { + // A source slug is a page's graph identity: refuse to take over a slug + // that lives anywhere but this wing's own sources/. + if existing, found, ferr := wiki.FindSlug(brainDir, wiki.Slug(source)); ferr != nil { + return Result{}, fmt.Errorf("check source slug: %w", ferr) + } else if own := "wiki/" + cfg.Wing + "/sources/" + wiki.Slug(source) + ".md"; found && existing != own { + return Result{}, fmt.Errorf("source slug %q is already used by %s", wiki.Slug(source), existing) + } inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing) } else { inventory, err = wiki.LoadInventory(brainDir) diff --git a/ingestion/internal/pipeline/resolve.go b/ingestion/internal/pipeline/resolve.go index e098ca1..21e04e3 100644 --- a/ingestion/internal/pipeline/resolve.go +++ b/ingestion/internal/pipeline/resolve.go @@ -56,17 +56,12 @@ func normalizeTitle(s string) string { } // pageTypeOf is the page's type: from its directory in the retired layout, or -// from `kind:` for a wing-layout facts page (concepts and entities share facts/). +// "sources" for wiki//sources/.md. Concepts and entities are always +// in their shared directories. func pageTypeOf(p wiki.Page) wiki.PageType { parts := strings.Split(filepath.ToSlash(p.Path), "/") - if len(parts) >= 4 && parts[0] == "wiki" { - switch parts[2] { - case "sources": - return wiki.PageTypeSource - case "facts": - pt, _ := wiki.KindFromContent(p.Content) - return pt - } + if len(parts) == 4 && parts[0] == "wiki" && parts[2] == "sources" { + return wiki.PageTypeSource } return pageTypeFromPath(p.Path) } diff --git a/ingestion/internal/pipeline/wing_layout_test.go b/ingestion/internal/pipeline/wing_layout_test.go index 0da280b..5f5b240 100644 --- a/ingestion/internal/pipeline/wing_layout_test.go +++ b/ingestion/internal/pipeline/wing_layout_test.go @@ -20,12 +20,14 @@ func wingCfg(wing string, pages ...RawPage) Config { } } -func sourceAndEntity() []RawPage { +func sourceEntityConcept() []RawPage { return []RawPage{ {Title: "Dark Factory Talk", Type: "source", Subtype: "video", Domain: "ai", - Content: "## Summary\n\nA talk.\n\n## Entities Mentioned\n\n[[Jev]]\n"}, + Content: "## Summary\n\nA talk.\n\n## Entities Mentioned\n\n[[Jev]]\n\n## Concepts Introduced or Reinforced\n\n[[Dark Factory]]\n"}, {Title: "Jev", Type: "entity", Subtype: "tool", Domain: "ai", Content: "## Definition\n\nA decision model.\n\n## Sources\n"}, + {Title: "Dark Factory", Type: "concept", Domain: "ai", + Content: "## Definition\n\nA fully automated build line.\n\n## Sources\n"}, } } @@ -41,73 +43,91 @@ func read(t *testing.T, dir, rel string) string { return string(b) } -func TestRun_WithAWingWritesIntoThatWingsHalls(t *testing.T) { +// With a wing, the SOURCE belongs to the wing; concepts and entities are the +// shared cross-wing layer (wiki/concepts, wiki/entities) because their slug is +// the graph's global identity and they are cited from many wings. +func TestRun_WithAWingTheSourceIsPerWingAndConceptsEntitiesAreShared(t *testing.T) { dir := t.TempDir() - _, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false) + _, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body of the talk", "dark-factory-talk", false) require.NoError(t, err) - src := read(t, dir, "wiki/radar/sources/dark-factory-talk.md") - assert.Contains(t, src, "wing: radar") + src := read(t, dir, "wiki/web-clip/sources/dark-factory-talk.md") + assert.Contains(t, src, "wing: web-clip") assert.Contains(t, src, "hall: sources") - - ent := read(t, dir, "wiki/radar/facts/jev.md") - assert.Contains(t, ent, "wing: radar") - assert.Contains(t, ent, "hall: facts") - assert.Contains(t, ent, "kind: entity") + assert.True(t, exists(dir, "wiki/entities/jev.md")) + assert.True(t, exists(dir, "wiki/concepts/dark-factory.md")) + assert.NotContains(t, read(t, dir, "wiki/entities/jev.md"), "wing:", "a shared page belongs to no single wing") + assert.False(t, exists(dir, "wiki/web-clip/facts"), "no per-wing copy of a shared page") + assert.False(t, exists(dir, "wiki/sources"), "the retired source layout is not used") } -func TestRun_WithAWingDoesNotTouchTheRetiredLayout(t *testing.T) { +func TestRun_WithAWingTheSharedPageLinksBackToTheSource(t *testing.T) { dir := t.TempDir() - _, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false) + _, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "dark-factory-talk", false) require.NoError(t, err) - for _, retired := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} { - assert.False(t, exists(dir, retired), "%s is the retired layout and must not be written", retired) - } + assert.Contains(t, read(t, dir, "wiki/entities/jev.md"), "[[dark-factory-talk|Dark Factory Talk]]") } -func TestRun_WithAWingTheSourceLinksBackFromTheEntity(t *testing.T) { +func TestRun_TheSameEntityCitedFromTwoWingsIsOnePageWithBothSources(t *testing.T) { dir := t.TempDir() - _, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false) - require.NoError(t, err) - assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "[[dark-factory-talk|Dark Factory Talk]]") -} - -func TestRun_WithAWingASecondSourceReusesTheSameEntityPage(t *testing.T) { - dir := t.TempDir() - _, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "first body", "talk-one", false) + _, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "first body", "talk-one", false) require.NoError(t, err) + // A wing must already exist to be named; create it the way a note would. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/homelab/facts"), 0o755)) again := []RawPage{ {Title: "Second Talk", Type: "source", Subtype: "video", Content: "## Summary\n\nMore.\n\n## Entities Mentioned\n\n[[The Jev]]\n"}, {Title: "The Jev", Type: "entity", Subtype: "tool", Content: "## Definition\n\nStill a decision model.\n\n## Sources\n"}, } - _, err = Run(context.Background(), wingCfg("radar", again...), dir, "second body", "talk-two", false) + _, err = Run(context.Background(), wingCfg("homelab", again...), dir, "second body", "talk-two", false) require.NoError(t, err) - files, err := filepath.Glob(filepath.Join(dir, "wiki/radar/facts/*.md")) + files, err := filepath.Glob(filepath.Join(dir, "wiki/entities/*.md")) require.NoError(t, err) - assert.Len(t, files, 1, "an entity matched by title (leading article ignored) is merged, not duplicated") - assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "talk-two") + assert.Len(t, files, 1, "one entity page, matched by title with the leading article ignored") + jev := read(t, dir, "wiki/entities/jev.md") + assert.Contains(t, jev, "talk-one") + assert.Contains(t, jev, "talk-two") + assert.False(t, exists(dir, "wiki/homelab/facts/jev.md"), "no second copy in the second wing") } -func TestRun_WithAWingNeverMergesIntoAnotherWingsPage(t *testing.T) { +// A source slug is the graph's identity too: a second wing must not take over +// the slug of a source that lives elsewhere. +func TestRun_ASourceSlugAlreadyUsedInAnotherWingIsRefused(t *testing.T) { dir := t.TempDir() - other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n" - require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644)) - - _, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body", "dark-factory-talk", false) + _, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "talk", false) require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/homelab/facts"), 0o755)) - assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "another wing's page is never read into or rewritten") - assert.True(t, exists(dir, "wiki/radar/facts/jev.md")) + _, err = Run(context.Background(), wingCfg("homelab", sourceEntityConcept()...), dir, "different body", "talk", false) + require.Error(t, err) + assert.Contains(t, err.Error(), "already used") + assert.False(t, exists(dir, "wiki/homelab/sources/talk.md")) +} + +func TestRun_ASourceSlugAlreadyInTheRetiredLayoutIsRefused(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/sources"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/sources/talk.md"), []byte("---\ntitle: Talk\n---\nold"), 0o644)) + + _, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "talk", false) + require.Error(t, err) + assert.Contains(t, err.Error(), "already used") +} + +func TestRun_ReIngestingTheSameSourceIntoItsOwnWingIsNotACollision(t *testing.T) { + dir := t.TempDir() + _, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body one", "talk", false) + require.NoError(t, err) + _, err = Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body two, edited", "talk", false) + require.NoError(t, err) } func TestRun_WithAWingRejectsANameThatIsNotAPlainWing(t *testing.T) { for _, bad := range []string{"../escape", "a/b", "Radar", "has space", "."} { t.Run(bad, func(t *testing.T) { dir := t.TempDir() - _, err := Run(context.Background(), wingCfg(bad, sourceAndEntity()...), dir, "body", "talk", false) + _, err := Run(context.Background(), wingCfg(bad, sourceEntityConcept()...), dir, "body", "talk", false) require.Error(t, err) assert.Len(t, listAll(t, dir), 1, "nothing is written for a rejected wing (only the temp dir itself exists)") }) @@ -120,19 +140,3 @@ func listAll(t *testing.T, dir string) []string { _ = filepath.WalkDir(dir, func(p string, _ os.DirEntry, _ error) error { out = append(out, p); return nil }) return out } - -// The dangerous path is the back-reference: a source that links [[Jev]] without -// proposing a Jev page makes the pipeline go and edit an existing Jev page. That -// lookup must be confined to the target wing. -func TestRun_WithAWingASourceLinkingToAnotherWingsEntityDoesNotEditIt(t *testing.T) { - dir := t.TempDir() - other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n\n## Sources\n" - require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644)) - - onlySource := sourceAndEntity()[:1] - _, err := Run(context.Background(), wingCfg("radar", onlySource...), dir, "body", "dark-factory-talk", false) - require.NoError(t, err) - - assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "no back-reference may be written into another wing") -} diff --git a/ingestion/internal/wiki/findslug_test.go b/ingestion/internal/wiki/findslug_test.go new file mode 100644 index 0000000..bb89795 --- /dev/null +++ b/ingestion/internal/wiki/findslug_test.go @@ -0,0 +1,52 @@ +package wiki + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func put(t *testing.T, dir, rel string) { + t.Helper() + full := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(t, os.WriteFile(full, []byte("---\ntitle: x\n---\n"), 0o644)) +} + +func TestFindSlug(t *testing.T) { + dir := t.TempDir() + put(t, dir, "wiki/sources/legacy-src.md") + put(t, dir, "wiki/concepts/a-concept.md") + put(t, dir, "wiki/entities/an-entity.md") + put(t, dir, "wiki/homelab/failures/in-a-hall.md") + put(t, dir, "wiki/web-clip/sources/a-clip.md") + + tests := []struct { + slug string + want string + }{ + {"legacy-src", "wiki/sources/legacy-src.md"}, + {"a-concept", "wiki/concepts/a-concept.md"}, + {"an-entity", "wiki/entities/an-entity.md"}, + {"in-a-hall", "wiki/homelab/failures/in-a-hall.md"}, + {"a-clip", "wiki/web-clip/sources/a-clip.md"}, + {"nothing-here", ""}, + } + for _, tt := range tests { + t.Run(tt.slug, func(t *testing.T) { + got, found, err := FindSlug(dir, tt.slug) + require.NoError(t, err) + assert.Equal(t, tt.want != "", found) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestFindSlug_NoWikiDirIsNotAFinding(t *testing.T) { + _, found, err := FindSlug(t.TempDir(), "x") + require.NoError(t, err) + assert.False(t, found) +} diff --git a/ingestion/internal/wiki/inventory.go b/ingestion/internal/wiki/inventory.go index c9e3d4e..b87f5f8 100644 --- a/ingestion/internal/wiki/inventory.go +++ b/ingestion/internal/wiki/inventory.go @@ -97,78 +97,97 @@ func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string return } -// LoadWingInventory returns the pages of ONE wing, grouped by type. Sources -// come from wiki//sources/; concepts and entities live together in -// wiki//facts/ and are told apart by their `kind:` frontmatter. A facts -// page without a kind (a hand-written note) is left out, so extraction can -// never merge into something it did not create. Other wings are never read. +// LoadWingInventory returns what an ingest into one wing can match against: +// that wing's sources (wiki//sources/) and the SHARED concepts and +// entities (wiki/concepts/, wiki/entities/), which belong to no single wing +// because their slug is the graph's global identity. Other wings' sources are +// not read. func LoadWingInventory(brainDir, wing string) (map[PageType][]Entry, error) { - result := map[PageType][]Entry{PageTypeConcept: {}, PageTypeEntity: {}, PageTypeSource: {}} + result, err := LoadInventory(brainDir) + if err != nil { + return nil, err + } + result[PageTypeSource] = []Entry{} // the retired source layout is not this wing's bfs, err := brainfs.Open(brainDir) if err != nil { return nil, fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() - for _, hall := range []string{"sources", "facts"} { - dir := "wiki/" + wing + "/" + hall - names, err := bfs.ReadDirRegular(dir) - if errors.Is(err, fs.ErrNotExist) { + dir := "wiki/" + wing + "/sources" + names, err := bfs.ReadDirRegular(dir) + if errors.Is(err, fs.ErrNotExist) { + return result, nil + } + if err != nil { + return nil, fmt.Errorf("read dir %s: %w", dir, err) + } + for _, name := range names { + if !strings.HasSuffix(name, ".md") { continue } - if err != nil { - return nil, fmt.Errorf("read dir %s: %w", dir, err) - } - for _, name := range names { - if !strings.HasSuffix(name, ".md") { - continue - } - rel := dir + "/" + name - slug := strings.TrimSuffix(name, ".md") - pt := PageTypeSource - if hall == "facts" { - var ok bool - if pt, ok = kindOf(bfs, rel); !ok { - continue - } - } - title, aliases := readFrontmatter(bfs, rel, slug) - result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt, Path: rel}) - } + rel := dir + "/" + name + slug := strings.TrimSuffix(name, ".md") + title, aliases := readFrontmatter(bfs, rel, slug) + result[PageTypeSource] = append(result[PageTypeSource], Entry{Slug: slug, Title: title, Aliases: aliases, Type: PageTypeSource, Path: rel}) } return result, nil } -// kindOf reads `kind:` from a facts page's frontmatter. -func kindOf(bfs *brainfs.Brain, rel string) (PageType, bool) { - content, err := bfs.ReadRegular(rel) +// FindSlug returns the brain-relative path of an existing page whose basename is +// slug, searching the retired dirs (wiki/{sources,concepts,entities}/) and every +// wing hall (wiki///). The slug is a page's graph identity, so two +// pages must not share one. +func FindSlug(brainDir, slug string) (string, bool, error) { + bfs, err := brainfs.Open(brainDir) if err != nil { - return "", false + return "", false, fmt.Errorf("open brain: %w", err) } - return KindFromContent(string(content)) -} - -// KindFromContent maps frontmatter `kind: concept|entity` to a PageType. -func KindFromContent(content string) (PageType, bool) { - inFM := false - for _, line := range strings.Split(content, "\n") { - if strings.TrimSpace(line) == "---" { - if inFM { - break + defer func() { _ = bfs.Close() }() + file := slug + ".md" + has := func(dir string) (bool, error) { + names, err := bfs.ReadDirRegular(dir) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + for _, n := range names { + if n == file { + return true, nil } - inFM = true + } + return false, nil + } + for _, d := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} { + if ok, err := has(d); err != nil { + return "", false, err + } else if ok { + return d + "/" + file, true, nil + } + } + wings, err := bfs.ReadDirDirs("wiki") + if errors.Is(err, fs.ErrNotExist) { + return "", false, nil + } + if err != nil { + return "", false, err + } + for _, w := range wings { + if w == "sources" || w == "concepts" || w == "entities" { continue } - if !inFM { + halls, err := bfs.ReadDirDirs("wiki/" + w) + if err != nil { continue } - if key, val, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "kind" { - switch strings.Trim(strings.TrimSpace(val), `"'`) { - case "concept": - return PageTypeConcept, true - case "entity": - return PageTypeEntity, true + for _, h := range halls { + if ok, err := has("wiki/" + w + "/" + h); err != nil { + return "", false, err + } else if ok { + return "wiki/" + w + "/" + h + "/" + file, true, nil } } } - return "", false + return "", false, nil }