feat(pipeline): per-wing sources, shared concepts/entities, one slug one page (#208)
With Config.Wing a source goes to wiki/<wing>/sources/ (wing/hall frontmatter); concepts and entities stay in the shared wiki/concepts and wiki/entities, so an entity cited from two wings is ONE page with both sources (the per-wing facts/ layout duplicated it). A source slug already used anywhere but its own wing's sources/ is refused (slug is the graph identity). wiki.FindSlug added. Mutation-checked: collision guard, shared inventory, per-wing source path. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,52 @@
|
||||
package wiki
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func put(t *testing.T, dir, rel string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, rel)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
||||
require.NoError(t, os.WriteFile(full, []byte("---\ntitle: x\n---\n"), 0o644))
|
||||
}
|
||||
|
||||
func TestFindSlug(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
put(t, dir, "wiki/sources/legacy-src.md")
|
||||
put(t, dir, "wiki/concepts/a-concept.md")
|
||||
put(t, dir, "wiki/entities/an-entity.md")
|
||||
put(t, dir, "wiki/homelab/failures/in-a-hall.md")
|
||||
put(t, dir, "wiki/web-clip/sources/a-clip.md")
|
||||
|
||||
tests := []struct {
|
||||
slug string
|
||||
want string
|
||||
}{
|
||||
{"legacy-src", "wiki/sources/legacy-src.md"},
|
||||
{"a-concept", "wiki/concepts/a-concept.md"},
|
||||
{"an-entity", "wiki/entities/an-entity.md"},
|
||||
{"in-a-hall", "wiki/homelab/failures/in-a-hall.md"},
|
||||
{"a-clip", "wiki/web-clip/sources/a-clip.md"},
|
||||
{"nothing-here", ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.slug, func(t *testing.T) {
|
||||
got, found, err := FindSlug(dir, tt.slug)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want != "", found)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindSlug_NoWikiDirIsNotAFinding(t *testing.T) {
|
||||
_, found, err := FindSlug(t.TempDir(), "x")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
}
|
||||
@@ -97,78 +97,97 @@ func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string
|
||||
return
|
||||
}
|
||||
|
||||
// LoadWingInventory returns the pages of ONE wing, grouped by type. Sources
|
||||
// come from wiki/<wing>/sources/; concepts and entities live together in
|
||||
// wiki/<wing>/facts/ and are told apart by their `kind:` frontmatter. A facts
|
||||
// page without a kind (a hand-written note) is left out, so extraction can
|
||||
// never merge into something it did not create. Other wings are never read.
|
||||
// LoadWingInventory returns what an ingest into one wing can match against:
|
||||
// that wing's sources (wiki/<wing>/sources/) and the SHARED concepts and
|
||||
// entities (wiki/concepts/, wiki/entities/), which belong to no single wing
|
||||
// because their slug is the graph's global identity. Other wings' sources are
|
||||
// not read.
|
||||
func LoadWingInventory(brainDir, wing string) (map[PageType][]Entry, error) {
|
||||
result := map[PageType][]Entry{PageTypeConcept: {}, PageTypeEntity: {}, PageTypeSource: {}}
|
||||
result, err := LoadInventory(brainDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[PageTypeSource] = []Entry{} // the retired source layout is not this wing's
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open brain: %w", err)
|
||||
}
|
||||
defer func() { _ = bfs.Close() }()
|
||||
for _, hall := range []string{"sources", "facts"} {
|
||||
dir := "wiki/" + wing + "/" + hall
|
||||
names, err := bfs.ReadDirRegular(dir)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
dir := "wiki/" + wing + "/sources"
|
||||
names, err := bfs.ReadDirRegular(dir)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return result, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dir %s: %w", dir, err)
|
||||
}
|
||||
for _, name := range names {
|
||||
if !strings.HasSuffix(name, ".md") {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dir %s: %w", dir, err)
|
||||
}
|
||||
for _, name := range names {
|
||||
if !strings.HasSuffix(name, ".md") {
|
||||
continue
|
||||
}
|
||||
rel := dir + "/" + name
|
||||
slug := strings.TrimSuffix(name, ".md")
|
||||
pt := PageTypeSource
|
||||
if hall == "facts" {
|
||||
var ok bool
|
||||
if pt, ok = kindOf(bfs, rel); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
title, aliases := readFrontmatter(bfs, rel, slug)
|
||||
result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt, Path: rel})
|
||||
}
|
||||
rel := dir + "/" + name
|
||||
slug := strings.TrimSuffix(name, ".md")
|
||||
title, aliases := readFrontmatter(bfs, rel, slug)
|
||||
result[PageTypeSource] = append(result[PageTypeSource], Entry{Slug: slug, Title: title, Aliases: aliases, Type: PageTypeSource, Path: rel})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// kindOf reads `kind:` from a facts page's frontmatter.
|
||||
func kindOf(bfs *brainfs.Brain, rel string) (PageType, bool) {
|
||||
content, err := bfs.ReadRegular(rel)
|
||||
// FindSlug returns the brain-relative path of an existing page whose basename is
|
||||
// slug, searching the retired dirs (wiki/{sources,concepts,entities}/) and every
|
||||
// wing hall (wiki/<wing>/<hall>/). The slug is a page's graph identity, so two
|
||||
// pages must not share one.
|
||||
func FindSlug(brainDir, slug string) (string, bool, error) {
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return "", false
|
||||
return "", false, fmt.Errorf("open brain: %w", err)
|
||||
}
|
||||
return KindFromContent(string(content))
|
||||
}
|
||||
|
||||
// KindFromContent maps frontmatter `kind: concept|entity` to a PageType.
|
||||
func KindFromContent(content string) (PageType, bool) {
|
||||
inFM := false
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
if strings.TrimSpace(line) == "---" {
|
||||
if inFM {
|
||||
break
|
||||
defer func() { _ = bfs.Close() }()
|
||||
file := slug + ".md"
|
||||
has := func(dir string) (bool, error) {
|
||||
names, err := bfs.ReadDirRegular(dir)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, n := range names {
|
||||
if n == file {
|
||||
return true, nil
|
||||
}
|
||||
inFM = true
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
for _, d := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} {
|
||||
if ok, err := has(d); err != nil {
|
||||
return "", false, err
|
||||
} else if ok {
|
||||
return d + "/" + file, true, nil
|
||||
}
|
||||
}
|
||||
wings, err := bfs.ReadDirDirs("wiki")
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, w := range wings {
|
||||
if w == "sources" || w == "concepts" || w == "entities" {
|
||||
continue
|
||||
}
|
||||
if !inFM {
|
||||
halls, err := bfs.ReadDirDirs("wiki/" + w)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if key, val, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "kind" {
|
||||
switch strings.Trim(strings.TrimSpace(val), `"'`) {
|
||||
case "concept":
|
||||
return PageTypeConcept, true
|
||||
case "entity":
|
||||
return PageTypeEntity, true
|
||||
for _, h := range halls {
|
||||
if ok, err := has("wiki/" + w + "/" + h); err != nil {
|
||||
return "", false, err
|
||||
} else if ok {
|
||||
return "wiki/" + w + "/" + h + "/" + file, true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user