feat(ingestion): place raw notes in a wing without human typing (#205 S1–S4, S6) #207

Merged
mathias merged 15 commits from wt/wing-layout into main 2026-10-20 06:21:45 +00:00
8 changed files with 334 additions and 25 deletions
Showing only changes of commit ad793f320d - Show all commits
+32 -2
View File
@@ -14,6 +14,13 @@ import (
// (derived from the filename, not the LLM title). Pages whose title resolves to an // (derived from the filename, not the LLM title). Pages whose title resolves to an
// empty slug are skipped and returned as warnings instead. // empty slug are skipped and returned as warnings instead.
func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) { func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) {
return BuildPagesIn("", rawPages, sourceSlug, date)
}
// BuildPagesIn is BuildPages for a wing: sources go to wiki/<wing>/sources/,
// concepts and entities to wiki/<wing>/facts/ with `kind:` recording which.
// An empty wing is the retired wiki/{sources,concepts,entities}/ layout.
func BuildPagesIn(wing string, rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) {
out := make([]wiki.Page, 0, len(rawPages)) out := make([]wiki.Page, 0, len(rawPages))
var warnings []string var warnings []string
for _, rp := range rawPages { for _, rp := range rawPages {
@@ -28,7 +35,7 @@ func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []str
warnings = append(warnings, fmt.Sprintf("skipped page with empty title (type: %s)", rp.Type)) warnings = append(warnings, fmt.Sprintf("skipped page with empty title (type: %s)", rp.Type))
continue continue
} }
out = append(out, buildPage(rp, sourceSlug, date)) out = append(out, buildPage(rp, sourceSlug, date, wing))
} }
return out, warnings return out, warnings
} }
@@ -42,7 +49,7 @@ func computeSlug(rp RawPage, sourceSlug string) string {
return wiki.Slug(rp.Title) return wiki.Slug(rp.Title)
} }
func buildPage(rp RawPage, sourceSlug, date string) wiki.Page { func buildPage(rp RawPage, sourceSlug, date, wing string) wiki.Page {
var slug, dir string var slug, dir string
switch rp.Type { switch rp.Type {
case "source": case "source":
@@ -59,8 +66,21 @@ func buildPage(rp RawPage, sourceSlug, date string) wiki.Page {
dir = "wiki/" + rp.Type dir = "wiki/" + rp.Type
} }
hall := ""
if wing != "" {
switch rp.Type {
case "source":
hall = "sources"
default:
hall = "facts"
}
dir = "wiki/" + wing + "/" + hall
}
path := dir + "/" + slug + ".md" path := dir + "/" + slug + ".md"
fm := buildFrontmatter(rp, date) fm := buildFrontmatter(rp, date)
if wing != "" {
fm = addWingFrontmatter(fm, wing, hall, rp.Type)
}
return wiki.Page{ return wiki.Page{
Path: path, Path: path,
@@ -122,6 +142,16 @@ func buildFrontmatter(rp RawPage, date string) string {
return sb.String() return sb.String()
} }
// addWingFrontmatter inserts wing/hall (and kind for facts pages) right after
// the opening fence of an already-built frontmatter block.
func addWingFrontmatter(fm, wing, hall, rpType string) string {
extra := "wing: " + wing + "\nhall: " + hall + "\n"
if hall == "facts" {
extra += "kind: " + rpType + "\n"
}
return strings.Replace(fm, "---\n", "---\n"+extra, 1)
}
func yamlScalar(s string) string { func yamlScalar(s string) string {
// A raw newline or CR would end the scalar for every line-oriented // A raw newline or CR would end the scalar for every line-oriented
// frontmatter reader and start a new key (PR #147 review: a title of // frontmatter reader and start a new key (PR #147 review: a title of
+35 -12
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex" "encoding/hex"
"errors" "errors"
"fmt" "fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"io/fs" "io/fs"
"os" "os"
@@ -26,6 +27,11 @@ type Config struct {
Complete CompleteFunc Complete CompleteFunc
ChunkSize int // 0 = no chunking ChunkSize int // 0 = no chunking
Schema string // overrides brain/schema.md when set (useful in tests) Schema string // overrides brain/schema.md when set (useful in tests)
// Wing, when set, writes into wiki/<wing>/{sources,facts}/ and scopes the
// inventory (title resolution, back-references) to that wing alone. Empty
// keeps the retired wiki/{sources,concepts,entities}/ layout.
Wing string
} }
// Result is the outcome of a pipeline run. // Result is the outcome of a pipeline run.
@@ -42,7 +48,16 @@ type Result struct {
// Run ingests content and writes structured wiki pages to brainDir/wiki/. // Run ingests content and writes structured wiki pages to brainDir/wiki/.
// In dry-run mode, pages are returned but not written to disk. // In dry-run mode, pages are returned but not written to disk.
func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryRun bool) (Result, error) { func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryRun bool) (Result, error) {
inventory, err := wiki.LoadInventory(brainDir) if cfg.Wing != "" && brain.Sanitise(cfg.Wing) != cfg.Wing {
return Result{}, fmt.Errorf("invalid wing %q: must be a plain lower-case wing name", cfg.Wing)
}
var inventory map[wiki.PageType][]wiki.Entry
var err error
if cfg.Wing != "" {
inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing)
} else {
inventory, err = wiki.LoadInventory(brainDir)
}
if err != nil { if err != nil {
return Result{}, fmt.Errorf("load inventory: %w", err) return Result{}, fmt.Errorf("load inventory: %w", err)
} }
@@ -60,7 +75,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
// The merge is already idempotent (#103), so this is cost and signal, not // The merge is already idempotent (#103), so this is cost and signal, not
// correctness — which is why it is safe for it to be an early return. // correctness — which is why it is safe for it to be an early return.
bodyHash := hashBody(content) bodyHash := hashBody(content)
if !dryRun && bodyHash != "" && bodyHash == storedSourceHash(brainDir, sourceSlug) { if !dryRun && bodyHash != "" && bodyHash == storedSourceHash(brainDir, cfg.Wing, sourceSlug) {
return Result{Skipped: true, Warnings: []string{ return Result{Skipped: true, Warnings: []string{
fmt.Sprintf("source %q unchanged since last ingest (hash %s) — extraction skipped", source, bodyHash[:12]), fmt.Sprintf("source %q unchanged since last ingest (hash %s) — extraction skipped", source, bodyHash[:12]),
}}, nil }}, nil
@@ -71,7 +86,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
// only exists once the model has produced a title. So page-aware // only exists once the model has produced a title. So page-aware
// extraction (#106) is scoped to the source page: it's the one page every // extraction (#106) is scoped to the source page: it's the one page every
// chunk's prompt can be given in advance. // chunk's prompt can be given in advance.
existingPage := readSourcePage(brainDir, sourceSlug) existingPage := readSourcePage(brainDir, cfg.Wing, sourceSlug)
chunks := Chunk(content, cfg.ChunkSize) chunks := Chunk(content, cfg.ChunkSize)
@@ -93,7 +108,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
meta.BodyHash = bodyHash meta.BodyHash = bodyHash
applySourceMeta(allRaw, meta) applySourceMeta(allRaw, meta)
return buildAndWrite(ctx, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun) return buildAndWrite(ctx, cfg.Wing, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun)
} }
// RunRaw runs the pipeline on pre-parsed RawPages, skipping the LLM extraction // RunRaw runs the pipeline on pre-parsed RawPages, skipping the LLM extraction
@@ -112,12 +127,12 @@ func RunRaw(ctx context.Context, brainDir, source string, rawPages []RawPage, dr
sourceSlug := wiki.Slug(source) sourceSlug := wiki.Slug(source)
date := time.Now().UTC().Format("2006-01-02") date := time.Now().UTC().Format("2006-01-02")
return buildAndWrite(ctx, rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun) return buildAndWrite(ctx, "", rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun)
} }
// buildAndWrite runs BuildPages through write for both Run and RunRaw. // buildAndWrite runs BuildPages through write for both Run and RunRaw.
func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) { func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) {
pages, buildWarnings := BuildPages(rawPages, sourceSlug, date) pages, buildWarnings := BuildPagesIn(wing, rawPages, sourceSlug, date)
warnings = append(warnings, buildWarnings...) warnings = append(warnings, buildWarnings...)
resolved := Resolve(pages, inventory) resolved := Resolve(pages, inventory)
canonicalized, linkWarnings := CanonicalizeLinks(resolved, inventory) canonicalized, linkWarnings := CanonicalizeLinks(resolved, inventory)
@@ -158,7 +173,11 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br
} }
if !dryRun { if !dryRun {
if err := wiki.RebuildIndex(brainDir, date); err != nil { if wing != "" {
if err := brain.BuildWingIndex(brainDir, wing); err != nil {
warnings = append(warnings, fmt.Sprintf("rebuild wing index: %v", err))
}
} else if err := wiki.RebuildIndex(brainDir, date); err != nil {
warnings = append(warnings, fmt.Sprintf("rebuild index: %v", err)) warnings = append(warnings, fmt.Sprintf("rebuild index: %v", err))
} }
if err := wiki.AppendLog(brainDir, source, written, warnings, date); err != nil { if err := wiki.AppendLog(brainDir, source, written, warnings, date); err != nil {
@@ -193,8 +212,12 @@ func hashBody(content string) string {
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet. // wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
// It reads through the brain root, regular files only (hyperguild#157): the // It reads through the brain root, regular files only (hyperguild#157): the
// content goes into the LLM prompt. // content goes into the LLM prompt.
func readSourcePage(brainDir, sourceSlug string) string { func readSourcePage(brainDir, wing, sourceSlug string) string {
raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md") rel := "wiki/sources/" + sourceSlug + ".md"
if wing != "" {
rel = "wiki/" + wing + "/sources/" + sourceSlug + ".md"
}
raw, err := readBrainFile(brainDir, rel)
if err != nil { if err != nil {
return "" return ""
} }
@@ -218,8 +241,8 @@ func readBrainFile(brainDir, rel string) ([]byte, error) {
// Absence always means "extract". That is the fail-safe direction: a missed // Absence always means "extract". That is the fail-safe direction: a missed
// skip costs one extraction, while a wrong skip loses an update silently — and // skip costs one extraction, while a wrong skip loses an update silently — and
// silence is the failure mode this repo keeps paying for. // silence is the failure mode this repo keeps paying for.
func storedSourceHash(brainDir, sourceSlug string) string { func storedSourceHash(brainDir, wing, sourceSlug string) string {
for _, line := range strings.Split(readSourcePage(brainDir, sourceSlug), "\n") { for _, line := range strings.Split(readSourcePage(brainDir, wing, sourceSlug), "\n") {
if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" { if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" {
return strings.TrimSpace(value) return strings.TrimSpace(value)
} }
+30 -5
View File
@@ -23,7 +23,7 @@ func injectSourceRefs(pages []wiki.Page, inventory map[wiki.PageType][]wiki.Entr
var sourceContent string var sourceContent string
for _, p := range pages { for _, p := range pages {
if strings.HasPrefix(p.Path, "wiki/sources/") && if isSourcePath(p.Path) &&
strings.TrimSuffix(filepath.Base(p.Path), ".md") == sourceSlug { strings.TrimSuffix(filepath.Base(p.Path), ".md") == sourceSlug {
sourceContent = p.Content sourceContent = p.Content
break break
@@ -35,7 +35,7 @@ func injectSourceRefs(pages []wiki.Page, inventory map[wiki.PageType][]wiki.Entr
bySlug := make(map[string]int, len(pages)) bySlug := make(map[string]int, len(pages))
for i, p := range pages { for i, p := range pages {
if !strings.HasPrefix(p.Path, "wiki/sources/") { if !isSourcePath(p.Path) {
bySlug[strings.TrimSuffix(filepath.Base(p.Path), ".md")] = i bySlug[strings.TrimSuffix(filepath.Base(p.Path), ".md")] = i
} }
} }
@@ -52,12 +52,13 @@ func injectSourceRefs(pages []wiki.Page, inventory map[wiki.PageType][]wiki.Entr
if !ok { if !ok {
continue continue
} }
b, err := readBrainFile(brainDir, "wiki/"+string(pt)+"/"+slug+".md") path := entryPath(slug, pt, inventory)
b, err := readBrainFile(brainDir, path)
if err != nil { if err != nil {
continue continue
} }
page := wiki.Page{ page := wiki.Page{
Path: "wiki/" + string(pt) + "/" + slug + ".md", Path: path,
Content: string(b), Content: string(b),
} }
pages = append(pages, addSourceRef(page, sourceRef)) pages = append(pages, addSourceRef(page, sourceRef))
@@ -88,7 +89,7 @@ func extractWikilinks(content string) map[string]bool {
// findSourcePage returns the slug and title of the first wiki/sources/ page in pages. // findSourcePage returns the slug and title of the first wiki/sources/ page in pages.
func findSourcePage(pages []wiki.Page) (slug, title string, found bool) { func findSourcePage(pages []wiki.Page) (slug, title string, found bool) {
for _, p := range pages { for _, p := range pages {
if strings.HasPrefix(p.Path, "wiki/sources/") { if isSourcePath(p.Path) {
slug = strings.TrimSuffix(filepath.Base(p.Path), ".md") slug = strings.TrimSuffix(filepath.Base(p.Path), ".md")
title = extractTitle(p.Content) title = extractTitle(p.Content)
if title == "" { if title == "" {
@@ -111,3 +112,27 @@ func findInInventory(slug string, inventory map[wiki.PageType][]wiki.Entry) (wik
} }
return "", false return "", false
} }
// isSourcePath reports whether path is a source page in either layout:
// wiki/sources/<slug>.md or wiki/<wing>/sources/<slug>.md.
func isSourcePath(path string) bool {
parts := strings.Split(filepath.ToSlash(path), "/")
switch {
case len(parts) == 3 && parts[0] == "wiki" && parts[1] == "sources":
return true
case len(parts) == 4 && parts[0] == "wiki" && parts[2] == "sources":
return true
}
return false
}
// entryPath is the brain-relative path of an inventory page: the recorded Path
// for a wing-layout entry, else the retired wiki/<type>/<slug>.md.
func entryPath(slug string, pt wiki.PageType, inventory map[wiki.PageType][]wiki.Entry) string {
for _, e := range inventory[pt] {
if e.Slug == slug && e.Path != "" {
return e.Path
}
}
return "wiki/" + string(pt) + "/" + slug + ".md"
}
+17 -1
View File
@@ -32,7 +32,7 @@ func Resolve(proposed []wiki.Page, inventory map[wiki.PageType][]wiki.Entry) []w
out := make([]wiki.Page, 0, len(proposed)) out := make([]wiki.Page, 0, len(proposed))
for _, page := range proposed { for _, page := range proposed {
pt := pageTypeFromPath(page.Path) pt := pageTypeOf(page)
title := extractTitle(page.Content) title := extractTitle(page.Content)
k := key{pt: pt, normalized: normalizeTitle(title)} k := key{pt: pt, normalized: normalizeTitle(title)}
if canonicalSlug, ok := lookup[k]; ok { if canonicalSlug, ok := lookup[k]; ok {
@@ -55,6 +55,22 @@ func normalizeTitle(s string) string {
return strings.Join(strings.Fields(s), " ") return strings.Join(strings.Fields(s), " ")
} }
// pageTypeOf is the page's type: from its directory in the retired layout, or
// from `kind:` for a wing-layout facts page (concepts and entities share facts/).
func pageTypeOf(p wiki.Page) wiki.PageType {
parts := strings.Split(filepath.ToSlash(p.Path), "/")
if len(parts) >= 4 && parts[0] == "wiki" {
switch parts[2] {
case "sources":
return wiki.PageTypeSource
case "facts":
pt, _ := wiki.KindFromContent(p.Content)
return pt
}
}
return pageTypeFromPath(p.Path)
}
// pageTypeFromPath extracts the wiki.PageType from a path like "wiki/entities/foo.md". // pageTypeFromPath extracts the wiki.PageType from a path like "wiki/entities/foo.md".
func pageTypeFromPath(path string) wiki.PageType { func pageTypeFromPath(path string) wiki.PageType {
parts := strings.Split(filepath.ToSlash(path), "/") parts := strings.Split(filepath.ToSlash(path), "/")
+5 -5
View File
@@ -35,17 +35,17 @@ func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T)
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md")) inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644))
assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control") assert.Equal(t, "zqinside source", readSourcePage(brain, "", "ok"), "positive control")
assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control") assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control")
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md"))) require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md")))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md"))) require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel) assert.NotContains(t, readSourcePage(brain, "", "out"), brainfstest.Sentinel)
assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel) assert.NotContains(t, readSourcePage(brain, "", "in"), brainfstest.Sentinel)
assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel) assert.NotContains(t, storedSourceHash(brain, "", "in"), brainfstest.Sentinel)
src := filepath.Join(brain, "wiki", "sources", "r.md") src := filepath.Join(brain, "wiki", "sources", "r.md")
read := func() string { return readSourcePage(brain, "r") } read := func() string { return readSourcePage(brain, "", "r") }
brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read) brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read)
brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read) brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() }) brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
@@ -0,0 +1,138 @@
package pipeline
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func wingCfg(wing string, pages ...RawPage) Config {
return Config{
Wing: wing,
Schema: "# Schema",
Complete: func(context.Context, string, string) (string, error) {
return mustJSON(pages), nil
},
}
}
func sourceAndEntity() []RawPage {
return []RawPage{
{Title: "Dark Factory Talk", Type: "source", Subtype: "video", Domain: "ai",
Content: "## Summary\n\nA talk.\n\n## Entities Mentioned\n\n[[Jev]]\n"},
{Title: "Jev", Type: "entity", Subtype: "tool", Domain: "ai",
Content: "## Definition\n\nA decision model.\n\n## Sources\n"},
}
}
func exists(dir, rel string) bool {
_, err := os.Stat(filepath.Join(dir, rel))
return err == nil
}
func read(t *testing.T, dir, rel string) string {
t.Helper()
b, err := os.ReadFile(filepath.Join(dir, rel))
require.NoError(t, err)
return string(b)
}
func TestRun_WithAWingWritesIntoThatWingsHalls(t *testing.T) {
dir := t.TempDir()
_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
require.NoError(t, err)
src := read(t, dir, "wiki/radar/sources/dark-factory-talk.md")
assert.Contains(t, src, "wing: radar")
assert.Contains(t, src, "hall: sources")
ent := read(t, dir, "wiki/radar/facts/jev.md")
assert.Contains(t, ent, "wing: radar")
assert.Contains(t, ent, "hall: facts")
assert.Contains(t, ent, "kind: entity")
}
func TestRun_WithAWingDoesNotTouchTheRetiredLayout(t *testing.T) {
dir := t.TempDir()
_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
require.NoError(t, err)
for _, retired := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} {
assert.False(t, exists(dir, retired), "%s is the retired layout and must not be written", retired)
}
}
func TestRun_WithAWingTheSourceLinksBackFromTheEntity(t *testing.T) {
dir := t.TempDir()
_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body of the talk", "dark-factory-talk", false)
require.NoError(t, err)
assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "[[dark-factory-talk|Dark Factory Talk]]")
}
func TestRun_WithAWingASecondSourceReusesTheSameEntityPage(t *testing.T) {
dir := t.TempDir()
_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "first body", "talk-one", false)
require.NoError(t, err)
again := []RawPage{
{Title: "Second Talk", Type: "source", Subtype: "video", Content: "## Summary\n\nMore.\n\n## Entities Mentioned\n\n[[The Jev]]\n"},
{Title: "The Jev", Type: "entity", Subtype: "tool", Content: "## Definition\n\nStill a decision model.\n\n## Sources\n"},
}
_, err = Run(context.Background(), wingCfg("radar", again...), dir, "second body", "talk-two", false)
require.NoError(t, err)
files, err := filepath.Glob(filepath.Join(dir, "wiki/radar/facts/*.md"))
require.NoError(t, err)
assert.Len(t, files, 1, "an entity matched by title (leading article ignored) is merged, not duplicated")
assert.Contains(t, read(t, dir, "wiki/radar/facts/jev.md"), "talk-two")
}
func TestRun_WithAWingNeverMergesIntoAnotherWingsPage(t *testing.T) {
dir := t.TempDir()
other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n"
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644))
_, err := Run(context.Background(), wingCfg("radar", sourceAndEntity()...), dir, "body", "dark-factory-talk", false)
require.NoError(t, err)
assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "another wing's page is never read into or rewritten")
assert.True(t, exists(dir, "wiki/radar/facts/jev.md"))
}
func TestRun_WithAWingRejectsANameThatIsNotAPlainWing(t *testing.T) {
for _, bad := range []string{"../escape", "a/b", "Radar", "has space", "."} {
t.Run(bad, func(t *testing.T) {
dir := t.TempDir()
_, err := Run(context.Background(), wingCfg(bad, sourceAndEntity()...), dir, "body", "talk", false)
require.Error(t, err)
assert.Len(t, listAll(t, dir), 1, "nothing is written for a rejected wing (only the temp dir itself exists)")
})
}
}
func listAll(t *testing.T, dir string) []string {
t.Helper()
var out []string
_ = filepath.WalkDir(dir, func(p string, _ os.DirEntry, _ error) error { out = append(out, p); return nil })
return out
}
// The dangerous path is the back-reference: a source that links [[Jev]] without
// proposing a Jev page makes the pipeline go and edit an existing Jev page. That
// lookup must be confined to the target wing.
func TestRun_WithAWingASourceLinkingToAnotherWingsEntityDoesNotEditIt(t *testing.T) {
dir := t.TempDir()
other := "---\ntitle: Jev\nwing: client-x\nhall: facts\nkind: entity\n---\n\n## Definition\n\nPrivate to another wing.\n\n## Sources\n"
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/client-x/facts"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/client-x/facts/jev.md"), []byte(other), 0o644))
onlySource := sourceAndEntity()[:1]
_, err := Run(context.Background(), wingCfg("radar", onlySource...), dir, "body", "dark-factory-talk", false)
require.NoError(t, err)
assert.Equal(t, other, read(t, dir, "wiki/client-x/facts/jev.md"), "no back-reference may be written into another wing")
}
+76
View File
@@ -96,3 +96,79 @@ func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string
} }
return return
} }
// LoadWingInventory returns the pages of ONE wing, grouped by type. Sources
// come from wiki/<wing>/sources/; concepts and entities live together in
// wiki/<wing>/facts/ and are told apart by their `kind:` frontmatter. A facts
// page without a kind (a hand-written note) is left out, so extraction can
// never merge into something it did not create. Other wings are never read.
func LoadWingInventory(brainDir, wing string) (map[PageType][]Entry, error) {
result := map[PageType][]Entry{PageTypeConcept: {}, PageTypeEntity: {}, PageTypeSource: {}}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
for _, hall := range []string{"sources", "facts"} {
dir := "wiki/" + wing + "/" + hall
names, err := bfs.ReadDirRegular(dir)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return nil, fmt.Errorf("read dir %s: %w", dir, err)
}
for _, name := range names {
if !strings.HasSuffix(name, ".md") {
continue
}
rel := dir + "/" + name
slug := strings.TrimSuffix(name, ".md")
pt := PageTypeSource
if hall == "facts" {
var ok bool
if pt, ok = kindOf(bfs, rel); !ok {
continue
}
}
title, aliases := readFrontmatter(bfs, rel, slug)
result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt, Path: rel})
}
}
return result, nil
}
// kindOf reads `kind:` from a facts page's frontmatter.
func kindOf(bfs *brainfs.Brain, rel string) (PageType, bool) {
content, err := bfs.ReadRegular(rel)
if err != nil {
return "", false
}
return KindFromContent(string(content))
}
// KindFromContent maps frontmatter `kind: concept|entity` to a PageType.
func KindFromContent(content string) (PageType, bool) {
inFM := false
for _, line := range strings.Split(content, "\n") {
if strings.TrimSpace(line) == "---" {
if inFM {
break
}
inFM = true
continue
}
if !inFM {
continue
}
if key, val, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "kind" {
switch strings.Trim(strings.TrimSpace(val), `"'`) {
case "concept":
return PageTypeConcept, true
case "entity":
return PageTypeEntity, true
}
}
}
return "", false
}
+1
View File
@@ -22,4 +22,5 @@ type Entry struct {
Title string Title string
Aliases []string Aliases []string
Type PageType Type PageType
Path string // brain-relative path; set for wing-layout entries
} }