feat(pipeline): write into wiki/<wing>/{sources,facts}/ when Config.Wing is set (#205 S2)

Sources -> wiki/<wing>/sources, concepts/entities -> wiki/<wing>/facts with
kind: frontmatter; inventory, title resolution and back-references are
confined to the target wing. Empty Wing keeps the retired layout unchanged.
Mutation-checked: widening the inventory beyond the wing fails the
cross-wing back-reference test; a non-plain wing name is rejected.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-20 06:07:14 +02:00
co-authored by Claude Sonnet 5.5
parent 62de8ac2e6
commit ad793f320d
8 changed files with 334 additions and 25 deletions
+35 -12
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"io/fs"
"os"
@@ -26,6 +27,11 @@ type Config struct {
Complete CompleteFunc
ChunkSize int // 0 = no chunking
Schema string // overrides brain/schema.md when set (useful in tests)
// Wing, when set, writes into wiki/<wing>/{sources,facts}/ and scopes the
// inventory (title resolution, back-references) to that wing alone. Empty
// keeps the retired wiki/{sources,concepts,entities}/ layout.
Wing string
}
// Result is the outcome of a pipeline run.
@@ -42,7 +48,16 @@ type Result struct {
// Run ingests content and writes structured wiki pages to brainDir/wiki/.
// In dry-run mode, pages are returned but not written to disk.
func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryRun bool) (Result, error) {
inventory, err := wiki.LoadInventory(brainDir)
if cfg.Wing != "" && brain.Sanitise(cfg.Wing) != cfg.Wing {
return Result{}, fmt.Errorf("invalid wing %q: must be a plain lower-case wing name", cfg.Wing)
}
var inventory map[wiki.PageType][]wiki.Entry
var err error
if cfg.Wing != "" {
inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing)
} else {
inventory, err = wiki.LoadInventory(brainDir)
}
if err != nil {
return Result{}, fmt.Errorf("load inventory: %w", err)
}
@@ -60,7 +75,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
// The merge is already idempotent (#103), so this is cost and signal, not
// correctness — which is why it is safe for it to be an early return.
bodyHash := hashBody(content)
if !dryRun && bodyHash != "" && bodyHash == storedSourceHash(brainDir, sourceSlug) {
if !dryRun && bodyHash != "" && bodyHash == storedSourceHash(brainDir, cfg.Wing, sourceSlug) {
return Result{Skipped: true, Warnings: []string{
fmt.Sprintf("source %q unchanged since last ingest (hash %s) — extraction skipped", source, bodyHash[:12]),
}}, nil
@@ -71,7 +86,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
// only exists once the model has produced a title. So page-aware
// extraction (#106) is scoped to the source page: it's the one page every
// chunk's prompt can be given in advance.
existingPage := readSourcePage(brainDir, sourceSlug)
existingPage := readSourcePage(brainDir, cfg.Wing, sourceSlug)
chunks := Chunk(content, cfg.ChunkSize)
@@ -93,7 +108,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
meta.BodyHash = bodyHash
applySourceMeta(allRaw, meta)
return buildAndWrite(ctx, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun)
return buildAndWrite(ctx, cfg.Wing, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun)
}
// RunRaw runs the pipeline on pre-parsed RawPages, skipping the LLM extraction
@@ -112,12 +127,12 @@ func RunRaw(ctx context.Context, brainDir, source string, rawPages []RawPage, dr
sourceSlug := wiki.Slug(source)
date := time.Now().UTC().Format("2006-01-02")
return buildAndWrite(ctx, rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun)
return buildAndWrite(ctx, "", rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun)
}
// buildAndWrite runs BuildPages through write for both Run and RunRaw.
func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) {
pages, buildWarnings := BuildPages(rawPages, sourceSlug, date)
func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) {
pages, buildWarnings := BuildPagesIn(wing, rawPages, sourceSlug, date)
warnings = append(warnings, buildWarnings...)
resolved := Resolve(pages, inventory)
canonicalized, linkWarnings := CanonicalizeLinks(resolved, inventory)
@@ -158,7 +173,11 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br
}
if !dryRun {
if err := wiki.RebuildIndex(brainDir, date); err != nil {
if wing != "" {
if err := brain.BuildWingIndex(brainDir, wing); err != nil {
warnings = append(warnings, fmt.Sprintf("rebuild wing index: %v", err))
}
} else if err := wiki.RebuildIndex(brainDir, date); err != nil {
warnings = append(warnings, fmt.Sprintf("rebuild index: %v", err))
}
if err := wiki.AppendLog(brainDir, source, written, warnings, date); err != nil {
@@ -193,8 +212,12 @@ func hashBody(content string) string {
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
// It reads through the brain root, regular files only (hyperguild#157): the
// content goes into the LLM prompt.
func readSourcePage(brainDir, sourceSlug string) string {
raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md")
func readSourcePage(brainDir, wing, sourceSlug string) string {
rel := "wiki/sources/" + sourceSlug + ".md"
if wing != "" {
rel = "wiki/" + wing + "/sources/" + sourceSlug + ".md"
}
raw, err := readBrainFile(brainDir, rel)
if err != nil {
return ""
}
@@ -218,8 +241,8 @@ func readBrainFile(brainDir, rel string) ([]byte, error) {
// Absence always means "extract". That is the fail-safe direction: a missed
// skip costs one extraction, while a wrong skip loses an update silently — and
// silence is the failure mode this repo keeps paying for.
func storedSourceHash(brainDir, sourceSlug string) string {
for _, line := range strings.Split(readSourcePage(brainDir, sourceSlug), "\n") {
func storedSourceHash(brainDir, wing, sourceSlug string) string {
for _, line := range strings.Split(readSourcePage(brainDir, wing, sourceSlug), "\n") {
if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" {
return strings.TrimSpace(value)
}