feat(pipeline): write into wiki/<wing>/{sources,facts}/ when Config.Wing is set (#205 S2)
Sources -> wiki/<wing>/sources, concepts/entities -> wiki/<wing>/facts with kind: frontmatter; inventory, title resolution and back-references are confined to the target wing. Empty Wing keeps the retired layout unchanged. Mutation-checked: widening the inventory beyond the wing fails the cross-wing back-reference test; a non-plain wing name is rejected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -26,6 +27,11 @@ type Config struct {
|
||||
Complete CompleteFunc
|
||||
ChunkSize int // 0 = no chunking
|
||||
Schema string // overrides brain/schema.md when set (useful in tests)
|
||||
|
||||
// Wing, when set, writes into wiki/<wing>/{sources,facts}/ and scopes the
|
||||
// inventory (title resolution, back-references) to that wing alone. Empty
|
||||
// keeps the retired wiki/{sources,concepts,entities}/ layout.
|
||||
Wing string
|
||||
}
|
||||
|
||||
// Result is the outcome of a pipeline run.
|
||||
@@ -42,7 +48,16 @@ type Result struct {
|
||||
// Run ingests content and writes structured wiki pages to brainDir/wiki/.
|
||||
// In dry-run mode, pages are returned but not written to disk.
|
||||
func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryRun bool) (Result, error) {
|
||||
inventory, err := wiki.LoadInventory(brainDir)
|
||||
if cfg.Wing != "" && brain.Sanitise(cfg.Wing) != cfg.Wing {
|
||||
return Result{}, fmt.Errorf("invalid wing %q: must be a plain lower-case wing name", cfg.Wing)
|
||||
}
|
||||
var inventory map[wiki.PageType][]wiki.Entry
|
||||
var err error
|
||||
if cfg.Wing != "" {
|
||||
inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing)
|
||||
} else {
|
||||
inventory, err = wiki.LoadInventory(brainDir)
|
||||
}
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("load inventory: %w", err)
|
||||
}
|
||||
@@ -60,7 +75,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
|
||||
// The merge is already idempotent (#103), so this is cost and signal, not
|
||||
// correctness — which is why it is safe for it to be an early return.
|
||||
bodyHash := hashBody(content)
|
||||
if !dryRun && bodyHash != "" && bodyHash == storedSourceHash(brainDir, sourceSlug) {
|
||||
if !dryRun && bodyHash != "" && bodyHash == storedSourceHash(brainDir, cfg.Wing, sourceSlug) {
|
||||
return Result{Skipped: true, Warnings: []string{
|
||||
fmt.Sprintf("source %q unchanged since last ingest (hash %s) — extraction skipped", source, bodyHash[:12]),
|
||||
}}, nil
|
||||
@@ -71,7 +86,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
|
||||
// only exists once the model has produced a title. So page-aware
|
||||
// extraction (#106) is scoped to the source page: it's the one page every
|
||||
// chunk's prompt can be given in advance.
|
||||
existingPage := readSourcePage(brainDir, sourceSlug)
|
||||
existingPage := readSourcePage(brainDir, cfg.Wing, sourceSlug)
|
||||
|
||||
chunks := Chunk(content, cfg.ChunkSize)
|
||||
|
||||
@@ -93,7 +108,7 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
|
||||
meta.BodyHash = bodyHash
|
||||
applySourceMeta(allRaw, meta)
|
||||
|
||||
return buildAndWrite(ctx, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun)
|
||||
return buildAndWrite(ctx, cfg.Wing, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun)
|
||||
}
|
||||
|
||||
// RunRaw runs the pipeline on pre-parsed RawPages, skipping the LLM extraction
|
||||
@@ -112,12 +127,12 @@ func RunRaw(ctx context.Context, brainDir, source string, rawPages []RawPage, dr
|
||||
sourceSlug := wiki.Slug(source)
|
||||
date := time.Now().UTC().Format("2006-01-02")
|
||||
|
||||
return buildAndWrite(ctx, rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun)
|
||||
return buildAndWrite(ctx, "", rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun)
|
||||
}
|
||||
|
||||
// buildAndWrite runs BuildPages through write for both Run and RunRaw.
|
||||
func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) {
|
||||
pages, buildWarnings := BuildPages(rawPages, sourceSlug, date)
|
||||
func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) {
|
||||
pages, buildWarnings := BuildPagesIn(wing, rawPages, sourceSlug, date)
|
||||
warnings = append(warnings, buildWarnings...)
|
||||
resolved := Resolve(pages, inventory)
|
||||
canonicalized, linkWarnings := CanonicalizeLinks(resolved, inventory)
|
||||
@@ -158,7 +173,11 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br
|
||||
}
|
||||
|
||||
if !dryRun {
|
||||
if err := wiki.RebuildIndex(brainDir, date); err != nil {
|
||||
if wing != "" {
|
||||
if err := brain.BuildWingIndex(brainDir, wing); err != nil {
|
||||
warnings = append(warnings, fmt.Sprintf("rebuild wing index: %v", err))
|
||||
}
|
||||
} else if err := wiki.RebuildIndex(brainDir, date); err != nil {
|
||||
warnings = append(warnings, fmt.Sprintf("rebuild index: %v", err))
|
||||
}
|
||||
if err := wiki.AppendLog(brainDir, source, written, warnings, date); err != nil {
|
||||
@@ -193,8 +212,12 @@ func hashBody(content string) string {
|
||||
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
|
||||
// It reads through the brain root, regular files only (hyperguild#157): the
|
||||
// content goes into the LLM prompt.
|
||||
func readSourcePage(brainDir, sourceSlug string) string {
|
||||
raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md")
|
||||
func readSourcePage(brainDir, wing, sourceSlug string) string {
|
||||
rel := "wiki/sources/" + sourceSlug + ".md"
|
||||
if wing != "" {
|
||||
rel = "wiki/" + wing + "/sources/" + sourceSlug + ".md"
|
||||
}
|
||||
raw, err := readBrainFile(brainDir, rel)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -218,8 +241,8 @@ func readBrainFile(brainDir, rel string) ([]byte, error) {
|
||||
// Absence always means "extract". That is the fail-safe direction: a missed
|
||||
// skip costs one extraction, while a wrong skip loses an update silently — and
|
||||
// silence is the failure mode this repo keeps paying for.
|
||||
func storedSourceHash(brainDir, sourceSlug string) string {
|
||||
for _, line := range strings.Split(readSourcePage(brainDir, sourceSlug), "\n") {
|
||||
func storedSourceHash(brainDir, wing, sourceSlug string) string {
|
||||
for _, line := range strings.Split(readSourcePage(brainDir, wing, sourceSlug), "\n") {
|
||||
if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user