hyperguild/ingestion/internal/pipeline/pipeline.go

// ingestion/internal/pipeline/pipeline.go
package pipeline

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
)

// CompleteFunc is the function signature for LLM calls.
type CompleteFunc func(ctx context.Context, system, user string) (string, error)

// Config holds pipeline configuration.
type Config struct {
	Complete  CompleteFunc
	ChunkSize int    // 0 = no chunking
	Schema    string // overrides brain/schema.md when set (useful in tests)
}

// Result is the outcome of a pipeline run.
type Result struct {
	Pages    []string // relative paths written (or would-be written in dry-run)
	Warnings []string
}

// Run ingests content and writes structured wiki pages to brainDir/wiki/.
// In dry-run mode, pages are returned but not written to disk.
func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryRun bool) (Result, error) {
	inventory, err := wiki.LoadInventory(brainDir)
	if err != nil {
		return Result{}, fmt.Errorf("load inventory: %w", err)
	}

	schema := cfg.Schema
	if schema == "" {
		schema = loadSchema(brainDir)
	}

	sourceSlug := wiki.Slug(source)
	date := time.Now().UTC().Format("2006-01-02")
	chunks := Chunk(content, cfg.ChunkSize)

	var allRaw []RawPage
	var allWarnings []string

	for _, chunk := range chunks {
		userPrompt := BuildPrompt(schema, source, chunk, inventory)
		output, err := cfg.Complete(ctx, systemPrompt, userPrompt)
		if err != nil {
			return Result{}, fmt.Errorf("LLM call: %w", err)
		}
		raw, warnings := ParseRawPages(output)
		allRaw = append(allRaw, raw...)
		allWarnings = append(allWarnings, warnings...)
	}

	applySourceMeta(allRaw, parseContentFrontmatter(content))

	return buildAndWrite(allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun)
}

// RunRaw runs the pipeline on pre-parsed RawPages, skipping the LLM extraction
// step. Use this when the caller has already produced the structured RawPage data
// (e.g. from a more capable model or manual curation).
func RunRaw(brainDir, source string, rawPages []RawPage, dryRun bool) (Result, error) {
	inventory, err := wiki.LoadInventory(brainDir)
	if err != nil {
		return Result{}, fmt.Errorf("load inventory: %w", err)
	}

	sourceSlug := wiki.Slug(source)
	date := time.Now().UTC().Format("2006-01-02")

	return buildAndWrite(rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun)
}

// buildAndWrite runs BuildPages through write for both Run and RunRaw.
func buildAndWrite(rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) {
	pages, buildWarnings := BuildPages(rawPages, sourceSlug, date)
	warnings = append(warnings, buildWarnings...)
	resolved := Resolve(pages, inventory)
	canonicalized, linkWarnings := CanonicalizeLinks(resolved, inventory)
	warnings = append(warnings, linkWarnings...)
	withRefs := injectSourceRefs(canonicalized, inventory, brainDir)
	merged := mergeAll(withRefs)

	var written []string
	for _, page := range merged {
		if !dryRun {
			dest := filepath.Join(brainDir, filepath.FromSlash(page.Path))
			if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
				return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
			}
			content, err := reviseOrCreate(dest, page)
			if err != nil {
				return Result{}, err
			}
			if err := os.WriteFile(dest, []byte(content), 0o644); err != nil {
				return Result{}, fmt.Errorf("write %s: %w", page.Path, err)
			}
		}
		written = append(written, page.Path)
	}

	if !dryRun {
		if err := wiki.RebuildIndex(brainDir, date); err != nil {
			warnings = append(warnings, fmt.Sprintf("rebuild index: %v", err))
		}
		if err := wiki.AppendLog(brainDir, source, written, warnings, date); err != nil {
			warnings = append(warnings, fmt.Sprintf("append log: %v", err))
		}
	}

	return Result{Pages: written, Warnings: warnings}, nil
}

// reviseOrCreate returns the bytes to write for one extracted page: merged into
// whatever is already on disk, or the page as-is when the path is new.
//
// This is the compile half of compile-on-ingest. Before it, the write was an
// unconditional os.WriteFile, so re-ingesting a source replaced its page
// wholesale and discarded everything accumulated since — including human edits
// (hyperguild#43: "re-ingest overwrites wholesale").
//
// The merge is deliberately asymmetric in the existing page's favour:
// wiki.Merge keeps the on-disk frontmatter and the on-disk version of any
// section both sides carry, adds sections only the extraction produced, and
// unions bullet lists. So a freshly extracted page can add to a note but never
// silently rewrite what a human put there — which is what makes it safe to run
// this on every ingest rather than behind a review gate.
//
// Known limitation: because the on-disk frontmatter wins, `last_updated` keeps
// its first-ingest date on a revised page. Left deliberately — the alternative
// is this function editing frontmatter it does not own, and no lint reads that
// field today.
func reviseOrCreate(dest string, page wiki.Page) (string, error) {
	existing, err := os.ReadFile(dest)
	switch {
	case os.IsNotExist(err):
		return page.Content, nil
	case err != nil:
		return "", fmt.Errorf("read existing %s: %w", page.Path, err)
	}
	return wiki.Merge(wiki.Page{Path: page.Path, Content: string(existing)}, page).Content, nil
}

// mergeAll deduplicates pages by path, merging content from later occurrences.
func mergeAll(pages []wiki.Page) []wiki.Page {
	order := make([]string, 0, len(pages))
	byPath := make(map[string]wiki.Page, len(pages))
	for _, p := range pages {
		if _, seen := byPath[p.Path]; !seen {
			order = append(order, p.Path)
			byPath[p.Path] = p
		} else {
			byPath[p.Path] = wiki.Merge(byPath[p.Path], p)
		}
	}
	result := make([]wiki.Page, 0, len(order))
	for _, path := range order {
		result = append(result, byPath[path])
	}
	return result
}

const defaultSchema = `# Brain Wiki Schema
Three page types: wiki/sources/, wiki/concepts/, wiki/entities/.
See brain/schema.md for the full schema.
`

func loadSchema(brainDir string) string {
	b, err := os.ReadFile(filepath.Join(brainDir, "schema.md"))
	if err != nil {
		return defaultSchema
	}
	return strings.TrimSpace(string(b))
}