hyperguild/ingestion/internal/pipeline/build.go

// ingestion/internal/pipeline/build.go
package pipeline

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
)

// BuildPages converts RawPages from the LLM into wiki.Pages with computed slugs,
// paths, and YAML frontmatter. sourceSlug is the slug of the source being ingested
// (derived from the filename, not the LLM title). Pages whose title resolves to an
// empty slug are skipped and returned as warnings instead.
func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) {
	return BuildPagesIn("", rawPages, sourceSlug, date)
}

// BuildPagesIn is BuildPages for a wing: sources go to wiki/<wing>/sources/,
// concepts and entities to wiki/<wing>/facts/ with `kind:` recording which.
// An empty wing is the retired wiki/{sources,concepts,entities}/ layout.
func BuildPagesIn(wing string, rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) {
	out := make([]wiki.Page, 0, len(rawPages))
	var warnings []string
	for _, rp := range rawPages {
		// The type becomes a directory under wiki/; "../raw" wrote outside it
		// (PR #147 review). Only a plain lower-case name is a page type.
		if !validPageType.MatchString(rp.Type) {
			warnings = append(warnings, fmt.Sprintf("skipped page with invalid type %q", rp.Type))
			continue
		}
		slug := computeSlug(rp, sourceSlug)
		if slug == "" {
			warnings = append(warnings, fmt.Sprintf("skipped page with empty title (type: %s)", rp.Type))
			continue
		}
		out = append(out, buildPage(rp, sourceSlug, date, wing))
	}
	return out, warnings
}

var validPageType = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)

func computeSlug(rp RawPage, sourceSlug string) string {
	if rp.Type == "source" {
		return sourceSlug
	}
	return wiki.Slug(rp.Title)
}

func buildPage(rp RawPage, sourceSlug, date, wing string) wiki.Page {
	var slug, dir string
	switch rp.Type {
	case "source":
		slug = sourceSlug
		dir = "wiki/sources"
	case "concept":
		slug = wiki.Slug(rp.Title)
		dir = "wiki/concepts"
	case "entity":
		slug = wiki.Slug(rp.Title)
		dir = "wiki/entities"
	default:
		slug = wiki.Slug(rp.Title)
		dir = "wiki/" + rp.Type
	}

	hall := ""
	if wing != "" {
		switch rp.Type {
		case "source":
			hall = "sources"
		default:
			hall = "facts"
		}
		dir = "wiki/" + wing + "/" + hall
	}
	path := dir + "/" + slug + ".md"
	fm := buildFrontmatter(rp, date)
	if wing != "" {
		fm = addWingFrontmatter(fm, wing, hall, rp.Type)
	}

	return wiki.Page{
		Path:    path,
		Content: fm + "\n" + rp.Content,
	}
}

func buildFrontmatter(rp RawPage, date string) string {
	var sb strings.Builder
	sb.WriteString("---\n")
	fmt.Fprintf(&sb, "title: %s\n", yamlScalar(rp.Title))

	switch rp.Type {
	case "source":
		subtype := rp.Subtype
		if subtype == "" {
			subtype = "article"
		}
		fmt.Fprintf(&sb, "type: %s\n", yamlScalar(subtype))
		if rp.Domain != "" {
			fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
		}
		fmt.Fprintf(&sb, "date_ingested: %s\n", date)
		fmt.Fprintf(&sb, "last_updated: %s\n", date)
		if rp.SourceHash != "" {
			fmt.Fprintf(&sb, "source_hash: %s\n", rp.SourceHash)
		}
		if rp.Source != "" {
			fmt.Fprintf(&sb, "source: %s\n", yamlScalar(rp.Source))
		}
		if rp.Author != "" {
			fmt.Fprintf(&sb, "author: %s\n", yamlScalar(rp.Author))
		}
		if rp.Published != "" {
			fmt.Fprintf(&sb, "published: %s\n", yamlScalar(rp.Published))
		}
	case "concept":
		if rp.Domain != "" {
			fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
		}
		fmt.Fprintf(&sb, "last_updated: %s\n", date)
	case "entity":
		if rp.Subtype != "" {
			fmt.Fprintf(&sb, "type: %s\n", yamlScalar(rp.Subtype))
		}
		if rp.Domain != "" {
			fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
		}
		fmt.Fprintf(&sb, "last_updated: %s\n", date)
	default:
		if rp.Domain != "" {
			fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
		}
		fmt.Fprintf(&sb, "last_updated: %s\n", date)
	}

	fmt.Fprintf(&sb, "aliases:\n  - %s\n", yamlScalar(rp.Title))
	sb.WriteString("---\n")
	return sb.String()
}

// addWingFrontmatter inserts wing/hall (and kind for facts pages) right after
// the opening fence of an already-built frontmatter block.
func addWingFrontmatter(fm, wing, hall, rpType string) string {
	extra := "wing: " + wing + "\nhall: " + hall + "\n"
	if hall == "facts" {
		extra += "kind: " + rpType + "\n"
	}
	return strings.Replace(fm, "---\n", "---\n"+extra, 1)
}

func yamlScalar(s string) string {
	// A raw newline or CR would end the scalar for every line-oriented
	// frontmatter reader and start a new key (PR #147 review: a title of
	// "x\norigin: static" forged provenance). Fold all line breaks to spaces.
	s = strings.NewReplacer("\r\n", " ", "\r", " ", "\n", " ").Replace(s)
	return "'" + strings.ReplaceAll(s, "'", "''") + "'"
}