mathias
a5a328fea5
Ingest now stamps source_hash — sha256 of the ingested body, frontmatter excluded — onto the source page, and compares it before doing any work. An unchanged source returns Skipped with no LLM call at all. This is cost and signal, not correctness: #103 already made a re-ingest byte-identical, so a repeated run was harmless but not free — a full extraction over every chunk to produce a result that merges to nothing. Building this first would have put idempotency in the guard rather than in the write, which is the wrong dependency direction. Body-only hashing on purpose. A re-crawl that stamps a new fetch date has not changed the source, and treating that as drift would re-extract the whole corpus every time a crawler ran. A missing or unreadable hash always means "extract": a missed skip costs one extraction, a wrong skip loses an update silently, and silence is the failure mode this repo keeps paying for. The skip is reported (Result.Skipped plus a warning naming the hash), not silent — a run that does nothing must say so, or "the pipeline is fine" and "the pipeline stopped working" look identical (#107). Storing the hash required settling who owns which frontmatter field. #103 made the on-disk block win wholesale, which protects human-set metadata from a regenerating writer but froze fields the generator legitimately owns. wiki.Merge now refreshes an explicit generatorOwned allowlist — last_updated and source_hash — and keeps everything else on disk. An allowlist rather than a carve-out, so a field nobody has thought about stays human-owned by default. That also retires the last_updated limitation #103 shipped with. Refs #105, #103, #107, brain#21.
119 lines
3.1 KiB
Go
119 lines
3.1 KiB
Go
// ingestion/internal/pipeline/build.go
|
|
package pipeline
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
|
)
|
|
|
|
// BuildPages converts RawPages from the LLM into wiki.Pages with computed slugs,
|
|
// paths, and YAML frontmatter. sourceSlug is the slug of the source being ingested
|
|
// (derived from the filename, not the LLM title). Pages whose title resolves to an
|
|
// empty slug are skipped and returned as warnings instead.
|
|
func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []string) {
|
|
out := make([]wiki.Page, 0, len(rawPages))
|
|
var warnings []string
|
|
for _, rp := range rawPages {
|
|
slug := computeSlug(rp, sourceSlug)
|
|
if slug == "" {
|
|
warnings = append(warnings, fmt.Sprintf("skipped page with empty title (type: %s)", rp.Type))
|
|
continue
|
|
}
|
|
out = append(out, buildPage(rp, sourceSlug, date))
|
|
}
|
|
return out, warnings
|
|
}
|
|
|
|
func computeSlug(rp RawPage, sourceSlug string) string {
|
|
if rp.Type == "source" {
|
|
return sourceSlug
|
|
}
|
|
return wiki.Slug(rp.Title)
|
|
}
|
|
|
|
func buildPage(rp RawPage, sourceSlug, date string) wiki.Page {
|
|
var slug, dir string
|
|
switch rp.Type {
|
|
case "source":
|
|
slug = sourceSlug
|
|
dir = "wiki/sources"
|
|
case "concept":
|
|
slug = wiki.Slug(rp.Title)
|
|
dir = "wiki/concepts"
|
|
case "entity":
|
|
slug = wiki.Slug(rp.Title)
|
|
dir = "wiki/entities"
|
|
default:
|
|
slug = wiki.Slug(rp.Title)
|
|
dir = "wiki/" + rp.Type
|
|
}
|
|
|
|
path := dir + "/" + slug + ".md"
|
|
fm := buildFrontmatter(rp, date)
|
|
|
|
return wiki.Page{
|
|
Path: path,
|
|
Content: fm + "\n" + rp.Content,
|
|
}
|
|
}
|
|
|
|
func buildFrontmatter(rp RawPage, date string) string {
|
|
var sb strings.Builder
|
|
sb.WriteString("---\n")
|
|
fmt.Fprintf(&sb, "title: %s\n", yamlScalar(rp.Title))
|
|
|
|
switch rp.Type {
|
|
case "source":
|
|
subtype := rp.Subtype
|
|
if subtype == "" {
|
|
subtype = "article"
|
|
}
|
|
fmt.Fprintf(&sb, "type: %s\n", yamlScalar(subtype))
|
|
if rp.Domain != "" {
|
|
fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
|
|
}
|
|
fmt.Fprintf(&sb, "date_ingested: %s\n", date)
|
|
fmt.Fprintf(&sb, "last_updated: %s\n", date)
|
|
if rp.SourceHash != "" {
|
|
fmt.Fprintf(&sb, "source_hash: %s\n", rp.SourceHash)
|
|
}
|
|
if rp.Source != "" {
|
|
fmt.Fprintf(&sb, "source: %s\n", yamlScalar(rp.Source))
|
|
}
|
|
if rp.Author != "" {
|
|
fmt.Fprintf(&sb, "author: %s\n", yamlScalar(rp.Author))
|
|
}
|
|
if rp.Published != "" {
|
|
fmt.Fprintf(&sb, "published: %s\n", yamlScalar(rp.Published))
|
|
}
|
|
case "concept":
|
|
if rp.Domain != "" {
|
|
fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
|
|
}
|
|
fmt.Fprintf(&sb, "last_updated: %s\n", date)
|
|
case "entity":
|
|
if rp.Subtype != "" {
|
|
fmt.Fprintf(&sb, "type: %s\n", yamlScalar(rp.Subtype))
|
|
}
|
|
if rp.Domain != "" {
|
|
fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
|
|
}
|
|
fmt.Fprintf(&sb, "last_updated: %s\n", date)
|
|
default:
|
|
if rp.Domain != "" {
|
|
fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))
|
|
}
|
|
fmt.Fprintf(&sb, "last_updated: %s\n", date)
|
|
}
|
|
|
|
fmt.Fprintf(&sb, "aliases:\n - %s\n", yamlScalar(rp.Title))
|
|
sb.WriteString("---\n")
|
|
return sb.String()
|
|
}
|
|
|
|
func yamlScalar(s string) string {
|
|
return "'" + strings.ReplaceAll(s, "'", "''") + "'"
|
|
}
|