fix(ingest): thread source/author/published frontmatter through extraction (#85)
pipeline.Run parses source/author/published from the raw content's own frontmatter and applies them deterministically to source-type RawPages after LLM extraction — never relies on the LLM to copy them through. buildFrontmatter now emits all three (source pages only) when present. Backfill of the 46 already-affected wiki/sources/*.md notes is a separate concern per the issue, not done here. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Roq1ajWKR5f1hG5Df9wC6A
This commit is contained in:
@@ -15,6 +15,14 @@ type RawPage struct {
|
||||
Subtype string `json:"subtype"` // entity: person|company|tool|model|framework|technology; source: article|pdf|book|video|note|project
|
||||
Domain string `json:"domain"`
|
||||
Content string `json:"content"` // Markdown body only — no frontmatter
|
||||
|
||||
// Source, Author, Published are deterministic passthrough from the raw
|
||||
// ingested content's own frontmatter (see parseContentFrontmatter) — never
|
||||
// set by the LLM. json:"-" keeps them immune to same-named keys the LLM
|
||||
// might emit. Only meaningful for Type == "source".
|
||||
Source string `json:"-"`
|
||||
Author string `json:"-"`
|
||||
Published string `json:"-"`
|
||||
}
|
||||
|
||||
// ParseRawPages parses LLM output as a JSON array of RawPage objects.
|
||||
@@ -98,6 +106,60 @@ func repairJSON(s string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sourceMeta is source/author/published pulled from the raw ingested
|
||||
// content's own frontmatter — deterministic passthrough, never LLM output.
|
||||
type sourceMeta struct {
|
||||
Source string
|
||||
Author string
|
||||
Published string
|
||||
}
|
||||
|
||||
// parseContentFrontmatter extracts source/author/published from a leading
|
||||
// "---\n...\n---" YAML block in raw ingested content. Only these three flat
|
||||
// scalar keys are recognised; anything else in the block is ignored. Returns
|
||||
// a zero-value sourceMeta if content has no frontmatter block.
|
||||
func parseContentFrontmatter(content string) sourceMeta {
|
||||
var meta sourceMeta
|
||||
if !strings.HasPrefix(content, "---\n") && !strings.HasPrefix(content, "---\r\n") {
|
||||
return meta
|
||||
}
|
||||
|
||||
lines := strings.Split(content, "\n")
|
||||
for _, line := range lines[1:] {
|
||||
if strings.TrimSpace(line) == "---" {
|
||||
break
|
||||
}
|
||||
key, val, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
val = strings.Trim(strings.TrimSpace(val), `"'`)
|
||||
switch key {
|
||||
case "source":
|
||||
meta.Source = val
|
||||
case "author":
|
||||
meta.Author = val
|
||||
case "published":
|
||||
meta.Published = val
|
||||
}
|
||||
}
|
||||
return meta
|
||||
}
|
||||
|
||||
// applySourceMeta deterministically overwrites Source/Author/Published on
|
||||
// every "source"-type page with meta — the LLM never controls these fields.
|
||||
func applySourceMeta(pages []RawPage, meta sourceMeta) {
|
||||
for i := range pages {
|
||||
if pages[i].Type != "source" {
|
||||
continue
|
||||
}
|
||||
pages[i].Source = meta.Source
|
||||
pages[i].Author = meta.Author
|
||||
pages[i].Published = meta.Published
|
||||
}
|
||||
}
|
||||
|
||||
func stripFences(s string) string {
|
||||
for _, prefix := range []string{"```json\n", "```json\r\n", "```\n", "```\r\n"} {
|
||||
if strings.HasPrefix(s, prefix) {
|
||||
|
||||
Reference in New Issue
Block a user