From 6ad275b5059838be1eaa40eaea0be7e197046a27 Mon Sep 17 00:00:00 2001 From: Mathias Date: Sun, 26 Jul 2026 23:42:33 +0200 Subject: [PATCH] fix(ingest): thread source/author/published frontmatter through extraction (#85) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipeline.Run parses source/author/published from the raw content's own frontmatter and applies them deterministically to source-type RawPages after LLM extraction — never relies on the LLM to copy them through. buildFrontmatter now emits all three (source pages only) when present. Backfill of the 46 already-affected wiki/sources/*.md notes is a separate concern per the issue, not done here. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Roq1ajWKR5f1hG5Df9wC6A --- ingestion/internal/pipeline/build.go | 9 +++ ingestion/internal/pipeline/build_test.go | 46 +++++++++++++++ ingestion/internal/pipeline/parse.go | 62 ++++++++++++++++++++ ingestion/internal/pipeline/pipeline.go | 2 + ingestion/internal/pipeline/pipeline_test.go | 34 +++++++++++ 5 files changed, 153 insertions(+) diff --git a/ingestion/internal/pipeline/build.go b/ingestion/internal/pipeline/build.go index 85fb965..c59ba8e 100644 --- a/ingestion/internal/pipeline/build.go +++ b/ingestion/internal/pipeline/build.go @@ -76,6 +76,15 @@ func buildFrontmatter(rp RawPage, date string) string { } fmt.Fprintf(&sb, "date_ingested: %s\n", date) fmt.Fprintf(&sb, "last_updated: %s\n", date) + if rp.Source != "" { + fmt.Fprintf(&sb, "source: %s\n", yamlScalar(rp.Source)) + } + if rp.Author != "" { + fmt.Fprintf(&sb, "author: %s\n", yamlScalar(rp.Author)) + } + if rp.Published != "" { + fmt.Fprintf(&sb, "published: %s\n", yamlScalar(rp.Published)) + } case "concept": if rp.Domain != "" { fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain)) diff --git a/ingestion/internal/pipeline/build_test.go b/ingestion/internal/pipeline/build_test.go index 482fd56..9524132 100644 --- a/ingestion/internal/pipeline/build_test.go +++ b/ingestion/internal/pipeline/build_test.go @@ -154,6 +154,52 @@ func TestBuildPages_EntityNoSubtype(t *testing.T) { assert.Contains(t, pages[0].Content, "title: 'Basecamp'") } +func TestBuildPages_SourcePageCarriesSourceAuthorPublished(t *testing.T) { + raw := []RawPage{ + { + Title: "Ornith", + Type: "source", + Subtype: "article", + Content: "## Summary\n\nAn agentic coding model.\n", + Source: "https://example.com/ornith", + Author: "Jane Doe", + Published: "2026-07-20", + }, + } + pages, warnings := BuildPages(raw, "ornith", "2026-07-26") + require.Len(t, pages, 1) + assert.Empty(t, warnings) + + p := pages[0] + assert.Contains(t, p.Content, "source: 'https://example.com/ornith'") + assert.Contains(t, p.Content, "author: 'Jane Doe'") + assert.Contains(t, p.Content, "published: '2026-07-20'") +} + +func TestBuildPages_SourcePageOmitsSourceAuthorPublishedWhenEmpty(t *testing.T) { + raw := []RawPage{ + {Title: "Shape Up", Type: "source", Subtype: "book", Content: "## Summary\n\nA book.\n"}, + } + pages, _ := BuildPages(raw, "shape-up", "2026-04-23") + require.Len(t, pages, 1) + assert.NotContains(t, pages[0].Content, "source:") + assert.NotContains(t, pages[0].Content, "author:") + assert.NotContains(t, pages[0].Content, "published:") +} + +func TestBuildPages_ConceptPageIgnoresSourceAuthorPublished(t *testing.T) { + // source/author/published are source-note-only metadata; a concept page + // shouldn't carry them even if somehow set on the RawPage. + raw := []RawPage{ + {Title: "Betting", Type: "concept", Content: "## Definition\n\nFoo.\n", Source: "x", Author: "y", Published: "z"}, + } + pages, _ := BuildPages(raw, "src", "2026-04-23") + require.Len(t, pages, 1) + assert.NotContains(t, pages[0].Content, "source:") + assert.NotContains(t, pages[0].Content, "author:") + assert.NotContains(t, pages[0].Content, "published:") +} + func TestBuildPages_EmptyTitleSkippedWithWarning(t *testing.T) { raw := []RawPage{ {Title: "", Type: "concept", Content: "## Definition\n\nFoo.\n"}, diff --git a/ingestion/internal/pipeline/parse.go b/ingestion/internal/pipeline/parse.go index ee4a23f..6b2afc5 100644 --- a/ingestion/internal/pipeline/parse.go +++ b/ingestion/internal/pipeline/parse.go @@ -15,6 +15,14 @@ type RawPage struct { Subtype string `json:"subtype"` // entity: person|company|tool|model|framework|technology; source: article|pdf|book|video|note|project Domain string `json:"domain"` Content string `json:"content"` // Markdown body only — no frontmatter + + // Source, Author, Published are deterministic passthrough from the raw + // ingested content's own frontmatter (see parseContentFrontmatter) — never + // set by the LLM. json:"-" keeps them immune to same-named keys the LLM + // might emit. Only meaningful for Type == "source". + Source string `json:"-"` + Author string `json:"-"` + Published string `json:"-"` } // ParseRawPages parses LLM output as a JSON array of RawPage objects. @@ -98,6 +106,60 @@ func repairJSON(s string) string { return b.String() } +// sourceMeta is source/author/published pulled from the raw ingested +// content's own frontmatter — deterministic passthrough, never LLM output. +type sourceMeta struct { + Source string + Author string + Published string +} + +// parseContentFrontmatter extracts source/author/published from a leading +// "---\n...\n---" YAML block in raw ingested content. Only these three flat +// scalar keys are recognised; anything else in the block is ignored. Returns +// a zero-value sourceMeta if content has no frontmatter block. +func parseContentFrontmatter(content string) sourceMeta { + var meta sourceMeta + if !strings.HasPrefix(content, "---\n") && !strings.HasPrefix(content, "---\r\n") { + return meta + } + + lines := strings.Split(content, "\n") + for _, line := range lines[1:] { + if strings.TrimSpace(line) == "---" { + break + } + key, val, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + val = strings.Trim(strings.TrimSpace(val), `"'`) + switch key { + case "source": + meta.Source = val + case "author": + meta.Author = val + case "published": + meta.Published = val + } + } + return meta +} + +// applySourceMeta deterministically overwrites Source/Author/Published on +// every "source"-type page with meta — the LLM never controls these fields. +func applySourceMeta(pages []RawPage, meta sourceMeta) { + for i := range pages { + if pages[i].Type != "source" { + continue + } + pages[i].Source = meta.Source + pages[i].Author = meta.Author + pages[i].Published = meta.Published + } +} + func stripFences(s string) string { for _, prefix := range []string{"```json\n", "```json\r\n", "```\n", "```\r\n"} { if strings.HasPrefix(s, prefix) { diff --git a/ingestion/internal/pipeline/pipeline.go b/ingestion/internal/pipeline/pipeline.go index 30d1cce..0385e7a 100644 --- a/ingestion/internal/pipeline/pipeline.go +++ b/ingestion/internal/pipeline/pipeline.go @@ -59,6 +59,8 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR allWarnings = append(allWarnings, warnings...) } + applySourceMeta(allRaw, parseContentFrontmatter(content)) + return buildAndWrite(allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun) } diff --git a/ingestion/internal/pipeline/pipeline_test.go b/ingestion/internal/pipeline/pipeline_test.go index 3467201..92c0be7 100644 --- a/ingestion/internal/pipeline/pipeline_test.go +++ b/ingestion/internal/pipeline/pipeline_test.go @@ -130,6 +130,40 @@ func TestRun_MergesDuplicatePaths(t *testing.T) { assert.Contains(t, string(content), "[[Baz]]") } +func TestRun_ThreadsSourceAuthorPublishedFromContentFrontmatter(t *testing.T) { + brainDir := t.TempDir() + for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} { + require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755)) + } + + llmResponse := mustJSON([]RawPage{{ + Title: "Ornith", + Type: "source", + Subtype: "article", + Content: "## Summary\n\nAn agentic coding model.\n", + }}) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}}, + }) + })) + defer srv.Close() + + cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete} + rawContent := "---\nsource: https://example.com/ornith\nauthor: Jane Doe\npublished: 2026-07-20\n---\n\nAn agentic coding model that runs on your laptop.\n" + + result, err := Run(context.Background(), cfg, brainDir, rawContent, "ornith", false) + require.NoError(t, err) + require.Len(t, result.Pages, 1) + + content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", "ornith.md")) + require.NoError(t, err) + assert.Contains(t, string(content), "source: 'https://example.com/ornith'") + assert.Contains(t, string(content), "author: 'Jane Doe'") + assert.Contains(t, string(content), "published: '2026-07-20'") +} + func mustJSON(v any) string { b, err := json.Marshal(v) if err != nil {