fix(ingest): thread source/author/published frontmatter through extraction (#85)

pipeline.Run parses source/author/published from the raw content's own
frontmatter and applies them deterministically to source-type RawPages
after LLM extraction — never relies on the LLM to copy them through.
buildFrontmatter now emits all three (source pages only) when present.

Backfill of the 46 already-affected wiki/sources/*.md notes is a
separate concern per the issue, not done here.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Roq1ajWKR5f1hG5Df9wC6A

ingestion/internal/pipeline/build.go

@@ -76,6 +76,15 @@ func buildFrontmatter(rp RawPage, date string) string {
 		}
 		fmt.Fprintf(&sb, "date_ingested: %s\n", date)
 		fmt.Fprintf(&sb, "last_updated: %s\n", date)
+		if rp.Source != "" {
+			fmt.Fprintf(&sb, "source: %s\n", yamlScalar(rp.Source))
+		}
+		if rp.Author != "" {
+			fmt.Fprintf(&sb, "author: %s\n", yamlScalar(rp.Author))
+		}
+		if rp.Published != "" {
+			fmt.Fprintf(&sb, "published: %s\n", yamlScalar(rp.Published))
+		}
 	case "concept":
 		if rp.Domain != "" {
 			fmt.Fprintf(&sb, "domain: %s\n", yamlScalar(rp.Domain))

ingestion/internal/pipeline/build_test.go

@@ -154,6 +154,52 @@ func TestBuildPages_EntityNoSubtype(t *testing.T) {
 	assert.Contains(t, pages[0].Content, "title: 'Basecamp'")
 }
 
+func TestBuildPages_SourcePageCarriesSourceAuthorPublished(t *testing.T) {
+	raw := []RawPage{
+		{
+			Title:     "Ornith",
+			Type:      "source",
+			Subtype:   "article",
+			Content:   "## Summary\n\nAn agentic coding model.\n",
+			Source:    "https://example.com/ornith",
+			Author:    "Jane Doe",
+			Published: "2026-07-20",
+		},
+	}
+	pages, warnings := BuildPages(raw, "ornith", "2026-07-26")
+	require.Len(t, pages, 1)
+	assert.Empty(t, warnings)
+
+	p := pages[0]
+	assert.Contains(t, p.Content, "source: 'https://example.com/ornith'")
+	assert.Contains(t, p.Content, "author: 'Jane Doe'")
+	assert.Contains(t, p.Content, "published: '2026-07-20'")
+}
+
+func TestBuildPages_SourcePageOmitsSourceAuthorPublishedWhenEmpty(t *testing.T) {
+	raw := []RawPage{
+		{Title: "Shape Up", Type: "source", Subtype: "book", Content: "## Summary\n\nA book.\n"},
+	}
+	pages, _ := BuildPages(raw, "shape-up", "2026-04-23")
+	require.Len(t, pages, 1)
+	assert.NotContains(t, pages[0].Content, "source:")
+	assert.NotContains(t, pages[0].Content, "author:")
+	assert.NotContains(t, pages[0].Content, "published:")
+}
+
+func TestBuildPages_ConceptPageIgnoresSourceAuthorPublished(t *testing.T) {
+	// source/author/published are source-note-only metadata; a concept page
+	// shouldn't carry them even if somehow set on the RawPage.
+	raw := []RawPage{
+		{Title: "Betting", Type: "concept", Content: "## Definition\n\nFoo.\n", Source: "x", Author: "y", Published: "z"},
+	}
+	pages, _ := BuildPages(raw, "src", "2026-04-23")
+	require.Len(t, pages, 1)
+	assert.NotContains(t, pages[0].Content, "source:")
+	assert.NotContains(t, pages[0].Content, "author:")
+	assert.NotContains(t, pages[0].Content, "published:")
+}
+
 func TestBuildPages_EmptyTitleSkippedWithWarning(t *testing.T) {
 	raw := []RawPage{
 		{Title: "", Type: "concept", Content: "## Definition\n\nFoo.\n"},

ingestion/internal/pipeline/parse.go

@@ -15,6 +15,14 @@ type RawPage struct {
 	Subtype string `json:"subtype"` // entity: person|company|tool|model|framework|technology; source: article|pdf|book|video|note|project
 	Domain  string `json:"domain"`
 	Content string `json:"content"` // Markdown body only — no frontmatter
+
+	// Source, Author, Published are deterministic passthrough from the raw
+	// ingested content's own frontmatter (see parseContentFrontmatter) — never
+	// set by the LLM. json:"-" keeps them immune to same-named keys the LLM
+	// might emit. Only meaningful for Type == "source".
+	Source    string `json:"-"`
+	Author    string `json:"-"`
+	Published string `json:"-"`
 }
 
 // ParseRawPages parses LLM output as a JSON array of RawPage objects.
@@ -98,6 +106,60 @@ func repairJSON(s string) string {
 	return b.String()
 }
 
+// sourceMeta is source/author/published pulled from the raw ingested
+// content's own frontmatter — deterministic passthrough, never LLM output.
+type sourceMeta struct {
+	Source    string
+	Author    string
+	Published string
+}
+
+// parseContentFrontmatter extracts source/author/published from a leading
+// "---\n...\n---" YAML block in raw ingested content. Only these three flat
+// scalar keys are recognised; anything else in the block is ignored. Returns
+// a zero-value sourceMeta if content has no frontmatter block.
+func parseContentFrontmatter(content string) sourceMeta {
+	var meta sourceMeta
+	if !strings.HasPrefix(content, "---\n") && !strings.HasPrefix(content, "---\r\n") {
+		return meta
+	}
+
+	lines := strings.Split(content, "\n")
+	for _, line := range lines[1:] {
+		if strings.TrimSpace(line) == "---" {
+			break
+		}
+		key, val, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		key = strings.TrimSpace(key)
+		val = strings.Trim(strings.TrimSpace(val), `"'`)
+		switch key {
+		case "source":
+			meta.Source = val
+		case "author":
+			meta.Author = val
+		case "published":
+			meta.Published = val
+		}
+	}
+	return meta
+}
+
+// applySourceMeta deterministically overwrites Source/Author/Published on
+// every "source"-type page with meta — the LLM never controls these fields.
+func applySourceMeta(pages []RawPage, meta sourceMeta) {
+	for i := range pages {
+		if pages[i].Type != "source" {
+			continue
+		}
+		pages[i].Source = meta.Source
+		pages[i].Author = meta.Author
+		pages[i].Published = meta.Published
+	}
+}
+
 func stripFences(s string) string {
 	for _, prefix := range []string{"```json\n", "```json\r\n", "```\n", "```\r\n"} {
 		if strings.HasPrefix(s, prefix) {

ingestion/internal/pipeline/pipeline.go

@@ -59,6 +59,8 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
 		allWarnings = append(allWarnings, warnings...)
 	}
 
+	applySourceMeta(allRaw, parseContentFrontmatter(content))
+
 	return buildAndWrite(allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun)
 }
 

ingestion/internal/pipeline/pipeline_test.go

@@ -130,6 +130,40 @@ func TestRun_MergesDuplicatePaths(t *testing.T) {
 	assert.Contains(t, string(content), "[[Baz]]")
 }
 
+func TestRun_ThreadsSourceAuthorPublishedFromContentFrontmatter(t *testing.T) {
+	brainDir := t.TempDir()
+	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
+		require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
+	}
+
+	llmResponse := mustJSON([]RawPage{{
+		Title:   "Ornith",
+		Type:    "source",
+		Subtype: "article",
+		Content: "## Summary\n\nAn agentic coding model.\n",
+	}})
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
+		})
+	}))
+	defer srv.Close()
+
+	cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
+	rawContent := "---\nsource: https://example.com/ornith\nauthor: Jane Doe\npublished: 2026-07-20\n---\n\nAn agentic coding model that runs on your laptop.\n"
+
+	result, err := Run(context.Background(), cfg, brainDir, rawContent, "ornith", false)
+	require.NoError(t, err)
+	require.Len(t, result.Pages, 1)
+
+	content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", "ornith.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(content), "source: 'https://example.com/ornith'")
+	assert.Contains(t, string(content), "author: 'Jane Doe'")
+	assert.Contains(t, string(content), "published: '2026-07-20'")
+}
+
 func mustJSON(v any) string {
 	b, err := json.Marshal(v)
 	if err != nil {