fix(ingest): thread source/author/published frontmatter through extraction (#85)
CI / Lint / Test / Vet (push) Failing after 1s
CI / Mirror to GitHub (push) Has been skipped

pipeline.Run parses source/author/published from the raw content's own
frontmatter and applies them deterministically to source-type RawPages
after LLM extraction — never relies on the LLM to copy them through.
buildFrontmatter now emits all three (source pages only) when present.

Backfill of the 46 already-affected wiki/sources/*.md notes is a
separate concern per the issue, not done here.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Roq1ajWKR5f1hG5Df9wC6A
This commit is contained in:
2026-07-26 23:42:33 +02:00
co-authored by Claude Sonnet 5
parent b600cc986c
commit 6ad275b505
5 changed files with 153 additions and 0 deletions
@@ -130,6 +130,40 @@ func TestRun_MergesDuplicatePaths(t *testing.T) {
assert.Contains(t, string(content), "[[Baz]]")
}
func TestRun_ThreadsSourceAuthorPublishedFromContentFrontmatter(t *testing.T) {
brainDir := t.TempDir()
for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
}
llmResponse := mustJSON([]RawPage{{
Title: "Ornith",
Type: "source",
Subtype: "article",
Content: "## Summary\n\nAn agentic coding model.\n",
}})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{
"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
})
}))
defer srv.Close()
cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
rawContent := "---\nsource: https://example.com/ornith\nauthor: Jane Doe\npublished: 2026-07-20\n---\n\nAn agentic coding model that runs on your laptop.\n"
result, err := Run(context.Background(), cfg, brainDir, rawContent, "ornith", false)
require.NoError(t, err)
require.Len(t, result.Pages, 1)
content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", "ornith.md"))
require.NoError(t, err)
assert.Contains(t, string(content), "source: 'https://example.com/ornith'")
assert.Contains(t, string(content), "author: 'Jane Doe'")
assert.Contains(t, string(content), "published: '2026-07-20'")
}
func mustJSON(v any) string {
b, err := json.Marshal(v)
if err != nil {