fix(ingest): thread source/author/published frontmatter through extraction (#85)
pipeline.Run parses source/author/published from the raw content's own frontmatter and applies them deterministically to source-type RawPages after LLM extraction — never relies on the LLM to copy them through. buildFrontmatter now emits all three (source pages only) when present. Backfill of the 46 already-affected wiki/sources/*.md notes is a separate concern per the issue, not done here. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Roq1ajWKR5f1hG5Df9wC6A
This commit is contained in:
@@ -130,6 +130,40 @@ func TestRun_MergesDuplicatePaths(t *testing.T) {
|
||||
assert.Contains(t, string(content), "[[Baz]]")
|
||||
}
|
||||
|
||||
func TestRun_ThreadsSourceAuthorPublishedFromContentFrontmatter(t *testing.T) {
|
||||
brainDir := t.TempDir()
|
||||
for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
|
||||
}
|
||||
|
||||
llmResponse := mustJSON([]RawPage{{
|
||||
Title: "Ornith",
|
||||
Type: "source",
|
||||
Subtype: "article",
|
||||
Content: "## Summary\n\nAn agentic coding model.\n",
|
||||
}})
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
|
||||
rawContent := "---\nsource: https://example.com/ornith\nauthor: Jane Doe\npublished: 2026-07-20\n---\n\nAn agentic coding model that runs on your laptop.\n"
|
||||
|
||||
result, err := Run(context.Background(), cfg, brainDir, rawContent, "ornith", false)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Pages, 1)
|
||||
|
||||
content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", "ornith.md"))
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(content), "source: 'https://example.com/ornith'")
|
||||
assert.Contains(t, string(content), "author: 'Jane Doe'")
|
||||
assert.Contains(t, string(content), "published: '2026-07-20'")
|
||||
}
|
||||
|
||||
func mustJSON(v any) string {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user