feat(pipeline): skip extraction when the source has not changed (#105) (#126)

Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/wiki/merge_test.go

@@ -174,3 +174,161 @@ title: X
 	assert.Contains(t, got, "Does this converge?", "existing question must survive")
 	assert.Contains(t, got, "What happens under load?", "new question must be added")
 }
+
+func TestMerge_GeneratorOwnedFrontmatterFieldsAreRefreshed(t *testing.T) {
+	// #103 made on-disk frontmatter win wholesale, which is what protects
+	// human-set fields from a regenerating writer. The cost was that fields the
+	// generator legitimately owns — when it last ran, what it last saw — froze
+	// at their first-ingest values. An explicit owned-set is the boundary:
+	// named fields the generator may refresh, everything else human-owned.
+	existing := Page{Path: "wiki/sources/x.md", Content: `---
+title: 'X'
+last_updated: 2026-01-01
+source_hash: aaaa
+half_life: durable
+claim_provenance: asserted
+---
+
+## Summary
+
+Original.
+`}
+	fresh := Page{Path: "wiki/sources/x.md", Content: `---
+title: 'Renamed by the model'
+last_updated: 2026-08-22
+source_hash: bbbb
+---
+
+## Summary
+
+Rewritten by the model.
+`}
+
+	got := Merge(existing, fresh).Content
+
+	assert.Contains(t, got, "last_updated: 2026-08-22", "generator owns last_updated")
+	assert.Contains(t, got, "source_hash: bbbb", "generator owns source_hash")
+	assert.Contains(t, got, "title: 'X'", "a human-set title must survive regeneration")
+	assert.Contains(t, got, "half_life: durable", "the claim envelope must survive regeneration")
+	assert.Contains(t, got, "claim_provenance: asserted")
+	assert.Contains(t, got, "Original.", "an existing section still wins over the model's rewrite")
+	assert.NotContains(t, got, "Renamed by the model")
+}
+
+func TestMerge_GeneratorOwnedFieldsAreAppendedWhenAPageNeverHadThemBefore(t *testing.T) {
+	// Found in review of PR #126: refreshOwned only replaced a generator-owned
+	// line that already existed in a's frontmatter — it never appended one a
+	// didn't have. Every page ingested before this feature shipped has no
+	// source_hash line at all, so on its first re-ingest after deploy, the
+	// fresh hash was silently dropped and the skip-on-unchanged feature this
+	// PR exists to build would never engage for the entire pre-existing
+	// corpus, permanently, with nothing telling anyone.
+	existing := Page{Path: "wiki/sources/x.md", Content: `---
+title: 'X'
+---
+
+## Summary
+
+Original.
+`}
+	fresh := Page{Path: "wiki/sources/x.md", Content: `---
+title: 'Renamed by the model'
+last_updated: 2026-08-22
+source_hash: bbbb
+---
+
+## Summary
+
+Rewritten by the model.
+`}
+
+	got := Merge(existing, fresh).Content
+
+	// Exact frontmatter block, not Contains: the second review round found
+	// that a substring check doesn't prove the fields landed inside a valid
+	// fence, in the right order, or that the fence itself wasn't corrupted —
+	// two independent mutations (closeIdx targeting the wrong fence, reversed
+	// sort) both survived a Contains-only version of this test.
+	want := "---\ntitle: 'X'\nlast_updated: 2026-08-22\nsource_hash: bbbb\n---\n\n## Summary\n\nOriginal.\n"
+	assert.Equal(t, want, got)
+}
+
+func TestMerge_GeneratorOwnedFieldsCreateAFrontmatterBlockWhenThePageHadNoneAtAll(t *testing.T) {
+	// Second review round on PR #126: when a has no frontmatter fence at all,
+	// the append fallback spliced raw, unfenced "key: value" lines straight
+	// into the body -- worse than the original bug, because the next
+	// parseSections call wouldn't recognize them as frontmatter (they don't
+	// start with `---`), so they'd be absorbed as ordinary preamble text and
+	// re-duplicate on every subsequent merge instead of updating in place.
+	existing := Page{Path: "wiki/sources/x.md", Content: "## Summary\n\nOriginal.\n"}
+	fresh := Page{Path: "wiki/sources/x.md", Content: `---
+last_updated: 2026-08-22
+source_hash: bbbb
+---
+
+## Summary
+
+Rewritten by the model.
+`}
+
+	got := Merge(existing, fresh).Content
+
+	want := "---\nlast_updated: 2026-08-22\nsource_hash: bbbb\n---\n\n## Summary\n\nOriginal.\n"
+	assert.Equal(t, want, got)
+}
+
+func TestMerge_GeneratorOwnedFieldsMixedReplaceAndAppend(t *testing.T) {
+	// a already has last_updated (replace in place) but not source_hash
+	// (append) -- the two code paths in the same call, together.
+	existing := Page{Path: "wiki/sources/x.md", Content: `---
+title: 'X'
+last_updated: 2026-01-01
+---
+
+## Summary
+
+Original.
+`}
+	fresh := Page{Path: "wiki/sources/x.md", Content: `---
+last_updated: 2026-08-22
+source_hash: bbbb
+---
+
+## Summary
+
+Rewritten by the model.
+`}
+
+	got := Merge(existing, fresh).Content
+
+	want := "---\ntitle: 'X'\nlast_updated: 2026-08-22\nsource_hash: bbbb\n---\n\n## Summary\n\nOriginal.\n"
+	assert.Equal(t, want, got)
+}
+
+func TestRefreshOwned_CommentLinesSurviveUnmangled(t *testing.T) {
+	// Third review round on PR #126: parseFrontmatterBlock dropped the
+	// sibling api package's comment guard (strings.HasPrefix(trimmed, "#")).
+	// Without it, a comment line containing a colon gets cut at the colon
+	// and re-rendered with collapsed whitespace instead of surviving
+	// verbatim.
+	a := "---\n#see also:   https://x/y\ntitle: 'X'\n---\n"
+	b := "---\nsource_hash: bbbb\n---\n"
+
+	got := refreshOwned(a, b)
+	assert.Equal(t, "---\n#see also:   https://x/y\ntitle: 'X'\nsource_hash: bbbb\n---\n", got)
+}
+
+func TestRefreshOwned_IndentedListItemsWithColonsAreNotMisparsedAsKeys(t *testing.T) {
+	// Same round: an indented YAML block-list item that happens to contain a
+	// colon (e.g. a title with a subtitle) is a continuation line, never a
+	// top-level key, regardless of what strings.Cut finds in it. Misparsing
+	// it as a key/value pair and re-rendering strips its indentation,
+	// de-indenting it out of the list it belongs to -- and refreshOwned
+	// re-renders this block on essentially every non-skipped ingest, so this
+	// is the normal path, not an edge case.
+	a := "---\naliases:\n  - 'Chapter 3: The Reckoning'\ntitle: 'X'\n---\n"
+	b := "---\nsource_hash: bbbb\n---\n"
+
+	got := refreshOwned(a, b)
+	assert.Equal(t, "---\naliases:\n  - 'Chapter 3: The Reckoning'\ntitle: 'X'\nsource_hash: bbbb\n---\n", got)
+}