feat(pipeline): feed the current source page into the extraction prompt (#106) (#132)

Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/wiki/merge.go

@@ -64,23 +64,50 @@ func MergeReport(a, b Page) (Page, []string) {
 	fmB, _, secsB := parseSections(b.Content)
 
 	var conflicts []string
-	idx := make(map[string]int, len(secsA))
+
+	// Sections in secsA sharing a normalized heading (pre-existing corpus
+	// drift — "## Notes" and "## NOTES" both already on disk, the exact
+	// pattern this normalization exists to catch going forward, found in
+	// review of PR #130/#132) are matched to b's occurrences of that same
+	// heading POSITIONALLY: b's 1st occurrence of the key pairs with a's
+	// 1st, b's 2nd with a's 2nd, and so on. A single "last write wins" (or
+	// even "first write wins") map from key to one index would instead
+	// route every one of b's occurrences to the SAME on-disk section —
+	// which either freezes every duplicate but one (never receiving
+	// updates again) or, worse, compares mismatched pairs and reports a
+	// disagreement between two sections that were never supposed to be
+	// compared, false-positiving on the plain self-merge case a revision
+	// pass produces when the model echoes a page back unchanged (found in
+	// review of PR #132).
+	byKey := make(map[string][]int, len(secsA))
 	for i, s := range secsA {
-		idx[s.heading] = i
+		key := normalizeHeading(s.heading)
+		byKey[key] = append(byKey[key], i)
 	}
+	cursor := make(map[string]int, len(byKey))
 
 	for _, sB := range secsB {
-		i, exists := idx[sB.heading]
-		if !exists {
-			idx[sB.heading] = len(secsA)
+		key := normalizeHeading(sB.heading)
+		indices := byKey[key]
+		pos := cursor[key]
+		cursor[key] = pos + 1
+
+		if pos >= len(indices) {
+			// b has more occurrences of this heading than a does (including
+			// the case where a has none at all) — the extra is new content,
+			// appended as its own section and registered so a THIRD
+			// occurrence of the same heading in b pairs with this one
+			// rather than appending yet another duplicate.
+			byKey[key] = append(byKey[key], len(secsA))
 			secsA = append(secsA, sB)
 			continue
 		}
+		i := indices[pos]
 		sA := secsA[i]
 		switch {
-		case bulletSections[sB.heading]:
+		case bulletSectionsNorm[key]:
 			secsA[i].content = mergeBullets(sA.content, sB.content)
-		case appendSections[sB.heading]:
+		case appendSectionsNorm[key]:
 			secsA[i].content = appendNew(sA.content, sB.content)
 		default:
 			// a wins — but say that b disagreed, rather than dropping it mute.
@@ -93,6 +120,34 @@ func MergeReport(a, b Page) (Page, []string) {
 	return Page{Path: a.Path, Content: rebuildContent(refreshOwned(fmA, fmB), preambleA, secsA)}, conflicts
 }
 
+// normalizeHeading returns the key Merge uses to decide "is this the same
+// section" — trimmed and case-folded, so "Key Claims" and "Key claims" match
+// (hyperguild#106: heading casing drift must not fork a section). It is used
+// only for that comparison; the on-disk section's original heading text
+// (secsA[i].heading, untouched here) is what actually gets written out, so a
+// human's heading casing is never silently rewritten by a merge.
+func normalizeHeading(h string) string {
+	return strings.ToLower(strings.TrimSpace(h))
+}
+
+// bulletSectionsNorm and appendSectionsNorm are bulletSections/appendSections
+// keyed by normalizeHeading instead of exact text, so a section's merge
+// strategy (union bullets, append) is chosen by the same normalized identity
+// Merge uses to match it — otherwise a heading that matched an on-disk
+// section (case-insensitively) could still fail the exact-string strategy
+// lookup and silently fall through to "keep a's version," dropping b's
+// content instead of unioning or appending it.
+var bulletSectionsNorm = normalizeSet(bulletSections)
+var appendSectionsNorm = normalizeSet(appendSections)
+
+func normalizeSet(names map[string]bool) map[string]bool {
+	out := make(map[string]bool, len(names))
+	for k := range names {
+		out[normalizeHeading(k)] = true
+	}
+	return out
+}
+
 // generatorOwned names the frontmatter fields the ingest pipeline may refresh
 // on a page it is revising. Everything else on disk wins.
 //

ingestion/internal/wiki/merge_test.go

@@ -305,6 +305,98 @@ Rewritten by the model.
 	assert.Equal(t, want, got)
 }
 
+// --- hyperguild#106: heading casing drift must not fork a section ----------
+// Page-aware extraction asks the model to revise the page it's given, but
+// nothing guarantees the model reproduces a heading's exact on-disk casing
+// ("Key Claims" vs "Key claims"). Without normalizing the match key, that
+// drift silently creates a near-duplicate section instead of updating the
+// original — the failure mode #106 exists to close.
+
+func TestMerge_HeadingCaseDriftMatchesSameSection(t *testing.T) {
+	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Key Claims\n\n- Appetite beats estimation.\n"}
+	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Key claims\n\n- New pattern holds under load.\n"}
+
+	got := Merge(a, b).Content
+
+	assert.Equal(t, 1, strings.Count(got, "## Key"),
+		"casing drift in the heading must not create a second section")
+	assert.Contains(t, got, "Appetite beats estimation.")
+	assert.Contains(t, got, "New pattern holds under load.",
+		"the differently-cased section's content must still be unioned in")
+	assert.Contains(t, got, "## Key Claims",
+		"the on-disk heading's original casing must be preserved in the output")
+	assert.NotContains(t, got, "## Key claims",
+		"the fresh extraction's heading casing must not overwrite the on-disk display text")
+}
+
+func TestMerge_HeadingWhitespaceDriftMatchesSameSection(t *testing.T) {
+	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Open Questions \n\n- Does appetite scale?\n"}
+	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n##  Open Questions\n\n- What about remote teams?\n"}
+
+	got := Merge(a, b).Content
+
+	assert.Equal(t, 1, strings.Count(got, "## Open Questions"),
+		"surrounding whitespace drift in the heading must not create a second section")
+	assert.Contains(t, got, "Does appetite scale?")
+	assert.Contains(t, got, "What about remote teams?")
+}
+
+func TestMerge_PreExistingCaseDuplicateHeadingsRouteToTheFirstOccurrence(t *testing.T) {
+	// Found in review of PR #130: normalizeHeading's idx map is built by
+	// iterating secsA with idx[normalizeHeading(s.heading)] = i -- last
+	// write wins. Before heading normalization existed, two on-disk
+	// sections differing only by case (real, pre-existing corpus drift --
+	// the exact pattern #106 itself cites) never collided, since idx keyed
+	// on the exact string. After normalization they collide, and every
+	// fresh update silently, deterministically routes to whichever
+	// duplicate happens to appear LAST in the file -- the first one is
+	// frozen forever, never receiving updates again, with nothing to say
+	// so. A Contains-only check doesn't catch this: nothing is literally
+	// deleted, a bullet just lands under the wrong of two still-present
+	// headings. Assert it lands under the FIRST occurrence instead —
+	// deterministic and matches "earliest is canonical," same reasoning
+	// as every other on-disk-wins default in this file.
+	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Key Claims\n\n- claim one\n\n## KEY CLAIMS\n\n- claim two\n"}
+	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Key Claims\n\n- claim one\n- fresh claim\n"}
+
+	got := Merge(a, b).Content
+
+	assert.Contains(t, got, "## Key Claims\n\n- claim one\n- fresh claim\n",
+		"the fresh bullet must land in the first-occurrence section, not silently route to the last duplicate")
+	assert.Contains(t, got, "## KEY CLAIMS\n\n- claim two\n",
+		"the second duplicate is untouched by this update, not merged into or overwritten")
+}
+
+func TestMergeReport_SelfMergeOfPreExistingDuplicateHeadingsReportsNoConflict(t *testing.T) {
+	// Found in review of PR #132: first-occurrence-wins routes ALL of b's
+	// occurrences of a normalized heading to a's FIRST occurrence, not to
+	// their own positional counterpart. So merging a page against itself
+	// (or an echoed-back copy, exactly what #106's revision prompt can
+	// produce) pairs b's second "## NOTES" against a's first "## Notes" —
+	// different content, same normalized key — and MergeReport wrongly
+	// reports a disagreement where none exists. Fixed by pairing
+	// positionally: b's Nth occurrence of a normalized heading matches a's
+	// Nth occurrence, not always the first.
+	page := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Notes\n\nFirst note.\n\n## NOTES\n\nSecond note.\n"}
+
+	_, conflicts := MergeReport(page, page)
+	assert.Empty(t, conflicts, "merging a page with pre-existing duplicate headings against itself must report no conflicts")
+}
+
+func TestMergeReport_DuplicateHeadingsPairPositionallyNotAllToFirst(t *testing.T) {
+	// Companion case: b has genuinely different content in its second
+	// occurrence of the normalized heading — that must compare against a's
+	// second occurrence (and report if they disagree), not against a's
+	// first (which would either false-positive if a's first differs, or
+	// false-negative-mask a real disagreement in the second pair).
+	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Notes\n\nFirst note.\n\n## NOTES\n\nSecond note.\n"}
+	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Notes\n\nFirst note.\n\n## NOTES\n\nSecond note, revised.\n"}
+
+	_, conflicts := MergeReport(a, b)
+	assert.Equal(t, []string{"NOTES"}, conflicts,
+		"only the second occurrence, which actually disagrees, is reported")
+}
+
 func TestRefreshOwned_CommentLinesSurviveUnmangled(t *testing.T) {
 	// Third review round on PR #126: parseFrontmatterBlock dropped the
 	// sibling api package's comment guard (strings.HasPrefix(trimmed, "#")).