fix(wiki): give the preamble its own field instead of overloading heading==""

Second review pass on PR #103 found a third bug in the same file: the
preamble fix from the prior commit represented "content before the first
## heading" as a section with heading=="" — which collides with a literal
degenerate `## ` (marker, no title text; strings.TrimPrefix also yields
"").  Two entries sharing one key in Merge's idx map means one loses
lookup visibility. Reproduced: a has only a preamble, b introduces a
genuinely new degenerate `## ` section — Merge's idx wrongly reports it
as "already exists" (it's really a's unrelated preamble) and never adds
it, so b's section vanishes from the output entirely.

Fixed by giving parseSections a dedicated preamble return value instead
of folding it into the sections slice. It can no longer collide with
anything, degenerate or otherwise, because it's no longer keyed by
heading at all. Merge and rebuildContent updated to thread it through
directly. No other caller of parseSections/rebuildContent in this repo.

Also fixed a doc-comment left misplaced by the prior commit (appendNew's
comment had drifted onto canonicalLine, inserted between them).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

ingestion/internal/wiki/merge.go

@@ -29,12 +29,16 @@ type section struct {
 }
 
 // Merge combines two Page values with the same path.
-// Frontmatter is taken from a. Sections are merged by strategy:
+// Frontmatter is taken from a. The preamble (content before the first ##
-// bullet sections union unique lines, append sections concatenate,
+// heading) is taken from a — same "on-disk wins" default every unlisted
-// all others keep a's version. Sections in b not present in a are appended.
+// named section gets, kept as a separate field rather than a section with
+// an empty heading so it can never collide with one (see parseSections).
+// Sections are merged by strategy: bullet sections union unique lines,
+// append sections concatenate, all others keep a's version. Sections in b
+// not present in a are appended.
 func Merge(a, b Page) Page {
-	fmA, secsA := parseSections(a.Content)
+	fmA, preambleA, secsA := parseSections(a.Content)
-	_, secsB := parseSections(b.Content)
+	_, _, secsB := parseSections(b.Content)
 
 	idx := make(map[string]int, len(secsA))
 	for i, s := range secsA {
@@ -57,10 +61,20 @@ func Merge(a, b Page) Page {
 		}
 	}
 
-	return Page{Path: a.Path, Content: rebuildContent(fmA, secsA)}
+	return Page{Path: a.Path, Content: rebuildContent(fmA, preambleA, secsA)}
 }
 
-func parseSections(markdown string) (frontmatter string, sections []section) {
+// parseSections splits markdown into its frontmatter, its preamble (content
+// before the first `## ` heading), and its `## `-headed sections.
+//
+// The preamble is returned separately rather than as a section with an
+// empty heading — a section keyed by heading=="" would collide with a
+// literal degenerate `## ` (marker, no title text; strings.TrimPrefix also
+// yields ""), which is a distinct, addressable section a page can genuinely
+// have, not the same thing as "no heading yet." Two entries sharing one key
+// in Merge's idx map means one of them silently loses lookup visibility —
+// found in review as real, reproducible data loss, not a hypothetical.
+func parseSections(markdown string) (frontmatter, preamble string, sections []section) {
 	lines := strings.Split(markdown, "\n")
 	i := 0
 
@@ -78,6 +92,22 @@ func parseSections(markdown string) (frontmatter string, sections []section) {
 		frontmatter = fmt.Sprintf("---\n%s\n---\n", strings.Join(fmLines, "\n"))
 	}
 
+	// Everything before the first `## ` heading — an intro paragraph, or, if
+	// the page has no headings at all, the entire body — is ordinary content,
+	// not something this parser is entitled to drop (hyperguild#103 review:
+	// dropping it meant Merge, one caller up, would silently replace the
+	// whole on-disk body with the fresh extraction whenever a page didn't
+	// follow the ## convention — exactly the class of bug this file exists
+	// to prevent).
+	var preambleLines []string
+	for ; i < len(lines); i++ {
+		if strings.HasPrefix(lines[i], "## ") {
+			break
+		}
+		preambleLines = append(preambleLines, lines[i])
+	}
+	preamble = strings.Join(preambleLines, "\n")
+
 	var cur *section
 	for ; i < len(lines); i++ {
 		line := lines[i]
@@ -104,9 +134,12 @@ func parseSections(markdown string) (frontmatter string, sections []section) {
 // by a blank line — invisible in a diff viewer, fatal to byte-stability. Compile-on-ingest
 // merges on every run, so "merging changes nothing when nothing changed" has to
 // hold at the byte level, not just the content level.
-func rebuildContent(frontmatter string, sections []section) string {
+func rebuildContent(frontmatter, preamble string, sections []section) string {
 	var sb strings.Builder
 	sb.WriteString(frontmatter)
+	if body := strings.Trim(preamble, "\n"); body != "" {
+		fmt.Fprintf(&sb, "\n%s\n", body)
+	}
 	for _, sec := range sections {
 		body := strings.Trim(sec.content, "\n")
 		fmt.Fprintf(&sb, "\n## %s\n\n%s\n", sec.heading, body)
@@ -114,6 +147,15 @@ func rebuildContent(frontmatter string, sections []section) string {
 	return sb.String()
 }
 
+// canonicalLine collapses a line to its comparison key: internal whitespace
+// runs (a re-extraction's bullet spacing, tabs vs spaces, trailing spaces)
+// must not make the same logical line look new. strings.Fields already
+// trims and splits on any whitespace run, so joining it back with single
+// spaces is exactly "same words, same order, whitespace-insensitive."
+func canonicalLine(line string) string {
+	return strings.Join(strings.Fields(line), " ")
+}
+
 // appendNew concatenates the lines of b that a does not already carry.
 //
 // Append sections used to concatenate wholesale, which is correct exactly once.
@@ -134,18 +176,18 @@ func rebuildContent(frontmatter string, sections []section) string {
 func appendNew(a, b string) string {
 	seen := make(map[string]bool)
 	for _, line := range strings.Split(a, "\n") {
-		if t := strings.TrimSpace(line); t != "" {
+		if c := canonicalLine(line); c != "" {
-			seen[t] = true
+			seen[c] = true
 		}
 	}
 
 	var add []string
 	for _, line := range strings.Split(b, "\n") {
-		t := strings.TrimSpace(line)
+		c := canonicalLine(line)
-		if t == "" || seen[t] {
+		if c == "" || seen[c] {
 			continue
 		}
-		seen[t] = true
+		seen[c] = true
 		add = append(add, line)
 	}
 	if len(add) == 0 {
@@ -158,11 +200,11 @@ func mergeBullets(a, b string) string {
 	seen := make(map[string]bool)
 	var lines []string
 	for _, line := range strings.Split(a+b, "\n") {
-		trimmed := strings.TrimSpace(line)
+		c := canonicalLine(line)
-		if trimmed == "" || seen[trimmed] {
+		if c == "" || seen[c] {
 			continue
 		}
-		seen[trimmed] = true
+		seen[c] = true
 		lines = append(lines, line)
 	}
 	return strings.Join(lines, "\n") + "\n"

ingestion/internal/wiki/merge_test.go

@@ -54,6 +54,77 @@ func TestMerge_KeepsFrontmatterFromA(t *testing.T) {
 	assert.NotContains(t, got.Content, "title: B")
 }
 
+// --- hyperguild#103 review: a page with content outside any `## ` section ---
+// must never lose that content on merge. parseSections used to only capture
+// text between `## ` headings — anything before the first heading (a common,
+// ordinary shape for a hand-written note: an intro paragraph before the first
+// section) or, worse, an entire page with no `## ` headings at all (a legacy
+// note, a stub, any hand-edit that doesn't follow the section convention)
+// vanished silently. Merge would then fill the page entirely from the fresh
+// extraction — the opposite of "never silently rewrite what a human put
+// there," the guarantee this whole file exists to provide.
+
+func TestMerge_PreservesOnDiskContentWithNoSectionHeadings(t *testing.T) {
+	a := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\nJust some free text a human wrote. No section headings at all.\n"}
+	b := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\n## Definition\n\nFresh extraction's take.\n"}
+
+	got := Merge(a, b)
+	assert.Contains(t, got.Content, "Just some free text a human wrote",
+		"on-disk content with no ## headings must survive the merge, not be replaced wholesale by the fresh extraction")
+}
+
+func TestMerge_DegenerateEmptyHeadingDoesNotCollideWithPreamble(t *testing.T) {
+	// Second review pass on #103: representing the preamble as a section with
+	// heading=="" collides with a literal `## ` (marker, no title text) —
+	// strings.TrimPrefix("## ", "## ") is also "". `a` has only a preamble
+	// (no degenerate heading of its own); `b` introduces a genuinely new
+	// degenerate `## ` section. Because both key to "", Merge's idx thinks
+	// b's section "already exists" in a (it's really just a's unrelated
+	// preamble) and never adds it — b's section vanishes from the output
+	// entirely, not merged, not kept, not appended.
+	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\nAn intro paragraph.\n\n## Definition\n\nA thing.\n"}
+	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## \n\nA genuinely new degenerate section's body.\n\n## Definition\n\nA thing.\n"}
+
+	got := Merge(a, b)
+	assert.Contains(t, got.Content, "An intro paragraph.", "a's own preamble must survive")
+	assert.Contains(t, got.Content, "A genuinely new degenerate section's body.",
+		"b's degenerate section is new content, not a collision with a's unrelated preamble — it must not be silently dropped")
+}
+
+func TestMerge_PreservesPreambleBeforeFirstHeading(t *testing.T) {
+	a := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\nAn intro paragraph before the first heading.\n\n## Definition\n\nA thing.\n"}
+	b := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\n## Definition\n\nA thing.\n"}
+
+	got := Merge(a, b)
+	assert.Contains(t, got.Content, "An intro paragraph before the first heading",
+		"text before the first ## heading is ordinary page content, not something the parser is entitled to drop")
+}
+
+// --- hyperguild#103 review: dedup must be whitespace-insensitive -----------
+// Page content is produced by an LLM per run, so byte-identical re-extraction
+// of "the same" logical line is not guaranteed — different bullet spacing or
+// trailing whitespace on an otherwise-identical line must still dedup, or
+// idempotency only holds when the extractor happens to reproduce whitespace
+// exactly, which is not a real guarantee.
+
+func TestMerge_AppendSectionDedupsAcrossWhitespaceVariants(t *testing.T) {
+	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Open Questions\n\n- Does this converge?\n"}
+	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Open Questions\n\n-   Does this converge?  \n"}
+
+	got := Merge(a, b)
+	assert.Equal(t, 1, strings.Count(got.Content, "Does this converge?"),
+		"same logical line with different whitespace must dedup, not duplicate")
+}
+
+func TestMerge_BulletSectionDedupsAcrossWhitespaceVariants(t *testing.T) {
+	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Related Concepts\n\n- [[bar|Bar]]\n"}
+	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Related Concepts\n\n-  [[bar|Bar]]\n"}
+
+	got := Merge(a, b)
+	assert.Equal(t, 1, strings.Count(got.Content, "[[bar|Bar]]"),
+		"same logical bullet with different whitespace must dedup, not duplicate")
+}
+
 // --- brain#21/B1: merging must be safe to run again -------------------------
 
 func TestMerge_AppendSectionDoesNotDuplicateContentAlreadyPresent(t *testing.T) {