hyperguild/ingestion/internal/wiki/merge_test.go

// ingestion/internal/wiki/merge_test.go
package wiki

import (
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestMerge_BulletSectionsUnion(t *testing.T) {
	a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Related Concepts\n\n- [[bar|Bar]]\n"}
	b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Related Concepts\n\n- [[bar|Bar]]\n- [[baz|Baz]]\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "[[bar|Bar]]")
	assert.Contains(t, got.Content, "[[baz|Baz]]")
	assert.Equal(t, 1, strings.Count(got.Content, "[[bar|Bar]]"))
}

func TestMerge_AppendSections(t *testing.T) {
	a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Evolving Notes\n\nFirst note.\n"}
	b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Evolving Notes\n\nSecond note.\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "First note.")
	assert.Contains(t, got.Content, "Second note.")
}

func TestMerge_KeepFirstForOtherSections(t *testing.T) {
	a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Definition\n\nFirst definition.\n"}
	b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Definition\n\nSecond definition.\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "First definition.")
	assert.NotContains(t, got.Content, "Second definition.")
}

func TestMerge_NewSectionFromB(t *testing.T) {
	a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Definition\n\nA thing.\n"}
	b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Why It Matters\n\nBecause reasons.\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "A thing.")
	assert.Contains(t, got.Content, "Because reasons.")
}

func TestMerge_KeepsFrontmatterFromA(t *testing.T) {
	a := Page{Path: "p.md", Content: "---\ntitle: A\nlast_updated: 2026-01-01\n---\n\n## Definition\n\nA.\n"}
	b := Page{Path: "p.md", Content: "---\ntitle: B\nlast_updated: 2026-06-01\n---\n\n## Definition\n\nB.\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "title: A")
	assert.NotContains(t, got.Content, "title: B")
}

// --- hyperguild#103 review: a page with content outside any `## ` section ---
// must never lose that content on merge. parseSections used to only capture
// text between `## ` headings — anything before the first heading (a common,
// ordinary shape for a hand-written note: an intro paragraph before the first
// section) or, worse, an entire page with no `## ` headings at all (a legacy
// note, a stub, any hand-edit that doesn't follow the section convention)
// vanished silently. Merge would then fill the page entirely from the fresh
// extraction — the opposite of "never silently rewrite what a human put
// there," the guarantee this whole file exists to provide.

func TestMerge_PreservesOnDiskContentWithNoSectionHeadings(t *testing.T) {
	a := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\nJust some free text a human wrote. No section headings at all.\n"}
	b := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\n## Definition\n\nFresh extraction's take.\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "Just some free text a human wrote",
		"on-disk content with no ## headings must survive the merge, not be replaced wholesale by the fresh extraction")
}

func TestMerge_DegenerateEmptyHeadingDoesNotCollideWithPreamble(t *testing.T) {
	// Second review pass on #103: representing the preamble as a section with
	// heading=="" collides with a literal `## ` (marker, no title text) —
	// strings.TrimPrefix("## ", "## ") is also "". `a` has only a preamble
	// (no degenerate heading of its own); `b` introduces a genuinely new
	// degenerate `## ` section. Because both key to "", Merge's idx thinks
	// b's section "already exists" in a (it's really just a's unrelated
	// preamble) and never adds it — b's section vanishes from the output
	// entirely, not merged, not kept, not appended.
	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\nAn intro paragraph.\n\n## Definition\n\nA thing.\n"}
	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## \n\nA genuinely new degenerate section's body.\n\n## Definition\n\nA thing.\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "An intro paragraph.", "a's own preamble must survive")
	assert.Contains(t, got.Content, "A genuinely new degenerate section's body.",
		"b's degenerate section is new content, not a collision with a's unrelated preamble — it must not be silently dropped")
}

func TestMerge_PreservesPreambleBeforeFirstHeading(t *testing.T) {
	a := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\nAn intro paragraph before the first heading.\n\n## Definition\n\nA thing.\n"}
	b := Page{Path: "wiki/x.md", Content: "---\ntitle: X\n---\n\n## Definition\n\nA thing.\n"}

	got := Merge(a, b)
	assert.Contains(t, got.Content, "An intro paragraph before the first heading",
		"text before the first ## heading is ordinary page content, not something the parser is entitled to drop")
}

// --- hyperguild#103 review: dedup must be whitespace-insensitive -----------
// Page content is produced by an LLM per run, so byte-identical re-extraction
// of "the same" logical line is not guaranteed — different bullet spacing or
// trailing whitespace on an otherwise-identical line must still dedup, or
// idempotency only holds when the extractor happens to reproduce whitespace
// exactly, which is not a real guarantee.

func TestMerge_AppendSectionDedupsAcrossWhitespaceVariants(t *testing.T) {
	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Open Questions\n\n- Does this converge?\n"}
	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Open Questions\n\n-   Does this converge?  \n"}

	got := Merge(a, b)
	assert.Equal(t, 1, strings.Count(got.Content, "Does this converge?"),
		"same logical line with different whitespace must dedup, not duplicate")
}

func TestMerge_BulletSectionDedupsAcrossWhitespaceVariants(t *testing.T) {
	a := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Related Concepts\n\n- [[bar|Bar]]\n"}
	b := Page{Path: "p.md", Content: "---\ntitle: X\n---\n\n## Related Concepts\n\n-  [[bar|Bar]]\n"}

	got := Merge(a, b)
	assert.Equal(t, 1, strings.Count(got.Content, "[[bar|Bar]]"),
		"same logical bullet with different whitespace must dedup, not duplicate")
}

// --- brain#21/B1: merging must be safe to run again -------------------------

func TestMerge_AppendSectionDoesNotDuplicateContentAlreadyPresent(t *testing.T) {
	// Compile-on-ingest merges a freshly extracted page into the one already
	// on disk. Bullet sections union, so they are naturally idempotent — but
	// append sections concatenate, so re-ingesting an unchanged source would
	// grow "Open Questions" on every run until the page is unreadable.
	page := Page{Path: "wiki/sources/x.md", Content: `---
title: X
---

## Open Questions

- Does this converge?
`}

	once := Merge(page, page)
	twice := Merge(once, page)

	assert.Equal(t, 1, strings.Count(once.Content, "Does this converge?"),
		"merging a page with itself must not duplicate an append section")
	assert.Equal(t, once.Content, twice.Content,
		"merge must be idempotent — a third pass may not keep growing the page")
}

func TestMerge_AppendSectionStillAccumulatesGenuinelyNewContent(t *testing.T) {
	// The dedup must not turn append sections into keep-a-only sections:
	// new material is exactly what compile-on-ingest exists to add.
	existing := Page{Path: "wiki/sources/x.md", Content: `---
title: X
---

## Open Questions

- Does this converge?
`}
	fresh := Page{Path: "wiki/sources/x.md", Content: `---
title: X
---

## Open Questions

- What happens under load?
`}

	got := Merge(existing, fresh).Content
	assert.Contains(t, got, "Does this converge?", "existing question must survive")
	assert.Contains(t, got, "What happens under load?", "new question must be added")
}

func TestMerge_GeneratorOwnedFrontmatterFieldsAreRefreshed(t *testing.T) {
	// #103 made on-disk frontmatter win wholesale, which is what protects
	// human-set fields from a regenerating writer. The cost was that fields the
	// generator legitimately owns — when it last ran, what it last saw — froze
	// at their first-ingest values. An explicit owned-set is the boundary:
	// named fields the generator may refresh, everything else human-owned.
	existing := Page{Path: "wiki/sources/x.md", Content: `---
title: 'X'
last_updated: 2026-01-01
source_hash: aaaa
half_life: durable
claim_provenance: asserted
---

## Summary

Original.
`}
	fresh := Page{Path: "wiki/sources/x.md", Content: `---
title: 'Renamed by the model'
last_updated: 2026-08-22
source_hash: bbbb
---

## Summary

Rewritten by the model.
`}

	got := Merge(existing, fresh).Content

	assert.Contains(t, got, "last_updated: 2026-08-22", "generator owns last_updated")
	assert.Contains(t, got, "source_hash: bbbb", "generator owns source_hash")
	assert.Contains(t, got, "title: 'X'", "a human-set title must survive regeneration")
	assert.Contains(t, got, "half_life: durable", "the claim envelope must survive regeneration")
	assert.Contains(t, got, "claim_provenance: asserted")
	assert.Contains(t, got, "Original.", "an existing section still wins over the model's rewrite")
	assert.NotContains(t, got, "Renamed by the model")
}

func TestMerge_GeneratorOwnedFieldsAreAppendedWhenAPageNeverHadThemBefore(t *testing.T) {
	// Found in review of PR #126: refreshOwned only replaced a generator-owned
	// line that already existed in a's frontmatter — it never appended one a
	// didn't have. Every page ingested before this feature shipped has no
	// source_hash line at all, so on its first re-ingest after deploy, the
	// fresh hash was silently dropped and the skip-on-unchanged feature this
	// PR exists to build would never engage for the entire pre-existing
	// corpus, permanently, with nothing telling anyone.
	existing := Page{Path: "wiki/sources/x.md", Content: `---
title: 'X'
---

## Summary

Original.
`}
	fresh := Page{Path: "wiki/sources/x.md", Content: `---
title: 'Renamed by the model'
last_updated: 2026-08-22
source_hash: bbbb
---

## Summary

Rewritten by the model.
`}

	got := Merge(existing, fresh).Content

	// Exact frontmatter block, not Contains: the second review round found
	// that a substring check doesn't prove the fields landed inside a valid
	// fence, in the right order, or that the fence itself wasn't corrupted —
	// two independent mutations (closeIdx targeting the wrong fence, reversed
	// sort) both survived a Contains-only version of this test.
	want := "---\ntitle: 'X'\nlast_updated: 2026-08-22\nsource_hash: bbbb\n---\n\n## Summary\n\nOriginal.\n"
	assert.Equal(t, want, got)
}

func TestMerge_GeneratorOwnedFieldsCreateAFrontmatterBlockWhenThePageHadNoneAtAll(t *testing.T) {
	// Second review round on PR #126: when a has no frontmatter fence at all,
	// the append fallback spliced raw, unfenced "key: value" lines straight
	// into the body -- worse than the original bug, because the next
	// parseSections call wouldn't recognize them as frontmatter (they don't
	// start with `---`), so they'd be absorbed as ordinary preamble text and
	// re-duplicate on every subsequent merge instead of updating in place.
	existing := Page{Path: "wiki/sources/x.md", Content: "## Summary\n\nOriginal.\n"}
	fresh := Page{Path: "wiki/sources/x.md", Content: `---
last_updated: 2026-08-22
source_hash: bbbb
---

## Summary

Rewritten by the model.
`}

	got := Merge(existing, fresh).Content

	want := "---\nlast_updated: 2026-08-22\nsource_hash: bbbb\n---\n\n## Summary\n\nOriginal.\n"
	assert.Equal(t, want, got)
}

func TestMerge_GeneratorOwnedFieldsMixedReplaceAndAppend(t *testing.T) {
	// a already has last_updated (replace in place) but not source_hash
	// (append) -- the two code paths in the same call, together.
	existing := Page{Path: "wiki/sources/x.md", Content: `---
title: 'X'
last_updated: 2026-01-01
---

## Summary

Original.
`}
	fresh := Page{Path: "wiki/sources/x.md", Content: `---
last_updated: 2026-08-22
source_hash: bbbb
---

## Summary

Rewritten by the model.
`}

	got := Merge(existing, fresh).Content

	want := "---\ntitle: 'X'\nlast_updated: 2026-08-22\nsource_hash: bbbb\n---\n\n## Summary\n\nOriginal.\n"
	assert.Equal(t, want, got)
}

func TestRefreshOwned_CommentLinesSurviveUnmangled(t *testing.T) {
	// Third review round on PR #126: parseFrontmatterBlock dropped the
	// sibling api package's comment guard (strings.HasPrefix(trimmed, "#")).
	// Without it, a comment line containing a colon gets cut at the colon
	// and re-rendered with collapsed whitespace instead of surviving
	// verbatim.
	a := "---\n#see also:   https://x/y\ntitle: 'X'\n---\n"
	b := "---\nsource_hash: bbbb\n---\n"

	got := refreshOwned(a, b)
	assert.Equal(t, "---\n#see also:   https://x/y\ntitle: 'X'\nsource_hash: bbbb\n---\n", got)
}

func TestRefreshOwned_IndentedListItemsWithColonsAreNotMisparsedAsKeys(t *testing.T) {
	// Same round: an indented YAML block-list item that happens to contain a
	// colon (e.g. a title with a subtitle) is a continuation line, never a
	// top-level key, regardless of what strings.Cut finds in it. Misparsing
	// it as a key/value pair and re-rendering strips its indentation,
	// de-indenting it out of the list it belongs to -- and refreshOwned
	// re-renders this block on essentially every non-skipped ingest, so this
	// is the normal path, not an edge case.
	a := "---\naliases:\n  - 'Chapter 3: The Reckoning'\ntitle: 'X'\n---\n"
	b := "---\nsource_hash: bbbb\n---\n"

	got := refreshOwned(a, b)
	assert.Equal(t, "---\naliases:\n  - 'Chapter 3: The Reckoning'\ntitle: 'X'\nsource_hash: bbbb\n---\n", got)
}