Ingest now stamps source_hash — sha256 of the ingested body, frontmatter excluded — onto the source page, and compares it before doing any work. An unchanged source returns Skipped with no LLM call at all. This is cost and signal, not correctness: #103 already made a re-ingest byte-identical, so a repeated run was harmless but not free — a full extraction over every chunk to produce a result that merges to nothing. Building this first would have put idempotency in the guard rather than in the write, which is the wrong dependency direction. Body-only hashing on purpose. A re-crawl that stamps a new fetch date has not changed the source, and treating that as drift would re-extract the whole corpus every time a crawler ran. A missing or unreadable hash always means "extract": a missed skip costs one extraction, a wrong skip loses an update silently, and silence is the failure mode this repo keeps paying for. The skip is reported (Result.Skipped plus a warning naming the hash), not silent — a run that does nothing must say so, or "the pipeline is fine" and "the pipeline stopped working" look identical (#107). Storing the hash required settling who owns which frontmatter field. #103 made the on-disk block win wholesale, which protects human-set metadata from a regenerating writer but froze fields the generator legitimately owns. wiki.Merge now refreshes an explicit generatorOwned allowlist — last_updated and source_hash — and keeps everything else on disk. An allowlist rather than a carve-out, so a field nobody has thought about stays human-owned by default. That also retires the last_updated limitation #103 shipped with. Refs #105, #103, #107, brain#21.
146 lines
5.0 KiB
Go
146 lines
5.0 KiB
Go
// ingestion/internal/wiki/merge_test.go
|
|
package wiki
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestMerge_BulletSectionsUnion(t *testing.T) {
|
|
a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Related Concepts\n\n- [[bar|Bar]]\n"}
|
|
b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Related Concepts\n\n- [[bar|Bar]]\n- [[baz|Baz]]\n"}
|
|
|
|
got := Merge(a, b)
|
|
assert.Contains(t, got.Content, "[[bar|Bar]]")
|
|
assert.Contains(t, got.Content, "[[baz|Baz]]")
|
|
assert.Equal(t, 1, strings.Count(got.Content, "[[bar|Bar]]"))
|
|
}
|
|
|
|
func TestMerge_AppendSections(t *testing.T) {
|
|
a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Evolving Notes\n\nFirst note.\n"}
|
|
b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Evolving Notes\n\nSecond note.\n"}
|
|
|
|
got := Merge(a, b)
|
|
assert.Contains(t, got.Content, "First note.")
|
|
assert.Contains(t, got.Content, "Second note.")
|
|
}
|
|
|
|
func TestMerge_KeepFirstForOtherSections(t *testing.T) {
|
|
a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Definition\n\nFirst definition.\n"}
|
|
b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Definition\n\nSecond definition.\n"}
|
|
|
|
got := Merge(a, b)
|
|
assert.Contains(t, got.Content, "First definition.")
|
|
assert.NotContains(t, got.Content, "Second definition.")
|
|
}
|
|
|
|
func TestMerge_NewSectionFromB(t *testing.T) {
|
|
a := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Definition\n\nA thing.\n"}
|
|
b := Page{Path: "wiki/concepts/foo.md", Content: "---\ntitle: Foo\n---\n\n## Why It Matters\n\nBecause reasons.\n"}
|
|
|
|
got := Merge(a, b)
|
|
assert.Contains(t, got.Content, "A thing.")
|
|
assert.Contains(t, got.Content, "Because reasons.")
|
|
}
|
|
|
|
func TestMerge_KeepsFrontmatterFromA(t *testing.T) {
|
|
a := Page{Path: "p.md", Content: "---\ntitle: A\nlast_updated: 2026-01-01\n---\n\n## Definition\n\nA.\n"}
|
|
b := Page{Path: "p.md", Content: "---\ntitle: B\nlast_updated: 2026-06-01\n---\n\n## Definition\n\nB.\n"}
|
|
|
|
got := Merge(a, b)
|
|
assert.Contains(t, got.Content, "title: A")
|
|
assert.NotContains(t, got.Content, "title: B")
|
|
}
|
|
|
|
// --- brain#21/B1: merging must be safe to run again -------------------------
|
|
|
|
func TestMerge_AppendSectionDoesNotDuplicateContentAlreadyPresent(t *testing.T) {
|
|
// Compile-on-ingest merges a freshly extracted page into the one already
|
|
// on disk. Bullet sections union, so they are naturally idempotent — but
|
|
// append sections concatenate, so re-ingesting an unchanged source would
|
|
// grow "Open Questions" on every run until the page is unreadable.
|
|
page := Page{Path: "wiki/sources/x.md", Content: `---
|
|
title: X
|
|
---
|
|
|
|
## Open Questions
|
|
|
|
- Does this converge?
|
|
`}
|
|
|
|
once := Merge(page, page)
|
|
twice := Merge(once, page)
|
|
|
|
assert.Equal(t, 1, strings.Count(once.Content, "Does this converge?"),
|
|
"merging a page with itself must not duplicate an append section")
|
|
assert.Equal(t, once.Content, twice.Content,
|
|
"merge must be idempotent — a third pass may not keep growing the page")
|
|
}
|
|
|
|
func TestMerge_AppendSectionStillAccumulatesGenuinelyNewContent(t *testing.T) {
|
|
// The dedup must not turn append sections into keep-a-only sections:
|
|
// new material is exactly what compile-on-ingest exists to add.
|
|
existing := Page{Path: "wiki/sources/x.md", Content: `---
|
|
title: X
|
|
---
|
|
|
|
## Open Questions
|
|
|
|
- Does this converge?
|
|
`}
|
|
fresh := Page{Path: "wiki/sources/x.md", Content: `---
|
|
title: X
|
|
---
|
|
|
|
## Open Questions
|
|
|
|
- What happens under load?
|
|
`}
|
|
|
|
got := Merge(existing, fresh).Content
|
|
assert.Contains(t, got, "Does this converge?", "existing question must survive")
|
|
assert.Contains(t, got, "What happens under load?", "new question must be added")
|
|
}
|
|
|
|
func TestMerge_GeneratorOwnedFrontmatterFieldsAreRefreshed(t *testing.T) {
|
|
// #103 made on-disk frontmatter win wholesale, which is what protects
|
|
// human-set fields from a regenerating writer. The cost was that fields the
|
|
// generator legitimately owns — when it last ran, what it last saw — froze
|
|
// at their first-ingest values. An explicit owned-set is the boundary:
|
|
// named fields the generator may refresh, everything else human-owned.
|
|
existing := Page{Path: "wiki/sources/x.md", Content: `---
|
|
title: 'X'
|
|
last_updated: 2026-01-01
|
|
source_hash: aaaa
|
|
half_life: durable
|
|
claim_provenance: asserted
|
|
---
|
|
|
|
## Summary
|
|
|
|
Original.
|
|
`}
|
|
fresh := Page{Path: "wiki/sources/x.md", Content: `---
|
|
title: 'Renamed by the model'
|
|
last_updated: 2026-08-22
|
|
source_hash: bbbb
|
|
---
|
|
|
|
## Summary
|
|
|
|
Rewritten by the model.
|
|
`}
|
|
|
|
got := Merge(existing, fresh).Content
|
|
|
|
assert.Contains(t, got, "last_updated: 2026-08-22", "generator owns last_updated")
|
|
assert.Contains(t, got, "source_hash: bbbb", "generator owns source_hash")
|
|
assert.Contains(t, got, "title: 'X'", "a human-set title must survive regeneration")
|
|
assert.Contains(t, got, "half_life: durable", "the claim envelope must survive regeneration")
|
|
assert.Contains(t, got, "claim_provenance: asserted")
|
|
assert.Contains(t, got, "Original.", "an existing section still wins over the model's rewrite")
|
|
assert.NotContains(t, got, "Renamed by the model")
|
|
}
|