feat(pipeline): skip extraction when the source has not changed (#105) #126

Merged
mathias merged 4 commits from fix/105-source-drift-on-main into main 2026-09-23 05:09:30 +00:00
2 changed files with 79 additions and 3 deletions
Showing only changes of commit 41a9112860 - Show all commits
+43 -3
View File
@@ -3,6 +3,7 @@ package wiki
import (
"fmt"
"sort"
"strings"
)
@@ -84,6 +85,14 @@ var generatorOwned = map[string]bool{
// refreshOwned returns a's frontmatter with the generator-owned fields taken
// from b where b supplies them. Fields absent from b keep a's value, so a
// generator that stops emitting a field cannot blank it.
//
// A field b supplies that a's frontmatter doesn't have yet — every page
// ingested before this mechanism existed has no source_hash line at all —
// is appended, not dropped. Dropping it silently defeats the whole reason a
// field is generator-owned: it would never reach the page it's meant to
// live on (found in review, PR #126: source_hash then never populates for
// the pre-existing corpus, so the skip-on-unchanged feature it exists to
// support never engages, permanently, with nothing to say so).
func refreshOwned(a, b string) string {
bVals := make(map[string]string)
for _, line := range strings.Split(b, "\n") {
@@ -97,8 +106,10 @@ func refreshOwned(a, b string) string {
return a
}
out := make([]string, 0, len(strings.Split(a, "\n")))
for _, line := range strings.Split(a, "\n") {
aLines := strings.Split(a, "\n")
out := make([]string, 0, len(aLines)+len(bVals))
closeIdx := -1
for _, line := range aLines {
key, _, found := strings.Cut(line, ":")
k := strings.TrimSpace(key)
if found && generatorOwned[k] {
@@ -108,9 +119,38 @@ func refreshOwned(a, b string) string {
continue
}
}
if strings.TrimSpace(line) == "---" {
closeIdx = len(out) // the fence's position in the OUTPUT slice
}
out = append(out, line)
}
return strings.Join(out, "\n")
if len(bVals) == 0 {
return strings.Join(out, "\n")
}
// Sorted for a deterministic, byte-stable result — map order is not.
remaining := make([]string, 0, len(bVals))
for k := range bVals {
remaining = append(remaining, k)
}
sort.Strings(remaining)
insert := make([]string, len(remaining))
for i, k := range remaining {
insert[i] = k + ": " + bVals[k]
}
if closeIdx < 0 {
// No closing fence found — malformed input for this well-formed-only
// helper; append at the end rather than lose the fields.
out = append(out, insert...)
return strings.Join(out, "\n")
}
final := make([]string, 0, len(out)+len(insert))
final = append(final, out[:closeIdx]...)
final = append(final, insert...)
final = append(final, out[closeIdx:]...)
return strings.Join(final, "\n")
}
// parseSections splits markdown into its frontmatter, its preamble (content
+36
View File
@@ -214,3 +214,39 @@ Rewritten by the model.
assert.Contains(t, got, "Original.", "an existing section still wins over the model's rewrite")
assert.NotContains(t, got, "Renamed by the model")
}
func TestMerge_GeneratorOwnedFieldsAreAppendedWhenAPageNeverHadThemBefore(t *testing.T) {
// Found in review of PR #126: refreshOwned only replaced a generator-owned
// line that already existed in a's frontmatter — it never appended one a
// didn't have. Every page ingested before this feature shipped has no
// source_hash line at all, so on its first re-ingest after deploy, the
// fresh hash was silently dropped and the skip-on-unchanged feature this
// PR exists to build would never engage for the entire pre-existing
// corpus, permanently, with nothing telling anyone.
existing := Page{Path: "wiki/sources/x.md", Content: `---
title: 'X'
---
## Summary
Original.
`}
fresh := Page{Path: "wiki/sources/x.md", Content: `---
title: 'Renamed by the model'
last_updated: 2026-08-22
source_hash: bbbb
---
## Summary
Rewritten by the model.
`}
got := Merge(existing, fresh).Content
assert.Contains(t, got, "source_hash: bbbb", "a generator-owned field a page never had before must be appended, not dropped")
assert.Contains(t, got, "last_updated: 2026-08-22")
assert.Contains(t, got, "title: 'X'", "a human-set title must still survive regeneration")
assert.Contains(t, got, "Original.")
assert.NotContains(t, got, "Renamed by the model")
}