feat(pipeline): report when a re-ingest disagrees with the page on disk (#104) #129
@@ -104,6 +104,24 @@ Two fields are **generator-owned** and do get refreshed on a revise:
|
|||||||
`last_updated` and `source_hash`. Everything else on disk wins. The boundary is
|
`last_updated` and `source_hash`. Everything else on disk wins. The boundary is
|
||||||
an explicit allowlist, so a field nobody has thought about stays human-owned.
|
an explicit allowlist, so a field nobody has thought about stays human-owned.
|
||||||
|
|
||||||
|
### Disagreement is reported, not resolved
|
||||||
|
|
||||||
|
When the fresh extraction produces a *different version* of a section that
|
||||||
|
already exists, the on-disk version still wins — but the run now says so:
|
||||||
|
|
||||||
|
```
|
||||||
|
wiki/sources/shape-up.md: source now disagrees on section "Summary" — on-disk version kept
|
||||||
|
```
|
||||||
|
|
||||||
|
That line lands in `log.md` with the rest of the run's warnings. Without it, "the
|
||||||
|
source is unchanged" and "the source changed its mind and we ignored it" look
|
||||||
|
identical from the outside.
|
||||||
|
|
||||||
|
This is deliberately **textual** disagreement, not semantic contradiction: cheap,
|
||||||
|
deterministic, no model on the write path. Deciding whether a disagreement is a
|
||||||
|
genuine contradiction stays with the Judge-gated `brain-enrich` pass, which is
|
||||||
|
approval-gated by design. Detection here, adjudication there.
|
||||||
|
|
||||||
## Re-ingesting an unchanged source is skipped
|
## Re-ingesting an unchanged source is skipped
|
||||||
|
|
||||||
Ingest stores `source_hash` — sha256 of the ingested **body**, frontmatter
|
Ingest stores `source_hash` — sha256 of the ingested **body**, frontmatter
|
||||||
|
|||||||
@@ -117,10 +117,18 @@ func buildAndWrite(rawPages []RawPage, sourceSlug, date, brainDir, source string
|
|||||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
||||||
return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
|
return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
|
||||||
}
|
}
|
||||||
content, err := reviseOrCreate(dest, page)
|
content, conflicts, err := reviseOrCreate(dest, page)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
|
for _, heading := range conflicts {
|
||||||
|
// Reported, not resolved. The on-disk version stands; whether
|
||||||
|
// the disagreement is a genuine contradiction is the
|
||||||
|
// Judge-gated enrich pass's call, not the write path's.
|
||||||
|
warnings = append(warnings, fmt.Sprintf(
|
||||||
|
"%s: source now disagrees on section %q — on-disk version kept",
|
||||||
|
page.Path, heading))
|
||||||
|
}
|
||||||
if err := os.WriteFile(dest, []byte(content), 0o644); err != nil {
|
if err := os.WriteFile(dest, []byte(content), 0o644); err != nil {
|
||||||
return Result{}, fmt.Errorf("write %s: %w", page.Path, err)
|
return Result{}, fmt.Errorf("write %s: %w", page.Path, err)
|
||||||
}
|
}
|
||||||
@@ -198,15 +206,16 @@ func storedSourceHash(brainDir, sourceSlug string) string {
|
|||||||
// its first-ingest date on a revised page. Left deliberately — the alternative
|
// its first-ingest date on a revised page. Left deliberately — the alternative
|
||||||
// is this function editing frontmatter it does not own, and no lint reads that
|
// is this function editing frontmatter it does not own, and no lint reads that
|
||||||
// field today.
|
// field today.
|
||||||
func reviseOrCreate(dest string, page wiki.Page) (string, error) {
|
func reviseOrCreate(dest string, page wiki.Page) (string, []string, error) {
|
||||||
existing, err := os.ReadFile(dest)
|
existing, err := os.ReadFile(dest)
|
||||||
switch {
|
switch {
|
||||||
case os.IsNotExist(err):
|
case os.IsNotExist(err):
|
||||||
return page.Content, nil
|
return page.Content, nil, nil
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return "", fmt.Errorf("read existing %s: %w", page.Path, err)
|
return "", nil, fmt.Errorf("read existing %s: %w", page.Path, err)
|
||||||
}
|
}
|
||||||
return wiki.Merge(wiki.Page{Path: page.Path, Content: string(existing)}, page).Content, nil
|
merged, conflicts := wiki.MergeReport(wiki.Page{Path: page.Path, Content: string(existing)}, page)
|
||||||
|
return merged.Content, conflicts, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// mergeAll deduplicates pages by path, merging content from later occurrences.
|
// mergeAll deduplicates pages by path, merging content from later occurrences.
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -293,3 +294,34 @@ func TestRun_FrontmatterOnlyChangeIsNotDrift(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(t, 1, calls, "only the body counts as the source")
|
assert.Equal(t, 1, calls, "only the body counts as the source")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRaw_ReportsWhenTheSourceDisagreesWithWhatIsOnDisk(t *testing.T) {
|
||||||
|
// The on-disk version still wins — that is what makes unattended ingest
|
||||||
|
// safe. But the run must say that the source now claims something else,
|
||||||
|
// or a changed source is indistinguishable from an unchanged one.
|
||||||
|
dir := t.TempDir()
|
||||||
|
page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Dir(page), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(page, []byte(`---
|
||||||
|
title: 'Shape Up'
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Appetite is fixed, scope is variable.
|
||||||
|
`), 0o644))
|
||||||
|
|
||||||
|
res, err := RunRaw(dir, "shape-up", []RawPage{{
|
||||||
|
Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product",
|
||||||
|
Content: "## Summary\n\nAppetite is negotiable and scope is fixed.\n",
|
||||||
|
}}, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
got, err := os.ReadFile(page)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Contains(t, string(got), "Appetite is fixed", "on-disk version wins")
|
||||||
|
|
||||||
|
joined := strings.Join(res.Warnings, " ")
|
||||||
|
assert.Contains(t, joined, "Summary", "the disagreeing section must be named")
|
||||||
|
assert.Contains(t, joined, "shape-up", "and the page it happened on")
|
||||||
|
}
|
||||||
|
|||||||
@@ -37,10 +37,33 @@ type section struct {
|
|||||||
// Sections are merged by strategy: bullet sections union unique lines,
|
// Sections are merged by strategy: bullet sections union unique lines,
|
||||||
// append sections concatenate, all others keep a's version. Sections in b
|
// append sections concatenate, all others keep a's version. Sections in b
|
||||||
// not present in a are appended.
|
// not present in a are appended.
|
||||||
|
// Merge discards any disagreement silently; prefer MergeReport where the
|
||||||
|
// caller can act on it.
|
||||||
func Merge(a, b Page) Page {
|
func Merge(a, b Page) Page {
|
||||||
|
page, _ := MergeReport(a, b)
|
||||||
|
return page
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeReport is Merge, plus the headings of sections where b carried a
|
||||||
|
// different version of a section a already had — and lost.
|
||||||
|
//
|
||||||
|
// That loss is the property which makes ingest safe to run unattended (#103):
|
||||||
|
// a regenerating writer can add to a note but never rewrite it. But the
|
||||||
|
// discarded text is also a signal, and until now it went in the bin: the source
|
||||||
|
// now says something different about a section we already hold. Nobody could
|
||||||
|
// tell "the source is unchanged" from "the source changed its mind and we
|
||||||
|
// ignored it".
|
||||||
|
//
|
||||||
|
// Deliberately NOT a semantic contradiction check. This reports textual
|
||||||
|
// disagreement, which is cheap, deterministic, and needs no model on the write
|
||||||
|
// path. Deciding whether a disagreement is a genuine contradiction stays with
|
||||||
|
// the Judge-gated enrich pass that already exists (brain-gardener#175), which
|
||||||
|
// is approval-gated by design — detection here, adjudication there.
|
||||||
|
func MergeReport(a, b Page) (Page, []string) {
|
||||||
fmA, preambleA, secsA := parseSections(a.Content)
|
fmA, preambleA, secsA := parseSections(a.Content)
|
||||||
fmB, _, secsB := parseSections(b.Content)
|
fmB, _, secsB := parseSections(b.Content)
|
||||||
|
|
||||||
|
var conflicts []string
|
||||||
idx := make(map[string]int, len(secsA))
|
idx := make(map[string]int, len(secsA))
|
||||||
for i, s := range secsA {
|
for i, s := range secsA {
|
||||||
idx[s.heading] = i
|
idx[s.heading] = i
|
||||||
@@ -59,10 +82,15 @@ func Merge(a, b Page) Page {
|
|||||||
secsA[i].content = mergeBullets(sA.content, sB.content)
|
secsA[i].content = mergeBullets(sA.content, sB.content)
|
||||||
case appendSections[sB.heading]:
|
case appendSections[sB.heading]:
|
||||||
secsA[i].content = appendNew(sA.content, sB.content)
|
secsA[i].content = appendNew(sA.content, sB.content)
|
||||||
|
default:
|
||||||
|
// a wins — but say that b disagreed, rather than dropping it mute.
|
||||||
|
if canonicalBlock(sA.content) != canonicalBlock(sB.content) {
|
||||||
|
conflicts = append(conflicts, sB.heading)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return Page{Path: a.Path, Content: rebuildContent(refreshOwned(fmA, fmB), preambleA, secsA)}
|
return Page{Path: a.Path, Content: rebuildContent(refreshOwned(fmA, fmB), preambleA, secsA)}, conflicts
|
||||||
}
|
}
|
||||||
|
|
||||||
// generatorOwned names the frontmatter fields the ingest pipeline may refresh
|
// generatorOwned names the frontmatter fields the ingest pipeline may refresh
|
||||||
@@ -301,6 +329,21 @@ func canonicalLine(line string) string {
|
|||||||
return strings.Join(strings.Fields(line), " ")
|
return strings.Join(strings.Fields(line), " ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// canonicalBlock applies canonicalLine per line of a multi-line section body
|
||||||
|
// — the same whitespace-insensitivity appendNew/mergeBullets already give a
|
||||||
|
// single line, extended to a whole section for MergeReport's disagreement
|
||||||
|
// check (found in review, PR #129: comparing raw strings.TrimSpace only
|
||||||
|
// tolerated leading/trailing whitespace, not a re-extraction's mid-line
|
||||||
|
// spacing, which every other comparison in this file already tolerates).
|
||||||
|
func canonicalBlock(block string) string {
|
||||||
|
lines := strings.Split(block, "\n")
|
||||||
|
out := make([]string, len(lines))
|
||||||
|
for i, l := range lines {
|
||||||
|
out[i] = canonicalLine(l)
|
||||||
|
}
|
||||||
|
return strings.Join(out, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
// appendNew concatenates the lines of b that a does not already carry.
|
// appendNew concatenates the lines of b that a does not already carry.
|
||||||
//
|
//
|
||||||
// Append sections used to concatenate wholesale, which is correct exactly once.
|
// Append sections used to concatenate wholesale, which is correct exactly once.
|
||||||
|
|||||||
@@ -332,3 +332,85 @@ func TestRefreshOwned_IndentedListItemsWithColonsAreNotMisparsedAsKeys(t *testin
|
|||||||
got := refreshOwned(a, b)
|
got := refreshOwned(a, b)
|
||||||
assert.Equal(t, "---\naliases:\n - 'Chapter 3: The Reckoning'\ntitle: 'X'\nsource_hash: bbbb\n---\n", got)
|
assert.Equal(t, "---\naliases:\n - 'Chapter 3: The Reckoning'\ntitle: 'X'\nsource_hash: bbbb\n---\n", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMerge_ReportsSectionsWhereTheFreshVersionWasDiscarded(t *testing.T) {
|
||||||
|
// The asymmetry that makes ingest safe (on-disk wins) also throws away a
|
||||||
|
// real signal: the source now says something different about a section we
|
||||||
|
// already have. Today that disagreement is silently discarded, so nobody
|
||||||
|
// can tell "the source is unchanged" from "the source changed its mind and
|
||||||
|
// we ignored it".
|
||||||
|
existing := Page{Path: "wiki/sources/x.md", Content: `---
|
||||||
|
title: 'X'
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Appetite is fixed, scope is variable.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
Six-week cycles.
|
||||||
|
`}
|
||||||
|
fresh := Page{Path: "wiki/sources/x.md", Content: `---
|
||||||
|
title: 'X'
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Appetite is negotiable and scope is fixed.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
Six-week cycles.
|
||||||
|
`}
|
||||||
|
|
||||||
|
got, conflicts := MergeReport(existing, fresh)
|
||||||
|
|
||||||
|
assert.Contains(t, got.Content, "Appetite is fixed", "on-disk version still wins")
|
||||||
|
assert.NotContains(t, got.Content, "negotiable")
|
||||||
|
assert.Equal(t, []string{"Summary"}, conflicts,
|
||||||
|
"only the section that actually disagreed is reported")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMerge_ReportsNothingWhenTheFreshVersionAgrees(t *testing.T) {
|
||||||
|
page := Page{Path: "wiki/sources/x.md", Content: `---
|
||||||
|
title: 'X'
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Appetite is fixed, scope is variable.
|
||||||
|
`}
|
||||||
|
|
||||||
|
_, conflicts := MergeReport(page, page)
|
||||||
|
assert.Empty(t, conflicts, "identical content is not a disagreement")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMerge_ReportsNothingForInternalWhitespaceVariantsOnly(t *testing.T) {
|
||||||
|
// Found in review of PR #129: the conflict comparison used raw
|
||||||
|
// strings.TrimSpace, which only tolerates LEADING/TRAILING whitespace —
|
||||||
|
// unlike appendNew/mergeBullets, which already canonicalize internal
|
||||||
|
// whitespace runs too (#103: a re-extraction's spacing isn't guaranteed
|
||||||
|
// byte-stable). A mid-line double-space from re-extraction would
|
||||||
|
// false-positive as a "disagreement" here even though every other
|
||||||
|
// whitespace-sensitive comparison in this file already tolerates it.
|
||||||
|
existing := Page{Path: "wiki/sources/x.md", Content: `---
|
||||||
|
title: 'X'
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Appetite is fixed, scope is variable.
|
||||||
|
`}
|
||||||
|
fresh := Page{Path: "wiki/sources/x.md", Content: `---
|
||||||
|
title: 'X'
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Appetite is fixed, scope is variable.
|
||||||
|
`}
|
||||||
|
|
||||||
|
_, conflicts := MergeReport(existing, fresh)
|
||||||
|
assert.Empty(t, conflicts, "internal whitespace differences alone are not a disagreement")
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user