hyperguild/ingestion/internal/wiki/merge.go

// ingestion/internal/wiki/merge.go
package wiki

import (
	"fmt"
	"sort"
	"strings"
)

var bulletSections = map[string]bool{
	"Related Concepts":                  true,
	"Related Entities":                  true,
	"Sources":                           true,
	"Key Claims":                        true,
	"Entities Mentioned":                true,
	"Concepts Introduced or Reinforced": true,
	"Chapters":                          true,
}

var appendSections = map[string]bool{
	"Evolving Notes":        true,
	"Updates":               true,
	"Open Questions Raised": true,
	"Open Questions":        true,
}

type section struct {
	heading string
	content string
}

// Merge combines two Page values with the same path.
// Frontmatter is taken from a. The preamble (content before the first ##
// heading) is taken from a — same "on-disk wins" default every unlisted
// named section gets, kept as a separate field rather than a section with
// an empty heading so it can never collide with one (see parseSections).
// Sections are merged by strategy: bullet sections union unique lines,
// append sections concatenate, all others keep a's version. Sections in b
// not present in a are appended.
func Merge(a, b Page) Page {
	fmA, preambleA, secsA := parseSections(a.Content)
	fmB, _, secsB := parseSections(b.Content)

	idx := make(map[string]int, len(secsA))
	for i, s := range secsA {
		idx[s.heading] = i
	}

	for _, sB := range secsB {
		i, exists := idx[sB.heading]
		if !exists {
			idx[sB.heading] = len(secsA)
			secsA = append(secsA, sB)
			continue
		}
		sA := secsA[i]
		switch {
		case bulletSections[sB.heading]:
			secsA[i].content = mergeBullets(sA.content, sB.content)
		case appendSections[sB.heading]:
			secsA[i].content = appendNew(sA.content, sB.content)
		}
	}

	return Page{Path: a.Path, Content: rebuildContent(refreshOwned(fmA, fmB), preambleA, secsA)}
}

// generatorOwned names the frontmatter fields the ingest pipeline may refresh
// on a page it is revising. Everything else on disk wins.
//
// #103 made the on-disk frontmatter win wholesale, which is what stops a
// regenerating writer from overwriting human-set metadata — a title someone
// fixed, or the claim-property envelope (brain#21). The cost was that fields
// the generator legitimately owns froze at their first-ingest values: when it
// last ran, and what it last saw.
//
// An explicit allowlist rather than a carve-out: the ownership boundary is
// nameable, and a field nobody has thought about stays human-owned by default,
// which is the safe direction.
var generatorOwned = map[string]bool{
	"last_updated": true,
	"source_hash":  true,
}

// refreshOwned returns a's frontmatter with the generator-owned fields taken
// from b where b supplies them. Fields absent from b keep a's value, so a
// generator that stops emitting a field cannot blank it.
//
// A field b supplies that a's frontmatter doesn't have yet — every page
// ingested before this mechanism existed has no source_hash line at all —
// is appended, not dropped. Dropping it silently defeats the whole reason a
// field is generator-owned: it would never reach the page it's meant to
// live on (found in review, PR #126: source_hash then never populates for
// the pre-existing corpus, so the skip-on-unchanged feature it exists to
// support never engages, permanently, with nothing to say so).
func refreshOwned(a, b string) string {
	bVals := make(map[string]string)
	for _, line := range strings.Split(b, "\n") {
		key, value, found := strings.Cut(line, ":")
		if !found || !generatorOwned[strings.TrimSpace(key)] {
			continue
		}
		bVals[strings.TrimSpace(key)] = strings.TrimSpace(value)
	}
	if len(bVals) == 0 {
		return a
	}

	aLines := strings.Split(a, "\n")
	out := make([]string, 0, len(aLines)+len(bVals))
	closeIdx := -1
	for _, line := range aLines {
		key, _, found := strings.Cut(line, ":")
		k := strings.TrimSpace(key)
		if found && generatorOwned[k] {
			if v, ok := bVals[k]; ok {
				out = append(out, k+": "+v)
				delete(bVals, k)
				continue
			}
		}
		if strings.TrimSpace(line) == "---" {
			closeIdx = len(out) // the fence's position in the OUTPUT slice
		}
		out = append(out, line)
	}

	if len(bVals) == 0 {
		return strings.Join(out, "\n")
	}

	// Sorted for a deterministic, byte-stable result — map order is not.
	remaining := make([]string, 0, len(bVals))
	for k := range bVals {
		remaining = append(remaining, k)
	}
	sort.Strings(remaining)
	insert := make([]string, len(remaining))
	for i, k := range remaining {
		insert[i] = k + ": " + bVals[k]
	}

	if closeIdx < 0 {
		// No closing fence found — malformed input for this well-formed-only
		// helper; append at the end rather than lose the fields.
		out = append(out, insert...)
		return strings.Join(out, "\n")
	}
	final := make([]string, 0, len(out)+len(insert))
	final = append(final, out[:closeIdx]...)
	final = append(final, insert...)
	final = append(final, out[closeIdx:]...)
	return strings.Join(final, "\n")
}

// parseSections splits markdown into its frontmatter, its preamble (content
// before the first `## ` heading), and its `## `-headed sections.
//
// The preamble is returned separately rather than as a section with an
// empty heading — a section keyed by heading=="" would collide with a
// literal degenerate `## ` (marker, no title text; strings.TrimPrefix also
// yields ""), which is a distinct, addressable section a page can genuinely
// have, not the same thing as "no heading yet." Two entries sharing one key
// in Merge's idx map means one of them silently loses lookup visibility —
// found in review as real, reproducible data loss, not a hypothetical.
func parseSections(markdown string) (frontmatter, preamble string, sections []section) {
	lines := strings.Split(markdown, "\n")
	i := 0

	if i < len(lines) && strings.TrimSpace(lines[i]) == "---" {
		i++
		var fmLines []string
		for i < len(lines) {
			if strings.TrimSpace(lines[i]) == "---" {
				i++
				break
			}
			fmLines = append(fmLines, lines[i])
			i++
		}
		frontmatter = fmt.Sprintf("---\n%s\n---\n", strings.Join(fmLines, "\n"))
	}

	// Everything before the first `## ` heading — an intro paragraph, or, if
	// the page has no headings at all, the entire body — is ordinary content,
	// not something this parser is entitled to drop (hyperguild#103 review:
	// dropping it meant Merge, one caller up, would silently replace the
	// whole on-disk body with the fresh extraction whenever a page didn't
	// follow the ## convention — exactly the class of bug this file exists
	// to prevent).
	var preambleLines []string
	for ; i < len(lines); i++ {
		if strings.HasPrefix(lines[i], "## ") {
			break
		}
		preambleLines = append(preambleLines, lines[i])
	}
	preamble = strings.Join(preambleLines, "\n")

	var cur *section
	for ; i < len(lines); i++ {
		line := lines[i]
		if strings.HasPrefix(line, "## ") {
			if cur != nil {
				sections = append(sections, *cur)
			}
			cur = &section{heading: strings.TrimPrefix(line, "## ")}
		} else if cur != nil {
			cur.content += line + "\n"
		}
	}
	if cur != nil {
		sections = append(sections, *cur)
	}
	return
}

// rebuildContent serialises sections back into a page.
//
// Each section body is normalised to exactly one trailing newline and no
// leading blank line. Without that, a section keeps whatever blank lines it was
// parsed with and this function adds its own, so every merge pass grows the page
// by a blank line — invisible in a diff viewer, fatal to byte-stability. Compile-on-ingest
// merges on every run, so "merging changes nothing when nothing changed" has to
// hold at the byte level, not just the content level.
func rebuildContent(frontmatter, preamble string, sections []section) string {
	var sb strings.Builder
	sb.WriteString(frontmatter)
	if body := strings.Trim(preamble, "\n"); body != "" {
		fmt.Fprintf(&sb, "\n%s\n", body)
	}
	for _, sec := range sections {
		body := strings.Trim(sec.content, "\n")
		fmt.Fprintf(&sb, "\n## %s\n\n%s\n", sec.heading, body)
	}
	return sb.String()
}

// canonicalLine collapses a line to its comparison key: internal whitespace
// runs (a re-extraction's bullet spacing, tabs vs spaces, trailing spaces)
// must not make the same logical line look new. strings.Fields already
// trims and splits on any whitespace run, so joining it back with single
// spaces is exactly "same words, same order, whitespace-insensitive."
func canonicalLine(line string) string {
	return strings.Join(strings.Fields(line), " ")
}

// appendNew concatenates the lines of b that a does not already carry.
//
// Append sections used to concatenate wholesale, which is correct exactly once.
// Compile-on-ingest merges a freshly extracted page into the one already on
// disk, so an unchanged source re-ingested would grow "Open Questions" and
// "Updates" on every run until the page is unreadable — and the drift guard
// that would otherwise catch it (hashing the source body) does not exist yet,
// so idempotency has to hold here rather than upstream.
//
// Line-level rather than block-level: the common case is a re-ingest carrying
// one genuinely new line alongside several it already has. A block-level
// "skip if identical" would duplicate the old lines every time that happened.
//
// The cost is that a line repeated verbatim on purpose collapses to one. For
// these sections — questions, updates, evolving notes — a repeat is noise
// rather than signal, and the failure is conservative: the page stops growing
// instead of growing wrongly.
func appendNew(a, b string) string {
	seen := make(map[string]bool)
	for _, line := range strings.Split(a, "\n") {
		if c := canonicalLine(line); c != "" {
			seen[c] = true
		}
	}

	var add []string
	for _, line := range strings.Split(b, "\n") {
		c := canonicalLine(line)
		if c == "" || seen[c] {
			continue
		}
		seen[c] = true
		add = append(add, line)
	}
	if len(add) == 0 {
		return a // nothing new — leave the section byte-identical
	}
	return strings.TrimRight(a, "\n") + "\n\n" + strings.Join(add, "\n") + "\n"
}

func mergeBullets(a, b string) string {
	seen := make(map[string]bool)
	var lines []string
	for _, line := range strings.Split(a+b, "\n") {
		c := canonicalLine(line)
		if c == "" || seen[c] {
			continue
		}
		seen[c] = true
		lines = append(lines, line)
	}
	return strings.Join(lines, "\n") + "\n"
}