hyperguild/ingestion/internal/wiki/merge.go

// ingestion/internal/wiki/merge.go
package wiki

import (
	"fmt"
	"strings"
)

var bulletSections = map[string]bool{
	"Related Concepts":                  true,
	"Related Entities":                  true,
	"Sources":                           true,
	"Key Claims":                        true,
	"Entities Mentioned":                true,
	"Concepts Introduced or Reinforced": true,
	"Chapters":                          true,
}

var appendSections = map[string]bool{
	"Evolving Notes":        true,
	"Updates":               true,
	"Open Questions Raised": true,
	"Open Questions":        true,
}

type section struct {
	heading string
	content string
}

// Merge combines two Page values with the same path.
// Frontmatter is taken from a. Sections are merged by strategy:
// bullet sections union unique lines, append sections concatenate,
// all others keep a's version. Sections in b not present in a are appended.
func Merge(a, b Page) Page {
	fmA, secsA := parseSections(a.Content)
	_, secsB := parseSections(b.Content)

	idx := make(map[string]int, len(secsA))
	for i, s := range secsA {
		idx[s.heading] = i
	}

	for _, sB := range secsB {
		i, exists := idx[sB.heading]
		if !exists {
			idx[sB.heading] = len(secsA)
			secsA = append(secsA, sB)
			continue
		}
		sA := secsA[i]
		switch {
		case bulletSections[sB.heading]:
			secsA[i].content = mergeBullets(sA.content, sB.content)
		case appendSections[sB.heading]:
			secsA[i].content = appendNew(sA.content, sB.content)
		}
	}

	return Page{Path: a.Path, Content: rebuildContent(fmA, secsA)}
}

func parseSections(markdown string) (frontmatter string, sections []section) {
	lines := strings.Split(markdown, "\n")
	i := 0

	if i < len(lines) && strings.TrimSpace(lines[i]) == "---" {
		i++
		var fmLines []string
		for i < len(lines) {
			if strings.TrimSpace(lines[i]) == "---" {
				i++
				break
			}
			fmLines = append(fmLines, lines[i])
			i++
		}
		frontmatter = fmt.Sprintf("---\n%s\n---\n", strings.Join(fmLines, "\n"))
	}

	// Everything before the first `## ` heading — an intro paragraph, or, if
	// the page has no headings at all, the entire body — is ordinary content,
	// not something this parser is entitled to drop. It becomes a section
	// with an empty heading, which Merge's switch never matches, so it falls
	// through to the same "keep a's version" default every other unlisted
	// section gets (hyperguild#103 review: dropping it here meant Merge, one
	// caller up, would silently replace the whole on-disk body with the
	// fresh extraction whenever a page didn't follow the ## convention —
	// exactly the class of bug this whole file exists to prevent).
	cur := &section{}
	for ; i < len(lines); i++ {
		line := lines[i]
		if strings.HasPrefix(line, "## ") {
			sections = append(sections, *cur)
			cur = &section{heading: strings.TrimPrefix(line, "## ")}
		} else {
			cur.content += line + "\n"
		}
	}
	sections = append(sections, *cur)
	return
}

// rebuildContent serialises sections back into a page.
//
// Each section body is normalised to exactly one trailing newline and no
// leading blank line. Without that, a section keeps whatever blank lines it was
// parsed with and this function adds its own, so every merge pass grows the page
// by a blank line — invisible in a diff viewer, fatal to byte-stability. Compile-on-ingest
// merges on every run, so "merging changes nothing when nothing changed" has to
// hold at the byte level, not just the content level.
func rebuildContent(frontmatter string, sections []section) string {
	var sb strings.Builder
	sb.WriteString(frontmatter)
	for _, sec := range sections {
		body := strings.Trim(sec.content, "\n")
		if sec.heading == "" {
			// The preamble pseudo-section: content before the first ##. No
			// heading of its own to re-emit, and — unlike a named section —
			// it is usually empty (most pages start directly with a ##), so
			// an empty one must produce nothing rather than a stray blank
			// line ahead of the first real heading.
			if body == "" {
				continue
			}
			fmt.Fprintf(&sb, "\n%s\n", body)
			continue
		}
		fmt.Fprintf(&sb, "\n## %s\n\n%s\n", sec.heading, body)
	}
	return sb.String()
}

// appendNew concatenates the lines of b that a does not already carry.
//
// Append sections used to concatenate wholesale, which is correct exactly once.
// Compile-on-ingest merges a freshly extracted page into the one already on
// disk, so an unchanged source re-ingested would grow "Open Questions" and
// "Updates" on every run until the page is unreadable — and the drift guard
// that would otherwise catch it (hashing the source body) does not exist yet,
// so idempotency has to hold here rather than upstream.
//
// Line-level rather than block-level: the common case is a re-ingest carrying
// one genuinely new line alongside several it already has. A block-level
// "skip if identical" would duplicate the old lines every time that happened.
//
// The cost is that a line repeated verbatim on purpose collapses to one. For
// these sections — questions, updates, evolving notes — a repeat is noise
// rather than signal, and the failure is conservative: the page stops growing
// instead of growing wrongly.
// canonicalLine collapses a line to its comparison key: internal whitespace
// runs (a re-extraction's bullet spacing, tabs vs spaces, trailing spaces)
// must not make the same logical line look new. strings.Fields already
// trims and splits on any whitespace run, so joining it back with single
// spaces is exactly "same words, same order, whitespace-insensitive."
func canonicalLine(line string) string {
	return strings.Join(strings.Fields(line), " ")
}

func appendNew(a, b string) string {
	seen := make(map[string]bool)
	for _, line := range strings.Split(a, "\n") {
		if c := canonicalLine(line); c != "" {
			seen[c] = true
		}
	}

	var add []string
	for _, line := range strings.Split(b, "\n") {
		c := canonicalLine(line)
		if c == "" || seen[c] {
			continue
		}
		seen[c] = true
		add = append(add, line)
	}
	if len(add) == 0 {
		return a // nothing new — leave the section byte-identical
	}
	return strings.TrimRight(a, "\n") + "\n\n" + strings.Join(add, "\n") + "\n"
}

func mergeBullets(a, b string) string {
	seen := make(map[string]bool)
	var lines []string
	for _, line := range strings.Split(a+b, "\n") {
		c := canonicalLine(line)
		if c == "" || seen[c] {
			continue
		}
		seen[c] = true
		lines = append(lines, line)
	}
	return strings.Join(lines, "\n") + "\n"
}