hyperguild/ingestion/internal/wiki/merge.go

// ingestion/internal/wiki/merge.go
package wiki

import (
	"fmt"
	"strings"
)

var bulletSections = map[string]bool{
	"Related Concepts":                  true,
	"Related Entities":                  true,
	"Sources":                           true,
	"Key Claims":                        true,
	"Entities Mentioned":                true,
	"Concepts Introduced or Reinforced": true,
	"Chapters":                          true,
}

var appendSections = map[string]bool{
	"Evolving Notes":        true,
	"Updates":               true,
	"Open Questions Raised": true,
	"Open Questions":        true,
}

type section struct {
	heading string
	content string
}

// Merge combines two Page values with the same path.
// Frontmatter is taken from a. Sections are merged by strategy:
// bullet sections union unique lines, append sections concatenate,
// all others keep a's version. Sections in b not present in a are appended.
func Merge(a, b Page) Page {
	fmA, secsA := parseSections(a.Content)
	_, secsB := parseSections(b.Content)

	idx := make(map[string]int, len(secsA))
	for i, s := range secsA {
		idx[s.heading] = i
	}

	for _, sB := range secsB {
		i, exists := idx[sB.heading]
		if !exists {
			idx[sB.heading] = len(secsA)
			secsA = append(secsA, sB)
			continue
		}
		sA := secsA[i]
		switch {
		case bulletSections[sB.heading]:
			secsA[i].content = mergeBullets(sA.content, sB.content)
		case appendSections[sB.heading]:
			secsA[i].content = appendNew(sA.content, sB.content)
		}
	}

	return Page{Path: a.Path, Content: rebuildContent(fmA, secsA)}
}

func parseSections(markdown string) (frontmatter string, sections []section) {
	lines := strings.Split(markdown, "\n")
	i := 0

	if i < len(lines) && strings.TrimSpace(lines[i]) == "---" {
		i++
		var fmLines []string
		for i < len(lines) {
			if strings.TrimSpace(lines[i]) == "---" {
				i++
				break
			}
			fmLines = append(fmLines, lines[i])
			i++
		}
		frontmatter = fmt.Sprintf("---\n%s\n---\n", strings.Join(fmLines, "\n"))
	}

	var cur *section
	for ; i < len(lines); i++ {
		line := lines[i]
		if strings.HasPrefix(line, "## ") {
			if cur != nil {
				sections = append(sections, *cur)
			}
			cur = &section{heading: strings.TrimPrefix(line, "## ")}
		} else if cur != nil {
			cur.content += line + "\n"
		}
	}
	if cur != nil {
		sections = append(sections, *cur)
	}
	return
}

// rebuildContent serialises sections back into a page.
//
// Each section body is normalised to exactly one trailing newline and no
// leading blank line. Without that, a section keeps whatever blank lines it was
// parsed with and this function adds its own, so every merge pass grows the page
// by a blank line — invisible in a diff viewer, fatal to byte-stability. Compile-on-ingest
// merges on every run, so "merging changes nothing when nothing changed" has to
// hold at the byte level, not just the content level.
func rebuildContent(frontmatter string, sections []section) string {
	var sb strings.Builder
	sb.WriteString(frontmatter)
	for _, sec := range sections {
		body := strings.Trim(sec.content, "\n")
		fmt.Fprintf(&sb, "\n## %s\n\n%s\n", sec.heading, body)
	}
	return sb.String()
}

// appendNew concatenates the lines of b that a does not already carry.
//
// Append sections used to concatenate wholesale, which is correct exactly once.
// Compile-on-ingest merges a freshly extracted page into the one already on
// disk, so an unchanged source re-ingested would grow "Open Questions" and
// "Updates" on every run until the page is unreadable — and the drift guard
// that would otherwise catch it (hashing the source body) does not exist yet,
// so idempotency has to hold here rather than upstream.
//
// Line-level rather than block-level: the common case is a re-ingest carrying
// one genuinely new line alongside several it already has. A block-level
// "skip if identical" would duplicate the old lines every time that happened.
//
// The cost is that a line repeated verbatim on purpose collapses to one. For
// these sections — questions, updates, evolving notes — a repeat is noise
// rather than signal, and the failure is conservative: the page stops growing
// instead of growing wrongly.
func appendNew(a, b string) string {
	seen := make(map[string]bool)
	for _, line := range strings.Split(a, "\n") {
		if t := strings.TrimSpace(line); t != "" {
			seen[t] = true
		}
	}

	var add []string
	for _, line := range strings.Split(b, "\n") {
		t := strings.TrimSpace(line)
		if t == "" || seen[t] {
			continue
		}
		seen[t] = true
		add = append(add, line)
	}
	if len(add) == 0 {
		return a // nothing new — leave the section byte-identical
	}
	return strings.TrimRight(a, "\n") + "\n\n" + strings.Join(add, "\n") + "\n"
}

func mergeBullets(a, b string) string {
	seen := make(map[string]bool)
	var lines []string
	for _, line := range strings.Split(a+b, "\n") {
		trimmed := strings.TrimSpace(line)
		if trimmed == "" || seen[trimmed] {
			continue
		}
		seen[trimmed] = true
		lines = append(lines, line)
	}
	return strings.Join(lines, "\n") + "\n"
}