// ingestion/internal/wiki/merge.go package wiki import ( "fmt" "strings" ) var bulletSections = map[string]bool{ "Related Concepts": true, "Related Entities": true, "Sources": true, "Key Claims": true, "Entities Mentioned": true, "Concepts Introduced or Reinforced": true, "Chapters": true, } var appendSections = map[string]bool{ "Evolving Notes": true, "Updates": true, "Open Questions Raised": true, "Open Questions": true, } type section struct { heading string content string } // Merge combines two Page values with the same path. // Frontmatter is taken from a. Sections are merged by strategy: // bullet sections union unique lines, append sections concatenate, // all others keep a's version. Sections in b not present in a are appended. func Merge(a, b Page) Page { fmA, secsA := parseSections(a.Content) _, secsB := parseSections(b.Content) idx := make(map[string]int, len(secsA)) for i, s := range secsA { idx[s.heading] = i } for _, sB := range secsB { i, exists := idx[sB.heading] if !exists { idx[sB.heading] = len(secsA) secsA = append(secsA, sB) continue } sA := secsA[i] switch { case bulletSections[sB.heading]: secsA[i].content = mergeBullets(sA.content, sB.content) case appendSections[sB.heading]: secsA[i].content = appendNew(sA.content, sB.content) } } return Page{Path: a.Path, Content: rebuildContent(fmA, secsA)} } func parseSections(markdown string) (frontmatter string, sections []section) { lines := strings.Split(markdown, "\n") i := 0 if i < len(lines) && strings.TrimSpace(lines[i]) == "---" { i++ var fmLines []string for i < len(lines) { if strings.TrimSpace(lines[i]) == "---" { i++ break } fmLines = append(fmLines, lines[i]) i++ } frontmatter = fmt.Sprintf("---\n%s\n---\n", strings.Join(fmLines, "\n")) } // Everything before the first `## ` heading — an intro paragraph, or, if // the page has no headings at all, the entire body — is ordinary content, // not something this parser is entitled to drop. It becomes a section // with an empty heading, which Merge's switch never matches, so it falls // through to the same "keep a's version" default every other unlisted // section gets (hyperguild#103 review: dropping it here meant Merge, one // caller up, would silently replace the whole on-disk body with the // fresh extraction whenever a page didn't follow the ## convention — // exactly the class of bug this whole file exists to prevent). cur := §ion{} for ; i < len(lines); i++ { line := lines[i] if strings.HasPrefix(line, "## ") { sections = append(sections, *cur) cur = §ion{heading: strings.TrimPrefix(line, "## ")} } else { cur.content += line + "\n" } } sections = append(sections, *cur) return } // rebuildContent serialises sections back into a page. // // Each section body is normalised to exactly one trailing newline and no // leading blank line. Without that, a section keeps whatever blank lines it was // parsed with and this function adds its own, so every merge pass grows the page // by a blank line — invisible in a diff viewer, fatal to byte-stability. Compile-on-ingest // merges on every run, so "merging changes nothing when nothing changed" has to // hold at the byte level, not just the content level. func rebuildContent(frontmatter string, sections []section) string { var sb strings.Builder sb.WriteString(frontmatter) for _, sec := range sections { body := strings.Trim(sec.content, "\n") if sec.heading == "" { // The preamble pseudo-section: content before the first ##. No // heading of its own to re-emit, and — unlike a named section — // it is usually empty (most pages start directly with a ##), so // an empty one must produce nothing rather than a stray blank // line ahead of the first real heading. if body == "" { continue } fmt.Fprintf(&sb, "\n%s\n", body) continue } fmt.Fprintf(&sb, "\n## %s\n\n%s\n", sec.heading, body) } return sb.String() } // appendNew concatenates the lines of b that a does not already carry. // // Append sections used to concatenate wholesale, which is correct exactly once. // Compile-on-ingest merges a freshly extracted page into the one already on // disk, so an unchanged source re-ingested would grow "Open Questions" and // "Updates" on every run until the page is unreadable — and the drift guard // that would otherwise catch it (hashing the source body) does not exist yet, // so idempotency has to hold here rather than upstream. // // Line-level rather than block-level: the common case is a re-ingest carrying // one genuinely new line alongside several it already has. A block-level // "skip if identical" would duplicate the old lines every time that happened. // // The cost is that a line repeated verbatim on purpose collapses to one. For // these sections — questions, updates, evolving notes — a repeat is noise // rather than signal, and the failure is conservative: the page stops growing // instead of growing wrongly. // canonicalLine collapses a line to its comparison key: internal whitespace // runs (a re-extraction's bullet spacing, tabs vs spaces, trailing spaces) // must not make the same logical line look new. strings.Fields already // trims and splits on any whitespace run, so joining it back with single // spaces is exactly "same words, same order, whitespace-insensitive." func canonicalLine(line string) string { return strings.Join(strings.Fields(line), " ") } func appendNew(a, b string) string { seen := make(map[string]bool) for _, line := range strings.Split(a, "\n") { if c := canonicalLine(line); c != "" { seen[c] = true } } var add []string for _, line := range strings.Split(b, "\n") { c := canonicalLine(line) if c == "" || seen[c] { continue } seen[c] = true add = append(add, line) } if len(add) == 0 { return a // nothing new — leave the section byte-identical } return strings.TrimRight(a, "\n") + "\n\n" + strings.Join(add, "\n") + "\n" } func mergeBullets(a, b string) string { seen := make(map[string]bool) var lines []string for _, line := range strings.Split(a+b, "\n") { c := canonicalLine(line) if c == "" || seen[c] { continue } seen[c] = true lines = append(lines, line) } return strings.Join(lines, "\n") + "\n" }