// ingestion/internal/wiki/merge.go package wiki import ( "fmt" "strings" ) var bulletSections = map[string]bool{ "Related Concepts": true, "Related Entities": true, "Sources": true, "Key Claims": true, "Entities Mentioned": true, "Concepts Introduced or Reinforced": true, "Chapters": true, } var appendSections = map[string]bool{ "Evolving Notes": true, "Updates": true, "Open Questions Raised": true, "Open Questions": true, } type section struct { heading string content string } // Merge combines two Page values with the same path. // Frontmatter is taken from a. The preamble (content before the first ## // heading) is taken from a — same "on-disk wins" default every unlisted // named section gets, kept as a separate field rather than a section with // an empty heading so it can never collide with one (see parseSections). // Sections are merged by strategy: bullet sections union unique lines, // append sections concatenate, all others keep a's version. Sections in b // not present in a are appended. func Merge(a, b Page) Page { fmA, preambleA, secsA := parseSections(a.Content) _, _, secsB := parseSections(b.Content) idx := make(map[string]int, len(secsA)) for i, s := range secsA { idx[s.heading] = i } for _, sB := range secsB { i, exists := idx[sB.heading] if !exists { idx[sB.heading] = len(secsA) secsA = append(secsA, sB) continue } sA := secsA[i] switch { case bulletSections[sB.heading]: secsA[i].content = mergeBullets(sA.content, sB.content) case appendSections[sB.heading]: secsA[i].content = appendNew(sA.content, sB.content) } } return Page{Path: a.Path, Content: rebuildContent(fmA, preambleA, secsA)} } // parseSections splits markdown into its frontmatter, its preamble (content // before the first `## ` heading), and its `## `-headed sections. // // The preamble is returned separately rather than as a section with an // empty heading — a section keyed by heading=="" would collide with a // literal degenerate `## ` (marker, no title text; strings.TrimPrefix also // yields ""), which is a distinct, addressable section a page can genuinely // have, not the same thing as "no heading yet." Two entries sharing one key // in Merge's idx map means one of them silently loses lookup visibility — // found in review as real, reproducible data loss, not a hypothetical. func parseSections(markdown string) (frontmatter, preamble string, sections []section) { lines := strings.Split(markdown, "\n") i := 0 if i < len(lines) && strings.TrimSpace(lines[i]) == "---" { i++ var fmLines []string for i < len(lines) { if strings.TrimSpace(lines[i]) == "---" { i++ break } fmLines = append(fmLines, lines[i]) i++ } frontmatter = fmt.Sprintf("---\n%s\n---\n", strings.Join(fmLines, "\n")) } // Everything before the first `## ` heading — an intro paragraph, or, if // the page has no headings at all, the entire body — is ordinary content, // not something this parser is entitled to drop (hyperguild#103 review: // dropping it meant Merge, one caller up, would silently replace the // whole on-disk body with the fresh extraction whenever a page didn't // follow the ## convention — exactly the class of bug this file exists // to prevent). var preambleLines []string for ; i < len(lines); i++ { if strings.HasPrefix(lines[i], "## ") { break } preambleLines = append(preambleLines, lines[i]) } preamble = strings.Join(preambleLines, "\n") var cur *section for ; i < len(lines); i++ { line := lines[i] if strings.HasPrefix(line, "## ") { if cur != nil { sections = append(sections, *cur) } cur = §ion{heading: strings.TrimPrefix(line, "## ")} } else if cur != nil { cur.content += line + "\n" } } if cur != nil { sections = append(sections, *cur) } return } // rebuildContent serialises sections back into a page. // // Each section body is normalised to exactly one trailing newline and no // leading blank line. Without that, a section keeps whatever blank lines it was // parsed with and this function adds its own, so every merge pass grows the page // by a blank line — invisible in a diff viewer, fatal to byte-stability. Compile-on-ingest // merges on every run, so "merging changes nothing when nothing changed" has to // hold at the byte level, not just the content level. func rebuildContent(frontmatter, preamble string, sections []section) string { var sb strings.Builder sb.WriteString(frontmatter) if body := strings.Trim(preamble, "\n"); body != "" { fmt.Fprintf(&sb, "\n%s\n", body) } for _, sec := range sections { body := strings.Trim(sec.content, "\n") fmt.Fprintf(&sb, "\n## %s\n\n%s\n", sec.heading, body) } return sb.String() } // canonicalLine collapses a line to its comparison key: internal whitespace // runs (a re-extraction's bullet spacing, tabs vs spaces, trailing spaces) // must not make the same logical line look new. strings.Fields already // trims and splits on any whitespace run, so joining it back with single // spaces is exactly "same words, same order, whitespace-insensitive." func canonicalLine(line string) string { return strings.Join(strings.Fields(line), " ") } // appendNew concatenates the lines of b that a does not already carry. // // Append sections used to concatenate wholesale, which is correct exactly once. // Compile-on-ingest merges a freshly extracted page into the one already on // disk, so an unchanged source re-ingested would grow "Open Questions" and // "Updates" on every run until the page is unreadable — and the drift guard // that would otherwise catch it (hashing the source body) does not exist yet, // so idempotency has to hold here rather than upstream. // // Line-level rather than block-level: the common case is a re-ingest carrying // one genuinely new line alongside several it already has. A block-level // "skip if identical" would duplicate the old lines every time that happened. // // The cost is that a line repeated verbatim on purpose collapses to one. For // these sections — questions, updates, evolving notes — a repeat is noise // rather than signal, and the failure is conservative: the page stops growing // instead of growing wrongly. func appendNew(a, b string) string { seen := make(map[string]bool) for _, line := range strings.Split(a, "\n") { if c := canonicalLine(line); c != "" { seen[c] = true } } var add []string for _, line := range strings.Split(b, "\n") { c := canonicalLine(line) if c == "" || seen[c] { continue } seen[c] = true add = append(add, line) } if len(add) == 0 { return a // nothing new — leave the section byte-identical } return strings.TrimRight(a, "\n") + "\n\n" + strings.Join(add, "\n") + "\n" } func mergeBullets(a, b string) string { seen := make(map[string]bool) var lines []string for _, line := range strings.Split(a+b, "\n") { c := canonicalLine(line) if c == "" || seen[c] { continue } seen[c] = true lines = append(lines, line) } return strings.Join(lines, "\n") + "\n" }