// Package textutil holds small text helpers shared across the ingestion // service. package textutil import "unicode/utf8" // Ellipsis marks a value that was cut short. const Ellipsis = "…" // Truncate shortens s to at most limit CHARACTERS, appending an ellipsis when // anything was dropped. // // It exists because four call sites did `s[:limit] + "…"` (#93). Both halves // of that are byte operations: len() counts bytes and a slice cuts bytes, so // on Swedish text — where å ä ö are two bytes and an em dash is three — the // cut lands mid-character and leaves an orphaned lead byte. The result is // invalid UTF-8. // // That failed loudly in exactly one place, wiki/index.md, which is written // raw to disk and then rejected by every strict decoder. The other three fed // MCP responses, where encoding/json silently substitutes U+FFFD — so those // had been quietly eating the last character of truncated excerpts with no // error anywhere. // // The marker is appended only when content was actually dropped: an ellipsis // on complete text is a lie about the content. func Truncate(s string, limit int) string { if limit <= 0 { return "" } if utf8.RuneCountInString(s) <= limit { return s } // Walk runes rather than converting to []rune: the input can be a whole // note body, and this runs on the read path. count := 0 for i := range s { if count == limit { return s[:i] + Ellipsis } count++ } return s + Ellipsis }