fix(vectorstore): hard-split oversized heading-less chunks

A doc with no headings and no blank-line paragraphs (JSON-lines, e.g.
wiki/telos/decisions/human-intent-column.md) survived both chunk passes whole
and was sent to nomic-embed over its context window → 'input length exceeds
the context length' (400, the steady embed errors=1). Add a final hard-split
pass (line then UTF-8 rune boundaries) so no chunk exceeds maxBytes. TDD.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

ingestion/internal/vectorstore/chunk.go

@@ -3,6 +3,7 @@ package vectorstore
 import (
 	"fmt"
 	"strings"
+	"unicode/utf8"
 )
 
 // NumberedChunk pairs a chunk's body with the storage path it will use
@@ -66,6 +67,70 @@ func ChunkMarkdown(content string, maxBytes int) []string {
 		}
 		out = append(out, splitAtParagraphs(s, maxBytes)...)
 	}
+
+	// Final guarantee: no chunk exceeds maxBytes. A single heading-less,
+	// paragraph-less block (JSON-lines, minified content) survives the two
+	// passes above whole — splitAtParagraphs emits an over-budget paragraph
+	// rather than truncating prose. Hard-split any such chunk at line/rune
+	// boundaries so the embedder never rejects an over-context chunk.
+	final := make([]string, 0, len(out))
+	for _, c := range out {
+		if len(c) <= maxBytes {
+			final = append(final, c)
+			continue
+		}
+		final = append(final, hardSplit(c, maxBytes)...)
+	}
+	return final
+}
+
+// hardSplit slices s into pieces no larger than maxBytes, breaking at line
+// boundaries where possible and otherwise mid-line at a UTF-8 rune boundary.
+// Last resort for content that has neither headings nor blank-line paragraphs.
+func hardSplit(s string, maxBytes int) []string {
+	var out []string
+	var cur strings.Builder
+	flush := func() {
+		if cur.Len() > 0 {
+			out = append(out, cur.String())
+			cur.Reset()
+		}
+	}
+	for _, line := range strings.SplitAfter(s, "\n") {
+		if line == "" {
+			continue
+		}
+		if len(line) > maxBytes {
+			flush()
+			out = append(out, runeSplit(line, maxBytes)...)
+			continue
+		}
+		if cur.Len() > 0 && cur.Len()+len(line) > maxBytes {
+			flush()
+		}
+		cur.WriteString(line)
+	}
+	flush()
+	return out
+}
+
+// runeSplit slices s into <=maxBytes pieces without splitting a UTF-8 rune.
+func runeSplit(s string, maxBytes int) []string {
+	var out []string
+	for len(s) > maxBytes {
+		cut := maxBytes
+		for cut > 0 && !utf8.RuneStart(s[cut]) {
+			cut--
+		}
+		if cut == 0 { // single rune wider than the budget; emit it whole
+			cut = maxBytes
+		}
+		out = append(out, s[:cut])
+		s = s[cut:]
+	}
+	if len(s) > 0 {
+		out = append(out, s)
+	}
 	return out
 }
 

ingestion/internal/vectorstore/chunk_test.go

@@ -30,6 +30,23 @@ func TestChunkMarkdown_SplitsAtHeadings(t *testing.T) {
 	}
 }
 
+func TestChunkMarkdown_HardSplitsHeadinglessOversizedBlock(t *testing.T) {
+	// A document with no headings and no blank-line paragraph breaks (e.g.
+	// JSON-lines like wiki/telos/decisions/human-intent-column.md). The old
+	// chunker emitted it as one over-budget chunk → nomic-embed returned
+	// "input length exceeds the context length" (400). Every chunk must now
+	// fit the budget, with no content lost.
+	maxBytes := 200
+	src := strings.Repeat("x", 1000) // one 1000-byte blob, no headings, no \n\n
+	out := vectorstore.ChunkMarkdown(src, maxBytes)
+
+	require.Greater(t, len(out), 1, "oversized blob must be split")
+	for i, c := range out {
+		assert.LessOrEqual(t, len(c), maxBytes, "chunk %d over budget: %d bytes", i, len(c))
+	}
+	assert.Equal(t, 1000, strings.Count(strings.Join(out, ""), "x"), "no content lost")
+}
+
 func TestChunkMarkdown_FurtherSplitsOversizedSection(t *testing.T) {
 	// One H2 section with 4 paragraphs of ~80 chars each, limit 100.
 	src := "## big\n\n" +