diff --git a/ingestion/internal/vectorstore/chunk.go b/ingestion/internal/vectorstore/chunk.go index b1e8aad..98f3554 100644 --- a/ingestion/internal/vectorstore/chunk.go +++ b/ingestion/internal/vectorstore/chunk.go @@ -3,6 +3,7 @@ package vectorstore import ( "fmt" "strings" + "unicode/utf8" ) // NumberedChunk pairs a chunk's body with the storage path it will use @@ -66,6 +67,70 @@ func ChunkMarkdown(content string, maxBytes int) []string { } out = append(out, splitAtParagraphs(s, maxBytes)...) } + + // Final guarantee: no chunk exceeds maxBytes. A single heading-less, + // paragraph-less block (JSON-lines, minified content) survives the two + // passes above whole — splitAtParagraphs emits an over-budget paragraph + // rather than truncating prose. Hard-split any such chunk at line/rune + // boundaries so the embedder never rejects an over-context chunk. + final := make([]string, 0, len(out)) + for _, c := range out { + if len(c) <= maxBytes { + final = append(final, c) + continue + } + final = append(final, hardSplit(c, maxBytes)...) + } + return final +} + +// hardSplit slices s into pieces no larger than maxBytes, breaking at line +// boundaries where possible and otherwise mid-line at a UTF-8 rune boundary. +// Last resort for content that has neither headings nor blank-line paragraphs. +func hardSplit(s string, maxBytes int) []string { + var out []string + var cur strings.Builder + flush := func() { + if cur.Len() > 0 { + out = append(out, cur.String()) + cur.Reset() + } + } + for _, line := range strings.SplitAfter(s, "\n") { + if line == "" { + continue + } + if len(line) > maxBytes { + flush() + out = append(out, runeSplit(line, maxBytes)...) + continue + } + if cur.Len() > 0 && cur.Len()+len(line) > maxBytes { + flush() + } + cur.WriteString(line) + } + flush() + return out +} + +// runeSplit slices s into <=maxBytes pieces without splitting a UTF-8 rune. +func runeSplit(s string, maxBytes int) []string { + var out []string + for len(s) > maxBytes { + cut := maxBytes + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + if cut == 0 { // single rune wider than the budget; emit it whole + cut = maxBytes + } + out = append(out, s[:cut]) + s = s[cut:] + } + if len(s) > 0 { + out = append(out, s) + } return out } diff --git a/ingestion/internal/vectorstore/chunk_test.go b/ingestion/internal/vectorstore/chunk_test.go index ab9414a..550b120 100644 --- a/ingestion/internal/vectorstore/chunk_test.go +++ b/ingestion/internal/vectorstore/chunk_test.go @@ -30,6 +30,23 @@ func TestChunkMarkdown_SplitsAtHeadings(t *testing.T) { } } +func TestChunkMarkdown_HardSplitsHeadinglessOversizedBlock(t *testing.T) { + // A document with no headings and no blank-line paragraph breaks (e.g. + // JSON-lines like wiki/telos/decisions/human-intent-column.md). The old + // chunker emitted it as one over-budget chunk → nomic-embed returned + // "input length exceeds the context length" (400). Every chunk must now + // fit the budget, with no content lost. + maxBytes := 200 + src := strings.Repeat("x", 1000) // one 1000-byte blob, no headings, no \n\n + out := vectorstore.ChunkMarkdown(src, maxBytes) + + require.Greater(t, len(out), 1, "oversized blob must be split") + for i, c := range out { + assert.LessOrEqual(t, len(c), maxBytes, "chunk %d over budget: %d bytes", i, len(c)) + } + assert.Equal(t, 1000, strings.Count(strings.Join(out, ""), "x"), "no content lost") +} + func TestChunkMarkdown_FurtherSplitsOversizedSection(t *testing.T) { // One H2 section with 4 paragraphs of ~80 chars each, limit 100. src := "## big\n\n" +