From 6f5fb481ad06becbda25e4db27eb0566025302ad Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 20 Oct 2026 20:45:00 +0200 Subject: [PATCH] fix(wing): client filter handles multi-word/\s/\b patterns, homoglyphs, marks, fillers; checked in bounded chunks (review of #211, pass 2) Views of each chunk: as is; NFKD + marks/format/fillers dropped + Cyrillic/Greek lookalikes mapped + dashes normalised, with whitespace collapsed (acme corp, \s) or removed (acme-\nclient); and a soft hyphen read as a dash. Chunks of 1 MiB with a 4 KiB overlap bound the folding's allocation (it was ~14x the input: 919 MiB for a 64 MiB note). Documented limits: leetspeak, punctuation splits, encodings and homoglyphs outside a small table. Mutation-checked. Co-Authored-By: Claude Sonnet 5.5 --- ingestion/internal/wing/router.go | 111 ++++++++++++++++++++++--- ingestion/internal/wing/router_test.go | 64 +++++++++++++- 2 files changed, 161 insertions(+), 14 deletions(-) diff --git a/ingestion/internal/wing/router.go b/ingestion/internal/wing/router.go index 8f628db..8d284ef 100644 --- a/ingestion/internal/wing/router.go +++ b/ingestion/internal/wing/router.go @@ -26,6 +26,12 @@ type Router struct { BrainDir string Default string ClientBlock *regexp.Regexp // nil = no content check + + // FilterChunk is how many bytes of a note the content filter folds and + // matches at a time (0 = 1 MiB). Folding allocates several times its input, + // so a 64 MiB note is checked in pieces (with an overlap, so a marker on a + // boundary is still found) instead of all at once. + FilterChunk int } // Resolve implements Placer. It never returns an error: anything it cannot @@ -47,7 +53,7 @@ func (r Router) Resolve(_ context.Context, n Note) Decision { case fm != "": return r.explicit(fm, "frontmatter") } - if r.ClientBlock != nil && matchesClient(r.ClientBlock, n.Text) { + if r.ClientBlock != nil && matchesClient(r.ClientBlock, n.Text, r.FilterChunk) { return exception("", 0, "text matches the client-content marker; not defaulted into a shared wing") } if r.Default == "" || !usableWingName(r.Default) { @@ -109,25 +115,104 @@ func isASCII(s string) bool { return true } -// matchesClient applies the client-content pattern to the text and to two -// folded forms of it, so the cheap evasions do not defeat it: NFKC (fullwidth -// letters), format characters removed (zero-width joiners, soft hyphens), every -// dash made "-", and whitespace removed ("acme-\nclient", "acme - client"). It catches what it names and nothing else; it is a filter, -// not a classifier. -func matchesClient(re *regexp.Regexp, text string) bool { - folded := xnorm.NFKC.String(text) +const ( + defaultFilterChunk = 1 << 20 + filterOverlap = 4096 +) + +// matchesClient applies the client-content pattern to the text in bounded +// chunks, each overlapping the next, so memory stays proportional to the chunk +// and a marker on a chunk edge is still found. +func matchesClient(re *regexp.Regexp, text string, chunk int) bool { + if chunk <= 0 { + chunk = defaultFilterChunk + } + overlap := filterOverlap + if overlap >= chunk { + overlap = chunk / 2 + } + for start := 0; start < len(text); { + end := start + chunk + if end >= len(text) { + end = len(text) + } + if matchesFolded(re, text[start:end]) { + return true + } + if end == len(text) { + return false + } + // A cut inside a rune is harmless: the overlap carries every marker + // shorter than it whole into the next chunk. + start = end - overlap + } + return false +} + +// matchesFolded matches the pattern against the chunk as it is and against +// folded forms, so the cheap evasions do not defeat it. Folding: NFKD (fullwidth +// letters, accents become base letter + mark), then marks, format characters and +// invisible fillers dropped, Cyrillic/Greek lookalikes mapped to Latin, every +// dash made "-". Views: whitespace collapsed to one space (so "acme corp" and +// \s work), whitespace removed (so "acme-\nclient" works), and the same with a +// soft hyphen read as a dash. It catches what it names and nothing else: +// leetspeak, punctuation splits (a.c.m.e), encodings and the many homoglyphs +// not in the small table below are not caught. It is a filter, not a classifier. +func matchesFolded(re *regexp.Regexp, chunk string) bool { + if re.MatchString(chunk) { + return true + } + decomposed := xnorm.NFKD.String(chunk) + return re.MatchString(fold(decomposed, false, true)) || + re.MatchString(fold(decomposed, false, false)) || + re.MatchString(fold(decomposed, true, false)) +} + +// fold builds one view. softAsDash reads U+00AD as "-" instead of dropping it; +// keepSpace collapses whitespace to a single space instead of removing it. +func fold(s string, softAsDash, keepSpace bool) string { var b strings.Builder - for _, r := range folded { + b.Grow(len(s)) + space := false + for _, r := range s { switch { - case unicode.Is(unicode.Cf, r): + case r == '\u00ad' && softAsDash: + b.WriteRune('-') + space = false continue + case unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Me, r) || unicode.Is(unicode.Mc, r) || + unicode.Is(unicode.Cf, r) || invisibleFiller[r]: + continue + case unicode.IsSpace(r) || unicode.Is(unicode.Zs, r): + if keepSpace && !space { + b.WriteByte(' ') + space = true + } + continue + } + space = false + switch { case unicode.Is(unicode.Pd, r) || r == '\u2212': b.WriteRune('-') default: + if l, ok := latinLookalike[r]; ok { + r = l + } b.WriteRune(r) } } - clean := b.String() - squeezed := strings.Join(strings.Fields(clean), "") - return re.MatchString(text) || re.MatchString(squeezed) + return b.String() +} + +// invisibleFiller are characters that render as nothing but are not format +// characters: Hangul fillers and the blank Braille pattern. +var invisibleFiller = map[rune]bool{0x115F: true, 0x1160: true, 0x3164: true, 0xFFA0: true, 0x2800: true} + +// latinLookalike maps the commonest Cyrillic and Greek letters that read as +// Latin ones. Deliberately small; see matchesFolded. +var latinLookalike = map[rune]rune{ + '\u0430': 'a', '\u0435': 'e', '\u043e': 'o', '\u0440': 'p', '\u0441': 'c', '\u0445': 'x', '\u0443': 'y', + '\u0456': 'i', '\u0458': 'j', '\u0455': 's', '\u04bb': 'h', '\u0501': 'd', '\u043a': 'k', '\u043d': 'h', '\u0442': 't', + '\u0410': 'A', '\u0412': 'B', '\u0415': 'E', '\u041a': 'K', '\u041c': 'M', '\u041d': 'H', '\u041e': 'O', '\u0420': 'P', '\u0421': 'C', '\u0422': 'T', '\u0425': 'X', + '\u03bf': 'o', '\u03bd': 'v', '\u03c1': 'p', '\u0391': 'A', '\u0392': 'B', '\u0395': 'E', '\u039f': 'O', '\u03a1': 'P', } diff --git a/ingestion/internal/wing/router_test.go b/ingestion/internal/wing/router_test.go index 68c9ed5..384192f 100644 --- a/ingestion/internal/wing/router_test.go +++ b/ingestion/internal/wing/router_test.go @@ -3,6 +3,7 @@ package wing_test import ( "context" "regexp" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -148,7 +149,7 @@ func TestRouter_FolderNamesAreAsciiAndNotReserved(t *testing.T) { dir string }{ {"Kelvin sign lowercases to an ASCII k", "Kubernetes"}, - {"cyrillic lookalike", "hоmelab"}, + {"cyrillic lookalike", "h\u043emelab"}, {"retired layout name: sources", "sources"}, {"retired layout name: concepts", "concepts"}, {"retired layout name: entities", "entities"}, @@ -172,3 +173,64 @@ func TestNewRouter_AReservedDefaultIsRefused(t *testing.T) { _, err := wing.NewRouter(routerBrain(t), "sources", "") assert.Error(t, err) } + +// From the second review of #211: the multi-word, \s and \b patterns an operator +// would actually write, and the next tier of evasions. +func TestRouter_ClientMarkerHandlesRealisticPatternsAndHarderEvasions(t *testing.T) { + tests := []struct { + name, pattern, text string + }{ + {"space in pattern vs NBSP", `acme corp`, "about acme\u00a0corp today"}, + {"space in pattern vs two spaces", `acme corp`, "about acme corp today"}, + {"space in pattern vs newline", `acme corp`, "about acme\ncorp today"}, + {"backslash-s pattern vs NBSP", `acme\s+corp`, "about acme\u00a0corp today"}, + {"word boundary vs a zero-width char inside", `\bacme\b`, "the ac\u200bme corp"}, + {"dash pattern vs a soft hyphen as the dash", `acme-client`, "acme\u00adclient"}, + {"cyrillic a", `acme-client`, "\u0430cme-client"}, + {"cyrillic e", `acme-client`, "acm\u0435-client"}, + {"cyrillic o and c", `coop`, "c\u043e\u043ep"}, + {"combining acute", `acme-client`, "a\u0301cme-client"}, + {"precomposed accent", `acme-client`, "\u00e1cme-client"}, + {"hangul filler", `acme-client`, "ac\u3164me-client"}, + {"halfwidth hangul filler", `acme-client`, "ac\uffa0me-client"}, + {"braille blank", `acme-client`, "ac\u2800me-client"}, + {"grapheme joiner", `acme-client`, "ac\u034fme-client"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, err := wing.NewRouter(routerBrain(t), "web-clip", tt.pattern) + assert.NoError(t, err) + assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: tt.text}).Source, "%q should be caught by %q", tt.text, tt.pattern) + }) + } + t.Run("ordinary text is not caught by a multi-word pattern", func(t *testing.T) { + r, _ := wing.NewRouter(routerBrain(t), "web-clip", `acme corp`) + assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: "acme is not corp, and corp is not acme"}).Source) + }) +} + +// The folding allocates several times the text, so a large note is checked in +// bounded chunks. A marker that straddles a chunk edge must still be found. +func TestRouter_ClientMarkerIsFoundAcrossAChunkBoundary(t *testing.T) { + r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`) + assert.NoError(t, err) + r.FilterChunk = 64 + for _, at := range []int{0, 30, 55, 58, 62, 64, 70, 120, 190} { + text := strings.Repeat("x", at) + "acme-client" + strings.Repeat("y", 200) + assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "marker at offset %d was missed", at) + } + clean := strings.Repeat("lorem ipsum ", 100) + assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source) +} + +func TestRouter_ClientMarkerChunkingDoesNotSplitAMultiByteRune(t *testing.T) { + r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`) + assert.NoError(t, err) + r.FilterChunk = 65 // odd, over two-byte runes: every boundary falls inside a rune unless it is moved + for pad := 0; pad < 8; pad++ { + text := strings.Repeat("\u00e9", 40+pad) + "acme-client" + strings.Repeat("\u00e9", 40) + assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "pad %d", pad) + } + clean := strings.Repeat("\u00e9", 300) + assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source, "no panic or false match on a text of 2-byte runes") +}