fix(wing): client filter handles multi-word/\s/\b patterns, homoglyphs, marks, fillers; checked in bounded chunks (review of #211, pass 2)
Views of each chunk: as is; NFKD + marks/format/fillers dropped + Cyrillic/Greek lookalikes mapped + dashes normalised, with whitespace collapsed (acme corp, \s) or removed (acme-\nclient); and a soft hyphen read as a dash. Chunks of 1 MiB with a 4 KiB overlap bound the folding's allocation (it was ~14x the input: 919 MiB for a 64 MiB note). Documented limits: leetspeak, punctuation splits, encodings and homoglyphs outside a small table. Mutation-checked. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ package wing_test
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -148,7 +149,7 @@ func TestRouter_FolderNamesAreAsciiAndNotReserved(t *testing.T) {
|
||||
dir string
|
||||
}{
|
||||
{"Kelvin sign lowercases to an ASCII k", "Kubernetes"},
|
||||
{"cyrillic lookalike", "hоmelab"},
|
||||
{"cyrillic lookalike", "h\u043emelab"},
|
||||
{"retired layout name: sources", "sources"},
|
||||
{"retired layout name: concepts", "concepts"},
|
||||
{"retired layout name: entities", "entities"},
|
||||
@@ -172,3 +173,64 @@ func TestNewRouter_AReservedDefaultIsRefused(t *testing.T) {
|
||||
_, err := wing.NewRouter(routerBrain(t), "sources", "")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// From the second review of #211: the multi-word, \s and \b patterns an operator
|
||||
// would actually write, and the next tier of evasions.
|
||||
func TestRouter_ClientMarkerHandlesRealisticPatternsAndHarderEvasions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name, pattern, text string
|
||||
}{
|
||||
{"space in pattern vs NBSP", `acme corp`, "about acme\u00a0corp today"},
|
||||
{"space in pattern vs two spaces", `acme corp`, "about acme corp today"},
|
||||
{"space in pattern vs newline", `acme corp`, "about acme\ncorp today"},
|
||||
{"backslash-s pattern vs NBSP", `acme\s+corp`, "about acme\u00a0corp today"},
|
||||
{"word boundary vs a zero-width char inside", `\bacme\b`, "the ac\u200bme corp"},
|
||||
{"dash pattern vs a soft hyphen as the dash", `acme-client`, "acme\u00adclient"},
|
||||
{"cyrillic a", `acme-client`, "\u0430cme-client"},
|
||||
{"cyrillic e", `acme-client`, "acm\u0435-client"},
|
||||
{"cyrillic o and c", `coop`, "c\u043e\u043ep"},
|
||||
{"combining acute", `acme-client`, "a\u0301cme-client"},
|
||||
{"precomposed accent", `acme-client`, "\u00e1cme-client"},
|
||||
{"hangul filler", `acme-client`, "ac\u3164me-client"},
|
||||
{"halfwidth hangul filler", `acme-client`, "ac\uffa0me-client"},
|
||||
{"braille blank", `acme-client`, "ac\u2800me-client"},
|
||||
{"grapheme joiner", `acme-client`, "ac\u034fme-client"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r, err := wing.NewRouter(routerBrain(t), "web-clip", tt.pattern)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: tt.text}).Source, "%q should be caught by %q", tt.text, tt.pattern)
|
||||
})
|
||||
}
|
||||
t.Run("ordinary text is not caught by a multi-word pattern", func(t *testing.T) {
|
||||
r, _ := wing.NewRouter(routerBrain(t), "web-clip", `acme corp`)
|
||||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: "acme is not corp, and corp is not acme"}).Source)
|
||||
})
|
||||
}
|
||||
|
||||
// The folding allocates several times the text, so a large note is checked in
|
||||
// bounded chunks. A marker that straddles a chunk edge must still be found.
|
||||
func TestRouter_ClientMarkerIsFoundAcrossAChunkBoundary(t *testing.T) {
|
||||
r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`)
|
||||
assert.NoError(t, err)
|
||||
r.FilterChunk = 64
|
||||
for _, at := range []int{0, 30, 55, 58, 62, 64, 70, 120, 190} {
|
||||
text := strings.Repeat("x", at) + "acme-client" + strings.Repeat("y", 200)
|
||||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "marker at offset %d was missed", at)
|
||||
}
|
||||
clean := strings.Repeat("lorem ipsum ", 100)
|
||||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source)
|
||||
}
|
||||
|
||||
func TestRouter_ClientMarkerChunkingDoesNotSplitAMultiByteRune(t *testing.T) {
|
||||
r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`)
|
||||
assert.NoError(t, err)
|
||||
r.FilterChunk = 65 // odd, over two-byte runes: every boundary falls inside a rune unless it is moved
|
||||
for pad := 0; pad < 8; pad++ {
|
||||
text := strings.Repeat("\u00e9", 40+pad) + "acme-client" + strings.Repeat("\u00e9", 40)
|
||||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "pad %d", pad)
|
||||
}
|
||||
clean := strings.Repeat("\u00e9", 300)
|
||||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source, "no panic or false match on a text of 2-byte runes")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user