ingestion/internal/wing/router.go

@@ -26,6 +26,12 @@ type Router struct {
 	BrainDir    string
 	Default     string
 	ClientBlock *regexp.Regexp // nil = no content check
+
+	// FilterChunk is how many bytes of a note the content filter folds and
+	// matches at a time (0 = 1 MiB). Folding allocates several times its input,
+	// so a 64 MiB note is checked in pieces (with an overlap, so a marker on a
+	// boundary is still found) instead of all at once.
+	FilterChunk int
 }
 
 // Resolve implements Placer. It never returns an error: anything it cannot
@@ -47,7 +53,7 @@ func (r Router) Resolve(_ context.Context, n Note) Decision {
 	case fm != "":
 		return r.explicit(fm, "frontmatter")
 	}
-	if r.ClientBlock != nil && matchesClient(r.ClientBlock, n.Text) {
+	if r.ClientBlock != nil && matchesClient(r.ClientBlock, n.Text, r.FilterChunk) {
 		return exception("", 0, "text matches the client-content marker; not defaulted into a shared wing")
 	}
 	if r.Default == "" || !usableWingName(r.Default) {
@@ -109,25 +115,104 @@ func isASCII(s string) bool {
 	return true
 }
 
-// matchesClient applies the client-content pattern to the text and to two
-// folded forms of it, so the cheap evasions do not defeat it: NFKC (fullwidth
-// letters), format characters removed (zero-width joiners, soft hyphens), every
-// dash made "-", and whitespace removed ("acme-\nclient", "acme - client"). It catches what it names and nothing else; it is a filter,
-// not a classifier.
-func matchesClient(re *regexp.Regexp, text string) bool {
-	folded := xnorm.NFKC.String(text)
+const (
+	defaultFilterChunk = 1 << 20
+	filterOverlap      = 4096
+)
+
+// matchesClient applies the client-content pattern to the text in bounded
+// chunks, each overlapping the next, so memory stays proportional to the chunk
+// and a marker on a chunk edge is still found.
+func matchesClient(re *regexp.Regexp, text string, chunk int) bool {
+	if chunk <= 0 {
+		chunk = defaultFilterChunk
+	}
+	overlap := filterOverlap
+	if overlap >= chunk {
+		overlap = chunk / 2
+	}
+	for start := 0; start < len(text); {
+		end := start + chunk
+		if end >= len(text) {
+			end = len(text)
+		}
+		if matchesFolded(re, text[start:end]) {
+			return true
+		}
+		if end == len(text) {
+			return false
+		}
+		// A cut inside a rune is harmless: the overlap carries every marker
+		// shorter than it whole into the next chunk.
+		start = end - overlap
+	}
+	return false
+}
+
+// matchesFolded matches the pattern against the chunk as it is and against
+// folded forms, so the cheap evasions do not defeat it. Folding: NFKD (fullwidth
+// letters, accents become base letter + mark), then marks, format characters and
+// invisible fillers dropped, Cyrillic/Greek lookalikes mapped to Latin, every
+// dash made "-". Views: whitespace collapsed to one space (so "acme corp" and
+// \s work), whitespace removed (so "acme-\nclient" works), and the same with a
+// soft hyphen read as a dash. It catches what it names and nothing else:
+// leetspeak, punctuation splits (a.c.m.e), encodings and the many homoglyphs
+// not in the small table below are not caught. It is a filter, not a classifier.
+func matchesFolded(re *regexp.Regexp, chunk string) bool {
+	if re.MatchString(chunk) {
+		return true
+	}
+	decomposed := xnorm.NFKD.String(chunk)
+	return re.MatchString(fold(decomposed, false, true)) ||
+		re.MatchString(fold(decomposed, false, false)) ||
+		re.MatchString(fold(decomposed, true, false))
+}
+
+// fold builds one view. softAsDash reads U+00AD as "-" instead of dropping it;
+// keepSpace collapses whitespace to a single space instead of removing it.
+func fold(s string, softAsDash, keepSpace bool) string {
 	var b strings.Builder
-	for _, r := range folded {
+	b.Grow(len(s))
+	space := false
+	for _, r := range s {
 		switch {
-		case unicode.Is(unicode.Cf, r):
+		case r == '\u00ad' && softAsDash:
+			b.WriteRune('-')
+			space = false
 			continue
+		case unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Me, r) || unicode.Is(unicode.Mc, r) ||
+			unicode.Is(unicode.Cf, r) || invisibleFiller[r]:
+			continue
+		case unicode.IsSpace(r) || unicode.Is(unicode.Zs, r):
+			if keepSpace && !space {
+				b.WriteByte(' ')
+				space = true
+			}
+			continue
+		}
+		space = false
+		switch {
 		case unicode.Is(unicode.Pd, r) || r == '\u2212':
 			b.WriteRune('-')
 		default:
+			if l, ok := latinLookalike[r]; ok {
+				r = l
+			}
 			b.WriteRune(r)
 		}
 	}
-	clean := b.String()
-	squeezed := strings.Join(strings.Fields(clean), "")
-	return re.MatchString(text) || re.MatchString(squeezed)
+	return b.String()
+}
+
+// invisibleFiller are characters that render as nothing but are not format
+// characters: Hangul fillers and the blank Braille pattern.
+var invisibleFiller = map[rune]bool{0x115F: true, 0x1160: true, 0x3164: true, 0xFFA0: true, 0x2800: true}
+
+// latinLookalike maps the commonest Cyrillic and Greek letters that read as
+// Latin ones. Deliberately small; see matchesFolded.
+var latinLookalike = map[rune]rune{
+	'\u0430': 'a', '\u0435': 'e', '\u043e': 'o', '\u0440': 'p', '\u0441': 'c', '\u0445': 'x', '\u0443': 'y',
+	'\u0456': 'i', '\u0458': 'j', '\u0455': 's', '\u04bb': 'h', '\u0501': 'd', '\u043a': 'k', '\u043d': 'h', '\u0442': 't',
+	'\u0410': 'A', '\u0412': 'B', '\u0415': 'E', '\u041a': 'K', '\u041c': 'M', '\u041d': 'H', '\u041e': 'O', '\u0420': 'P', '\u0421': 'C', '\u0422': 'T', '\u0425': 'X',
+	'\u03bf': 'o', '\u03bd': 'v', '\u03c1': 'p', '\u0391': 'A', '\u0392': 'B', '\u0395': 'E', '\u039f': 'O', '\u03a1': 'P',
 }

ingestion/internal/wing/router_test.go

@@ -3,6 +3,7 @@ package wing_test
 import (
 	"context"
 	"regexp"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -148,7 +149,7 @@ func TestRouter_FolderNamesAreAsciiAndNotReserved(t *testing.T) {
 		dir  string
 	}{
 		{"Kelvin sign lowercases to an ASCII k", "Kubernetes"},
-		{"cyrillic lookalike", "hоmelab"},
+		{"cyrillic lookalike", "h\u043emelab"},
 		{"retired layout name: sources", "sources"},
 		{"retired layout name: concepts", "concepts"},
 		{"retired layout name: entities", "entities"},
@@ -172,3 +173,64 @@ func TestNewRouter_AReservedDefaultIsRefused(t *testing.T) {
 	_, err := wing.NewRouter(routerBrain(t), "sources", "")
 	assert.Error(t, err)
 }
+
+// From the second review of #211: the multi-word, \s and \b patterns an operator
+// would actually write, and the next tier of evasions.
+func TestRouter_ClientMarkerHandlesRealisticPatternsAndHarderEvasions(t *testing.T) {
+	tests := []struct {
+		name, pattern, text string
+	}{
+		{"space in pattern vs NBSP", `acme corp`, "about acme\u00a0corp today"},
+		{"space in pattern vs two spaces", `acme corp`, "about acme  corp today"},
+		{"space in pattern vs newline", `acme corp`, "about acme\ncorp today"},
+		{"backslash-s pattern vs NBSP", `acme\s+corp`, "about acme\u00a0corp today"},
+		{"word boundary vs a zero-width char inside", `\bacme\b`, "the ac\u200bme corp"},
+		{"dash pattern vs a soft hyphen as the dash", `acme-client`, "acme\u00adclient"},
+		{"cyrillic a", `acme-client`, "\u0430cme-client"},
+		{"cyrillic e", `acme-client`, "acm\u0435-client"},
+		{"cyrillic o and c", `coop`, "c\u043e\u043ep"},
+		{"combining acute", `acme-client`, "a\u0301cme-client"},
+		{"precomposed accent", `acme-client`, "\u00e1cme-client"},
+		{"hangul filler", `acme-client`, "ac\u3164me-client"},
+		{"halfwidth hangul filler", `acme-client`, "ac\uffa0me-client"},
+		{"braille blank", `acme-client`, "ac\u2800me-client"},
+		{"grapheme joiner", `acme-client`, "ac\u034fme-client"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r, err := wing.NewRouter(routerBrain(t), "web-clip", tt.pattern)
+			assert.NoError(t, err)
+			assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: tt.text}).Source, "%q should be caught by %q", tt.text, tt.pattern)
+		})
+	}
+	t.Run("ordinary text is not caught by a multi-word pattern", func(t *testing.T) {
+		r, _ := wing.NewRouter(routerBrain(t), "web-clip", `acme corp`)
+		assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: "acme is not corp, and corp is not acme"}).Source)
+	})
+}
+
+// The folding allocates several times the text, so a large note is checked in
+// bounded chunks. A marker that straddles a chunk edge must still be found.
+func TestRouter_ClientMarkerIsFoundAcrossAChunkBoundary(t *testing.T) {
+	r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`)
+	assert.NoError(t, err)
+	r.FilterChunk = 64
+	for _, at := range []int{0, 30, 55, 58, 62, 64, 70, 120, 190} {
+		text := strings.Repeat("x", at) + "acme-client" + strings.Repeat("y", 200)
+		assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "marker at offset %d was missed", at)
+	}
+	clean := strings.Repeat("lorem ipsum ", 100)
+	assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source)
+}
+
+func TestRouter_ClientMarkerChunkingDoesNotSplitAMultiByteRune(t *testing.T) {
+	r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`)
+	assert.NoError(t, err)
+	r.FilterChunk = 65 // odd, over two-byte runes: every boundary falls inside a rune unless it is moved
+	for pad := 0; pad < 8; pad++ {
+		text := strings.Repeat("\u00e9", 40+pad) + "acme-client" + strings.Repeat("\u00e9", 40)
+		assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "pad %d", pad)
+	}
+	clean := strings.Repeat("\u00e9", 300)
+	assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source, "no panic or false match on a text of 2-byte runes")
+}