mathiasandClaude Sonnet 5.5
6f5fb481ad
Views of each chunk: as is; NFKD + marks/format/fillers dropped + Cyrillic/Greek lookalikes mapped + dashes normalised, with whitespace collapsed (acme corp, \s) or removed (acme-\nclient); and a soft hyphen read as a dash. Chunks of 1 MiB with a 4 KiB overlap bound the folding's allocation (it was ~14x the input: 919 MiB for a 64 MiB note). Documented limits: leetspeak, punctuation splits, encodings and homoglyphs outside a small table. Mutation-checked. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
237 lines
10 KiB
Go
237 lines
10 KiB
Go
package wing_test
|
||
|
||
import (
|
||
"context"
|
||
"regexp"
|
||
"strings"
|
||
"testing"
|
||
|
||
"github.com/stretchr/testify/assert"
|
||
|
||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||
)
|
||
|
||
func routerBrain(t *testing.T) string {
|
||
dir := t.TempDir()
|
||
writePage(t, dir, "wiki/homelab/facts/a.md", "x")
|
||
writePage(t, dir, "wiki/rikatillsammans/facts/b.md", "x")
|
||
return dir
|
||
}
|
||
|
||
func TestRouter_Decisions(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
note wing.Note
|
||
want wing.Source
|
||
wantWing string
|
||
}{
|
||
{"no signal goes to the default wing", wing.Note{Text: "a talk about agents"}, wing.Default, "web-clip"},
|
||
{"a subfolder naming an existing wing places there", wing.Note{Dir: "homelab", Text: "x"}, wing.Explicit, "homelab"},
|
||
{"a deliberate subfolder into a client wing is the owner's call and is placed", wing.Note{Dir: "rikatillsammans", Text: "x"}, wing.Explicit, "rikatillsammans"},
|
||
{"frontmatter wing places like a subfolder", wing.Note{FrontmatterWing: "homelab", Text: "x"}, wing.Explicit, "homelab"},
|
||
{"subfolder is case-insensitive", wing.Note{Dir: "HomeLab", Text: "x"}, wing.Explicit, "homelab"},
|
||
{"a typo never creates a wing", wing.Note{Dir: "hoemlab", Text: "x"}, wing.Exception, "hoemlab"},
|
||
{"conflicting signals are parked", wing.Note{Dir: "homelab", FrontmatterWing: "rikatillsammans", Text: "x"}, wing.Exception, ""},
|
||
{"the default wing may be named explicitly even before it exists", wing.Note{Dir: "web-clip", Text: "x"}, wing.Explicit, "web-clip"},
|
||
{"client marker with no signal is parked, not defaulted", wing.Note{Text: "notes from the ACME-CLIENT kickoff"}, wing.Exception, ""},
|
||
{"client marker does not block a deliberate subfolder", wing.Note{Dir: "homelab", Text: "ACME-CLIENT"}, wing.Explicit, "homelab"},
|
||
}
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
r := wing.Router{BrainDir: routerBrain(t), Default: "web-clip", ClientBlock: regexp.MustCompile(`(?i)acme-client`)}
|
||
d := r.Resolve(context.Background(), tt.note)
|
||
assert.Equal(t, tt.want, d.Source)
|
||
assert.Equal(t, tt.wantWing, d.Wing)
|
||
assert.NotEmpty(t, d.Reason)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestRouter_ReadsTheWingsFreshOnEveryDecision(t *testing.T) {
|
||
dir := routerBrain(t)
|
||
r := wing.Router{BrainDir: dir, Default: "web-clip"}
|
||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Dir: "newwing", Text: "x"}).Source)
|
||
|
||
writePage(t, dir, "wiki/newwing/facts/c.md", "x") // a wing created after startup
|
||
assert.Equal(t, wing.Explicit, r.Resolve(context.Background(), wing.Note{Dir: "newwing", Text: "x"}).Source,
|
||
"a wing made while the server runs is usable without a restart")
|
||
}
|
||
|
||
func TestRouter_AnUnreadableBrainParksInsteadOfGuessing(t *testing.T) {
|
||
r := wing.Router{BrainDir: t.TempDir() + "/missing", Default: "web-clip"}
|
||
d := r.Resolve(context.Background(), wing.Note{Dir: "homelab", Text: "x"})
|
||
assert.Equal(t, wing.Exception, d.Source)
|
||
}
|
||
|
||
func TestRouter_WithoutADefaultAnUnsignalledNoteIsParked(t *testing.T) {
|
||
r := wing.Router{BrainDir: routerBrain(t)}
|
||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: "x"}).Source)
|
||
}
|
||
|
||
func TestRouter_ADefaultThatIsNotAPlainNameIsRefused(t *testing.T) {
|
||
r := wing.Router{BrainDir: routerBrain(t), Default: "../escape"}
|
||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: "x"}).Source)
|
||
}
|
||
|
||
func TestRouter_ANameThatIsNotPlainIsRefusedAsSuch(t *testing.T) {
|
||
for _, bad := range []string{"../x", "a/b", "has space", "dot.ted"} {
|
||
t.Run(bad, func(t *testing.T) {
|
||
// Default equals the bad name: the "named explicitly" shortcut for the
|
||
// default wing must not wave a malformed name through.
|
||
r := wing.Router{BrainDir: routerBrain(t), Default: bad}
|
||
d := r.Resolve(context.Background(), wing.Note{Dir: bad, Text: "x"})
|
||
assert.Equal(t, wing.Exception, d.Source)
|
||
assert.Empty(t, d.Wing, "a malformed name is never echoed as a proposal")
|
||
assert.Contains(t, d.Reason, "not a usable wing name")
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestNewRouter(t *testing.T) {
|
||
dir := routerBrain(t)
|
||
t.Run("valid", func(t *testing.T) {
|
||
r, err := wing.NewRouter(dir, "web-clip", "acme-client|globex")
|
||
assert.NoError(t, err)
|
||
d := r.Resolve(context.Background(), wing.Note{Text: "kickoff with GLOBEX"})
|
||
assert.Equal(t, wing.Exception, d.Source, "the marker is matched case-insensitively")
|
||
})
|
||
t.Run("no marker pattern means no content check", func(t *testing.T) {
|
||
r, err := wing.NewRouter(dir, "web-clip", "")
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: "anything"}).Source)
|
||
})
|
||
t.Run("a default that is not a plain wing name is a startup error", func(t *testing.T) {
|
||
_, err := wing.NewRouter(dir, "Web Clip", "")
|
||
assert.Error(t, err)
|
||
})
|
||
t.Run("an empty default is a startup error", func(t *testing.T) {
|
||
_, err := wing.NewRouter(dir, "", "")
|
||
assert.Error(t, err)
|
||
})
|
||
t.Run("a pattern that does not compile is a startup error", func(t *testing.T) {
|
||
_, err := wing.NewRouter(dir, "web-clip", "([unclosed")
|
||
assert.Error(t, err, "a client filter that silently does nothing is worse than none")
|
||
})
|
||
}
|
||
|
||
// From the review of #211: the content filter is the only guard on the default
|
||
// lane, so it must survive the cheap evasions.
|
||
func TestRouter_ClientMarkerSurvivesCheapEvasions(t *testing.T) {
|
||
r := wing.Router{BrainDir: routerBrain(t), Default: "web-clip", ClientBlock: regexp.MustCompile(`(?i)acme-client`)}
|
||
tests := []struct{ name, text string }{
|
||
{"non-breaking hyphen", "notes on acme‑client"},
|
||
{"zero-width joiner inside", "notes on acme\u200d-client"},
|
||
{"zero-width space inside", "notes on ac\u200bme-client"},
|
||
{"split across lines", "notes on acme-\nclient"},
|
||
{"spaced hyphen", "notes on acme - client"},
|
||
{"fullwidth letters", "notes on ａｃｍｅ-client"},
|
||
{"soft hyphen", "notes on acme\u00ad-client"},
|
||
{"en dash", "notes on acme–client"},
|
||
}
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
d := r.Resolve(context.Background(), wing.Note{Text: tt.text})
|
||
assert.Equal(t, wing.Exception, d.Source, tt.text)
|
||
})
|
||
}
|
||
t.Run("an unrelated text still files", func(t *testing.T) {
|
||
d := r.Resolve(context.Background(), wing.Note{Text: "a talk about agents and a client-server model"})
|
||
assert.Equal(t, wing.Default, d.Source)
|
||
})
|
||
}
|
||
|
||
func TestRouter_FolderNamesAreAsciiAndNotReserved(t *testing.T) {
|
||
dir := routerBrain(t)
|
||
writePage(t, dir, "wiki/kubernetes/facts/a.md", "x")
|
||
r := wing.Router{BrainDir: dir, Default: "web-clip"}
|
||
tests := []struct {
|
||
name string
|
||
dir string
|
||
}{
|
||
{"Kelvin sign lowercases to an ASCII k", "Kubernetes"},
|
||
{"cyrillic lookalike", "h\u043emelab"},
|
||
{"retired layout name: sources", "sources"},
|
||
{"retired layout name: concepts", "concepts"},
|
||
{"retired layout name: entities", "entities"},
|
||
{"wiki", "wiki"},
|
||
}
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
d := r.Resolve(context.Background(), wing.Note{Dir: tt.dir, Text: "x"})
|
||
assert.Equal(t, wing.Exception, d.Source)
|
||
assert.Empty(t, d.Wing, "never proposed as a wing")
|
||
})
|
||
}
|
||
t.Run("plain ASCII case folding still works", func(t *testing.T) {
|
||
d := r.Resolve(context.Background(), wing.Note{Dir: "KubernetEs", Text: "x"})
|
||
assert.Equal(t, wing.Explicit, d.Source)
|
||
assert.Equal(t, "kubernetes", d.Wing)
|
||
})
|
||
}
|
||
|
||
func TestNewRouter_AReservedDefaultIsRefused(t *testing.T) {
|
||
_, err := wing.NewRouter(routerBrain(t), "sources", "")
|
||
assert.Error(t, err)
|
||
}
|
||
|
||
// From the second review of #211: the multi-word, \s and \b patterns an operator
|
||
// would actually write, and the next tier of evasions.
|
||
func TestRouter_ClientMarkerHandlesRealisticPatternsAndHarderEvasions(t *testing.T) {
|
||
tests := []struct {
|
||
name, pattern, text string
|
||
}{
|
||
{"space in pattern vs NBSP", `acme corp`, "about acme\u00a0corp today"},
|
||
{"space in pattern vs two spaces", `acme corp`, "about acme corp today"},
|
||
{"space in pattern vs newline", `acme corp`, "about acme\ncorp today"},
|
||
{"backslash-s pattern vs NBSP", `acme\s+corp`, "about acme\u00a0corp today"},
|
||
{"word boundary vs a zero-width char inside", `\bacme\b`, "the ac\u200bme corp"},
|
||
{"dash pattern vs a soft hyphen as the dash", `acme-client`, "acme\u00adclient"},
|
||
{"cyrillic a", `acme-client`, "\u0430cme-client"},
|
||
{"cyrillic e", `acme-client`, "acm\u0435-client"},
|
||
{"cyrillic o and c", `coop`, "c\u043e\u043ep"},
|
||
{"combining acute", `acme-client`, "a\u0301cme-client"},
|
||
{"precomposed accent", `acme-client`, "\u00e1cme-client"},
|
||
{"hangul filler", `acme-client`, "ac\u3164me-client"},
|
||
{"halfwidth hangul filler", `acme-client`, "ac\uffa0me-client"},
|
||
{"braille blank", `acme-client`, "ac\u2800me-client"},
|
||
{"grapheme joiner", `acme-client`, "ac\u034fme-client"},
|
||
}
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
r, err := wing.NewRouter(routerBrain(t), "web-clip", tt.pattern)
|
||
assert.NoError(t, err)
|
||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: tt.text}).Source, "%q should be caught by %q", tt.text, tt.pattern)
|
||
})
|
||
}
|
||
t.Run("ordinary text is not caught by a multi-word pattern", func(t *testing.T) {
|
||
r, _ := wing.NewRouter(routerBrain(t), "web-clip", `acme corp`)
|
||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: "acme is not corp, and corp is not acme"}).Source)
|
||
})
|
||
}
|
||
|
||
// The folding allocates several times the text, so a large note is checked in
|
||
// bounded chunks. A marker that straddles a chunk edge must still be found.
|
||
func TestRouter_ClientMarkerIsFoundAcrossAChunkBoundary(t *testing.T) {
|
||
r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`)
|
||
assert.NoError(t, err)
|
||
r.FilterChunk = 64
|
||
for _, at := range []int{0, 30, 55, 58, 62, 64, 70, 120, 190} {
|
||
text := strings.Repeat("x", at) + "acme-client" + strings.Repeat("y", 200)
|
||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "marker at offset %d was missed", at)
|
||
}
|
||
clean := strings.Repeat("lorem ipsum ", 100)
|
||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source)
|
||
}
|
||
|
||
func TestRouter_ClientMarkerChunkingDoesNotSplitAMultiByteRune(t *testing.T) {
|
||
r, err := wing.NewRouter(routerBrain(t), "web-clip", `acme-client`)
|
||
assert.NoError(t, err)
|
||
r.FilterChunk = 65 // odd, over two-byte runes: every boundary falls inside a rune unless it is moved
|
||
for pad := 0; pad < 8; pad++ {
|
||
text := strings.Repeat("\u00e9", 40+pad) + "acme-client" + strings.Repeat("\u00e9", 40)
|
||
assert.Equal(t, wing.Exception, r.Resolve(context.Background(), wing.Note{Text: text}).Source, "pad %d", pad)
|
||
}
|
||
clean := strings.Repeat("\u00e9", 300)
|
||
assert.Equal(t, wing.Default, r.Resolve(context.Background(), wing.Note{Text: clean}).Source, "no panic or false match on a text of 2-byte runes")
|
||
}
|