perf(tools): linear duplicate-key scan keyed on SimpleFold orbit
PR #66 review round 2: the pairwise EqualFold loop was quadratic, 7s of CPU for ~90k keys under the 1 MiB body cap. foldKey maps each rune to the smallest member of its unicode.SimpleFold orbit, equal exactly when strings.EqualFold is, so the scan is a map lookup again. The scaling test fails on the previous code (7.1s). Refs #65 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,42 @@
|
|||||||
|
package tools
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// foldKey must agree with strings.EqualFold, which is how encoding/json matches.
|
||||||
|
func TestFoldKeyAgreesWithEqualFold(t *testing.T) {
|
||||||
|
pairs := [][2]string{{"branch", "Branch"}, {"sha", "ſha"}, {"k", "K"}, {"ss", "ß"}, {"a", "b"}, {"Σ", "ς"}}
|
||||||
|
for _, p := range pairs {
|
||||||
|
if got, want := foldKey(p[0]) == foldKey(p[1]), strings.EqualFold(p[0], p[1]); got != want {
|
||||||
|
t.Errorf("%q vs %q: foldKey equal=%v, EqualFold=%v", p[0], p[1], got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The duplicate scan is linear in the number of keys: 90k keys (about the
|
||||||
|
// 1 MiB body cap) must scan well under a second.
|
||||||
|
func TestRejectAmbiguousKeysScalesLinearly(t *testing.T) {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("{")
|
||||||
|
for i := 0; i < 90_000; i++ {
|
||||||
|
if i > 0 {
|
||||||
|
b.WriteString(",")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, `"k%d":1`, i)
|
||||||
|
}
|
||||||
|
b.WriteString("}")
|
||||||
|
var dst struct {
|
||||||
|
Owner string `json:"owner"`
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
if err := rejectAmbiguousKeys([]byte(b.String()), &dst); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d := time.Since(start); d > 2*time.Second {
|
||||||
|
t.Errorf("scan of 90k keys took %v", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
+25
-8
@@ -8,6 +8,7 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"git.d-ma.be/mathias/gitea-mcp/internal/gitea"
|
"git.d-ma.be/mathias/gitea-mcp/internal/gitea"
|
||||||
"git.d-ma.be/mathias/gitea-mcp/internal/registry"
|
"git.d-ma.be/mathias/gitea-mcp/internal/registry"
|
||||||
@@ -84,21 +85,18 @@ func scanJSONValue(dec *json.Decoder) error {
|
|||||||
}
|
}
|
||||||
switch d {
|
switch d {
|
||||||
case '{':
|
case '{':
|
||||||
var seen []string
|
seen := map[string]string{}
|
||||||
for dec.More() {
|
for dec.More() {
|
||||||
kt, err := dec.Token()
|
kt, err := dec.Token()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
k, _ := kt.(string)
|
k, _ := kt.(string)
|
||||||
// EqualFold, not ToLower: encoding/json folds the same way, so
|
fk := foldKey(k)
|
||||||
// long s (U+017F) matches s and the Kelvin sign matches k.
|
if prev, dup := seen[fk]; dup {
|
||||||
for _, prev := range seen {
|
return fmt.Errorf("duplicate argument keys %q and %q (keys must be unique ignoring case): %w", prev, k, gitea.ErrValidation)
|
||||||
if strings.EqualFold(prev, k) {
|
|
||||||
return fmt.Errorf("duplicate argument keys %q and %q (keys must be unique ignoring case): %w", prev, k, gitea.ErrValidation)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
seen = append(seen, k)
|
seen[fk] = k
|
||||||
if err := scanJSONValue(dec); err != nil {
|
if err := scanJSONValue(dec); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -114,6 +112,25 @@ func scanJSONValue(dec *json.Decoder) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// foldKey maps every rune to the smallest member of its unicode.SimpleFold
|
||||||
|
// orbit, so foldKey(a) == foldKey(b) exactly when strings.EqualFold(a, b) —
|
||||||
|
// the folding encoding/json uses (long s == s, Kelvin sign == k). A map keyed
|
||||||
|
// on it keeps the duplicate scan linear (PR #66 review: a pairwise EqualFold
|
||||||
|
// loop cost 7s of CPU on ~90k keys).
|
||||||
|
func foldKey(s string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range s {
|
||||||
|
min := r
|
||||||
|
for f := unicode.SimpleFold(r); f != r; f = unicode.SimpleFold(f) {
|
||||||
|
if f < min {
|
||||||
|
min = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteRune(min)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
// jsonFieldNames returns the json tag names of dst's top-level struct fields.
|
// jsonFieldNames returns the json tag names of dst's top-level struct fields.
|
||||||
func jsonFieldNames(dst any) []string {
|
func jsonFieldNames(dst any) []string {
|
||||||
v := reflect.ValueOf(dst)
|
v := reflect.ValueOf(dst)
|
||||||
|
|||||||
Reference in New Issue
Block a user