mathiasandClaude Sonnet 5.5
00fd884c6c
Keeps the word as written, keys it without case/punctuation/hyphens/plus, joins a lone dash or full stop to its neighbour so it is not a word the engines can disagree about. Red: package reconcile does not exist. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
67 lines
2.1 KiB
Go
67 lines
2.1 KiB
Go
package reconcile_test
|
||
|
||
import (
|
||
"testing"
|
||
|
||
"github.com/stretchr/testify/assert"
|
||
|
||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||
)
|
||
|
||
func texts(ts []reconcile.Token) []string {
|
||
out := make([]string, len(ts))
|
||
for i, t := range ts {
|
||
out[i] = t.Text
|
||
}
|
||
return out
|
||
}
|
||
|
||
func keys(ts []reconcile.Token) []string {
|
||
out := make([]string, len(ts))
|
||
for i, t := range ts {
|
||
out[i] = t.Key
|
||
}
|
||
return out
|
||
}
|
||
|
||
func TestTokenize_KeepsTheWordAsWrittenAndKeysItWithoutCaseOrPunctuation(t *testing.T) {
|
||
got := reconcile.Tokenize("Hej, allihopa! Vi går igenom ÅÄÖ.")
|
||
|
||
assert.Equal(t, []string{"Hej,", "allihopa!", "Vi", "går", "igenom", "ÅÄÖ."}, texts(got),
|
||
"the original spelling and punctuation are kept for the merged text")
|
||
assert.Equal(t, []string{"hej", "allihopa", "vi", "går", "igenom", "åäö"}, keys(got),
|
||
"the key is lower case with punctuation removed, Swedish letters lowered correctly")
|
||
}
|
||
|
||
func TestTokenize_KeysIgnoreHyphensAndPlusSoOnlyWordsDecideEquality(t *testing.T) {
|
||
got := reconcile.Tokenize("end-to-end T+1 5 000 kr")
|
||
|
||
assert.Equal(t, []string{"endtoend", "t1", "5", "000", "kr"}, keys(got))
|
||
}
|
||
|
||
func TestTokenize_APunctuationOnlyWordJoinsThePreviousWordInsteadOfBecomingOne(t *testing.T) {
|
||
got := reconcile.Tokenize("Vi går – sen tillbaka . Slut")
|
||
|
||
assert.Equal(t, []string{"Vi", "går –", "sen", "tillbaka .", "Slut"}, texts(got),
|
||
"a dash or a lone full stop is not a word the engines can disagree about")
|
||
assert.Equal(t, []string{"vi", "går", "sen", "tillbaka", "slut"}, keys(got))
|
||
}
|
||
|
||
func TestTokenize_LeadingPunctuationOnlyWordJoinsTheNextWord(t *testing.T) {
|
||
got := reconcile.Tokenize("— Hej då")
|
||
|
||
assert.Equal(t, []string{"— Hej", "då"}, texts(got))
|
||
}
|
||
|
||
func TestTokenize_NewlinesAndRunsOfSpaceAreJustSeparators(t *testing.T) {
|
||
got := reconcile.Tokenize(" Hej\n allihopa \t Vi ")
|
||
|
||
assert.Equal(t, []string{"Hej", "allihopa", "Vi"}, texts(got))
|
||
}
|
||
|
||
func TestTokenize_EmptyAndPunctuationOnlyInputGiveNoTokens(t *testing.T) {
|
||
assert.Empty(t, reconcile.Tokenize(""))
|
||
assert.Empty(t, reconcile.Tokenize(" \n "))
|
||
assert.Empty(t, reconcile.Tokenize("— . !"))
|
||
}
|