package reconcile_test import ( "testing" "github.com/stretchr/testify/assert" "git.d-ma.be/mathias/tapir/internal/reconcile" ) func texts(ts []reconcile.Token) []string { out := make([]string, len(ts)) for i, t := range ts { out[i] = t.Text } return out } func keys(ts []reconcile.Token) []string { out := make([]string, len(ts)) for i, t := range ts { out[i] = t.Key } return out } func TestTokenize_KeepsTheWordAsWrittenAndKeysItWithoutCaseOrPunctuation(t *testing.T) { got := reconcile.Tokenize("Hej, allihopa! Vi går igenom ÅÄÖ.") assert.Equal(t, []string{"Hej,", "allihopa!", "Vi", "går", "igenom", "ÅÄÖ."}, texts(got), "the original spelling and punctuation are kept for the merged text") assert.Equal(t, []string{"hej", "allihopa", "vi", "går", "igenom", "åäö"}, keys(got), "the key is lower case with punctuation removed, Swedish letters lowered correctly") } func TestTokenize_KeysIgnoreHyphensAndPlusSoOnlyWordsDecideEquality(t *testing.T) { got := reconcile.Tokenize("end-to-end T+1 5 000 kr") assert.Equal(t, []string{"endtoend", "t1", "5", "000", "kr"}, keys(got)) } func TestTokenize_APunctuationOnlyWordJoinsThePreviousWordInsteadOfBecomingOne(t *testing.T) { got := reconcile.Tokenize("Vi går – sen tillbaka . Slut") assert.Equal(t, []string{"Vi", "går –", "sen", "tillbaka .", "Slut"}, texts(got), "a dash or a lone full stop is not a word the engines can disagree about") assert.Equal(t, []string{"vi", "går", "sen", "tillbaka", "slut"}, keys(got)) } func TestTokenize_LeadingPunctuationOnlyWordJoinsTheNextWord(t *testing.T) { got := reconcile.Tokenize("— Hej då") assert.Equal(t, []string{"— Hej", "då"}, texts(got)) } func TestTokenize_NewlinesAndRunsOfSpaceAreJustSeparators(t *testing.T) { got := reconcile.Tokenize(" Hej\n allihopa \t Vi ") assert.Equal(t, []string{"Hej", "allihopa", "Vi"}, texts(got)) } func TestTokenize_EmptyAndPunctuationOnlyInputGiveNoTokens(t *testing.T) { assert.Empty(t, reconcile.Tokenize("")) assert.Empty(t, reconcile.Tokenize(" \n ")) assert.Empty(t, reconcile.Tokenize("— . !")) }