test(reconcile): tokenising for word-level comparison (red)

Keeps the word as written, keys it without case/punctuation/hyphens/plus, joins a
lone dash or full stop to its neighbour so it is not a word the engines can
disagree about. Red: package reconcile does not exist.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/reconcile/tokens_test.go

@@ -0,0 +1,66 @@
+package reconcile_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"git.d-ma.be/mathias/tapir/internal/reconcile"
+)
+
+func texts(ts []reconcile.Token) []string {
+	out := make([]string, len(ts))
+	for i, t := range ts {
+		out[i] = t.Text
+	}
+	return out
+}
+
+func keys(ts []reconcile.Token) []string {
+	out := make([]string, len(ts))
+	for i, t := range ts {
+		out[i] = t.Key
+	}
+	return out
+}
+
+func TestTokenize_KeepsTheWordAsWrittenAndKeysItWithoutCaseOrPunctuation(t *testing.T) {
+	got := reconcile.Tokenize("Hej, allihopa! Vi går igenom ÅÄÖ.")
+
+	assert.Equal(t, []string{"Hej,", "allihopa!", "Vi", "går", "igenom", "ÅÄÖ."}, texts(got),
+		"the original spelling and punctuation are kept for the merged text")
+	assert.Equal(t, []string{"hej", "allihopa", "vi", "går", "igenom", "åäö"}, keys(got),
+		"the key is lower case with punctuation removed, Swedish letters lowered correctly")
+}
+
+func TestTokenize_KeysIgnoreHyphensAndPlusSoOnlyWordsDecideEquality(t *testing.T) {
+	got := reconcile.Tokenize("end-to-end T+1 5 000 kr")
+
+	assert.Equal(t, []string{"endtoend", "t1", "5", "000", "kr"}, keys(got))
+}
+
+func TestTokenize_APunctuationOnlyWordJoinsThePreviousWordInsteadOfBecomingOne(t *testing.T) {
+	got := reconcile.Tokenize("Vi går – sen tillbaka . Slut")
+
+	assert.Equal(t, []string{"Vi", "går –", "sen", "tillbaka .", "Slut"}, texts(got),
+		"a dash or a lone full stop is not a word the engines can disagree about")
+	assert.Equal(t, []string{"vi", "går", "sen", "tillbaka", "slut"}, keys(got))
+}
+
+func TestTokenize_LeadingPunctuationOnlyWordJoinsTheNextWord(t *testing.T) {
+	got := reconcile.Tokenize("— Hej då")
+
+	assert.Equal(t, []string{"— Hej", "då"}, texts(got))
+}
+
+func TestTokenize_NewlinesAndRunsOfSpaceAreJustSeparators(t *testing.T) {
+	got := reconcile.Tokenize("  Hej\n  allihopa \t Vi ")
+
+	assert.Equal(t, []string{"Hej", "allihopa", "Vi"}, texts(got))
+}
+
+func TestTokenize_EmptyAndPunctuationOnlyInputGiveNoTokens(t *testing.T) {
+	assert.Empty(t, reconcile.Tokenize(""))
+	assert.Empty(t, reconcile.Tokenize("   \n "))
+	assert.Empty(t, reconcile.Tokenize("— . !"))
+}