internal/reconcile/tokens.go

@@ -0,0 +1,57 @@
+// Package reconcile merges two transcripts of the same speech: it finds where they
+// disagree word by word, resolves the trivial disagreements itself, and asks a LOCAL
+// model to choose between the two readings of the rest (ADR-034). It depends on
+// nothing outside the standard library and the domain; the model, the engines and
+// the store are wired in from outside.
+package reconcile
+
+import (
+	"strings"
+	"unicode"
+)
+
+// Token is one word of a transcript. Text is the word as written, with its casing
+// and punctuation, so the merged transcript reads naturally; Key is what two engines
+// are compared on: lower case, letters and digits only.
+type Token struct {
+	Text string
+	Key  string
+}
+
+// Tokenize splits s on whitespace. A "word" that has no letter or digit (a dash, a
+// lone full stop) is not something two engines can disagree about, so it is joined
+// to the neighbouring word instead of becoming a token of its own.
+func Tokenize(s string) []Token {
+	var out []Token
+	pending := ""
+	for _, w := range strings.Fields(s) {
+		k := key(w)
+		if k == "" {
+			if len(out) > 0 {
+				out[len(out)-1].Text += " " + w
+			} else if pending == "" {
+				pending = w
+			} else {
+				pending += " " + w
+			}
+			continue
+		}
+		text := w
+		if pending != "" {
+			text = pending + " " + w
+			pending = ""
+		}
+		out = append(out, Token{Text: text, Key: k})
+	}
+	return out
+}
+
+func key(w string) string {
+	var b strings.Builder
+	for _, r := range w {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
+			b.WriteRune(unicode.ToLower(r))
+		}
+	}
+	return b.String()
+}