// Package reconcile merges two transcripts of the same speech: it finds where they
// disagree word by word, resolves the trivial disagreements itself, and asks a LOCAL
// model to choose between the two readings of the rest (ADR-034). It depends on
// nothing outside the standard library and the domain; the model, the engines and
// the store are wired in from outside.
package reconcile

import (
	"strings"
	"unicode"
)

// Token is one word of a transcript. Text is the word as written, with its casing
// and punctuation, so the merged transcript reads naturally; Key is what two engines
// are compared on: lower case, letters and digits only.
type Token struct {
	Text string
	Key  string
}

// Tokenize splits s on whitespace. A "word" that has no letter or digit (a dash, a
// lone full stop) is not something two engines can disagree about, so it is joined
// to the neighbouring word instead of becoming a token of its own.
func Tokenize(s string) []Token {
	var out []Token
	pending := ""
	for _, w := range strings.Fields(s) {
		k := key(w)
		if k == "" {
			if len(out) > 0 {
				out[len(out)-1].Text += " " + w
			} else if pending == "" {
				pending = w
			} else {
				pending += " " + w
			}
			continue
		}
		text := w
		if pending != "" {
			text = pending + " " + w
			pending = ""
		}
		out = append(out, Token{Text: text, Key: k})
	}
	return out
}

func key(w string) string {
	var b strings.Builder
	for _, r := range w {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			b.WriteRune(unicode.ToLower(r))
		}
	}
	return b.String()
}