58 lines
1.5 KiB
Go
58 lines
1.5 KiB
Go
// Package reconcile merges two transcripts of the same speech: it finds where they
|
|
// disagree word by word, resolves the trivial disagreements itself, and asks a LOCAL
|
|
// model to choose between the two readings of the rest (ADR-034). It depends on
|
|
// nothing outside the standard library and the domain; the model, the engines and
|
|
// the store are wired in from outside.
|
|
package reconcile
|
|
|
|
import (
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
// Token is one word of a transcript. Text is the word as written, with its casing
|
|
// and punctuation, so the merged transcript reads naturally; Key is what two engines
|
|
// are compared on: lower case, letters and digits only.
|
|
type Token struct {
|
|
Text string
|
|
Key string
|
|
}
|
|
|
|
// Tokenize splits s on whitespace. A "word" that has no letter or digit (a dash, a
|
|
// lone full stop) is not something two engines can disagree about, so it is joined
|
|
// to the neighbouring word instead of becoming a token of its own.
|
|
func Tokenize(s string) []Token {
|
|
var out []Token
|
|
pending := ""
|
|
for _, w := range strings.Fields(s) {
|
|
k := key(w)
|
|
if k == "" {
|
|
if len(out) > 0 {
|
|
out[len(out)-1].Text += " " + w
|
|
} else if pending == "" {
|
|
pending = w
|
|
} else {
|
|
pending += " " + w
|
|
}
|
|
continue
|
|
}
|
|
text := w
|
|
if pending != "" {
|
|
text = pending + " " + w
|
|
pending = ""
|
|
}
|
|
out = append(out, Token{Text: text, Key: k})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func key(w string) string {
|
|
var b strings.Builder
|
|
for _, r := range w {
|
|
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
|
b.WriteRune(unicode.ToLower(r))
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|