package reconcile import "strings" // Dispute is one place where the two transcripts disagree. BaseStart:BaseEnd and // AltStart:AltEnd are token ranges into the two Tokenize results; an empty range // means that engine heard nothing there. Base and Alt are the two readings as each // engine wrote them, so a replaced span takes the alternative's own spelling. type Dispute struct { BaseStart, BaseEnd int AltStart, AltEnd int Base, Alt string } // FindDisputes aligns base against alt word by word and returns the disagreements. // Differences separated by a single shared word are merged into one dispute: for a // reader that is one decision, and asking about "b" and "d" separately in "b c d" // hides that "c" sits between them. Two shared words keep disputes apart. func FindDisputes(base, alt []Token) []Dispute { a, b := keysOf(base), keysOf(alt) ops := Diff(a, b) var out []Dispute for i := 0; i < len(ops); i++ { if ops[i].Kind == Equal { continue } cur := ops[i] for i+2 < len(ops) && ops[i+1].Kind == Equal && ops[i+1].AEnd-ops[i+1].AStart == 1 && ops[i+2].Kind != Equal { cur.AEnd, cur.BEnd = ops[i+2].AEnd, ops[i+2].BEnd i += 2 } out = append(out, Dispute{ BaseStart: cur.AStart, BaseEnd: cur.AEnd, AltStart: cur.BStart, AltEnd: cur.BEnd, Base: join(base[cur.AStart:cur.AEnd]), Alt: join(alt[cur.BStart:cur.BEnd]), }) } return out } // fillers are Swedish discourse particles. A dispute that is only one or two of them // on one side changes no meaning, so it is resolved without asking a model. The list // is deliberately short and leaves out words that can be an answer ("nej") or that // carry content. var fillers = map[string]bool{ "ju": true, "då": true, "liksom": true, "alltså": true, "så": true, "ja": true, "eh": true, "ehm": true, "öh": true, "mm": true, "hm": true, "typ": true, "va": true, "asså": true, } // maxFillerRun is how many filler words may be waved through together. const maxFillerRun = 2 // AutoResolve reports whether a dispute needs no model, in which case the base // reading is kept: either the two sides are the same letters once spaces are removed // (a compound written as one word or two), or one side is empty and the other is a // short run of filler words. func AutoResolve(d Dispute) bool { baseKeys, altKeys := keysOfText(d.Base), keysOfText(d.Alt) if strings.Join(baseKeys, "") == strings.Join(altKeys, "") { return true } return (len(baseKeys) == 0 && onlyFillers(altKeys)) || (len(altKeys) == 0 && onlyFillers(baseKeys)) } func onlyFillers(keys []string) bool { if len(keys) == 0 || len(keys) > maxFillerRun { return false } for _, k := range keys { if !fillers[k] { return false } } return true } // ContextAround returns up to window words of the base before and after the dispute, // as the base wrote them. func ContextAround(base []Token, d Dispute, window int) (before, after string) { return join(base[max(0, d.BaseStart-window):d.BaseStart]), join(base[d.BaseEnd:min(len(base), d.BaseEnd+window)]) } func keysOf(ts []Token) []string { out := make([]string, len(ts)) for i, t := range ts { out[i] = t.Key } return out } func keysOfText(s string) []string { return keysOf(Tokenize(s)) } func join(ts []Token) string { parts := make([]string, len(ts)) for i, t := range ts { parts[i] = t.Text } return strings.Join(parts, " ") }