diff --git a/internal/reconcile/disputes.go b/internal/reconcile/disputes.go new file mode 100644 index 0000000..27b0be5 --- /dev/null +++ b/internal/reconcile/disputes.go @@ -0,0 +1,100 @@ +package reconcile + +import "strings" + +// Dispute is one place where the two transcripts disagree. BaseStart:BaseEnd and +// AltStart:AltEnd are token ranges into the two Tokenize results; an empty range +// means that engine heard nothing there. Base and Alt are the two readings as each +// engine wrote them, so a replaced span takes the alternative's own spelling. +type Dispute struct { + BaseStart, BaseEnd int + AltStart, AltEnd int + Base, Alt string +} + +// FindDisputes aligns base against alt word by word and returns the disagreements. +// Differences separated by a single shared word are merged into one dispute: for a +// reader that is one decision, and asking about "b" and "d" separately in "b c d" +// hides that "c" sits between them. Two shared words keep disputes apart. +func FindDisputes(base, alt []Token) []Dispute { + a, b := keysOf(base), keysOf(alt) + ops := Diff(a, b) + var out []Dispute + for i := 0; i < len(ops); i++ { + if ops[i].Kind == Equal { + continue + } + cur := ops[i] + for i+2 < len(ops) && ops[i+1].Kind == Equal && ops[i+1].AEnd-ops[i+1].AStart == 1 && ops[i+2].Kind != Equal { + cur.AEnd, cur.BEnd = ops[i+2].AEnd, ops[i+2].BEnd + i += 2 + } + out = append(out, Dispute{ + BaseStart: cur.AStart, BaseEnd: cur.AEnd, + AltStart: cur.BStart, AltEnd: cur.BEnd, + Base: join(base[cur.AStart:cur.AEnd]), + Alt: join(alt[cur.BStart:cur.BEnd]), + }) + } + return out +} + +// fillers are Swedish discourse particles. A dispute that is only one or two of them +// on one side changes no meaning, so it is resolved without asking a model. The list +// is deliberately short and leaves out words that can be an answer ("nej") or that +// carry content. +var fillers = map[string]bool{ + "ju": true, "då": true, "liksom": true, "alltså": true, "så": true, "ja": true, + "eh": true, "ehm": true, "öh": true, "mm": true, "hm": true, "typ": true, "va": true, "asså": true, +} + +// maxFillerRun is how many filler words may be waved through together. +const maxFillerRun = 2 + +// AutoResolve reports whether a dispute needs no model, in which case the base +// reading is kept: either the two sides are the same letters once spaces are removed +// (a compound written as one word or two), or one side is empty and the other is a +// short run of filler words. +func AutoResolve(d Dispute) bool { + baseKeys, altKeys := keysOfText(d.Base), keysOfText(d.Alt) + if strings.Join(baseKeys, "") == strings.Join(altKeys, "") { + return true + } + return (len(baseKeys) == 0 && onlyFillers(altKeys)) || (len(altKeys) == 0 && onlyFillers(baseKeys)) +} + +func onlyFillers(keys []string) bool { + if len(keys) == 0 || len(keys) > maxFillerRun { + return false + } + for _, k := range keys { + if !fillers[k] { + return false + } + } + return true +} + +// ContextAround returns up to window words of the base before and after the dispute, +// as the base wrote them. +func ContextAround(base []Token, d Dispute, window int) (before, after string) { + return join(base[max(0, d.BaseStart-window):d.BaseStart]), join(base[d.BaseEnd:min(len(base), d.BaseEnd+window)]) +} + +func keysOf(ts []Token) []string { + out := make([]string, len(ts)) + for i, t := range ts { + out[i] = t.Key + } + return out +} + +func keysOfText(s string) []string { return keysOf(Tokenize(s)) } + +func join(ts []Token) string { + parts := make([]string, len(ts)) + for i, t := range ts { + parts[i] = t.Text + } + return strings.Join(parts, " ") +}