feat(reconcile): find disputes, resolve trivial ones without a model, give context
Disputes carry both readings with their own spelling; differences separated by one shared word merge into one decision. Compound splits/joins and one or two filler particles are resolved without a model; 'nej', numbers and content words never are. Mutation-checked (merge distance, filler limit, 'nej', compound rule). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,100 @@
|
|||||||
|
package reconcile
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// Dispute is one place where the two transcripts disagree. BaseStart:BaseEnd and
|
||||||
|
// AltStart:AltEnd are token ranges into the two Tokenize results; an empty range
|
||||||
|
// means that engine heard nothing there. Base and Alt are the two readings as each
|
||||||
|
// engine wrote them, so a replaced span takes the alternative's own spelling.
|
||||||
|
type Dispute struct {
|
||||||
|
BaseStart, BaseEnd int
|
||||||
|
AltStart, AltEnd int
|
||||||
|
Base, Alt string
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindDisputes aligns base against alt word by word and returns the disagreements.
|
||||||
|
// Differences separated by a single shared word are merged into one dispute: for a
|
||||||
|
// reader that is one decision, and asking about "b" and "d" separately in "b c d"
|
||||||
|
// hides that "c" sits between them. Two shared words keep disputes apart.
|
||||||
|
func FindDisputes(base, alt []Token) []Dispute {
|
||||||
|
a, b := keysOf(base), keysOf(alt)
|
||||||
|
ops := Diff(a, b)
|
||||||
|
var out []Dispute
|
||||||
|
for i := 0; i < len(ops); i++ {
|
||||||
|
if ops[i].Kind == Equal {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cur := ops[i]
|
||||||
|
for i+2 < len(ops) && ops[i+1].Kind == Equal && ops[i+1].AEnd-ops[i+1].AStart == 1 && ops[i+2].Kind != Equal {
|
||||||
|
cur.AEnd, cur.BEnd = ops[i+2].AEnd, ops[i+2].BEnd
|
||||||
|
i += 2
|
||||||
|
}
|
||||||
|
out = append(out, Dispute{
|
||||||
|
BaseStart: cur.AStart, BaseEnd: cur.AEnd,
|
||||||
|
AltStart: cur.BStart, AltEnd: cur.BEnd,
|
||||||
|
Base: join(base[cur.AStart:cur.AEnd]),
|
||||||
|
Alt: join(alt[cur.BStart:cur.BEnd]),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// fillers are Swedish discourse particles. A dispute that is only one or two of them
|
||||||
|
// on one side changes no meaning, so it is resolved without asking a model. The list
|
||||||
|
// is deliberately short and leaves out words that can be an answer ("nej") or that
|
||||||
|
// carry content.
|
||||||
|
var fillers = map[string]bool{
|
||||||
|
"ju": true, "då": true, "liksom": true, "alltså": true, "så": true, "ja": true,
|
||||||
|
"eh": true, "ehm": true, "öh": true, "mm": true, "hm": true, "typ": true, "va": true, "asså": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxFillerRun is how many filler words may be waved through together.
|
||||||
|
const maxFillerRun = 2
|
||||||
|
|
||||||
|
// AutoResolve reports whether a dispute needs no model, in which case the base
|
||||||
|
// reading is kept: either the two sides are the same letters once spaces are removed
|
||||||
|
// (a compound written as one word or two), or one side is empty and the other is a
|
||||||
|
// short run of filler words.
|
||||||
|
func AutoResolve(d Dispute) bool {
|
||||||
|
baseKeys, altKeys := keysOfText(d.Base), keysOfText(d.Alt)
|
||||||
|
if strings.Join(baseKeys, "") == strings.Join(altKeys, "") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return (len(baseKeys) == 0 && onlyFillers(altKeys)) || (len(altKeys) == 0 && onlyFillers(baseKeys))
|
||||||
|
}
|
||||||
|
|
||||||
|
func onlyFillers(keys []string) bool {
|
||||||
|
if len(keys) == 0 || len(keys) > maxFillerRun {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
if !fillers[k] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContextAround returns up to window words of the base before and after the dispute,
|
||||||
|
// as the base wrote them.
|
||||||
|
func ContextAround(base []Token, d Dispute, window int) (before, after string) {
|
||||||
|
return join(base[max(0, d.BaseStart-window):d.BaseStart]), join(base[d.BaseEnd:min(len(base), d.BaseEnd+window)])
|
||||||
|
}
|
||||||
|
|
||||||
|
func keysOf(ts []Token) []string {
|
||||||
|
out := make([]string, len(ts))
|
||||||
|
for i, t := range ts {
|
||||||
|
out[i] = t.Key
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func keysOfText(s string) []string { return keysOf(Tokenize(s)) }
|
||||||
|
|
||||||
|
func join(ts []Token) string {
|
||||||
|
parts := make([]string, len(ts))
|
||||||
|
for i, t := range ts {
|
||||||
|
parts[i] = t.Text
|
||||||
|
}
|
||||||
|
return strings.Join(parts, " ")
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user