feat(reconcile): find disputes, resolve trivial ones without a model, give context
Disputes carry both readings with their own spelling; differences separated by one shared word merge into one decision. Compound splits/joins and one or two filler particles are resolved without a model; 'nej', numbers and content words never are. Mutation-checked (merge distance, filler limit, 'nej', compound rule). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
package reconcile
|
||||
|
||||
import "strings"
|
||||
|
||||
// Dispute is one place where the two transcripts disagree. BaseStart:BaseEnd and
|
||||
// AltStart:AltEnd are token ranges into the two Tokenize results; an empty range
|
||||
// means that engine heard nothing there. Base and Alt are the two readings as each
|
||||
// engine wrote them, so a replaced span takes the alternative's own spelling.
|
||||
type Dispute struct {
|
||||
BaseStart, BaseEnd int
|
||||
AltStart, AltEnd int
|
||||
Base, Alt string
|
||||
}
|
||||
|
||||
// FindDisputes aligns base against alt word by word and returns the disagreements.
|
||||
// Differences separated by a single shared word are merged into one dispute: for a
|
||||
// reader that is one decision, and asking about "b" and "d" separately in "b c d"
|
||||
// hides that "c" sits between them. Two shared words keep disputes apart.
|
||||
func FindDisputes(base, alt []Token) []Dispute {
|
||||
a, b := keysOf(base), keysOf(alt)
|
||||
ops := Diff(a, b)
|
||||
var out []Dispute
|
||||
for i := 0; i < len(ops); i++ {
|
||||
if ops[i].Kind == Equal {
|
||||
continue
|
||||
}
|
||||
cur := ops[i]
|
||||
for i+2 < len(ops) && ops[i+1].Kind == Equal && ops[i+1].AEnd-ops[i+1].AStart == 1 && ops[i+2].Kind != Equal {
|
||||
cur.AEnd, cur.BEnd = ops[i+2].AEnd, ops[i+2].BEnd
|
||||
i += 2
|
||||
}
|
||||
out = append(out, Dispute{
|
||||
BaseStart: cur.AStart, BaseEnd: cur.AEnd,
|
||||
AltStart: cur.BStart, AltEnd: cur.BEnd,
|
||||
Base: join(base[cur.AStart:cur.AEnd]),
|
||||
Alt: join(alt[cur.BStart:cur.BEnd]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fillers are Swedish discourse particles. A dispute that is only one or two of them
|
||||
// on one side changes no meaning, so it is resolved without asking a model. The list
|
||||
// is deliberately short and leaves out words that can be an answer ("nej") or that
|
||||
// carry content.
|
||||
var fillers = map[string]bool{
|
||||
"ju": true, "då": true, "liksom": true, "alltså": true, "så": true, "ja": true,
|
||||
"eh": true, "ehm": true, "öh": true, "mm": true, "hm": true, "typ": true, "va": true, "asså": true,
|
||||
}
|
||||
|
||||
// maxFillerRun is how many filler words may be waved through together.
|
||||
const maxFillerRun = 2
|
||||
|
||||
// AutoResolve reports whether a dispute needs no model, in which case the base
|
||||
// reading is kept: either the two sides are the same letters once spaces are removed
|
||||
// (a compound written as one word or two), or one side is empty and the other is a
|
||||
// short run of filler words.
|
||||
func AutoResolve(d Dispute) bool {
|
||||
baseKeys, altKeys := keysOfText(d.Base), keysOfText(d.Alt)
|
||||
if strings.Join(baseKeys, "") == strings.Join(altKeys, "") {
|
||||
return true
|
||||
}
|
||||
return (len(baseKeys) == 0 && onlyFillers(altKeys)) || (len(altKeys) == 0 && onlyFillers(baseKeys))
|
||||
}
|
||||
|
||||
func onlyFillers(keys []string) bool {
|
||||
if len(keys) == 0 || len(keys) > maxFillerRun {
|
||||
return false
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !fillers[k] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ContextAround returns up to window words of the base before and after the dispute,
|
||||
// as the base wrote them.
|
||||
func ContextAround(base []Token, d Dispute, window int) (before, after string) {
|
||||
return join(base[max(0, d.BaseStart-window):d.BaseStart]), join(base[d.BaseEnd:min(len(base), d.BaseEnd+window)])
|
||||
}
|
||||
|
||||
func keysOf(ts []Token) []string {
|
||||
out := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = t.Key
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysOfText(s string) []string { return keysOf(Tokenize(s)) }
|
||||
|
||||
func join(ts []Token) string {
|
||||
parts := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
parts[i] = t.Text
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
Reference in New Issue
Block a user