diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index ba604a7..f1e3956 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -6,11 +6,17 @@ import ( "strings" "git.d-ma.be/mathias/tapir/internal/domain" + "git.d-ma.be/mathias/tapir/internal/progress" ) -// MaxContested bounds the stored list of contested spots. The counts on the -// Reconciliation are always exact; only this list is capped. -const MaxContested = 200 +// MaxContested is the default bound on the stored list of judged spots. A 90-minute +// recording has a few hundred, so this is a guard against a runaway, not a budget. The +// counts on the Reconciliation are always exact; only the list is capped. +const MaxContested = 2000 + +// beforeWords is how many words of left context a stored spot carries: enough to find +// the place again, small enough to stay out of the way. +const beforeWords = 8 // Options configure Reconcile. Zero Window and BreakAfter take their defaults. type Options struct { @@ -20,6 +26,8 @@ type Options struct { // BreakAfter is how many consecutive model errors stop the reconciliation, so a // model that is down costs three timeouts, not one per dispute. BreakAfter int + // MaxContested caps the stored judged spots; zero takes the package default. + MaxContested int // BaseEngine, AltEngine and Reconciler are recorded on the result. BaseEngine, AltEngine, Reconciler string } @@ -31,22 +39,50 @@ type Options struct { // and the result says why. The only error it returns is a cancelled context, which // is the job stopping, not a verdict on the model. func Reconcile(ctx context.Context, base, alt domain.Transcription, adj Adjudicator, o Options) (domain.Transcription, error) { - window, breakAfter := o.Window, o.BreakAfter + window, breakAfter, maxSpots := o.Window, o.BreakAfter, o.MaxContested if window <= 0 { window = 100 } if breakAfter <= 0 { breakAfter = 3 } + if maxSpots <= 0 { + maxSpots = MaxContested + } + rep := progress.From(ctx) + + rep.Begin("compare", 0) bt, at := Tokenize(base.Text), Tokenize(alt.Text) ds := FindDisputes(bt, at) + rep.Done("compare") + + // The deciding phase counts what the MODEL decides: the trivial disputes are settled + // without it, so counting them would leave the bar short of its end. + deciding := 0 + for _, d := range ds { + if !AutoResolve(d) { + deciding++ + } + } rec := &domain.Reconciliation{ Status: domain.ReconcileDone, Base: o.BaseEngine, Alt: o.AltEngine, Reconciler: o.Reconciler, Disputes: len(ds), } seconds := positioner(base, len(bt)) + spot := func(d Dispute, used string, unsure bool) { + before, _ := ContextAround(bt, d, window) + if w := strings.Fields(before); len(w) > beforeWords { + before = strings.Join(w[len(w)-beforeWords:], " ") + } + if len(rec.Contested) < maxSpots { + rec.Contested = append(rec.Contested, domain.ContestedSpot{ + Seconds: seconds(d.BaseStart), Base: d.Base, Alt: d.Alt, Used: used, Unsure: unsure, Before: before, + }) + } + } + rep.Begin("decide", deciding) verdicts := make([]Verdict, len(ds)) - consecutive := 0 + consecutive, handled := 0, 0 for i, d := range ds { if AutoResolve(d) { rec.AutoResolved++ @@ -65,7 +101,9 @@ func Reconcile(ctx context.Context, base, alt domain.Transcription, adj Adjudica rec.Errors++ consecutive++ if consecutive >= breakAfter { - return skipped(base, rec, fmt.Sprintf("the model failed %d times in a row: %v", consecutive, err)), nil + failure := fmt.Errorf("the model failed %d times in a row: %w", consecutive, err) + rep.Fail("decide", failure) + return skipped(base, rec, failure.Error()), nil } v = VerdictUnsure } else { @@ -76,30 +114,94 @@ func Reconcile(ctx context.Context, base, alt domain.Transcription, adj Adjudica switch v { case VerdictAlt: rec.Changed++ - addContested(rec, domain.ContestedSpot{Seconds: seconds(d.BaseStart), Base: d.Base, Alt: d.Alt, Used: "alt"}) + spot(d, "alt", false) case VerdictUnsure: rec.Unsure++ - addContested(rec, domain.ContestedSpot{Seconds: seconds(d.BaseStart), Base: d.Base, Alt: d.Alt, Used: "base", Unsure: true}) + spot(d, "base", true) + default: + spot(d, "base", false) } + handled++ + rep.Advance("decide", handled) + } + rep.Done("decide") + + out := base + out.Text, out.Segments = merge(base, bt, at, ds, verdicts) + out.Reconciliation = rec + return out, nil +} + +// merge applies the verdicts: the base's words stand except where the alternative's +// reading was chosen. It returns the merged text and the base's segments carrying the +// merged words (timing is the base's). Segments are rewritten only when their words line +// up with the text; a word wrapped across a segment edge makes them differ, and guessing +// would put words in the wrong place, so then the base's segments are returned as they +// were (ADR-035). The base's own segments are never edited in place. +func merge(base domain.Transcription, bt, at []Token, ds []Dispute, verdicts []Verdict) (string, []domain.Segment) { + segOf, aligned := segmentIndex(base.Segments, len(bt)) + var perSeg [][]string + var changed map[int]bool + if aligned { + perSeg, changed = make([][]string, len(base.Segments)), map[int]bool{} } var parts []string + keep := func(from, to int) { + for i := from; i < to; i++ { + parts = append(parts, bt[i].Text) + if aligned { + perSeg[segOf[i]] = append(perSeg[segOf[i]], bt[i].Text) + } + } + } pos := 0 for i, d := range ds { - parts = appendTexts(parts, bt[pos:d.BaseStart]) + keep(pos, d.BaseStart) if verdicts[i] == VerdictAlt { - parts = appendTexts(parts, at[d.AltStart:d.AltEnd]) + for _, t := range at[d.AltStart:d.AltEnd] { + parts = append(parts, t.Text) + } + if aligned { + anchor := min(d.BaseStart, len(bt)-1) + for _, t := range at[d.AltStart:d.AltEnd] { + perSeg[segOf[anchor]] = append(perSeg[segOf[anchor]], t.Text) + } + changed[segOf[anchor]] = true + for k := d.BaseStart; k < d.BaseEnd; k++ { + changed[segOf[k]] = true + } + } } else { - parts = appendTexts(parts, bt[d.BaseStart:d.BaseEnd]) + keep(d.BaseStart, d.BaseEnd) } pos = d.BaseEnd } - parts = appendTexts(parts, bt[pos:]) + keep(pos, len(bt)) - out := base - out.Text = strings.Join(parts, " ") - out.Reconciliation = rec - return out, nil + if !aligned { + return strings.Join(parts, " "), base.Segments + } + segs := append([]domain.Segment(nil), base.Segments...) + for k := range changed { + segs[k].Text = strings.Join(perSeg[k], " ") + } + return strings.Join(parts, " "), segs +} + +// segmentIndex maps each base token to the segment it came from. aligned is false when +// there are no segments or their word count is not the text's. +func segmentIndex(segs []domain.Segment, tokens int) (segOf []int, aligned bool) { + if len(segs) == 0 || tokens == 0 { + return nil, false + } + segOf = make([]int, 0, tokens) + for k, s := range segs { + for range Tokenize(s.Text) { + segOf = append(segOf, k) + } + } + return segOf, len(segOf) == tokens } // skipped is the base text, none of the partial changes, and the reason. @@ -112,13 +214,6 @@ func skipped(base domain.Transcription, rec *domain.Reconciliation, reason strin return out } -func appendTexts(dst []string, ts []Token) []string { - for _, t := range ts { - dst = append(dst, t.Text) - } - return dst -} - // positioner maps a token index of the base to the second it starts at: through the // segments when there are any, proportionally otherwise. func positioner(base domain.Transcription, tokens int) func(int) float64 { @@ -144,9 +239,3 @@ func positioner(base domain.Transcription, tokens int) func(int) float64 { return base.Segments[k].Start } } - -func addContested(rec *domain.Reconciliation, c domain.ContestedSpot) { - if len(rec.Contested) < MaxContested { - rec.Contested = append(rec.Contested, c) - } -}