From b20558ffa61dcf936aa3bf3db0eb93dac9efdd9a Mon Sep 17 00:00:00 2001 From: Mathias Date: Thu, 1 Oct 2026 23:19:00 +0200 Subject: [PATCH] test(reconcile): merging two transcripts, and failing soft (red) Identical texts and trivial disputes cost no model call; an alt verdict takes the alternative's own words, base/unsure keep the base and only unsure is flagged; insertions and deletions land in the right place; the reading order alternates; the model gets the base's context. Three consecutive model errors stop the whole reconciliation (base text, none of the partial changes, reason recorded) while a success in between resets the breaker; a cancelled job is an error, not 'skipped'. Spots carry the second they start at; the stored list is capped, the counts are not. Red: Reconcile, Options, MaxContested and domain.Reconcile* are undefined. Co-Authored-By: Claude Sonnet 5.5 --- internal/reconcile/reconcile_test.go | 279 +++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 internal/reconcile/reconcile_test.go diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go new file mode 100644 index 0000000..d97a44f --- /dev/null +++ b/internal/reconcile/reconcile_test.go @@ -0,0 +1,279 @@ +package reconcile_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "git.d-ma.be/mathias/tapir/internal/domain" + "git.d-ma.be/mathias/tapir/internal/reconcile" +) + +// scripted is an Adjudicator that answers from a function and records every request. +type scripted struct { + fn func(i int, r reconcile.Request) (reconcile.Verdict, error) + calls []reconcile.Request +} + +func (s *scripted) Adjudicate(_ context.Context, r reconcile.Request) (reconcile.Verdict, error) { + i := len(s.calls) + s.calls = append(s.calls, r) + return s.fn(i, r) +} + +func always(v reconcile.Verdict) *scripted { + return &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return v, nil }} +} + +func tr(text string) domain.Transcription { + return domain.Transcription{Text: text, AudioSeconds: 100, Endpoint: "e"} +} + +var opts = reconcile.Options{Window: 100, BreakAfter: 3, Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"} + +func merge(t *testing.T, base, alt domain.Transcription, adj reconcile.Adjudicator) domain.Transcription { + t.Helper() + out, err := reconcile.Reconcile(context.Background(), base, alt, adj, opts) + require.NoError(t, err) + return out +} + +func TestReconcile_IdenticalTextsNeedNoModelAndComeBackUnchanged(t *testing.T) { + adj := always(reconcile.VerdictAlt) + + out := merge(t, tr("Vi går igenom läget."), tr("vi går igenom läget"), adj) + + assert.Equal(t, "Vi går igenom läget.", out.Text, "the base's own spelling and punctuation stand") + assert.Empty(t, adj.calls) + require.NotNil(t, out.Reconciliation) + assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status) + assert.Zero(t, out.Reconciliation.Disputes) +} + +func TestReconcile_TrivialDisputesAreResolvedWithoutAskingTheModel(t *testing.T) { + adj := always(reconcile.VerdictAlt) + + out := merge(t, tr("i crossborder delen ska vi ju betala"), tr("i crossborderdelen ska vi betala"), adj) + + assert.Empty(t, adj.calls, "a compound split and a filler particle are not worth a model call") + assert.Equal(t, "i crossborder delen ska vi ju betala", out.Text, "the base reading is kept") + assert.Equal(t, 2, out.Reconciliation.Disputes) + assert.Equal(t, 2, out.Reconciliation.AutoResolved) + assert.Zero(t, out.Reconciliation.Adjudicated) +} + +func TestReconcile_AltVerdictReplacesTheSpanWithTheAlternativesOwnWords(t *testing.T) { + out := merge(t, tr("Totalt 1015 kronor. Det ska betalas."), tr("Totalt 1050 kronor. Det ska betalas."), always(reconcile.VerdictAlt)) + + assert.Equal(t, "Totalt 1050 kronor. Det ska betalas.", out.Text) + assert.Equal(t, 1, out.Reconciliation.Changed) + assert.Equal(t, 1, out.Reconciliation.Adjudicated) +} + +func TestReconcile_BaseAndUnsureVerdictsKeepTheBaseButOnlyUnsureIsFlagged(t *testing.T) { + base := tr("Totalt 1015 kronor till SEB.") + alt := tr("Totalt 1050 kronor till SCB.") + + keptBase := merge(t, base, alt, always(reconcile.VerdictBase)) + unsure := merge(t, base, alt, always(reconcile.VerdictUnsure)) + + assert.Equal(t, "Totalt 1015 kronor till SEB.", keptBase.Text) + assert.Zero(t, keptBase.Reconciliation.Unsure) + assert.Empty(t, keptBase.Reconciliation.Contested, "a confident 'the base is right' is not a spot worth the reader's time") + assert.Equal(t, "Totalt 1015 kronor till SEB.", unsure.Text, "unsure keeps the base") + assert.Equal(t, 2, unsure.Reconciliation.Unsure) + require.Len(t, unsure.Reconciliation.Contested, 2) + assert.True(t, unsure.Reconciliation.Contested[0].Unsure) + assert.Equal(t, "base", unsure.Reconciliation.Contested[0].Used) +} + +func TestReconcile_ChangedSpotsAreRecordedWithBothReadings(t *testing.T) { + out := merge(t, tr("det blir T+1 imorgon"), tr("det blir en T imorgon"), always(reconcile.VerdictAlt)) + + require.Len(t, out.Reconciliation.Contested, 1) + c := out.Reconciliation.Contested[0] + assert.Equal(t, "T+1", c.Base) + assert.Equal(t, "en T", c.Alt) + assert.Equal(t, "alt", c.Used) + assert.False(t, c.Unsure) +} + +func TestReconcile_AWordOnlyTheAlternativeHeardIsInsertedWhereItBelongs(t *testing.T) { + out := merge(t, tr("Man får tillbaka ett förslag."), tr("Man får tillbaka ett coolt förslag."), always(reconcile.VerdictAlt)) + + assert.Equal(t, "Man får tillbaka ett coolt förslag.", out.Text) +} + +func TestReconcile_AWordOnlyTheBaseHeardCanBeRemoved(t *testing.T) { + out := merge(t, tr("Man får tillbaka ett coolt förslag."), tr("Man får tillbaka ett förslag."), always(reconcile.VerdictAlt)) + + assert.Equal(t, "Man får tillbaka ett förslag.", out.Text) +} + +func TestReconcile_SeveralDisputesAreAppliedIndependentlyAndInOrder(t *testing.T) { + adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) { + if r.Alt == "omvårdade" || r.Alt == "SCB" { + return reconcile.VerdictAlt, nil + } + return reconcile.VerdictBase, nil + }} + + out := merge(t, tr("kunderna onboardade nu och sen betalas till SEB med 1050 kronor"), + tr("kunderna omvårdade nu och sen betalas till SCB med 2050 kronor"), adj) + + assert.Equal(t, "kunderna omvårdade nu och sen betalas till SCB med 1050 kronor", out.Text) + assert.Equal(t, 3, out.Reconciliation.Disputes) + assert.Equal(t, 2, out.Reconciliation.Changed) +} + +func TestReconcile_TheReadingOrderAlternatesFromOneDisputeToTheNext(t *testing.T) { + adj := always(reconcile.VerdictBase) + + merge(t, tr("a x b y c z d"), tr("a p b q c r d"), adj) + + require.Len(t, adj.calls, 3) + assert.False(t, adj.calls[0].AltFirst) + assert.True(t, adj.calls[1].AltFirst) + assert.False(t, adj.calls[2].AltFirst, "alternating averages out position bias across a transcript") +} + +func TestReconcile_TheModelGetsTheContextWindowFromTheBase(t *testing.T) { + adj := always(reconcile.VerdictBase) + o := opts + o.Window = 2 + + _, err := reconcile.Reconcile(context.Background(), tr("ett två tre fyra fem sex sju åtta nio"), tr("ett två tre fyra FEMM sex sju åtta nio"), adj, o) + + require.NoError(t, err) + require.Len(t, adj.calls, 1) + assert.Equal(t, "tre fyra", adj.calls[0].Before) + assert.Equal(t, "sex sju", adj.calls[0].After) + assert.Equal(t, "fem", adj.calls[0].Base) + assert.Equal(t, "FEMM", adj.calls[0].Alt) +} + +func TestReconcile_OneModelErrorLeavesThatSpotUnsureAndCarriesOn(t *testing.T) { + boom := errors.New("timeout") + adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) { + if i == 0 { + return reconcile.VerdictUnsure, boom + } + return reconcile.VerdictAlt, nil + }} + + out := merge(t, tr("a x b y c"), tr("a p b q c"), adj) + + assert.Equal(t, "a x b q c", out.Text, "the first spot stayed as the base, the second was decided") + assert.Equal(t, 1, out.Reconciliation.Errors) + assert.Equal(t, 1, out.Reconciliation.Unsure) + assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status) +} + +func TestReconcile_ConsecutiveModelErrorsStopTheWholeReconciliationInsteadOfWaitingOutEveryTimeout(t *testing.T) { + boom := errors.New("gateway down") + adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return reconcile.VerdictUnsure, boom }} + a, b := make([]string, 0, 20), make([]string, 0, 20) + for i := 0; i < 10; i++ { + a = append(a, fmt.Sprintf("w%d", i), "same") + b = append(b, fmt.Sprintf("v%d", i), "same") + } + + out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj) + + assert.Len(t, adj.calls, 3, "three consecutive failures and it stops asking") + assert.Equal(t, strings.Join(a, " "), out.Text, "the base text, with none of the partial changes") + assert.Equal(t, domain.ReconcileSkipped, out.Reconciliation.Status) + assert.Contains(t, out.Reconciliation.Reason, "gateway down") +} + +func TestReconcile_ASuccessBetweenFailuresResetsTheBreaker(t *testing.T) { + boom := errors.New("blip") + adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) { + if i%3 == 2 { + return reconcile.VerdictBase, nil + } + return reconcile.VerdictUnsure, boom + }} + a, b := make([]string, 0, 20), make([]string, 0, 20) + for i := 0; i < 9; i++ { + a = append(a, fmt.Sprintf("w%d", i), "same") + b = append(b, fmt.Sprintf("v%d", i), "same") + } + + out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj) + + assert.Len(t, adj.calls, 9, "two failures then a success never reach three in a row") + assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status) + assert.Equal(t, 6, out.Reconciliation.Errors) +} + +func TestReconcile_ACancelledJobIsAnErrorNotASkippedReconciliation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { + cancel() + return reconcile.VerdictUnsure, ctx.Err() + }} + + _, err := reconcile.Reconcile(ctx, tr("a x b"), tr("a y b"), adj, opts) + + assert.ErrorIs(t, err, context.Canceled, "a shutdown must not be recorded as 'the model was unavailable'") +} + +func TestReconcile_KeepsTheBaseSegmentsDurationAndNamesTheEngines(t *testing.T) { + base := tr("a x b") + base.Segments = []domain.Segment{{Start: 0, End: 4, Text: "a x b"}} + base.AudioSeconds = 3821.5 + + out := merge(t, base, tr("a y b"), always(reconcile.VerdictAlt)) + + assert.Equal(t, base.Segments, out.Segments, "the timed segments are the base engine's, untouched") + assert.InDelta(t, 3821.5, out.AudioSeconds, 1e-9) + assert.Equal(t, "iguana/pianissimo", out.Reconciliation.Base) + assert.Equal(t, "iguana/kb-whisper", out.Reconciliation.Alt) + assert.Equal(t, "iguana/gemma4-26b", out.Reconciliation.Reconciler) +} + +func TestReconcile_ASpotCarriesTheSecondItStartsAt(t *testing.T) { + base := tr("ett två tre. fyra fem sex. sju åtta nio.") + base.Segments = []domain.Segment{ + {Start: 0, End: 10, Text: "ett två tre."}, + {Start: 10, End: 20, Text: "fyra fem sex."}, + {Start: 20, End: 30, Text: "sju åtta nio."}, + } + alt := tr("ett två tre. fyra femm sex. sju åtta nio.") + + out := merge(t, base, alt, always(reconcile.VerdictUnsure)) + + require.Len(t, out.Reconciliation.Contested, 1) + assert.InDelta(t, 10.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "'fem' is in the second segment, which starts at 10 s") +} + +func TestReconcile_WithNoSegmentsASpotIsPlacedProportionally(t *testing.T) { + base := tr("a b c d e f g h i j") + base.AudioSeconds = 100 + alt := tr("a b c d e f g h i J2") + + out := merge(t, base, alt, always(reconcile.VerdictUnsure)) + + require.Len(t, out.Reconciliation.Contested, 1) + assert.InDelta(t, 90.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "token 9 of 10 in 100 s") +} + +func TestReconcile_ContestedSpotsAreCappedButTheCountsAreExact(t *testing.T) { + a, b := make([]string, 0, 600), make([]string, 0, 600) + for i := 0; i < 300; i++ { + a = append(a, fmt.Sprintf("w%d", i), "same", "same2") + b = append(b, fmt.Sprintf("v%d", i), "same", "same2") + } + + out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), always(reconcile.VerdictUnsure)) + + assert.Equal(t, 300, out.Reconciliation.Unsure) + assert.Equal(t, 300, out.Reconciliation.Disputes) + assert.Len(t, out.Reconciliation.Contested, reconcile.MaxContested, "the stored list is bounded; the counts are not") +}