mathiasandClaude Sonnet 5.5
a1e16fa658
Disputes separated by a single shared word are merged into one decision by design, so several of my fixtures were secretly one dispute. Corrected before any implementation existed; the rule is unchanged. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
280 lines
11 KiB
Go
280 lines
11 KiB
Go
package reconcile_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
|
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
|
)
|
|
|
|
// scripted is an Adjudicator that answers from a function and records every request.
|
|
type scripted struct {
|
|
fn func(i int, r reconcile.Request) (reconcile.Verdict, error)
|
|
calls []reconcile.Request
|
|
}
|
|
|
|
func (s *scripted) Adjudicate(_ context.Context, r reconcile.Request) (reconcile.Verdict, error) {
|
|
i := len(s.calls)
|
|
s.calls = append(s.calls, r)
|
|
return s.fn(i, r)
|
|
}
|
|
|
|
func always(v reconcile.Verdict) *scripted {
|
|
return &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return v, nil }}
|
|
}
|
|
|
|
func tr(text string) domain.Transcription {
|
|
return domain.Transcription{Text: text, AudioSeconds: 100, Endpoint: "e"}
|
|
}
|
|
|
|
var opts = reconcile.Options{Window: 100, BreakAfter: 3, Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}
|
|
|
|
func merge(t *testing.T, base, alt domain.Transcription, adj reconcile.Adjudicator) domain.Transcription {
|
|
t.Helper()
|
|
out, err := reconcile.Reconcile(context.Background(), base, alt, adj, opts)
|
|
require.NoError(t, err)
|
|
return out
|
|
}
|
|
|
|
func TestReconcile_IdenticalTextsNeedNoModelAndComeBackUnchanged(t *testing.T) {
|
|
adj := always(reconcile.VerdictAlt)
|
|
|
|
out := merge(t, tr("Vi går igenom läget."), tr("vi går igenom läget"), adj)
|
|
|
|
assert.Equal(t, "Vi går igenom läget.", out.Text, "the base's own spelling and punctuation stand")
|
|
assert.Empty(t, adj.calls)
|
|
require.NotNil(t, out.Reconciliation)
|
|
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
|
assert.Zero(t, out.Reconciliation.Disputes)
|
|
}
|
|
|
|
func TestReconcile_TrivialDisputesAreResolvedWithoutAskingTheModel(t *testing.T) {
|
|
adj := always(reconcile.VerdictAlt)
|
|
|
|
out := merge(t, tr("i crossborder delen ska vi ju betala"), tr("i crossborderdelen ska vi betala"), adj)
|
|
|
|
assert.Empty(t, adj.calls, "a compound split and a filler particle are not worth a model call")
|
|
assert.Equal(t, "i crossborder delen ska vi ju betala", out.Text, "the base reading is kept")
|
|
assert.Equal(t, 2, out.Reconciliation.Disputes)
|
|
assert.Equal(t, 2, out.Reconciliation.AutoResolved)
|
|
assert.Zero(t, out.Reconciliation.Adjudicated)
|
|
}
|
|
|
|
func TestReconcile_AltVerdictReplacesTheSpanWithTheAlternativesOwnWords(t *testing.T) {
|
|
out := merge(t, tr("Totalt 1015 kronor. Det ska betalas."), tr("Totalt 1050 kronor. Det ska betalas."), always(reconcile.VerdictAlt))
|
|
|
|
assert.Equal(t, "Totalt 1050 kronor. Det ska betalas.", out.Text)
|
|
assert.Equal(t, 1, out.Reconciliation.Changed)
|
|
assert.Equal(t, 1, out.Reconciliation.Adjudicated)
|
|
}
|
|
|
|
func TestReconcile_BaseAndUnsureVerdictsKeepTheBaseButOnlyUnsureIsFlagged(t *testing.T) {
|
|
base := tr("Totalt 1015 kronor till SEB.")
|
|
alt := tr("Totalt 1050 kronor till SCB.")
|
|
|
|
keptBase := merge(t, base, alt, always(reconcile.VerdictBase))
|
|
unsure := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
|
|
|
assert.Equal(t, "Totalt 1015 kronor till SEB.", keptBase.Text)
|
|
assert.Zero(t, keptBase.Reconciliation.Unsure)
|
|
assert.Empty(t, keptBase.Reconciliation.Contested, "a confident 'the base is right' is not a spot worth the reader's time")
|
|
assert.Equal(t, "Totalt 1015 kronor till SEB.", unsure.Text, "unsure keeps the base")
|
|
assert.Equal(t, 2, unsure.Reconciliation.Unsure)
|
|
require.Len(t, unsure.Reconciliation.Contested, 2)
|
|
assert.True(t, unsure.Reconciliation.Contested[0].Unsure)
|
|
assert.Equal(t, "base", unsure.Reconciliation.Contested[0].Used)
|
|
}
|
|
|
|
func TestReconcile_ChangedSpotsAreRecordedWithBothReadings(t *testing.T) {
|
|
out := merge(t, tr("det blir T+1 imorgon"), tr("det blir en T imorgon"), always(reconcile.VerdictAlt))
|
|
|
|
require.Len(t, out.Reconciliation.Contested, 1)
|
|
c := out.Reconciliation.Contested[0]
|
|
assert.Equal(t, "T+1", c.Base)
|
|
assert.Equal(t, "en T", c.Alt)
|
|
assert.Equal(t, "alt", c.Used)
|
|
assert.False(t, c.Unsure)
|
|
}
|
|
|
|
func TestReconcile_AWordOnlyTheAlternativeHeardIsInsertedWhereItBelongs(t *testing.T) {
|
|
out := merge(t, tr("Man får tillbaka ett förslag."), tr("Man får tillbaka ett coolt förslag."), always(reconcile.VerdictAlt))
|
|
|
|
assert.Equal(t, "Man får tillbaka ett coolt förslag.", out.Text)
|
|
}
|
|
|
|
func TestReconcile_AWordOnlyTheBaseHeardCanBeRemoved(t *testing.T) {
|
|
out := merge(t, tr("Man får tillbaka ett coolt förslag."), tr("Man får tillbaka ett förslag."), always(reconcile.VerdictAlt))
|
|
|
|
assert.Equal(t, "Man får tillbaka ett förslag.", out.Text)
|
|
}
|
|
|
|
func TestReconcile_SeveralDisputesAreAppliedIndependentlyAndInOrder(t *testing.T) {
|
|
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
|
if r.Alt == "omvårdade" || r.Alt == "SCB" {
|
|
return reconcile.VerdictAlt, nil
|
|
}
|
|
return reconcile.VerdictBase, nil
|
|
}}
|
|
|
|
out := merge(t, tr("kunderna onboardade nu och sen betalas till SEB via ombud med 1050 kronor"),
|
|
tr("kunderna omvårdade nu och sen betalas till SCB via ombud med 2050 kronor"), adj)
|
|
|
|
assert.Equal(t, "kunderna omvårdade nu och sen betalas till SCB via ombud med 1050 kronor", out.Text)
|
|
assert.Equal(t, 3, out.Reconciliation.Disputes)
|
|
assert.Equal(t, 2, out.Reconciliation.Changed)
|
|
}
|
|
|
|
func TestReconcile_TheReadingOrderAlternatesFromOneDisputeToTheNext(t *testing.T) {
|
|
adj := always(reconcile.VerdictBase)
|
|
|
|
merge(t, tr("a x s1 s2 y t1 t2 z d"), tr("a p s1 s2 q t1 t2 r d"), adj)
|
|
|
|
require.Len(t, adj.calls, 3)
|
|
assert.False(t, adj.calls[0].AltFirst)
|
|
assert.True(t, adj.calls[1].AltFirst)
|
|
assert.False(t, adj.calls[2].AltFirst, "alternating averages out position bias across a transcript")
|
|
}
|
|
|
|
func TestReconcile_TheModelGetsTheContextWindowFromTheBase(t *testing.T) {
|
|
adj := always(reconcile.VerdictBase)
|
|
o := opts
|
|
o.Window = 2
|
|
|
|
_, err := reconcile.Reconcile(context.Background(), tr("ett två tre fyra fem sex sju åtta nio"), tr("ett två tre fyra FEMM sex sju åtta nio"), adj, o)
|
|
|
|
require.NoError(t, err)
|
|
require.Len(t, adj.calls, 1)
|
|
assert.Equal(t, "tre fyra", adj.calls[0].Before)
|
|
assert.Equal(t, "sex sju", adj.calls[0].After)
|
|
assert.Equal(t, "fem", adj.calls[0].Base)
|
|
assert.Equal(t, "FEMM", adj.calls[0].Alt)
|
|
}
|
|
|
|
func TestReconcile_OneModelErrorLeavesThatSpotUnsureAndCarriesOn(t *testing.T) {
|
|
boom := errors.New("timeout")
|
|
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
|
if i == 0 {
|
|
return reconcile.VerdictUnsure, boom
|
|
}
|
|
return reconcile.VerdictAlt, nil
|
|
}}
|
|
|
|
out := merge(t, tr("a x s1 s2 y c"), tr("a p s1 s2 q c"), adj)
|
|
|
|
assert.Equal(t, "a x s1 s2 q c", out.Text, "the first spot stayed as the base, the second was decided")
|
|
assert.Equal(t, 1, out.Reconciliation.Errors)
|
|
assert.Equal(t, 1, out.Reconciliation.Unsure)
|
|
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
|
}
|
|
|
|
func TestReconcile_ConsecutiveModelErrorsStopTheWholeReconciliationInsteadOfWaitingOutEveryTimeout(t *testing.T) {
|
|
boom := errors.New("gateway down")
|
|
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return reconcile.VerdictUnsure, boom }}
|
|
a, b := make([]string, 0, 20), make([]string, 0, 20)
|
|
for i := 0; i < 10; i++ {
|
|
a = append(a, fmt.Sprintf("w%d", i), "s", "t")
|
|
b = append(b, fmt.Sprintf("v%d", i), "s", "t")
|
|
}
|
|
|
|
out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)
|
|
|
|
assert.Len(t, adj.calls, 3, "three consecutive failures and it stops asking")
|
|
assert.Equal(t, strings.Join(a, " "), out.Text, "the base text, with none of the partial changes")
|
|
assert.Equal(t, domain.ReconcileSkipped, out.Reconciliation.Status)
|
|
assert.Contains(t, out.Reconciliation.Reason, "gateway down")
|
|
}
|
|
|
|
func TestReconcile_ASuccessBetweenFailuresResetsTheBreaker(t *testing.T) {
|
|
boom := errors.New("blip")
|
|
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
|
if i%3 == 2 {
|
|
return reconcile.VerdictBase, nil
|
|
}
|
|
return reconcile.VerdictUnsure, boom
|
|
}}
|
|
a, b := make([]string, 0, 20), make([]string, 0, 20)
|
|
for i := 0; i < 9; i++ {
|
|
a = append(a, fmt.Sprintf("w%d", i), "s", "t")
|
|
b = append(b, fmt.Sprintf("v%d", i), "s", "t")
|
|
}
|
|
|
|
out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)
|
|
|
|
assert.Len(t, adj.calls, 9, "two failures then a success never reach three in a row")
|
|
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
|
assert.Equal(t, 6, out.Reconciliation.Errors)
|
|
}
|
|
|
|
func TestReconcile_ACancelledJobIsAnErrorNotASkippedReconciliation(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
|
cancel()
|
|
return reconcile.VerdictUnsure, ctx.Err()
|
|
}}
|
|
|
|
_, err := reconcile.Reconcile(ctx, tr("a x b"), tr("a y b"), adj, opts)
|
|
|
|
assert.ErrorIs(t, err, context.Canceled, "a shutdown must not be recorded as 'the model was unavailable'")
|
|
}
|
|
|
|
func TestReconcile_KeepsTheBaseSegmentsDurationAndNamesTheEngines(t *testing.T) {
|
|
base := tr("a x b")
|
|
base.Segments = []domain.Segment{{Start: 0, End: 4, Text: "a x b"}}
|
|
base.AudioSeconds = 3821.5
|
|
|
|
out := merge(t, base, tr("a y b"), always(reconcile.VerdictAlt))
|
|
|
|
assert.Equal(t, base.Segments, out.Segments, "the timed segments are the base engine's, untouched")
|
|
assert.InDelta(t, 3821.5, out.AudioSeconds, 1e-9)
|
|
assert.Equal(t, "iguana/pianissimo", out.Reconciliation.Base)
|
|
assert.Equal(t, "iguana/kb-whisper", out.Reconciliation.Alt)
|
|
assert.Equal(t, "iguana/gemma4-26b", out.Reconciliation.Reconciler)
|
|
}
|
|
|
|
func TestReconcile_ASpotCarriesTheSecondItStartsAt(t *testing.T) {
|
|
base := tr("ett två tre. fyra fem sex. sju åtta nio.")
|
|
base.Segments = []domain.Segment{
|
|
{Start: 0, End: 10, Text: "ett två tre."},
|
|
{Start: 10, End: 20, Text: "fyra fem sex."},
|
|
{Start: 20, End: 30, Text: "sju åtta nio."},
|
|
}
|
|
alt := tr("ett två tre. fyra femm sex. sju åtta nio.")
|
|
|
|
out := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
|
|
|
require.Len(t, out.Reconciliation.Contested, 1)
|
|
assert.InDelta(t, 10.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "'fem' is in the second segment, which starts at 10 s")
|
|
}
|
|
|
|
func TestReconcile_WithNoSegmentsASpotIsPlacedProportionally(t *testing.T) {
|
|
base := tr("a b c d e f g h i j")
|
|
base.AudioSeconds = 100
|
|
alt := tr("a b c d e f g h i J2")
|
|
|
|
out := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
|
|
|
require.Len(t, out.Reconciliation.Contested, 1)
|
|
assert.InDelta(t, 90.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "token 9 of 10 in 100 s")
|
|
}
|
|
|
|
func TestReconcile_ContestedSpotsAreCappedButTheCountsAreExact(t *testing.T) {
|
|
a, b := make([]string, 0, 600), make([]string, 0, 600)
|
|
for i := 0; i < 300; i++ {
|
|
a = append(a, fmt.Sprintf("w%d", i), "same", "same2")
|
|
b = append(b, fmt.Sprintf("v%d", i), "same", "same2")
|
|
}
|
|
|
|
out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), always(reconcile.VerdictUnsure))
|
|
|
|
assert.Equal(t, 300, out.Reconciliation.Unsure)
|
|
assert.Equal(t, 300, out.Reconciliation.Disputes)
|
|
assert.Len(t, out.Reconciliation.Contested, reconcile.MaxContested, "the stored list is bounded; the counts are not")
|
|
}
|