tapir/internal/reconcile/reconcile_test.go

package reconcile_test

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"git.d-ma.be/mathias/tapir/internal/domain"
	"git.d-ma.be/mathias/tapir/internal/reconcile"
)

// scripted is an Adjudicator that answers from a function and records every request.
type scripted struct {
	fn    func(i int, r reconcile.Request) (reconcile.Verdict, error)
	calls []reconcile.Request
}

func (s *scripted) Adjudicate(_ context.Context, r reconcile.Request) (reconcile.Verdict, error) {
	i := len(s.calls)
	s.calls = append(s.calls, r)
	return s.fn(i, r)
}

func always(v reconcile.Verdict) *scripted {
	return &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return v, nil }}
}

func tr(text string) domain.Transcription {
	return domain.Transcription{Text: text, AudioSeconds: 100, Endpoint: "e"}
}

var opts = reconcile.Options{Window: 100, BreakAfter: 3, Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}

func merge(t *testing.T, base, alt domain.Transcription, adj reconcile.Adjudicator) domain.Transcription {
	t.Helper()
	out, err := reconcile.Reconcile(context.Background(), base, alt, adj, opts)
	require.NoError(t, err)
	return out
}

func TestReconcile_IdenticalTextsNeedNoModelAndComeBackUnchanged(t *testing.T) {
	adj := always(reconcile.VerdictAlt)

	out := merge(t, tr("Vi går igenom läget."), tr("vi går igenom läget"), adj)

	assert.Equal(t, "Vi går igenom läget.", out.Text, "the base's own spelling and punctuation stand")
	assert.Empty(t, adj.calls)
	require.NotNil(t, out.Reconciliation)
	assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
	assert.Zero(t, out.Reconciliation.Disputes)
}

func TestReconcile_TrivialDisputesAreResolvedWithoutAskingTheModel(t *testing.T) {
	adj := always(reconcile.VerdictAlt)

	out := merge(t, tr("i takrenns delen ska vi ju betala"), tr("i takrennsdelen ska vi betala"), adj)

	assert.Empty(t, adj.calls, "a compound split and a filler particle are not worth a model call")
	assert.Equal(t, "i takrenns delen ska vi ju betala", out.Text, "the base reading is kept")
	assert.Equal(t, 2, out.Reconciliation.Disputes)
	assert.Equal(t, 2, out.Reconciliation.AutoResolved)
	assert.Zero(t, out.Reconciliation.Adjudicated)
}

func TestReconcile_AltVerdictReplacesTheSpanWithTheAlternativesOwnWords(t *testing.T) {
	out := merge(t, tr("Totalt 1015 kronor. Det ska betalas."), tr("Totalt 1050 kronor. Det ska betalas."), always(reconcile.VerdictAlt))

	assert.Equal(t, "Totalt 1050 kronor. Det ska betalas.", out.Text)
	assert.Equal(t, 1, out.Reconciliation.Changed)
	assert.Equal(t, 1, out.Reconciliation.Adjudicated)
}

func TestReconcile_BaseAndUnsureVerdictsKeepTheBaseButOnlyUnsureIsFlagged(t *testing.T) {
	base := tr("Totalt 1015 kronor till Axel.")
	alt := tr("Totalt 1050 kronor till Alex.")

	keptBase := merge(t, base, alt, always(reconcile.VerdictBase))
	unsure := merge(t, base, alt, always(reconcile.VerdictUnsure))

	assert.Equal(t, "Totalt 1015 kronor till Axel.", keptBase.Text)
	assert.Zero(t, keptBase.Reconciliation.Unsure)
	require.Len(t, keptBase.Reconciliation.Contested, 2,
		"every judged spot is stored, kept ones too: without them the reconciler cannot be compared with either engine alone (ADR-035)")
	for _, c := range keptBase.Reconciliation.Contested {
		assert.Equal(t, "base", c.Used)
		assert.False(t, c.Unsure, "a confident 'the base is right' is stored but is not an uncertain spot")
	}
	assert.Equal(t, "Totalt 1015 kronor till Axel.", unsure.Text, "unsure keeps the base")
	assert.Equal(t, 2, unsure.Reconciliation.Unsure)
	require.Len(t, unsure.Reconciliation.Contested, 2)
	assert.True(t, unsure.Reconciliation.Contested[0].Unsure)
	assert.Equal(t, "base", unsure.Reconciliation.Contested[0].Used)
}

func TestReconcile_ChangedSpotsAreRecordedWithBothReadings(t *testing.T) {
	out := merge(t, tr("det blir A+2 imorgon"), tr("det blir en A imorgon"), always(reconcile.VerdictAlt))

	require.Len(t, out.Reconciliation.Contested, 1)
	c := out.Reconciliation.Contested[0]
	assert.Equal(t, "A+2", c.Base)
	assert.Equal(t, "en A", c.Alt)
	assert.Equal(t, "alt", c.Used)
	assert.False(t, c.Unsure)
}

func TestReconcile_AWordOnlyTheAlternativeHeardIsInsertedWhereItBelongs(t *testing.T) {
	out := merge(t, tr("Man får tillbaka ett förslag."), tr("Man får tillbaka ett coolt förslag."), always(reconcile.VerdictAlt))

	assert.Equal(t, "Man får tillbaka ett coolt förslag.", out.Text)
}

func TestReconcile_AWordOnlyTheBaseHeardCanBeRemoved(t *testing.T) {
	out := merge(t, tr("Man får tillbaka ett coolt förslag."), tr("Man får tillbaka ett förslag."), always(reconcile.VerdictAlt))

	assert.Equal(t, "Man får tillbaka ett förslag.", out.Text)
}

func TestReconcile_SeveralDisputesAreAppliedIndependentlyAndInOrder(t *testing.T) {
	adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
		if r.Alt == "mölade" || r.Alt == "Alex" {
			return reconcile.VerdictAlt, nil
		}
		return reconcile.VerdictBase, nil
	}}

	out := merge(t, tr("kunderna målade nu och sen betalas till Axel via ombud med 1050 kronor"),
		tr("kunderna mölade nu och sen betalas till Alex via ombud med 2050 kronor"), adj)

	assert.Equal(t, "kunderna mölade nu och sen betalas till Alex via ombud med 1050 kronor", out.Text)
	assert.Equal(t, 3, out.Reconciliation.Disputes)
	assert.Equal(t, 2, out.Reconciliation.Changed)
}

func TestReconcile_TheReadingOrderAlternatesFromOneDisputeToTheNext(t *testing.T) {
	adj := always(reconcile.VerdictBase)

	merge(t, tr("a x s1 s2 y t1 t2 z d"), tr("a p s1 s2 q t1 t2 r d"), adj)

	require.Len(t, adj.calls, 3)
	assert.False(t, adj.calls[0].AltFirst)
	assert.True(t, adj.calls[1].AltFirst)
	assert.False(t, adj.calls[2].AltFirst, "alternating averages out position bias across a transcript")
}

func TestReconcile_TheModelGetsTheContextWindowFromTheBase(t *testing.T) {
	adj := always(reconcile.VerdictBase)
	o := opts
	o.Window = 2

	_, err := reconcile.Reconcile(context.Background(), tr("ett två tre fyra fem sex sju åtta nio"), tr("ett två tre fyra FEMM sex sju åtta nio"), adj, o)

	require.NoError(t, err)
	require.Len(t, adj.calls, 1)
	assert.Equal(t, "tre fyra", adj.calls[0].Before)
	assert.Equal(t, "sex sju", adj.calls[0].After)
	assert.Equal(t, "fem", adj.calls[0].Base)
	assert.Equal(t, "FEMM", adj.calls[0].Alt)
}

func TestReconcile_OneModelErrorLeavesThatSpotUnsureAndCarriesOn(t *testing.T) {
	boom := errors.New("timeout")
	adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
		if i == 0 {
			return reconcile.VerdictUnsure, boom
		}
		return reconcile.VerdictAlt, nil
	}}

	out := merge(t, tr("a x s1 s2 y c"), tr("a p s1 s2 q c"), adj)

	assert.Equal(t, "a x s1 s2 q c", out.Text, "the first spot stayed as the base, the second was decided")
	assert.Equal(t, 1, out.Reconciliation.Errors)
	assert.Equal(t, 1, out.Reconciliation.Unsure)
	assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
}

func TestReconcile_ConsecutiveModelErrorsStopTheWholeReconciliationInsteadOfWaitingOutEveryTimeout(t *testing.T) {
	boom := errors.New("gateway down")
	adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return reconcile.VerdictUnsure, boom }}
	a, b := make([]string, 0, 20), make([]string, 0, 20)
	for i := 0; i < 10; i++ {
		a = append(a, fmt.Sprintf("w%d", i), "s", "t")
		b = append(b, fmt.Sprintf("v%d", i), "s", "t")
	}

	out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)

	assert.Len(t, adj.calls, 3, "three consecutive failures and it stops asking")
	assert.Equal(t, strings.Join(a, " "), out.Text, "the base text, with none of the partial changes")
	assert.Equal(t, domain.ReconcileSkipped, out.Reconciliation.Status)
	assert.Contains(t, out.Reconciliation.Reason, "3 times in a row")
	assert.NotContains(t, out.Reconciliation.Reason, "gateway down", "the reason is shown on the page: a kind, not the model's own text")
}

func TestReconcile_ASuccessBetweenFailuresResetsTheBreaker(t *testing.T) {
	boom := errors.New("blip")
	adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
		if i%3 == 2 {
			return reconcile.VerdictBase, nil
		}
		return reconcile.VerdictUnsure, boom
	}}
	a, b := make([]string, 0, 20), make([]string, 0, 20)
	for i := 0; i < 9; i++ {
		a = append(a, fmt.Sprintf("w%d", i), "s", "t")
		b = append(b, fmt.Sprintf("v%d", i), "s", "t")
	}

	out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)

	assert.Len(t, adj.calls, 9, "two failures then a success never reach three in a row")
	assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
	assert.Equal(t, 6, out.Reconciliation.Errors)
}

func TestReconcile_ACancelledJobIsAnErrorNotASkippedReconciliation(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
		cancel()
		return reconcile.VerdictUnsure, ctx.Err()
	}}

	_, err := reconcile.Reconcile(ctx, tr("a x b"), tr("a y b"), adj, opts)

	assert.ErrorIs(t, err, context.Canceled, "a shutdown must not be recorded as 'the model was unavailable'")
}

func TestReconcile_KeepsTheBaseTimingAndDurationAndNamesTheEngines(t *testing.T) {
	base := tr("a x b")
	base.Segments = []domain.Segment{{Start: 0, End: 4, Text: "a x b"}}
	base.AudioSeconds = 3821.5

	out := merge(t, base, tr("a y b"), always(reconcile.VerdictAlt))

	require.Len(t, out.Segments, 1)
	assert.InDelta(t, 0.0, out.Segments[0].Start, 1e-9, "timing is the base engine's")
	assert.InDelta(t, 4.0, out.Segments[0].End, 1e-9)
	assert.InDelta(t, 3821.5, out.AudioSeconds, 1e-9)
	assert.Equal(t, "iguana/pianissimo", out.Reconciliation.Base)
	assert.Equal(t, "iguana/kb-whisper", out.Reconciliation.Alt)
	assert.Equal(t, "iguana/gemma4-26b", out.Reconciliation.Reconciler)
}

func TestReconcile_ASpotCarriesTheSecondItStartsAt(t *testing.T) {
	base := tr("ett två tre. fyra fem sex. sju åtta nio.")
	base.Segments = []domain.Segment{
		{Start: 0, End: 10, Text: "ett två tre."},
		{Start: 10, End: 20, Text: "fyra fem sex."},
		{Start: 20, End: 30, Text: "sju åtta nio."},
	}
	alt := tr("ett två tre. fyra femm sex. sju åtta nio.")

	out := merge(t, base, alt, always(reconcile.VerdictUnsure))

	require.Len(t, out.Reconciliation.Contested, 1)
	assert.InDelta(t, 10.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "'fem' is in the second segment, which starts at 10 s")
}

func TestReconcile_WithNoSegmentsASpotIsPlacedProportionally(t *testing.T) {
	base := tr("a b c d e f g h i j")
	base.AudioSeconds = 100
	alt := tr("a b c d e f g h i J2")

	out := merge(t, base, alt, always(reconcile.VerdictUnsure))

	require.Len(t, out.Reconciliation.Contested, 1)
	assert.InDelta(t, 90.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "token 9 of 10 in 100 s")
}

func TestReconcile_JudgedSpotsAreCappedButTheCountsAreExact(t *testing.T) {
	a, b := make([]string, 0, 600), make([]string, 0, 600)
	for i := 0; i < 300; i++ {
		a = append(a, fmt.Sprintf("w%d", i), "same", "same2")
		b = append(b, fmt.Sprintf("v%d", i), "same", "same2")
	}
	capped := opts
	capped.MaxContested = 50

	out, err := reconcile.Reconcile(context.Background(), tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), always(reconcile.VerdictUnsure), capped)

	require.NoError(t, err)
	assert.Equal(t, 300, out.Reconciliation.Unsure)
	assert.Equal(t, 300, out.Reconciliation.Disputes)
	assert.Len(t, out.Reconciliation.Contested, 50, "the stored list is bounded; the counts are not")
}

func TestReconcile_TheDefaultCapIsAFewThousandSoAWholeMeetingIsKept(t *testing.T) {
	assert.Equal(t, 2000, reconcile.MaxContested, "a 90-minute recording has a few hundred judged spots; the cap is a guard, not a budget")
}

func TestReconcile_AVerdictReturnedAlongsideAnErrorIsIgnored(t *testing.T) {
	// An adjudicator that returns a verdict AND an error must not have the verdict used:
	// the contract is that an error means no decision.
	adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
		return reconcile.VerdictAlt, errors.New("half a reply")
	}}

	out := merge(t, tr("a x b"), tr("a y b"), adj)

	assert.Equal(t, "a x b", out.Text, "no decision means the base stands")
	assert.Zero(t, out.Reconciliation.Changed)
	assert.Equal(t, 1, out.Reconciliation.Unsure)
}