From ad39b6154ce0e498ff88395f7a10732351e15dde Mon Sep 17 00:00:00 2001 From: Mathias Date: Thu, 1 Oct 2026 23:20:40 +0200 Subject: [PATCH] test(transcribe): two engines, then a reconciler -- and every way it can fail softly (red) Both run in order (the fast one first) and are reconciled; identical texts cost no model call; one engine down uses the other and says so, with the cloud fallback untouched; only when BOTH local engines fail does the fallback run, and the row says berget served it; both failing with no fallback reports both errors; a cancelled job never reaches the cloud; a model that is down yields the base text unreconciled, not a failed job. Red: transcribe.Ensemble is undefined. Co-Authored-By: Claude Sonnet 5.5 --- internal/adapters/transcribe/ensemble_test.go | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 internal/adapters/transcribe/ensemble_test.go diff --git a/internal/adapters/transcribe/ensemble_test.go b/internal/adapters/transcribe/ensemble_test.go new file mode 100644 index 0000000..e0735e8 --- /dev/null +++ b/internal/adapters/transcribe/ensemble_test.go @@ -0,0 +1,166 @@ +package transcribe_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "git.d-ma.be/mathias/tapir/internal/adapters/transcribe" + "git.d-ma.be/mathias/tapir/internal/domain" + "git.d-ma.be/mathias/tapir/internal/reconcile" +) + +// orderedBackend records, in a shared slice, when it was called. +type orderedBackend struct { + name string + text string + err error + calls *[]string +} + +func (o *orderedBackend) Name() string { return o.name } +func (o *orderedBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) { + *o.calls = append(*o.calls, o.name) + if o.err != nil { + return domain.Transcription{}, o.err + } + return domain.Transcription{Text: o.text, Endpoint: o.name, AudioSeconds: 10}, nil +} + +type verdictAdj struct { + v reconcile.Verdict + err error + calls int +} + +func (a *verdictAdj) Adjudicate(context.Context, reconcile.Request) (reconcile.Verdict, error) { + a.calls++ + return a.v, a.err +} + +var ensembleOpts = reconcile.Options{Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"} + +type rig struct { + e *transcribe.Ensemble + calls *[]string + adj *verdictAdj +} + +func newEnsemble(baseText, altText string, baseErr, altErr error, withFallback bool, adj *verdictAdj) *rig { + calls := &[]string{} + base := &orderedBackend{name: "iguana/pianissimo", text: baseText, err: baseErr, calls: calls} + alt := &orderedBackend{name: "iguana/kb-whisper", text: altText, err: altErr, calls: calls} + var fb transcribe.Backend + if withFallback { + fb = &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls} + } + return &rig{e: transcribe.NewEnsemble(base, alt, fb, adj, ensembleOpts), calls: calls, adj: adj} +} + +func TestEnsemble_BothEnginesRunInOrderAndTheirDisagreementIsReconciled(t *testing.T) { + adj := &verdictAdj{v: reconcile.VerdictAlt} + r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, true, adj) + + got, err := r.e.Transcribe(context.Background(), "/a.wav") + + require.NoError(t, err) + assert.Equal(t, []string{"iguana/pianissimo", "iguana/kb-whisper"}, *r.calls, + "the 40-second engine runs first, so a failure there is known early; the fallback is not touched") + assert.Equal(t, "Totalt 1050 kronor idag.", got.Text) + assert.Equal(t, "iguana/pianissimo + iguana/kb-whisper, reconciled by iguana/gemma4-26b", got.Endpoint, + "every contributor is recorded: which engines touched the audio is a quality and a privacy fact") + require.NotNil(t, got.Reconciliation) + assert.Equal(t, domain.ReconcileDone, got.Reconciliation.Status) +} + +func TestEnsemble_IdenticalTextsNeverCallTheModel(t *testing.T) { + adj := &verdictAdj{v: reconcile.VerdictAlt} + r := newEnsemble("Vi går igenom läget.", "Vi går igenom läget.", nil, nil, false, adj) + + got, err := r.e.Transcribe(context.Background(), "/a.wav") + + require.NoError(t, err) + assert.Zero(t, adj.calls) + assert.Equal(t, "Vi går igenom läget.", got.Text) +} + +func TestEnsemble_WhenOneEngineFailsTheOtherIsUsedAndItIsSaidSo(t *testing.T) { + tests := []struct { + name string + baseErr error + altErr error + wantText string + wantEndpoint string + wantReasonHas string + }{ + {"pianissimo down", errors.New("pianissimo wedged"), nil, "alt text", "iguana/kb-whisper", "iguana/pianissimo failed"}, + {"kb-whisper down", nil, errors.New("metal oom"), "base text", "iguana/pianissimo", "iguana/kb-whisper failed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + adj := &verdictAdj{v: reconcile.VerdictAlt} + r := newEnsemble("base text", "alt text", tt.baseErr, tt.altErr, true, adj) + + got, err := r.e.Transcribe(context.Background(), "/a.wav") + + require.NoError(t, err) + assert.Equal(t, tt.wantText, got.Text) + assert.Equal(t, tt.wantEndpoint, got.Endpoint) + assert.Zero(t, adj.calls, "there is nothing to reconcile") + require.NotNil(t, got.Reconciliation) + assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status) + assert.Contains(t, got.Reconciliation.Reason, tt.wantReasonHas) + assert.NotContains(t, *r.calls, "berget/kb-whisper", "one local engine working means audio does not leave the homelab") + }) + } +} + +func TestEnsemble_OnlyWhenBothLocalEnginesFailDoesTheCloudFallbackRun(t *testing.T) { + r := newEnsemble("", "", errors.New("a"), errors.New("b"), true, &verdictAdj{}) + + got, err := r.e.Transcribe(context.Background(), "/a.wav") + + require.NoError(t, err) + assert.Equal(t, "fallback text", got.Text) + assert.Equal(t, "berget/kb-whisper", got.Endpoint, "the row must say berget served it: the audio left the homelab") + assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status) + assert.Contains(t, got.Reconciliation.Reason, "both local engines failed") +} + +func TestEnsemble_BothFailingWithNoFallbackReportsBothErrors(t *testing.T) { + r := newEnsemble("", "", errors.New("pianissimo down"), errors.New("whisper down"), false, &verdictAdj{}) + + _, err := r.e.Transcribe(context.Background(), "/a.wav") + + require.Error(t, err) + assert.Contains(t, err.Error(), "pianissimo down") + assert.Contains(t, err.Error(), "whisper down") +} + +func TestEnsemble_ACancelledJobNeverReachesTheCloud(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r := newEnsemble("", "", context.Canceled, context.Canceled, true, &verdictAdj{}) + + _, err := r.e.Transcribe(ctx, "/a.wav") + + assert.ErrorIs(t, err, context.Canceled) + assert.NotContains(t, *r.calls, "berget/kb-whisper", "a cancelled job must not ship its audio to a cloud endpoint as a last act") +} + +func TestEnsemble_AModelThatIsDownGivesTheBaseTextUnreconciledNotAFailedJob(t *testing.T) { + adj := &verdictAdj{err: errors.New("gateway down")} + // Four separate disagreements, each separated by two shared words, so the + // breaker trips after three consecutive failures. + r := newEnsemble("a1 s t b1 s t c1 s t d1", "a2 s t b2 s t c2 s t d2", nil, nil, false, adj) + + got, err := r.e.Transcribe(context.Background(), "/a.wav") + + require.NoError(t, err) + assert.Equal(t, "a1 s t b1 s t c1 s t d1", got.Text) + assert.Equal(t, domain.ReconcileSkipped, got.Reconciliation.Status) + assert.Equal(t, 3, adj.calls) +}