package transcribe_test import ( "context" "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "git.d-ma.be/mathias/tapir/internal/adapters/transcribe" "git.d-ma.be/mathias/tapir/internal/domain" "git.d-ma.be/mathias/tapir/internal/reconcile" ) // orderedBackend records, in a shared slice, when it was called. type orderedBackend struct { name string text string err error calls *[]string } func (o *orderedBackend) Name() string { return o.name } func (o *orderedBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) { *o.calls = append(*o.calls, o.name) if o.err != nil { return domain.Transcription{}, o.err } return domain.Transcription{Text: o.text, Endpoint: o.name, AudioSeconds: 10}, nil } type verdictAdj struct { v reconcile.Verdict err error calls int } func (a *verdictAdj) Adjudicate(context.Context, reconcile.Request) (reconcile.Verdict, error) { a.calls++ return a.v, a.err } var ensembleOpts = reconcile.Options{Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"} type rig struct { e *transcribe.Ensemble calls *[]string adj *verdictAdj } func newEnsemble(baseText, altText string, baseErr, altErr error, withFallback bool, adj *verdictAdj) *rig { calls := &[]string{} base := &orderedBackend{name: "iguana/pianissimo", text: baseText, err: baseErr, calls: calls} alt := &orderedBackend{name: "iguana/kb-whisper", text: altText, err: altErr, calls: calls} var fb transcribe.Backend if withFallback { fb = &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls} } return &rig{e: transcribe.NewEnsemble(base, alt, fb, adj, ensembleOpts), calls: calls, adj: adj} } func TestEnsemble_BothEnginesRunInOrderAndTheirDisagreementIsReconciled(t *testing.T) { adj := &verdictAdj{v: reconcile.VerdictAlt} r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, true, adj) got, err := r.e.Transcribe(context.Background(), "/a.wav") require.NoError(t, err) assert.Equal(t, []string{"iguana/pianissimo", "iguana/kb-whisper"}, *r.calls, "the 40-second engine runs first, so a failure there is known early; the fallback is not touched") assert.Equal(t, "Totalt 1050 kronor idag.", got.Text) assert.Equal(t, "iguana/pianissimo + iguana/kb-whisper, reconciled by iguana/gemma4-26b", got.Endpoint, "every contributor is recorded: which engines touched the audio is a quality and a privacy fact") require.NotNil(t, got.Reconciliation) assert.Equal(t, domain.ReconcileDone, got.Reconciliation.Status) } func TestEnsemble_IdenticalTextsNeverCallTheModel(t *testing.T) { adj := &verdictAdj{v: reconcile.VerdictAlt} r := newEnsemble("Vi går igenom läget.", "Vi går igenom läget.", nil, nil, false, adj) got, err := r.e.Transcribe(context.Background(), "/a.wav") require.NoError(t, err) assert.Zero(t, adj.calls) assert.Equal(t, "Vi går igenom läget.", got.Text) } func TestEnsemble_WhenOneEngineFailsTheOtherIsUsedAndItIsSaidSo(t *testing.T) { tests := []struct { name string baseErr error altErr error wantText string wantEndpoint string wantReasonHas string }{ {"pianissimo down", errors.New("pianissimo wedged"), nil, "alt text", "iguana/kb-whisper", "iguana/pianissimo failed"}, {"kb-whisper down", nil, errors.New("metal oom"), "base text", "iguana/pianissimo", "iguana/kb-whisper failed"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { adj := &verdictAdj{v: reconcile.VerdictAlt} r := newEnsemble("base text", "alt text", tt.baseErr, tt.altErr, true, adj) got, err := r.e.Transcribe(context.Background(), "/a.wav") require.NoError(t, err) assert.Equal(t, tt.wantText, got.Text) assert.Equal(t, tt.wantEndpoint, got.Endpoint) assert.Zero(t, adj.calls, "there is nothing to reconcile") require.NotNil(t, got.Reconciliation) assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status) assert.Contains(t, got.Reconciliation.Reason, tt.wantReasonHas) assert.NotContains(t, *r.calls, "berget/kb-whisper", "one local engine working means audio does not leave the homelab") }) } } func TestEnsemble_OnlyWhenBothLocalEnginesFailDoesTheCloudFallbackRun(t *testing.T) { r := newEnsemble("", "", errors.New("a"), errors.New("b"), true, &verdictAdj{}) got, err := r.e.Transcribe(context.Background(), "/a.wav") require.NoError(t, err) assert.Equal(t, "fallback text", got.Text) assert.Equal(t, "berget/kb-whisper", got.Endpoint, "the row must say berget served it: the audio left the homelab") assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status) assert.Contains(t, got.Reconciliation.Reason, "both local engines failed") } func TestEnsemble_BothFailingWithNoFallbackReportsBothErrors(t *testing.T) { r := newEnsemble("", "", errors.New("pianissimo down"), errors.New("whisper down"), false, &verdictAdj{}) _, err := r.e.Transcribe(context.Background(), "/a.wav") require.Error(t, err) assert.Contains(t, err.Error(), "pianissimo down") assert.Contains(t, err.Error(), "whisper down") } func TestEnsemble_ACancelledJobNeverReachesTheCloud(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() r := newEnsemble("", "", context.Canceled, context.Canceled, true, &verdictAdj{}) _, err := r.e.Transcribe(ctx, "/a.wav") assert.ErrorIs(t, err, context.Canceled) assert.NotContains(t, *r.calls, "berget/kb-whisper", "a cancelled job must not ship its audio to a cloud endpoint as a last act") } func TestEnsemble_AModelThatIsDownGivesTheBaseTextUnreconciledNotAFailedJob(t *testing.T) { adj := &verdictAdj{err: errors.New("gateway down")} // Four separate disagreements, each separated by two shared words, so the // breaker trips after three consecutive failures. r := newEnsemble("a1 s t b1 s t c1 s t d1", "a2 s t b2 s t c2 s t d2", nil, nil, false, adj) got, err := r.e.Transcribe(context.Background(), "/a.wav") require.NoError(t, err) assert.Equal(t, "a1 s t b1 s t c1 s t d1", got.Text) assert.Equal(t, domain.ReconcileSkipped, got.Reconciliation.Status) assert.Equal(t, 3, adj.calls) } // cancelling is a backend that cancels the job's context while it runs, the way a // shutdown arrives in the middle of a long transcription. type cancelling struct { name string cancel context.CancelFunc calls *[]string } func (c *cancelling) Name() string { return c.name } func (c *cancelling) Transcribe(_ context.Context, _ string) (domain.Transcription, error) { *c.calls = append(*c.calls, c.name) c.cancel() return domain.Transcription{}, context.Canceled } func TestEnsemble_AShutdownDuringTheFirstEngineStopsBeforeTheSecondOrTheCloud(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) calls := &[]string{} base := &cancelling{name: "iguana/pianissimo", cancel: cancel, calls: calls} alt := &orderedBackend{name: "iguana/kb-whisper", text: "alt", calls: calls} fb := &orderedBackend{name: "berget/kb-whisper", text: "fb", calls: calls} _, err := transcribe.NewEnsemble(base, alt, fb, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav") assert.ErrorIs(t, err, context.Canceled) assert.Equal(t, []string{"iguana/pianissimo"}, *calls, "no second engine and no cloud once the job is cancelled") } func TestEnsemble_AShutdownDuringTheSecondEngineIsAnErrorNotASingleEngineSuccess(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) calls := &[]string{} base := &orderedBackend{name: "iguana/pianissimo", text: "base", calls: calls} alt := &cancelling{name: "iguana/kb-whisper", cancel: cancel, calls: calls} got, err := transcribe.NewEnsemble(base, alt, nil, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav") assert.ErrorIs(t, err, context.Canceled, "a cancelled job must not be recorded as 'kb-whisper failed'") assert.Empty(t, got.Text) } // The base engine is Swedish-only. A recording that is not Swedish must never reach it, and // there is no second opinion to reconcile: the other engine transcribes alone (tapir#61). func autoCtx() context.Context { return domain.WithLanguage(context.Background(), domain.LangAuto) } func TestEnsemble_AnAutoLanguageUploadNeverReachesTheBaseEngineAndIsNotReconciled(t *testing.T) { adj := &verdictAdj{v: reconcile.VerdictAlt} r := newEnsemble("base text", "alt text", nil, nil, true, adj) got, err := r.e.Transcribe(autoCtx(), "/a.wav") require.NoError(t, err) assert.Equal(t, []string{"iguana/kb-whisper"}, *r.calls, "only the other engine is called") assert.Zero(t, adj.calls, "nothing is adjudicated") assert.Equal(t, "alt text", got.Text) assert.Equal(t, "iguana/kb-whisper", got.Endpoint) assert.Nil(t, got.Reconciliation, "reconciliation does not apply, so none is recorded") } func TestEnsemble_AnAutoLanguageUploadFallsBackWhenItsEngineFailsAndStillSkipsTheBaseEngine(t *testing.T) { r := newEnsemble("base text", "alt text", nil, errors.New("engine down"), true, &verdictAdj{}) got, err := r.e.Transcribe(autoCtx(), "/a.wav") require.NoError(t, err) assert.Equal(t, []string{"iguana/kb-whisper", "berget/kb-whisper"}, *r.calls) assert.Equal(t, "berget/kb-whisper", got.Endpoint) } func TestEnsemble_AnAutoLanguageUploadWithNoFallbackFailsWithoutTouchingTheBaseEngine(t *testing.T) { r := newEnsemble("base text", "alt text", nil, errors.New("engine down"), false, &verdictAdj{}) _, err := r.e.Transcribe(autoCtx(), "/a.wav") require.Error(t, err) assert.Equal(t, []string{"iguana/kb-whisper"}, *r.calls) } func TestEnsemble_ACancelledAutoLanguageJobDoesNotReachTheFallback(t *testing.T) { r := newEnsemble("base text", "alt text", nil, errors.New("engine down"), true, &verdictAdj{}) ctx, cancel := context.WithCancel(autoCtx()) cancel() _, err := r.e.Transcribe(ctx, "/a.wav") require.Error(t, err) assert.NotContains(t, *r.calls, "berget/kb-whisper", "a stopped job must not ship its audio to a cloud endpoint") }