internal/adapters/transcribe/ensemble_test.go

@@ -0,0 +1,166 @@
+package transcribe_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
+	"git.d-ma.be/mathias/tapir/internal/domain"
+	"git.d-ma.be/mathias/tapir/internal/reconcile"
+)
+
+// orderedBackend records, in a shared slice, when it was called.
+type orderedBackend struct {
+	name  string
+	text  string
+	err   error
+	calls *[]string
+}
+
+func (o *orderedBackend) Name() string { return o.name }
+func (o *orderedBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
+	*o.calls = append(*o.calls, o.name)
+	if o.err != nil {
+		return domain.Transcription{}, o.err
+	}
+	return domain.Transcription{Text: o.text, Endpoint: o.name, AudioSeconds: 10}, nil
+}
+
+type verdictAdj struct {
+	v     reconcile.Verdict
+	err   error
+	calls int
+}
+
+func (a *verdictAdj) Adjudicate(context.Context, reconcile.Request) (reconcile.Verdict, error) {
+	a.calls++
+	return a.v, a.err
+}
+
+var ensembleOpts = reconcile.Options{Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}
+
+type rig struct {
+	e     *transcribe.Ensemble
+	calls *[]string
+	adj   *verdictAdj
+}
+
+func newEnsemble(baseText, altText string, baseErr, altErr error, withFallback bool, adj *verdictAdj) *rig {
+	calls := &[]string{}
+	base := &orderedBackend{name: "iguana/pianissimo", text: baseText, err: baseErr, calls: calls}
+	alt := &orderedBackend{name: "iguana/kb-whisper", text: altText, err: altErr, calls: calls}
+	var fb transcribe.Backend
+	if withFallback {
+		fb = &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
+	}
+	return &rig{e: transcribe.NewEnsemble(base, alt, fb, adj, ensembleOpts), calls: calls, adj: adj}
+}
+
+func TestEnsemble_BothEnginesRunInOrderAndTheirDisagreementIsReconciled(t *testing.T) {
+	adj := &verdictAdj{v: reconcile.VerdictAlt}
+	r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, true, adj)
+
+	got, err := r.e.Transcribe(context.Background(), "/a.wav")
+
+	require.NoError(t, err)
+	assert.Equal(t, []string{"iguana/pianissimo", "iguana/kb-whisper"}, *r.calls,
+		"the 40-second engine runs first, so a failure there is known early; the fallback is not touched")
+	assert.Equal(t, "Totalt 1050 kronor idag.", got.Text)
+	assert.Equal(t, "iguana/pianissimo + iguana/kb-whisper, reconciled by iguana/gemma4-26b", got.Endpoint,
+		"every contributor is recorded: which engines touched the audio is a quality and a privacy fact")
+	require.NotNil(t, got.Reconciliation)
+	assert.Equal(t, domain.ReconcileDone, got.Reconciliation.Status)
+}
+
+func TestEnsemble_IdenticalTextsNeverCallTheModel(t *testing.T) {
+	adj := &verdictAdj{v: reconcile.VerdictAlt}
+	r := newEnsemble("Vi går igenom läget.", "Vi går igenom läget.", nil, nil, false, adj)
+
+	got, err := r.e.Transcribe(context.Background(), "/a.wav")
+
+	require.NoError(t, err)
+	assert.Zero(t, adj.calls)
+	assert.Equal(t, "Vi går igenom läget.", got.Text)
+}
+
+func TestEnsemble_WhenOneEngineFailsTheOtherIsUsedAndItIsSaidSo(t *testing.T) {
+	tests := []struct {
+		name          string
+		baseErr       error
+		altErr        error
+		wantText      string
+		wantEndpoint  string
+		wantReasonHas string
+	}{
+		{"pianissimo down", errors.New("pianissimo wedged"), nil, "alt text", "iguana/kb-whisper", "iguana/pianissimo failed"},
+		{"kb-whisper down", nil, errors.New("metal oom"), "base text", "iguana/pianissimo", "iguana/kb-whisper failed"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			adj := &verdictAdj{v: reconcile.VerdictAlt}
+			r := newEnsemble("base text", "alt text", tt.baseErr, tt.altErr, true, adj)
+
+			got, err := r.e.Transcribe(context.Background(), "/a.wav")
+
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantText, got.Text)
+			assert.Equal(t, tt.wantEndpoint, got.Endpoint)
+			assert.Zero(t, adj.calls, "there is nothing to reconcile")
+			require.NotNil(t, got.Reconciliation)
+			assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
+			assert.Contains(t, got.Reconciliation.Reason, tt.wantReasonHas)
+			assert.NotContains(t, *r.calls, "berget/kb-whisper", "one local engine working means audio does not leave the homelab")
+		})
+	}
+}
+
+func TestEnsemble_OnlyWhenBothLocalEnginesFailDoesTheCloudFallbackRun(t *testing.T) {
+	r := newEnsemble("", "", errors.New("a"), errors.New("b"), true, &verdictAdj{})
+
+	got, err := r.e.Transcribe(context.Background(), "/a.wav")
+
+	require.NoError(t, err)
+	assert.Equal(t, "fallback text", got.Text)
+	assert.Equal(t, "berget/kb-whisper", got.Endpoint, "the row must say berget served it: the audio left the homelab")
+	assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
+	assert.Contains(t, got.Reconciliation.Reason, "both local engines failed")
+}
+
+func TestEnsemble_BothFailingWithNoFallbackReportsBothErrors(t *testing.T) {
+	r := newEnsemble("", "", errors.New("pianissimo down"), errors.New("whisper down"), false, &verdictAdj{})
+
+	_, err := r.e.Transcribe(context.Background(), "/a.wav")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "pianissimo down")
+	assert.Contains(t, err.Error(), "whisper down")
+}
+
+func TestEnsemble_ACancelledJobNeverReachesTheCloud(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	r := newEnsemble("", "", context.Canceled, context.Canceled, true, &verdictAdj{})
+
+	_, err := r.e.Transcribe(ctx, "/a.wav")
+
+	assert.ErrorIs(t, err, context.Canceled)
+	assert.NotContains(t, *r.calls, "berget/kb-whisper", "a cancelled job must not ship its audio to a cloud endpoint as a last act")
+}
+
+func TestEnsemble_AModelThatIsDownGivesTheBaseTextUnreconciledNotAFailedJob(t *testing.T) {
+	adj := &verdictAdj{err: errors.New("gateway down")}
+	// Four separate disagreements, each separated by two shared words, so the
+	// breaker trips after three consecutive failures.
+	r := newEnsemble("a1 s t b1 s t c1 s t d1", "a2 s t b2 s t c2 s t d2", nil, nil, false, adj)
+
+	got, err := r.e.Transcribe(context.Background(), "/a.wav")
+
+	require.NoError(t, err)
+	assert.Equal(t, "a1 s t b1 s t c1 s t d1", got.Text)
+	assert.Equal(t, domain.ReconcileSkipped, got.Reconciliation.Status)
+	assert.Equal(t, 3, adj.calls)
+}