mathiasandClaude Sonnet 5.5
ed9d76b0cf
The other engine transcribes alone, then the cloud fallback if it fails and the job is still live. A cancelled job never reaches the fallback (mutation-checked). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
256 lines
9.8 KiB
Go
256 lines
9.8 KiB
Go
package transcribe_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
|
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
|
)
|
|
|
|
// orderedBackend records, in a shared slice, when it was called.
|
|
type orderedBackend struct {
|
|
name string
|
|
text string
|
|
err error
|
|
calls *[]string
|
|
}
|
|
|
|
func (o *orderedBackend) Name() string { return o.name }
|
|
func (o *orderedBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
|
*o.calls = append(*o.calls, o.name)
|
|
if o.err != nil {
|
|
return domain.Transcription{}, o.err
|
|
}
|
|
return domain.Transcription{Text: o.text, Endpoint: o.name, AudioSeconds: 10}, nil
|
|
}
|
|
|
|
type verdictAdj struct {
|
|
v reconcile.Verdict
|
|
err error
|
|
calls int
|
|
}
|
|
|
|
func (a *verdictAdj) Adjudicate(context.Context, reconcile.Request) (reconcile.Verdict, error) {
|
|
a.calls++
|
|
return a.v, a.err
|
|
}
|
|
|
|
var ensembleOpts = reconcile.Options{Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}
|
|
|
|
type rig struct {
|
|
e *transcribe.Ensemble
|
|
calls *[]string
|
|
adj *verdictAdj
|
|
}
|
|
|
|
func newEnsemble(baseText, altText string, baseErr, altErr error, withFallback bool, adj *verdictAdj) *rig {
|
|
calls := &[]string{}
|
|
base := &orderedBackend{name: "iguana/pianissimo", text: baseText, err: baseErr, calls: calls}
|
|
alt := &orderedBackend{name: "iguana/kb-whisper", text: altText, err: altErr, calls: calls}
|
|
var fb transcribe.Backend
|
|
if withFallback {
|
|
fb = &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
|
|
}
|
|
return &rig{e: transcribe.NewEnsemble(base, alt, fb, adj, ensembleOpts), calls: calls, adj: adj}
|
|
}
|
|
|
|
func TestEnsemble_BothEnginesRunInOrderAndTheirDisagreementIsReconciled(t *testing.T) {
|
|
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
|
r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, true, adj)
|
|
|
|
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"iguana/pianissimo", "iguana/kb-whisper"}, *r.calls,
|
|
"the 40-second engine runs first, so a failure there is known early; the fallback is not touched")
|
|
assert.Equal(t, "Totalt 1050 kronor idag.", got.Text)
|
|
assert.Equal(t, "iguana/pianissimo + iguana/kb-whisper, reconciled by iguana/gemma4-26b", got.Endpoint,
|
|
"every contributor is recorded: which engines touched the audio is a quality and a privacy fact")
|
|
require.NotNil(t, got.Reconciliation)
|
|
assert.Equal(t, domain.ReconcileDone, got.Reconciliation.Status)
|
|
}
|
|
|
|
func TestEnsemble_IdenticalTextsNeverCallTheModel(t *testing.T) {
|
|
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
|
r := newEnsemble("Vi går igenom läget.", "Vi går igenom läget.", nil, nil, false, adj)
|
|
|
|
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
|
|
|
require.NoError(t, err)
|
|
assert.Zero(t, adj.calls)
|
|
assert.Equal(t, "Vi går igenom läget.", got.Text)
|
|
}
|
|
|
|
func TestEnsemble_WhenOneEngineFailsTheOtherIsUsedAndItIsSaidSo(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
baseErr error
|
|
altErr error
|
|
wantText string
|
|
wantEndpoint string
|
|
wantReasonHas string
|
|
}{
|
|
{"pianissimo down", errors.New("pianissimo wedged"), nil, "alt text", "iguana/kb-whisper", "iguana/pianissimo failed"},
|
|
{"kb-whisper down", nil, errors.New("metal oom"), "base text", "iguana/pianissimo", "iguana/kb-whisper failed"},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
|
r := newEnsemble("base text", "alt text", tt.baseErr, tt.altErr, true, adj)
|
|
|
|
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, tt.wantText, got.Text)
|
|
assert.Equal(t, tt.wantEndpoint, got.Endpoint)
|
|
assert.Zero(t, adj.calls, "there is nothing to reconcile")
|
|
require.NotNil(t, got.Reconciliation)
|
|
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
|
assert.Contains(t, got.Reconciliation.Reason, tt.wantReasonHas)
|
|
assert.NotContains(t, *r.calls, "berget/kb-whisper", "one local engine working means audio does not leave the homelab")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEnsemble_OnlyWhenBothLocalEnginesFailDoesTheCloudFallbackRun(t *testing.T) {
|
|
r := newEnsemble("", "", errors.New("a"), errors.New("b"), true, &verdictAdj{})
|
|
|
|
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "fallback text", got.Text)
|
|
assert.Equal(t, "berget/kb-whisper", got.Endpoint, "the row must say berget served it: the audio left the homelab")
|
|
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
|
assert.Contains(t, got.Reconciliation.Reason, "both local engines failed")
|
|
}
|
|
|
|
func TestEnsemble_BothFailingWithNoFallbackReportsBothErrors(t *testing.T) {
|
|
r := newEnsemble("", "", errors.New("pianissimo down"), errors.New("whisper down"), false, &verdictAdj{})
|
|
|
|
_, err := r.e.Transcribe(context.Background(), "/a.wav")
|
|
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "pianissimo down")
|
|
assert.Contains(t, err.Error(), "whisper down")
|
|
}
|
|
|
|
func TestEnsemble_ACancelledJobNeverReachesTheCloud(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
r := newEnsemble("", "", context.Canceled, context.Canceled, true, &verdictAdj{})
|
|
|
|
_, err := r.e.Transcribe(ctx, "/a.wav")
|
|
|
|
assert.ErrorIs(t, err, context.Canceled)
|
|
assert.NotContains(t, *r.calls, "berget/kb-whisper", "a cancelled job must not ship its audio to a cloud endpoint as a last act")
|
|
}
|
|
|
|
func TestEnsemble_AModelThatIsDownGivesTheBaseTextUnreconciledNotAFailedJob(t *testing.T) {
|
|
adj := &verdictAdj{err: errors.New("gateway down")}
|
|
// Four separate disagreements, each separated by two shared words, so the
|
|
// breaker trips after three consecutive failures.
|
|
r := newEnsemble("a1 s t b1 s t c1 s t d1", "a2 s t b2 s t c2 s t d2", nil, nil, false, adj)
|
|
|
|
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "a1 s t b1 s t c1 s t d1", got.Text)
|
|
assert.Equal(t, domain.ReconcileSkipped, got.Reconciliation.Status)
|
|
assert.Equal(t, 3, adj.calls)
|
|
}
|
|
|
|
// cancelling is a backend that cancels the job's context while it runs, the way a
|
|
// shutdown arrives in the middle of a long transcription.
|
|
type cancelling struct {
|
|
name string
|
|
cancel context.CancelFunc
|
|
calls *[]string
|
|
}
|
|
|
|
func (c *cancelling) Name() string { return c.name }
|
|
func (c *cancelling) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
|
*c.calls = append(*c.calls, c.name)
|
|
c.cancel()
|
|
return domain.Transcription{}, context.Canceled
|
|
}
|
|
|
|
func TestEnsemble_AShutdownDuringTheFirstEngineStopsBeforeTheSecondOrTheCloud(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
calls := &[]string{}
|
|
base := &cancelling{name: "iguana/pianissimo", cancel: cancel, calls: calls}
|
|
alt := &orderedBackend{name: "iguana/kb-whisper", text: "alt", calls: calls}
|
|
fb := &orderedBackend{name: "berget/kb-whisper", text: "fb", calls: calls}
|
|
|
|
_, err := transcribe.NewEnsemble(base, alt, fb, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav")
|
|
|
|
assert.ErrorIs(t, err, context.Canceled)
|
|
assert.Equal(t, []string{"iguana/pianissimo"}, *calls, "no second engine and no cloud once the job is cancelled")
|
|
}
|
|
|
|
func TestEnsemble_AShutdownDuringTheSecondEngineIsAnErrorNotASingleEngineSuccess(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
calls := &[]string{}
|
|
base := &orderedBackend{name: "iguana/pianissimo", text: "base", calls: calls}
|
|
alt := &cancelling{name: "iguana/kb-whisper", cancel: cancel, calls: calls}
|
|
|
|
got, err := transcribe.NewEnsemble(base, alt, nil, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav")
|
|
|
|
assert.ErrorIs(t, err, context.Canceled, "a cancelled job must not be recorded as 'kb-whisper failed'")
|
|
assert.Empty(t, got.Text)
|
|
}
|
|
|
|
// The base engine is Swedish-only. A recording that is not Swedish must never reach it, and
|
|
// there is no second opinion to reconcile: the other engine transcribes alone (tapir#61).
|
|
|
|
func autoCtx() context.Context { return domain.WithLanguage(context.Background(), domain.LangAuto) }
|
|
|
|
func TestEnsemble_AnAutoLanguageUploadNeverReachesTheBaseEngineAndIsNotReconciled(t *testing.T) {
|
|
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
|
r := newEnsemble("base text", "alt text", nil, nil, true, adj)
|
|
|
|
got, err := r.e.Transcribe(autoCtx(), "/a.wav")
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"iguana/kb-whisper"}, *r.calls, "only the other engine is called")
|
|
assert.Zero(t, adj.calls, "nothing is adjudicated")
|
|
assert.Equal(t, "alt text", got.Text)
|
|
assert.Equal(t, "iguana/kb-whisper", got.Endpoint)
|
|
assert.Nil(t, got.Reconciliation, "reconciliation does not apply, so none is recorded")
|
|
}
|
|
|
|
func TestEnsemble_AnAutoLanguageUploadFallsBackWhenItsEngineFailsAndStillSkipsTheBaseEngine(t *testing.T) {
|
|
r := newEnsemble("base text", "alt text", nil, errors.New("engine down"), true, &verdictAdj{})
|
|
|
|
got, err := r.e.Transcribe(autoCtx(), "/a.wav")
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"iguana/kb-whisper", "berget/kb-whisper"}, *r.calls)
|
|
assert.Equal(t, "berget/kb-whisper", got.Endpoint)
|
|
}
|
|
|
|
func TestEnsemble_AnAutoLanguageUploadWithNoFallbackFailsWithoutTouchingTheBaseEngine(t *testing.T) {
|
|
r := newEnsemble("base text", "alt text", nil, errors.New("engine down"), false, &verdictAdj{})
|
|
|
|
_, err := r.e.Transcribe(autoCtx(), "/a.wav")
|
|
|
|
require.Error(t, err)
|
|
assert.Equal(t, []string{"iguana/kb-whisper"}, *r.calls)
|
|
}
|
|
|
|
func TestEnsemble_ACancelledAutoLanguageJobDoesNotReachTheFallback(t *testing.T) {
|
|
r := newEnsemble("base text", "alt text", nil, errors.New("engine down"), true, &verdictAdj{})
|
|
ctx, cancel := context.WithCancel(autoCtx())
|
|
cancel()
|
|
|
|
_, err := r.e.Transcribe(ctx, "/a.wav")
|
|
|
|
require.Error(t, err)
|
|
assert.NotContains(t, *r.calls, "berget/kb-whisper", "a stopped job must not ship its audio to a cloud endpoint")
|
|
}
|