test(transcribe): gateway client and iguana->berget chain (red)
Gateway: field is `file`, no Expect header, a computed Content-Length (not chunked, not buffered -- a 90 min WAV is ~170 MB), verbose_json normalised with the last segment clamped to the audio duration, errors independent of the body, an empty transcript is a failure. Chain: ANY primary failure falls back and the row records the fallback endpoint; a cancelled job never falls back to the cloud. Red: package transcribe does not exist. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,99 @@
|
|||||||
|
package transcribe_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeBackend struct {
|
||||||
|
name string
|
||||||
|
text string
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeBackend) Name() string { return f.name }
|
||||||
|
func (f *fakeBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||||
|
f.calls++
|
||||||
|
if f.err != nil {
|
||||||
|
return domain.Transcription{}, f.err
|
||||||
|
}
|
||||||
|
return domain.Transcription{Text: f.text, Endpoint: f.name}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_PrimarySuccessNeverTouchesTheFallback(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", text: "primary text"}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||||
|
|
||||||
|
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "primary text", got.Text)
|
||||||
|
assert.Equal(t, "iguana/kb-whisper", got.Endpoint)
|
||||||
|
assert.Zero(t, fallback.calls, "audio must not leave the homelab when the local endpoint worked")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_AnyPrimaryFailureFallsBackAndRecordsTheFallbackEndpoint(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{"server error", &transcribe.StatusError{Code: 500}},
|
||||||
|
{"gateway timeout", &transcribe.StatusError{Code: 504}},
|
||||||
|
{"client error", &transcribe.StatusError{Code: 400}},
|
||||||
|
{"transport failure", errors.New("dial tcp: connection refused")},
|
||||||
|
{"empty transcript", transcribe.ErrEmptyTranscript},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: tt.err}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||||
|
|
||||||
|
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "fallback text", got.Text)
|
||||||
|
assert.Equal(t, "berget/kb-whisper", got.Endpoint,
|
||||||
|
"the row must say berget served it: that means the audio left the homelab")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_BothFailingReportsBothErrors(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", err: errors.New("berget down")}
|
||||||
|
|
||||||
|
_, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "iguana down")
|
||||||
|
assert.Contains(t, err.Error(), "berget down")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_WithoutAFallbackAPrimaryFailureIsTheResult(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||||
|
|
||||||
|
_, err := transcribe.NewChain(primary, nil).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "iguana down")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_ACancelledJobDoesNotFallBack(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: context.Canceled}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", text: "x"}
|
||||||
|
|
||||||
|
_, err := transcribe.NewChain(primary, fallback).Transcribe(ctx, "/a.wav")
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, context.Canceled)
|
||||||
|
assert.Zero(t, fallback.calls, "a cancelled job must not ship the audio to a cloud endpoint")
|
||||||
|
}
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
package transcribe_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The httptest server stands in for the LiteLLM gateway: it is the process
|
||||||
|
// boundary, so a fake here is legitimate. The JSON it returns mirrors what the
|
||||||
|
// real gateway returned for iguana/kb-whisper and berget/kb-whisper on 2026-10-01
|
||||||
|
// (docs/research/audio-length-ceiling-2026-10-01.md). All text is synthetic.
|
||||||
|
|
||||||
|
const verboseJSON = `{
|
||||||
|
"text": " God morgon allihopa.\n Vi går igenom läget.\n",
|
||||||
|
"language": "swedish",
|
||||||
|
"duration": 4.25,
|
||||||
|
"segments": [
|
||||||
|
{"id": 0, "start": 0.0, "end": 1.0, "text": " God morgon allihopa."},
|
||||||
|
{"id": 1, "start": 1.0, "end": 9.0, "text": " Vi går igenom läget."}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
func wavFile(t *testing.T, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(t.TempDir(), "a.wav")
|
||||||
|
require.NoError(t, os.WriteFile(p, []byte(content), 0o600))
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
type seen struct {
|
||||||
|
method, path, auth, expect, contentType string
|
||||||
|
contentLength int64
|
||||||
|
transferEncoding []string
|
||||||
|
fields map[string]string
|
||||||
|
file string
|
||||||
|
fileField string
|
||||||
|
bodyLen int
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve starts a gateway fake that records the request it received and replies
|
||||||
|
// with status and body.
|
||||||
|
func serve(t *testing.T, status int, reply string, got *seen) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got != nil {
|
||||||
|
got.method, got.path = r.Method, r.URL.Path
|
||||||
|
got.auth = r.Header.Get("Authorization")
|
||||||
|
got.expect = r.Header.Get("Expect")
|
||||||
|
got.contentType = r.Header.Get("Content-Type")
|
||||||
|
got.contentLength = r.ContentLength
|
||||||
|
got.transferEncoding = r.TransferEncoding
|
||||||
|
raw, _ := io.ReadAll(r.Body)
|
||||||
|
got.bodyLen = len(raw)
|
||||||
|
r.Body = io.NopCloser(bytes.NewReader(raw))
|
||||||
|
require.NoError(t, r.ParseMultipartForm(32<<20))
|
||||||
|
got.fields = map[string]string{}
|
||||||
|
for k, v := range r.MultipartForm.Value {
|
||||||
|
got.fields[k] = v[0]
|
||||||
|
}
|
||||||
|
for field, fhs := range r.MultipartForm.File {
|
||||||
|
got.fileField = field
|
||||||
|
f, _ := fhs[0].Open()
|
||||||
|
b, _ := io.ReadAll(f)
|
||||||
|
got.file = string(b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_, _ = io.WriteString(w, reply)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_SendsTheRequestTheGatewayExpects(t *testing.T) {
|
||||||
|
var got seen
|
||||||
|
srv := serve(t, 200, verboseJSON, &got)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "secret-key", "iguana/kb-whisper", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "RIFF-synthetic-pcm"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, http.MethodPost, got.method)
|
||||||
|
assert.Equal(t, "/v1/audio/transcriptions", got.path)
|
||||||
|
assert.Equal(t, "Bearer secret-key", got.auth)
|
||||||
|
assert.Equal(t, "file", got.fileField, "the multipart field is `file`; `audio_file` is rejected as 'Invalid request'")
|
||||||
|
assert.Equal(t, "RIFF-synthetic-pcm", got.file)
|
||||||
|
assert.Equal(t, "iguana/kb-whisper", got.fields["model"])
|
||||||
|
assert.Equal(t, "verbose_json", got.fields["response_format"])
|
||||||
|
assert.Equal(t, "sv", got.fields["language"])
|
||||||
|
assert.Empty(t, got.expect, "no Expect: 100-continue: it trips the proxy for any body over 1 KB")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_SendsAKnownContentLengthNotChunked(t *testing.T) {
|
||||||
|
var got seen
|
||||||
|
srv := serve(t, 200, verboseJSON, &got)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "0123456789"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, got.transferEncoding, "a 90-minute WAV is ~170 MB: stream it with a computed length, not chunked and not buffered")
|
||||||
|
assert.Equal(t, int64(got.bodyLen), got.contentLength)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_NormalisesTheVerboseJSONResponse(t *testing.T) {
|
||||||
|
srv := serve(t, 200, verboseJSON, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
||||||
|
|
||||||
|
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "God morgon allihopa.\n Vi går igenom läget.", got.Text)
|
||||||
|
assert.Equal(t, "iguana/kb-whisper", got.Endpoint, "the serving endpoint is part of the result")
|
||||||
|
assert.InDelta(t, 4.25, got.AudioSeconds, 1e-9)
|
||||||
|
assert.Equal(t, []domain.Segment{
|
||||||
|
{Start: 0, End: 1, Text: "God morgon allihopa."},
|
||||||
|
// The final segment ended at 9.0 s for 4.25 s of audio: whisper's last
|
||||||
|
// window can run past the end. Clamped to the audio duration.
|
||||||
|
{Start: 1, End: 4.25, Text: "Vi går igenom läget."},
|
||||||
|
}, got.Segments)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_ErrorsDoNotDependOnTheBody(t *testing.T) {
|
||||||
|
// {"error":"failed to process audio"} has had at least five distinct causes, so
|
||||||
|
// no code may branch on it. Two different bodies must be treated identically.
|
||||||
|
for _, reply := range []string{`{"error":"failed to process audio"}`, `{"error":"something else entirely"}`} {
|
||||||
|
srv := serve(t, 500, reply, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
var se *transcribe.StatusError
|
||||||
|
require.ErrorAs(t, err, &se)
|
||||||
|
assert.Equal(t, 500, se.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_AnEmptyTranscriptIsAFailureNotASuccess(t *testing.T) {
|
||||||
|
srv := serve(t, 200, `{"text":" ","language":"swedish","duration":3,"segments":[]}`, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, transcribe.ErrEmptyTranscript)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_MalformedJSONIsAnError(t *testing.T) {
|
||||||
|
srv := serve(t, 200, `not json`, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_MissingAudioFileIsAnError(t *testing.T) {
|
||||||
|
g := transcribe.NewGateway("http://127.0.0.1:1/v1", "k", "m", http.DefaultClient)
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), filepath.Join(t.TempDir(), "nope.wav"))
|
||||||
|
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user