mathiasandClaude Sonnet 5.5
500e11962b
Found by comparing the first real one-hour transcript against pianissimo: whisper.cpp wraps at ~60 chars and can wrap INSIDE a word. The continuation segment has no leading space, the text field joins segments with newlines, and a segment can be a lone '.'. Measured 2026-10-01: 7% of segments (96/1332) in a synthetic hour, 42/1302 lines in the real one. My earlier tests used short sentences and could not see it. berget's segments are whole sentences with no leading space, so the two backends need different joins. Red: Text still joins by newline. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
233 lines
9.0 KiB
Go
233 lines
9.0 KiB
Go
package transcribe_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
|
)
|
|
|
|
// The httptest server stands in for the LiteLLM gateway: it is the process
|
|
// boundary, so a fake here is legitimate. The JSON it returns mirrors what the
|
|
// real gateway returned for iguana/kb-whisper and berget/kb-whisper on 2026-10-01
|
|
// (docs/research/audio-length-ceiling-2026-10-01.md). All text is synthetic.
|
|
|
|
const verboseJSON = `{
|
|
"text": " God morgon allihopa.\n Vi går igenom läget.\n",
|
|
"language": "swedish",
|
|
"duration": 4.25,
|
|
"segments": [
|
|
{"id": 0, "start": 0.0, "end": 1.0, "text": " God morgon allihopa."},
|
|
{"id": 1, "start": 1.0, "end": 9.0, "text": " Vi går igenom läget."}
|
|
]
|
|
}`
|
|
|
|
func wavFile(t *testing.T, content string) string {
|
|
t.Helper()
|
|
p := filepath.Join(t.TempDir(), "a.wav")
|
|
require.NoError(t, os.WriteFile(p, []byte(content), 0o600))
|
|
return p
|
|
}
|
|
|
|
type seen struct {
|
|
method, path, auth, expect, contentType string
|
|
contentLength int64
|
|
transferEncoding []string
|
|
fields map[string]string
|
|
file string
|
|
fileField string
|
|
bodyLen int
|
|
}
|
|
|
|
// serve starts a gateway fake that records the request it received and replies
|
|
// with status and body.
|
|
func serve(t *testing.T, status int, reply string, got *seen) *httptest.Server {
|
|
t.Helper()
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if got != nil {
|
|
got.method, got.path = r.Method, r.URL.Path
|
|
got.auth = r.Header.Get("Authorization")
|
|
got.expect = r.Header.Get("Expect")
|
|
got.contentType = r.Header.Get("Content-Type")
|
|
got.contentLength = r.ContentLength
|
|
got.transferEncoding = r.TransferEncoding
|
|
raw, _ := io.ReadAll(r.Body)
|
|
got.bodyLen = len(raw)
|
|
r.Body = io.NopCloser(bytes.NewReader(raw))
|
|
require.NoError(t, r.ParseMultipartForm(32<<20))
|
|
got.fields = map[string]string{}
|
|
for k, v := range r.MultipartForm.Value {
|
|
got.fields[k] = v[0]
|
|
}
|
|
for field, fhs := range r.MultipartForm.File {
|
|
got.fileField = field
|
|
f, _ := fhs[0].Open()
|
|
b, _ := io.ReadAll(f)
|
|
got.file = string(b)
|
|
}
|
|
}
|
|
w.WriteHeader(status)
|
|
_, _ = io.WriteString(w, reply)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func TestGateway_SendsTheRequestTheGatewayExpects(t *testing.T) {
|
|
var got seen
|
|
srv := serve(t, 200, verboseJSON, &got)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "secret-key", "iguana/kb-whisper", srv.Client())
|
|
|
|
_, err := g.Transcribe(context.Background(), wavFile(t, "RIFF-synthetic-pcm"))
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, http.MethodPost, got.method)
|
|
assert.Equal(t, "/v1/audio/transcriptions", got.path)
|
|
assert.Equal(t, "Bearer secret-key", got.auth)
|
|
assert.Equal(t, "file", got.fileField, "the multipart field is `file`; `audio_file` is rejected as 'Invalid request'")
|
|
assert.Equal(t, "RIFF-synthetic-pcm", got.file)
|
|
assert.Equal(t, "iguana/kb-whisper", got.fields["model"])
|
|
assert.Equal(t, "verbose_json", got.fields["response_format"])
|
|
assert.Equal(t, "sv", got.fields["language"])
|
|
assert.Empty(t, got.expect, "no Expect: 100-continue: it trips the proxy for any body over 1 KB")
|
|
}
|
|
|
|
func TestGateway_SendsAKnownContentLengthNotChunked(t *testing.T) {
|
|
var got seen
|
|
srv := serve(t, 200, verboseJSON, &got)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
|
|
|
_, err := g.Transcribe(context.Background(), wavFile(t, "0123456789"))
|
|
|
|
require.NoError(t, err)
|
|
assert.Empty(t, got.transferEncoding, "a 90-minute WAV is ~170 MB: stream it with a computed length, not chunked and not buffered")
|
|
assert.Equal(t, int64(got.bodyLen), got.contentLength)
|
|
}
|
|
|
|
func TestGateway_NormalisesTheVerboseJSONResponse(t *testing.T) {
|
|
srv := serve(t, 200, verboseJSON, nil)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
|
|
|
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "God morgon allihopa. Vi går igenom läget.", got.Text)
|
|
assert.Equal(t, "iguana/kb-whisper", got.Endpoint, "the serving endpoint is part of the result")
|
|
assert.InDelta(t, 4.25, got.AudioSeconds, 1e-9)
|
|
assert.Equal(t, []domain.Segment{
|
|
{Start: 0, End: 1, Text: "God morgon allihopa."},
|
|
// The final segment ended at 9.0 s for 4.25 s of audio: whisper's last
|
|
// window can run past the end. Clamped to the audio duration.
|
|
{Start: 1, End: 4.25, Text: "Vi går igenom läget."},
|
|
}, got.Segments)
|
|
}
|
|
|
|
func TestGateway_ErrorsDoNotDependOnTheBody(t *testing.T) {
|
|
// {"error":"failed to process audio"} has had at least five distinct causes, so
|
|
// no code may branch on it. Two different bodies must be treated identically.
|
|
for _, reply := range []string{`{"error":"failed to process audio"}`, `{"error":"something else entirely"}`} {
|
|
srv := serve(t, 500, reply, nil)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
|
|
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
|
|
|
var se *transcribe.StatusError
|
|
require.ErrorAs(t, err, &se)
|
|
assert.Equal(t, 500, se.Code)
|
|
}
|
|
}
|
|
|
|
func TestGateway_AnEmptyTranscriptIsAFailureNotASuccess(t *testing.T) {
|
|
srv := serve(t, 200, `{"text":" ","language":"swedish","duration":3,"segments":[]}`, nil)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
|
|
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
|
|
|
assert.ErrorIs(t, err, transcribe.ErrEmptyTranscript)
|
|
}
|
|
|
|
func TestGateway_MalformedJSONIsAnError(t *testing.T) {
|
|
srv := serve(t, 200, `not json`, nil)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
|
|
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
|
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestGateway_MissingAudioFileIsAnError(t *testing.T) {
|
|
g := transcribe.NewGateway("http://127.0.0.1:1/v1", "k", "m", http.DefaultClient)
|
|
|
|
_, err := g.Transcribe(context.Background(), filepath.Join(t.TempDir(), "nope.wav"))
|
|
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
// whisper.cpp wraps its output at ~60 characters, and a wrap can land INSIDE a word:
|
|
// the continuation segment has no leading space, the `text` field joins segments
|
|
// with newlines, and a segment can be a lone ".". Measured on real gateway output
|
|
// 2026-10-01: 7% of a one-hour recording's segments were such fragments. Joining by
|
|
// newline hands the chat model broken words ("nyt\nt"), so Text is rebuilt from the
|
|
// segments: a segment that begins with a space starts a new word, one that does not
|
|
// continues the previous one.
|
|
func TestGateway_RejoinsWordsWhisperCppSplitAcrossSegments(t *testing.T) {
|
|
reply := `{
|
|
"text": " Vi ligger just nu 12 procent över den ursprungliga kalkylen\n.\n Det kräver ett nyt\nt godkännande från beställaren.\n Framförallt på arbetspl\natsen.\n",
|
|
"language": "swedish", "duration": 20,
|
|
"segments": [
|
|
{"start": 0, "end": 4, "text": " Vi ligger just nu 12 procent över den ursprungliga kalkylen"},
|
|
{"start": 4, "end": 5, "text": "."},
|
|
{"start": 5, "end": 8, "text": " Det kräver ett nyt"},
|
|
{"start": 8, "end": 11, "text": "t godkännande från beställaren."},
|
|
{"start": 11, "end": 14, "text": " Framförallt på arbetspl"},
|
|
{"start": 14, "end": 16, "text": "atsen."}
|
|
]}`
|
|
srv := serve(t, 200, reply, nil)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
|
|
|
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "Vi ligger just nu 12 procent över den ursprungliga kalkylen. "+
|
|
"Det kräver ett nytt godkännande från beställaren. Framförallt på arbetsplatsen.", got.Text)
|
|
assert.Len(t, got.Segments, 6, "the timed segments are kept as the backend gave them (trimmed)")
|
|
assert.Equal(t, ".", got.Segments[1].Text)
|
|
}
|
|
|
|
func TestGateway_SentenceLevelSegmentsWithNoLeadingSpaceAreJoinedWithASpace(t *testing.T) {
|
|
// berget/kb-whisper returns whole sentences with no leading space; rebuilding by
|
|
// concatenation would glue them ("allihopa!Vi").
|
|
reply := `{"text": "God morgon allihopa!\nVi går igenom läget.", "language": "sv", "duration": 5,
|
|
"segments": [
|
|
{"start": 0.05, "end": 1.2, "text": "God morgon allihopa!"},
|
|
{"start": 1.6, "end": 4.2, "text": "Vi går igenom läget."}
|
|
]}`
|
|
srv := serve(t, 200, reply, nil)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "berget/kb-whisper", srv.Client())
|
|
|
|
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "God morgon allihopa! Vi går igenom läget.", got.Text)
|
|
}
|
|
|
|
func TestGateway_WithNoSegmentsTheTextFieldIsUsedWithWhitespaceCollapsed(t *testing.T) {
|
|
srv := serve(t, 200, `{"text": " Hej allihopa.\n Vi börjar. ", "duration": 3, "segments": []}`, nil)
|
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
|
|
|
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
|
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "Hej allihopa. Vi börjar.", got.Text)
|
|
}
|