Upload a recording (audio, or video with the camera off), get a private Swedish transcript, and chat about it. - ADR-033: RLS-scoped private transcript store; ffprobe/ffmpeg intake; gateway transcription (iguana kb-whisper, berget fallback); in-process single-replica worker; chat over the transcript (local models only). - ADR-034: optional two-engine mode (pianissimo + kb-whisper) reconciled by a LOCAL model; off unless TAPIR_RECONCILE_BASE_MODEL is set. - ADR-035: real progress (phases, running/quiet/lost), phase logging, every judged spot stored, merged segments. - Hardening from four adversarial reviews: errors logged and stored as kinds never text; per-engine deadlines; converted-length check; bounded queue; client hang-ups are not errors; strict local-alias check. - Dockerfile: static ffmpeg/ffprobe 7.0.2 pinned by sha256 (#34). Built and inspected by hand; CI does not build the image on a PR, so its first CI build is this merge. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set in the deployment. Closes #33, #34, #42, #43, #44. Co-authored-by: Mathias <mthbqv@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// Backend is one transcription endpoint. *Gateway satisfies it; Name is the value
|
||||
// recorded as the serving endpoint.
|
||||
type Backend interface {
|
||||
Name() string
|
||||
Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error)
|
||||
}
|
||||
|
||||
// Chain tries a primary (local) backend and, if it fails for ANY reason, a
|
||||
// fallback. The fallback exists because iguana can wedge permanently on a Metal
|
||||
// OOM (infra#273) — which is also why a fallback is a data-egress decision: the
|
||||
// berget endpoint is a cloud service, so the result records which one served it.
|
||||
type Chain struct {
|
||||
primary Backend
|
||||
fallback Backend // nil disables the fallback
|
||||
}
|
||||
|
||||
// NewChain builds a Chain. A nil fallback means a primary failure is the result.
|
||||
func NewChain(primary, fallback Backend) *Chain {
|
||||
return &Chain{primary: primary, fallback: fallback}
|
||||
}
|
||||
|
||||
// Transcribe returns the first backend's success. It falls back on any primary
|
||||
// error — transport, timeout, non-2xx, empty transcript — and never inspects a
|
||||
// response body to decide. A cancelled context stops the chain: a job the user or
|
||||
// shutdown cancelled must not ship its audio to a cloud endpoint as a last act.
|
||||
func (c *Chain) Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error) {
|
||||
t, err := c.primary.Transcribe(ctx, wavPath)
|
||||
if err == nil {
|
||||
t.Endpoint = c.primary.Name()
|
||||
return t, nil
|
||||
}
|
||||
if c.fallback == nil || ctx.Err() != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("%s: %w", c.primary.Name(), err)
|
||||
}
|
||||
t, ferr := c.fallback.Transcribe(ctx, wavPath)
|
||||
if ferr != nil {
|
||||
return domain.Transcription{}, errors.Join(
|
||||
fmt.Errorf("%s: %w", c.primary.Name(), err),
|
||||
fmt.Errorf("%s: %w", c.fallback.Name(), ferr))
|
||||
}
|
||||
t.Endpoint = c.fallback.Name()
|
||||
return t, nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
type fakeBackend struct {
|
||||
name string
|
||||
text string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeBackend) Name() string { return f.name }
|
||||
func (f *fakeBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return domain.Transcription{}, f.err
|
||||
}
|
||||
return domain.Transcription{Text: f.text, Endpoint: f.name}, nil
|
||||
}
|
||||
|
||||
func TestChain_PrimarySuccessNeverTouchesTheFallback(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", text: "primary text"}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||
|
||||
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "primary text", got.Text)
|
||||
assert.Equal(t, "iguana/kb-whisper", got.Endpoint)
|
||||
assert.Zero(t, fallback.calls, "audio must not leave the homelab when the local endpoint worked")
|
||||
}
|
||||
|
||||
func TestChain_AnyPrimaryFailureFallsBackAndRecordsTheFallbackEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"server error", &transcribe.StatusError{Code: 500}},
|
||||
{"gateway timeout", &transcribe.StatusError{Code: 504}},
|
||||
{"client error", &transcribe.StatusError{Code: 400}},
|
||||
{"transport failure", errors.New("dial tcp: connection refused")},
|
||||
{"empty transcript", transcribe.ErrEmptyTranscript},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: tt.err}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||
|
||||
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fallback text", got.Text)
|
||||
assert.Equal(t, "berget/kb-whisper", got.Endpoint,
|
||||
"the row must say berget served it: that means the audio left the homelab")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChain_BothFailingReportsBothErrors(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", err: errors.New("berget down")}
|
||||
|
||||
_, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "iguana down")
|
||||
assert.Contains(t, err.Error(), "berget down")
|
||||
}
|
||||
|
||||
func TestChain_WithoutAFallbackAPrimaryFailureIsTheResult(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||
|
||||
_, err := transcribe.NewChain(primary, nil).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "iguana down")
|
||||
}
|
||||
|
||||
func TestChain_ACancelledJobDoesNotFallBack(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: context.Canceled}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", text: "x"}
|
||||
|
||||
_, err := transcribe.NewChain(primary, fallback).Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.Zero(t, fallback.calls, "a cancelled job must not ship the audio to a cloud endpoint")
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// Ensemble transcribes the same WAV with two engines, one after the other, and has a
|
||||
// local model reconcile their disagreements (ADR-034). It satisfies uploads.Transcriber,
|
||||
// so the worker neither knows nor cares that two engines were involved.
|
||||
//
|
||||
// It degrades in steps and never turns a recoverable problem into a failed job:
|
||||
// one engine down → the other's text, said so on the result; the model down → the
|
||||
// base text unreconciled; both engines down → the cloud fallback, if configured, and
|
||||
// recorded as such. A cancelled job stops everything and never reaches the cloud.
|
||||
type Ensemble struct {
|
||||
base, alt Backend
|
||||
fallback Backend // nil = no cloud fallback
|
||||
adj reconcile.Adjudicator
|
||||
opts reconcile.Options
|
||||
}
|
||||
|
||||
// NewEnsemble builds an Ensemble. base is the engine whose text stands where the two
|
||||
// disagree and is the faster one, so it runs first; alt supplies the other reading.
|
||||
func NewEnsemble(base, alt, fallback Backend, adj reconcile.Adjudicator, opts reconcile.Options) *Ensemble {
|
||||
return &Ensemble{base: base, alt: alt, fallback: fallback, adj: adj, opts: opts}
|
||||
}
|
||||
|
||||
// Plan implements uploads.Planner: the phases this ensemble reports, so the page can
|
||||
// show them before they run. The fallback is a phase only when one is configured.
|
||||
func (e *Ensemble) Plan() []progress.Planned {
|
||||
plan := []progress.Planned{
|
||||
{Key: "engine_a", Label: "Transcribing with " + e.base.Name()},
|
||||
{Key: "engine_b", Label: "Transcribing with " + e.alt.Name()},
|
||||
{Key: "compare", Label: "Comparing the two transcripts"},
|
||||
{Key: "decide", Label: "Deciding the differences"},
|
||||
}
|
||||
if e.fallback != nil {
|
||||
plan = append(plan, progress.Planned{Key: "fallback", Label: "Cloud fallback (" + e.fallback.Name() + ")"})
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
// Transcribe implements uploads.Transcriber. Each phase is reported through the
|
||||
// progress.Reporter in ctx (a no-op when nobody listens).
|
||||
func (e *Ensemble) Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error) {
|
||||
rep := progress.From(ctx)
|
||||
b, berr := e.run(ctx, rep, "engine_a", e.base, wavPath)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
a, aerr := e.run(ctx, rep, "engine_b", e.alt, wavPath)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
|
||||
switch {
|
||||
case berr == nil && aerr == nil:
|
||||
out, err := reconcile.Reconcile(ctx, b, a, e.adj, e.opts)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
out.Endpoint = fmt.Sprintf("%s + %s, reconciled by %s", e.base.Name(), e.alt.Name(), e.opts.Reconciler)
|
||||
if rc := out.Reconciliation; rc != nil {
|
||||
rep.Note("reconciled", "status", string(rc.Status), "disputes", rc.Disputes, "auto_resolved", rc.AutoResolved,
|
||||
"adjudicated", rc.Adjudicated, "changed", rc.Changed, "unsure", rc.Unsure, "errors", rc.Errors)
|
||||
}
|
||||
return out, nil
|
||||
case berr == nil:
|
||||
return e.single(b, e.base.Name(), fmt.Sprintf("%s failed (%s)", e.alt.Name(), errkind.Of(aerr))), nil
|
||||
case aerr == nil:
|
||||
return e.single(a, e.alt.Name(), fmt.Sprintf("%s failed (%s)", e.base.Name(), errkind.Of(berr))), nil
|
||||
}
|
||||
|
||||
both := errors.Join(fmt.Errorf("%s: %w", e.base.Name(), berr), fmt.Errorf("%s: %w", e.alt.Name(), aerr))
|
||||
if e.fallback == nil {
|
||||
return domain.Transcription{}, both
|
||||
}
|
||||
f, ferr := e.run(ctx, rep, "fallback", e.fallback, wavPath)
|
||||
if ferr != nil {
|
||||
return domain.Transcription{}, errors.Join(both, fmt.Errorf("%s: %w", e.fallback.Name(), ferr))
|
||||
}
|
||||
return e.single(f, e.fallback.Name(), "both local engines failed"), nil
|
||||
}
|
||||
|
||||
// run is one engine call, reported as a phase. A job that was stopped is not a broken
|
||||
// engine, so a cancelled context is never reported as a failed phase.
|
||||
func (e *Ensemble) run(ctx context.Context, rep progress.Reporter, key string, be Backend, wav string) (domain.Transcription, error) {
|
||||
rep.Begin(key, 0)
|
||||
t, err := be.Transcribe(ctx, wav)
|
||||
switch {
|
||||
case err == nil:
|
||||
rep.Done(key)
|
||||
case ctx.Err() == nil:
|
||||
rep.Fail(key, err)
|
||||
}
|
||||
return t, err
|
||||
}
|
||||
|
||||
// single is a transcript from one engine only: nothing to reconcile, and the result
|
||||
// says why, because a reader should know the second opinion was missing.
|
||||
func (e *Ensemble) single(t domain.Transcription, engine, reason string) domain.Transcription {
|
||||
t.Endpoint = engine
|
||||
t.Reconciliation = &domain.Reconciliation{
|
||||
Status: domain.ReconcileSingle, Reason: reason, Base: e.opts.BaseEngine, Alt: e.opts.AltEngine, Reconciler: e.opts.Reconciler,
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// What the Ensemble reports to the page and the log (ADR-035). The reporter here is a
|
||||
// recording double carried in the context, exactly as the worker's Tracker is.
|
||||
|
||||
type recReporter struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
notes []map[string]any
|
||||
}
|
||||
|
||||
func (r *recReporter) add(e string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.events = append(r.events, e)
|
||||
}
|
||||
|
||||
func (r *recReporter) Begin(key string, _ int) { r.add("begin:" + key) }
|
||||
func (r *recReporter) Advance(key string, _ int) { r.add("advance:" + key) }
|
||||
func (r *recReporter) Done(key string) { r.add("done:" + key) }
|
||||
func (r *recReporter) Fail(key string, _ error) { r.add("fail:" + key) }
|
||||
func (r *recReporter) Note(msg string, kv ...any) {
|
||||
m := map[string]any{"msg": msg}
|
||||
for i := 0; i+1 < len(kv); i += 2 {
|
||||
m[kv[i].(string)] = kv[i+1]
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.notes = append(r.notes, m)
|
||||
r.mu.Unlock()
|
||||
r.add("note:" + msg)
|
||||
}
|
||||
|
||||
func (r *recReporter) joined() string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return strings.Join(r.events, ",")
|
||||
}
|
||||
|
||||
func TestEnsemblePlan_DeclaresBothEnginesAndTheReconcileStepsByName(t *testing.T) {
|
||||
r := newEnsemble("a", "a", nil, nil, false, &verdictAdj{})
|
||||
|
||||
plan := r.e.Plan()
|
||||
|
||||
require.Len(t, plan, 4)
|
||||
assert.Equal(t, []string{"engine_a", "engine_b", "compare", "decide"}, []string{plan[0].Key, plan[1].Key, plan[2].Key, plan[3].Key})
|
||||
assert.Contains(t, plan[0].Label, "iguana/pianissimo", "the page says which engine is running")
|
||||
assert.Contains(t, plan[1].Label, "iguana/kb-whisper")
|
||||
}
|
||||
|
||||
func TestEnsemblePlan_TheCloudFallbackIsAStepOnlyWhenOneIsConfigured(t *testing.T) {
|
||||
with := newEnsemble("a", "a", nil, nil, true, &verdictAdj{}).e.Plan()
|
||||
without := newEnsemble("a", "a", nil, nil, false, &verdictAdj{}).e.Plan()
|
||||
|
||||
assert.Equal(t, "fallback", with[len(with)-1].Key)
|
||||
assert.Len(t, without, 4, "no fallback configured: the page must not promise one")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_BothEnginesAreReportedInTheOrderTheyRun(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, false, &verdictAdj{v: reconcile.VerdictAlt})
|
||||
|
||||
_, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
got := rep.joined()
|
||||
a, aDone := strings.Index(got, "begin:engine_a"), strings.Index(got, "done:engine_a")
|
||||
b, bDone := strings.Index(got, "begin:engine_b"), strings.Index(got, "done:engine_b")
|
||||
require.True(t, a >= 0 && aDone > a && b > aDone && bDone > b, "events out of order: %s", got)
|
||||
assert.NotContains(t, got, "fail:")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_AnEngineThatFailsIsShownFailedAndTheOtherStillRuns(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("text", "text", errors.New("pianissimo down"), nil, false, &verdictAdj{})
|
||||
|
||||
got, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err, "one engine down is a degraded result, not a failed job")
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
ev := rep.joined()
|
||||
assert.Contains(t, ev, "fail:engine_a")
|
||||
assert.Contains(t, ev, "done:engine_b")
|
||||
assert.NotContains(t, ev, "begin:compare", "nothing to compare with one engine")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_TheFallbackIsReportedOnlyWhenBothEnginesFailed(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("t", "t", errors.New("a down"), errors.New("b down"), true, &verdictAdj{})
|
||||
|
||||
_, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
ev := rep.joined()
|
||||
assert.Contains(t, ev, "fail:engine_a")
|
||||
assert.Contains(t, ev, "fail:engine_b")
|
||||
assert.Contains(t, ev, "begin:fallback")
|
||||
assert.Contains(t, ev, "done:fallback")
|
||||
|
||||
rep2 := &recReporter{}
|
||||
ok := newEnsemble("x", "x", nil, nil, true, &verdictAdj{})
|
||||
_, err = ok.e.Transcribe(progress.With(context.Background(), rep2), "/a.wav")
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, rep2.joined(), "fallback", "a healthy run never touches the cloud step")
|
||||
}
|
||||
|
||||
// cancellingBackend ends the job in the middle of its own call, like a shutdown.
|
||||
type cancellingBackend struct {
|
||||
name string
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (c cancellingBackend) Name() string { return c.name }
|
||||
func (c cancellingBackend) Transcribe(ctx context.Context, _ string) (domain.Transcription, error) {
|
||||
c.cancel()
|
||||
return domain.Transcription{}, ctx.Err()
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_AShutdownMidEngineIsNotReportedAsAFailedEngine(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
ctx, cancel := context.WithCancel(progress.With(context.Background(), rep))
|
||||
calls := &[]string{}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: "x", calls: calls}
|
||||
e := transcribe.NewEnsemble(cancellingBackend{name: "iguana/pianissimo", cancel: cancel}, alt, nil, &verdictAdj{}, ensembleOpts)
|
||||
|
||||
_, err := e.Transcribe(ctx, "/a.wav")
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.NotContains(t, rep.joined(), "fail:", "a stopped job is not a broken engine: the page must not blame one")
|
||||
assert.Empty(t, *calls, "and the second engine never starts")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_TheReconcileCountsAreNotedForTheOperator(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, false, &verdictAdj{v: reconcile.VerdictAlt})
|
||||
|
||||
_, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rep.notes, 1)
|
||||
n := rep.notes[0]
|
||||
assert.Equal(t, "reconciled", n["msg"])
|
||||
assert.Equal(t, 1, n["disputes"])
|
||||
assert.Equal(t, 1, n["changed"])
|
||||
assert.Equal(t, 0, n["unsure"])
|
||||
assert.Equal(t, 0, n["errors"])
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// When one engine is down the page says so and why; the "why" is a kind, not the
|
||||
// engine's error text, which can carry provider output (adversarial review F3).
|
||||
|
||||
func TestEnsemble_AFailedEnginesReasonIsAKindNotItsErrorText(t *testing.T) {
|
||||
r := newEnsemble("text", "text", nil, errors.New("kb-whisper said: zebrakaka secret words"), false, &verdictAdj{v: reconcile.VerdictAlt})
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, got.Reconciliation.Reason, "zebrakaka")
|
||||
assert.Contains(t, got.Reconciliation.Reason, "iguana/kb-whisper failed", "which engine failed is still said")
|
||||
}
|
||||
|
||||
func TestEnsemble_AnHTTPFailureShowsItsStatusCodeAsTheKind(t *testing.T) {
|
||||
r := newEnsemble("text", "text", &transcribe.StatusError{Code: 502, Snippet: "zebrakaka"}, nil, false, &verdictAdj{})
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, got.Reconciliation.Reason, "http_502")
|
||||
assert.NotContains(t, got.Reconciliation.Reason, "zebrakaka")
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// orderedBackend records, in a shared slice, when it was called.
|
||||
type orderedBackend struct {
|
||||
name string
|
||||
text string
|
||||
err error
|
||||
calls *[]string
|
||||
}
|
||||
|
||||
func (o *orderedBackend) Name() string { return o.name }
|
||||
func (o *orderedBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||
*o.calls = append(*o.calls, o.name)
|
||||
if o.err != nil {
|
||||
return domain.Transcription{}, o.err
|
||||
}
|
||||
return domain.Transcription{Text: o.text, Endpoint: o.name, AudioSeconds: 10}, nil
|
||||
}
|
||||
|
||||
type verdictAdj struct {
|
||||
v reconcile.Verdict
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (a *verdictAdj) Adjudicate(context.Context, reconcile.Request) (reconcile.Verdict, error) {
|
||||
a.calls++
|
||||
return a.v, a.err
|
||||
}
|
||||
|
||||
var ensembleOpts = reconcile.Options{Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}
|
||||
|
||||
type rig struct {
|
||||
e *transcribe.Ensemble
|
||||
calls *[]string
|
||||
adj *verdictAdj
|
||||
}
|
||||
|
||||
func newEnsemble(baseText, altText string, baseErr, altErr error, withFallback bool, adj *verdictAdj) *rig {
|
||||
calls := &[]string{}
|
||||
base := &orderedBackend{name: "iguana/pianissimo", text: baseText, err: baseErr, calls: calls}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: altText, err: altErr, calls: calls}
|
||||
var fb transcribe.Backend
|
||||
if withFallback {
|
||||
fb = &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
|
||||
}
|
||||
return &rig{e: transcribe.NewEnsemble(base, alt, fb, adj, ensembleOpts), calls: calls, adj: adj}
|
||||
}
|
||||
|
||||
func TestEnsemble_BothEnginesRunInOrderAndTheirDisagreementIsReconciled(t *testing.T) {
|
||||
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
||||
r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, true, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"iguana/pianissimo", "iguana/kb-whisper"}, *r.calls,
|
||||
"the 40-second engine runs first, so a failure there is known early; the fallback is not touched")
|
||||
assert.Equal(t, "Totalt 1050 kronor idag.", got.Text)
|
||||
assert.Equal(t, "iguana/pianissimo + iguana/kb-whisper, reconciled by iguana/gemma4-26b", got.Endpoint,
|
||||
"every contributor is recorded: which engines touched the audio is a quality and a privacy fact")
|
||||
require.NotNil(t, got.Reconciliation)
|
||||
assert.Equal(t, domain.ReconcileDone, got.Reconciliation.Status)
|
||||
}
|
||||
|
||||
func TestEnsemble_IdenticalTextsNeverCallTheModel(t *testing.T) {
|
||||
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
||||
r := newEnsemble("Vi går igenom läget.", "Vi går igenom läget.", nil, nil, false, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, adj.calls)
|
||||
assert.Equal(t, "Vi går igenom läget.", got.Text)
|
||||
}
|
||||
|
||||
func TestEnsemble_WhenOneEngineFailsTheOtherIsUsedAndItIsSaidSo(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
baseErr error
|
||||
altErr error
|
||||
wantText string
|
||||
wantEndpoint string
|
||||
wantReasonHas string
|
||||
}{
|
||||
{"pianissimo down", errors.New("pianissimo wedged"), nil, "alt text", "iguana/kb-whisper", "iguana/pianissimo failed"},
|
||||
{"kb-whisper down", nil, errors.New("metal oom"), "base text", "iguana/pianissimo", "iguana/kb-whisper failed"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
||||
r := newEnsemble("base text", "alt text", tt.baseErr, tt.altErr, true, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.wantText, got.Text)
|
||||
assert.Equal(t, tt.wantEndpoint, got.Endpoint)
|
||||
assert.Zero(t, adj.calls, "there is nothing to reconcile")
|
||||
require.NotNil(t, got.Reconciliation)
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
assert.Contains(t, got.Reconciliation.Reason, tt.wantReasonHas)
|
||||
assert.NotContains(t, *r.calls, "berget/kb-whisper", "one local engine working means audio does not leave the homelab")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsemble_OnlyWhenBothLocalEnginesFailDoesTheCloudFallbackRun(t *testing.T) {
|
||||
r := newEnsemble("", "", errors.New("a"), errors.New("b"), true, &verdictAdj{})
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fallback text", got.Text)
|
||||
assert.Equal(t, "berget/kb-whisper", got.Endpoint, "the row must say berget served it: the audio left the homelab")
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
assert.Contains(t, got.Reconciliation.Reason, "both local engines failed")
|
||||
}
|
||||
|
||||
func TestEnsemble_BothFailingWithNoFallbackReportsBothErrors(t *testing.T) {
|
||||
r := newEnsemble("", "", errors.New("pianissimo down"), errors.New("whisper down"), false, &verdictAdj{})
|
||||
|
||||
_, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "pianissimo down")
|
||||
assert.Contains(t, err.Error(), "whisper down")
|
||||
}
|
||||
|
||||
func TestEnsemble_ACancelledJobNeverReachesTheCloud(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
r := newEnsemble("", "", context.Canceled, context.Canceled, true, &verdictAdj{})
|
||||
|
||||
_, err := r.e.Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.NotContains(t, *r.calls, "berget/kb-whisper", "a cancelled job must not ship its audio to a cloud endpoint as a last act")
|
||||
}
|
||||
|
||||
func TestEnsemble_AModelThatIsDownGivesTheBaseTextUnreconciledNotAFailedJob(t *testing.T) {
|
||||
adj := &verdictAdj{err: errors.New("gateway down")}
|
||||
// Four separate disagreements, each separated by two shared words, so the
|
||||
// breaker trips after three consecutive failures.
|
||||
r := newEnsemble("a1 s t b1 s t c1 s t d1", "a2 s t b2 s t c2 s t d2", nil, nil, false, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "a1 s t b1 s t c1 s t d1", got.Text)
|
||||
assert.Equal(t, domain.ReconcileSkipped, got.Reconciliation.Status)
|
||||
assert.Equal(t, 3, adj.calls)
|
||||
}
|
||||
|
||||
// cancelling is a backend that cancels the job's context while it runs, the way a
|
||||
// shutdown arrives in the middle of a long transcription.
|
||||
type cancelling struct {
|
||||
name string
|
||||
cancel context.CancelFunc
|
||||
calls *[]string
|
||||
}
|
||||
|
||||
func (c *cancelling) Name() string { return c.name }
|
||||
func (c *cancelling) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||
*c.calls = append(*c.calls, c.name)
|
||||
c.cancel()
|
||||
return domain.Transcription{}, context.Canceled
|
||||
}
|
||||
|
||||
func TestEnsemble_AShutdownDuringTheFirstEngineStopsBeforeTheSecondOrTheCloud(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := &[]string{}
|
||||
base := &cancelling{name: "iguana/pianissimo", cancel: cancel, calls: calls}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: "alt", calls: calls}
|
||||
fb := &orderedBackend{name: "berget/kb-whisper", text: "fb", calls: calls}
|
||||
|
||||
_, err := transcribe.NewEnsemble(base, alt, fb, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, []string{"iguana/pianissimo"}, *calls, "no second engine and no cloud once the job is cancelled")
|
||||
}
|
||||
|
||||
func TestEnsemble_AShutdownDuringTheSecondEngineIsAnErrorNotASingleEngineSuccess(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := &[]string{}
|
||||
base := &orderedBackend{name: "iguana/pianissimo", text: "base", calls: calls}
|
||||
alt := &cancelling{name: "iguana/kb-whisper", cancel: cancel, calls: calls}
|
||||
|
||||
got, err := transcribe.NewEnsemble(base, alt, nil, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled, "a cancelled job must not be recorded as 'kb-whisper failed'")
|
||||
assert.Empty(t, got.Text)
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// Package transcribe turns a WAV file into a timed Swedish transcript by calling
|
||||
// the LiteLLM gateway's OpenAI-compatible /audio/transcriptions endpoint
|
||||
// (ADR-033). A Gateway is one model alias; a Chain tries a local one and falls
|
||||
// back to a second.
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// ErrEmptyTranscript is returned when the endpoint answered 200 but with no text.
|
||||
// An empty transcript is not a success: chat over it could only answer "nothing".
|
||||
var ErrEmptyTranscript = errors.New("transcribe: endpoint returned an empty transcript")
|
||||
|
||||
// StatusError is a non-2xx answer from the gateway. Nothing may branch on the
|
||||
// body — {"error":"failed to process audio"} has had at least five distinct
|
||||
// causes — so only the status code is exposed for logic; Snippet is for logs.
|
||||
type StatusError struct {
|
||||
Code int
|
||||
Snippet string
|
||||
}
|
||||
|
||||
// HTTPStatus implements errkind.HTTPStatuser: the code, never the snippet.
|
||||
func (e *StatusError) HTTPStatus() int { return e.Code }
|
||||
|
||||
func (e *StatusError) Error() string {
|
||||
return fmt.Sprintf("transcribe: gateway returned HTTP %d: %s", e.Code, e.Snippet)
|
||||
}
|
||||
|
||||
// Gateway transcribes through one model alias on the gateway. baseURL is the
|
||||
// gateway's /v1 root — the in-cluster NodePort, not the public hostname, whose
|
||||
// ingress rejects bodies over ~8-10 MB with 413 (docs/research/audio-length-ceiling).
|
||||
type Gateway struct {
|
||||
baseURL string
|
||||
key string
|
||||
model string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewGateway builds a Gateway. The client should carry no overall Timeout: a 90
|
||||
// minute recording takes minutes, so the deadline belongs to the caller's context.
|
||||
func NewGateway(baseURL, key, model string, client *http.Client) *Gateway {
|
||||
return &Gateway{baseURL: strings.TrimRight(baseURL, "/"), key: key, model: model, client: client}
|
||||
}
|
||||
|
||||
// Name is the model alias; it is what gets recorded as the serving endpoint.
|
||||
func (g *Gateway) Name() string { return g.model }
|
||||
|
||||
// Transcribe uploads the WAV at path and returns the normalised transcription.
|
||||
// The body is streamed from disk with a COMPUTED Content-Length: a 90 minute WAV
|
||||
// is ~170 MB, so it is neither buffered in memory nor sent chunked.
|
||||
func (g *Gateway) Transcribe(ctx context.Context, path string) (domain.Transcription, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: open audio: %w", err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: stat audio: %w", err)
|
||||
}
|
||||
|
||||
var prefix bytes.Buffer
|
||||
mw := multipart.NewWriter(&prefix)
|
||||
for _, kv := range [][2]string{{"model", g.model}, {"response_format", "verbose_json"}, {"language", "sv"}} {
|
||||
if err := mw.WriteField(kv[0], kv[1]); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
}
|
||||
// The multipart field is `file`; `audio_file` is rejected as "Invalid request".
|
||||
if _, err := mw.CreateFormFile("file", "audio.wav"); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
suffix := []byte("\r\n--" + mw.Boundary() + "--\r\n")
|
||||
|
||||
body := io.MultiReader(bytes.NewReader(prefix.Bytes()), f, bytes.NewReader(suffix))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.baseURL+"/audio/transcriptions", body)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
req.ContentLength = int64(prefix.Len()) + info.Size() + int64(len(suffix))
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer "+g.key)
|
||||
// No "Expect: 100-continue": it trips the proxy for any body over 1 KB.
|
||||
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
snip, _ := io.ReadAll(io.LimitReader(resp.Body, 200))
|
||||
return domain.Transcription{}, &StatusError{Code: resp.StatusCode, Snippet: strings.TrimSpace(string(snip))}
|
||||
}
|
||||
|
||||
t, err := parseVerbose(resp.Body)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||
}
|
||||
t.Endpoint = g.model
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// verboseJSON is the subset of the verbose_json response used. Both backends
|
||||
// return it; they differ in segment granularity (iguana: whole seconds and
|
||||
// sub-sentence pieces, berget: sub-second sentences), in the language token, and in
|
||||
// whether a segment's text starts with a space.
|
||||
type verboseJSON struct {
|
||||
Text string `json:"text"`
|
||||
Duration float64 `json:"duration"`
|
||||
Segments []struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
|
||||
// parseVerbose normalises a verbose_json body. Whisper's final window can end past
|
||||
// the audio (9.0 s for 4.25 s of audio was observed), so segment ends are clamped
|
||||
// to the reported duration. Text is rebuilt from the segments, not taken from the
|
||||
// response's `text` field: see joinSegments.
|
||||
func parseVerbose(r io.Reader) (domain.Transcription, error) {
|
||||
var v verboseJSON
|
||||
if err := json.NewDecoder(r).Decode(&v); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
raw := make([]string, 0, len(v.Segments))
|
||||
for _, s := range v.Segments {
|
||||
raw = append(raw, s.Text)
|
||||
}
|
||||
text := joinSegments(raw)
|
||||
if text == "" {
|
||||
text = collapseSpace(v.Text)
|
||||
}
|
||||
if text == "" {
|
||||
return domain.Transcription{}, ErrEmptyTranscript
|
||||
}
|
||||
t := domain.Transcription{Text: text, AudioSeconds: v.Duration}
|
||||
for _, s := range v.Segments {
|
||||
seg := domain.Segment{Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text)}
|
||||
if v.Duration > 0 {
|
||||
seg.End = min(seg.End, v.Duration)
|
||||
seg.Start = min(seg.Start, seg.End)
|
||||
}
|
||||
t.Segments = append(t.Segments, seg)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// joinSegments rebuilds the running text from raw (untrimmed) segment texts.
|
||||
//
|
||||
// whisper.cpp wraps its output at ~60 characters and a wrap can land INSIDE a word
|
||||
// ("nyt" | "t godkännande"): a segment that begins with a space starts a new word,
|
||||
// one that does not continues the previous one, and the response's own `text`
|
||||
// field joins them with newlines, which breaks the word. So when any segment
|
||||
// carries a leading space the raw texts are concatenated as they are. Backends
|
||||
// that return whole sentences with no leading space (berget) are joined with a
|
||||
// space instead — concatenating those would glue sentences together.
|
||||
func joinSegments(raw []string) string {
|
||||
leadingSpace := false
|
||||
for _, s := range raw {
|
||||
if strings.HasPrefix(s, " ") {
|
||||
leadingSpace = true
|
||||
break
|
||||
}
|
||||
}
|
||||
sep := " "
|
||||
if leadingSpace {
|
||||
sep = ""
|
||||
}
|
||||
return collapseSpace(strings.Join(raw, sep))
|
||||
}
|
||||
|
||||
// collapseSpace turns every run of whitespace (including newlines) into one space.
|
||||
func collapseSpace(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
)
|
||||
|
||||
// The httptest server stands in for the LiteLLM gateway: it is the process
|
||||
// boundary, so a fake here is legitimate. The JSON it returns mirrors what the
|
||||
// real gateway returned for iguana/kb-whisper and berget/kb-whisper on 2026-10-01
|
||||
// (docs/research/audio-length-ceiling-2026-10-01.md). All text is synthetic.
|
||||
|
||||
const verboseJSON = `{
|
||||
"text": " God morgon allihopa.\n Vi går igenom läget.\n",
|
||||
"language": "swedish",
|
||||
"duration": 4.25,
|
||||
"segments": [
|
||||
{"id": 0, "start": 0.0, "end": 1.0, "text": " God morgon allihopa."},
|
||||
{"id": 1, "start": 1.0, "end": 9.0, "text": " Vi går igenom läget."}
|
||||
]
|
||||
}`
|
||||
|
||||
func wavFile(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "a.wav")
|
||||
require.NoError(t, os.WriteFile(p, []byte(content), 0o600))
|
||||
return p
|
||||
}
|
||||
|
||||
type seen struct {
|
||||
method, path, auth, expect, contentType string
|
||||
contentLength int64
|
||||
transferEncoding []string
|
||||
fields map[string]string
|
||||
file string
|
||||
fileField string
|
||||
bodyLen int
|
||||
}
|
||||
|
||||
// serve starts a gateway fake that records the request it received and replies
|
||||
// with status and body.
|
||||
func serve(t *testing.T, status int, reply string, got *seen) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got != nil {
|
||||
got.method, got.path = r.Method, r.URL.Path
|
||||
got.auth = r.Header.Get("Authorization")
|
||||
got.expect = r.Header.Get("Expect")
|
||||
got.contentType = r.Header.Get("Content-Type")
|
||||
got.contentLength = r.ContentLength
|
||||
got.transferEncoding = r.TransferEncoding
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
got.bodyLen = len(raw)
|
||||
r.Body = io.NopCloser(bytes.NewReader(raw))
|
||||
require.NoError(t, r.ParseMultipartForm(32<<20))
|
||||
got.fields = map[string]string{}
|
||||
for k, v := range r.MultipartForm.Value {
|
||||
got.fields[k] = v[0]
|
||||
}
|
||||
for field, fhs := range r.MultipartForm.File {
|
||||
got.fileField = field
|
||||
f, _ := fhs[0].Open()
|
||||
b, _ := io.ReadAll(f)
|
||||
got.file = string(b)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, reply)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestGateway_SendsTheRequestTheGatewayExpects(t *testing.T) {
|
||||
var got seen
|
||||
srv := serve(t, 200, verboseJSON, &got)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "secret-key", "iguana/kb-whisper", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "RIFF-synthetic-pcm"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.MethodPost, got.method)
|
||||
assert.Equal(t, "/v1/audio/transcriptions", got.path)
|
||||
assert.Equal(t, "Bearer secret-key", got.auth)
|
||||
assert.Equal(t, "file", got.fileField, "the multipart field is `file`; `audio_file` is rejected as 'Invalid request'")
|
||||
assert.Equal(t, "RIFF-synthetic-pcm", got.file)
|
||||
assert.Equal(t, "iguana/kb-whisper", got.fields["model"])
|
||||
assert.Equal(t, "verbose_json", got.fields["response_format"])
|
||||
assert.Equal(t, "sv", got.fields["language"])
|
||||
assert.Empty(t, got.expect, "no Expect: 100-continue: it trips the proxy for any body over 1 KB")
|
||||
}
|
||||
|
||||
func TestGateway_SendsAKnownContentLengthNotChunked(t *testing.T) {
|
||||
var got seen
|
||||
srv := serve(t, 200, verboseJSON, &got)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "0123456789"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got.transferEncoding, "a 90-minute WAV is ~170 MB: stream it with a computed length, not chunked and not buffered")
|
||||
assert.Equal(t, int64(got.bodyLen), got.contentLength)
|
||||
}
|
||||
|
||||
func TestGateway_NormalisesTheVerboseJSONResponse(t *testing.T) {
|
||||
srv := serve(t, 200, verboseJSON, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "God morgon allihopa. Vi går igenom läget.", got.Text)
|
||||
assert.Equal(t, "iguana/kb-whisper", got.Endpoint, "the serving endpoint is part of the result")
|
||||
assert.InDelta(t, 4.25, got.AudioSeconds, 1e-9)
|
||||
assert.Equal(t, []domain.Segment{
|
||||
{Start: 0, End: 1, Text: "God morgon allihopa."},
|
||||
// The final segment ended at 9.0 s for 4.25 s of audio: whisper's last
|
||||
// window can run past the end. Clamped to the audio duration.
|
||||
{Start: 1, End: 4.25, Text: "Vi går igenom läget."},
|
||||
}, got.Segments)
|
||||
}
|
||||
|
||||
func TestGateway_ErrorsDoNotDependOnTheBody(t *testing.T) {
|
||||
// {"error":"failed to process audio"} has had at least five distinct causes, so
|
||||
// no code may branch on it. Two different bodies must be treated identically.
|
||||
for _, reply := range []string{`{"error":"failed to process audio"}`, `{"error":"something else entirely"}`} {
|
||||
srv := serve(t, 500, reply, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
var se *transcribe.StatusError
|
||||
require.ErrorAs(t, err, &se)
|
||||
assert.Equal(t, 500, se.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGateway_AnEmptyTranscriptIsAFailureNotASuccess(t *testing.T) {
|
||||
srv := serve(t, 200, `{"text":" ","language":"swedish","duration":3,"segments":[]}`, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
assert.ErrorIs(t, err, transcribe.ErrEmptyTranscript)
|
||||
}
|
||||
|
||||
func TestGateway_MalformedJSONIsAnError(t *testing.T) {
|
||||
srv := serve(t, 200, `not json`, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestGateway_MissingAudioFileIsAnError(t *testing.T) {
|
||||
g := transcribe.NewGateway("http://127.0.0.1:1/v1", "k", "m", http.DefaultClient)
|
||||
|
||||
_, err := g.Transcribe(context.Background(), filepath.Join(t.TempDir(), "nope.wav"))
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// whisper.cpp wraps its output at ~60 characters, and a wrap can land INSIDE a word:
|
||||
// the continuation segment has no leading space, the `text` field joins segments
|
||||
// with newlines, and a segment can be a lone ".". Measured on real gateway output
|
||||
// 2026-10-01: 7% of a one-hour recording's segments were such fragments. Joining by
|
||||
// newline hands the chat model broken words ("nyt\nt"), so Text is rebuilt from the
|
||||
// segments: a segment that begins with a space starts a new word, one that does not
|
||||
// continues the previous one.
|
||||
func TestGateway_RejoinsWordsWhisperCppSplitAcrossSegments(t *testing.T) {
|
||||
reply := `{
|
||||
"text": " Vi ligger just nu 12 procent över den ursprungliga kalkylen\n.\n Det kräver ett nyt\nt godkännande från beställaren.\n Framförallt på arbetspl\natsen.\n",
|
||||
"language": "swedish", "duration": 20,
|
||||
"segments": [
|
||||
{"start": 0, "end": 4, "text": " Vi ligger just nu 12 procent över den ursprungliga kalkylen"},
|
||||
{"start": 4, "end": 5, "text": "."},
|
||||
{"start": 5, "end": 8, "text": " Det kräver ett nyt"},
|
||||
{"start": 8, "end": 11, "text": "t godkännande från beställaren."},
|
||||
{"start": 11, "end": 14, "text": " Framförallt på arbetspl"},
|
||||
{"start": 14, "end": 16, "text": "atsen."}
|
||||
]}`
|
||||
srv := serve(t, 200, reply, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Vi ligger just nu 12 procent över den ursprungliga kalkylen. "+
|
||||
"Det kräver ett nytt godkännande från beställaren. Framförallt på arbetsplatsen.", got.Text)
|
||||
assert.Len(t, got.Segments, 6, "the timed segments are kept as the backend gave them (trimmed)")
|
||||
assert.Equal(t, ".", got.Segments[1].Text)
|
||||
}
|
||||
|
||||
func TestGateway_SentenceLevelSegmentsWithNoLeadingSpaceAreJoinedWithASpace(t *testing.T) {
|
||||
// berget/kb-whisper returns whole sentences with no leading space; rebuilding by
|
||||
// concatenation would glue them ("allihopa!Vi").
|
||||
reply := `{"text": "God morgon allihopa!\nVi går igenom läget.", "language": "sv", "duration": 5,
|
||||
"segments": [
|
||||
{"start": 0.05, "end": 1.2, "text": "God morgon allihopa!"},
|
||||
{"start": 1.6, "end": 4.2, "text": "Vi går igenom läget."}
|
||||
]}`
|
||||
srv := serve(t, 200, reply, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "berget/kb-whisper", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "God morgon allihopa! Vi går igenom läget.", got.Text)
|
||||
}
|
||||
|
||||
func TestGateway_WithNoSegmentsTheTextFieldIsUsedWithWhitespaceCollapsed(t *testing.T) {
|
||||
srv := serve(t, 200, `{"text": " Hej allihopa.\n Vi börjar. ", "duration": 3, "segments": []}`, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Hej allihopa. Vi börjar.", got.Text)
|
||||
}
|
||||
|
||||
func TestStatusError_ExposesItsCodeAsAKindAndNotItsSnippet(t *testing.T) {
|
||||
err := fmt.Errorf("transcribe: %w", &transcribe.StatusError{Code: 502, Snippet: "echoed: Anders ansvarar"})
|
||||
|
||||
assert.Equal(t, "http_502", errkind.Of(err))
|
||||
}
|
||||
|
||||
func TestGateway_AnErrorBodyIsBoundedToTwoHundredBytesInTheStatusError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte(strings.Repeat("x", 100_000)))
|
||||
}))
|
||||
defer srv.Close()
|
||||
wav := filepath.Join(t.TempDir(), "a.wav")
|
||||
require.NoError(t, os.WriteFile(wav, []byte("RIFF....WAVEfmt "), 0o600))
|
||||
|
||||
_, err := transcribe.NewGateway(srv.URL, "", "iguana/kb-whisper", srv.Client()).Transcribe(context.Background(), wav)
|
||||
|
||||
var se *transcribe.StatusError
|
||||
require.ErrorAs(t, err, &se)
|
||||
assert.LessOrEqual(t, len(se.Snippet), 200, "a gateway's whole body must not become part of an error")
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
)
|
||||
|
||||
// TestGateway_LiveGateway is an opt-in check against the REAL gateway, because
|
||||
// the httptest fake above can only prove what we think the gateway accepts. It
|
||||
// is skipped unless TAPIR_LIVE_WAV names a 16 kHz mono WAV and the gateway
|
||||
// coordinates are set:
|
||||
//
|
||||
// TAPIR_LIVE_WAV=/path/a.wav TAPIR_LIVE_GATEWAY_URL=http://koala:30401/v1 \
|
||||
// TAPIR_LIVE_MODEL=iguana/kb-whisper op run --env-file ~/.op-env -- \
|
||||
// go test ./internal/adapters/transcribe -run LiveGateway -v
|
||||
//
|
||||
// Use SYNTHETIC audio only: this repo is public and the gateway may be a cloud
|
||||
// endpoint (#33).
|
||||
func TestGateway_LiveGateway(t *testing.T) {
|
||||
wav := os.Getenv("TAPIR_LIVE_WAV")
|
||||
url := os.Getenv("TAPIR_LIVE_GATEWAY_URL")
|
||||
if wav == "" || url == "" {
|
||||
t.Skip("set TAPIR_LIVE_WAV and TAPIR_LIVE_GATEWAY_URL to run against the real gateway")
|
||||
}
|
||||
model := os.Getenv("TAPIR_LIVE_MODEL")
|
||||
if model == "" {
|
||||
model = "iguana/kb-whisper"
|
||||
}
|
||||
key := os.Getenv("LITELLM_MASTER_KEY")
|
||||
require.NotEmpty(t, key, "LITELLM_MASTER_KEY must be set (via op run)")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute)
|
||||
defer cancel()
|
||||
got, err := transcribe.NewGateway(url, key, model, &http.Client{}).Transcribe(ctx, wav)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, got.Text)
|
||||
assert.NotEmpty(t, got.Segments)
|
||||
assert.Equal(t, model, got.Endpoint)
|
||||
t.Logf("endpoint=%s audio_s=%.1f segments=%d chars=%d", got.Endpoint, got.AudioSeconds, len(got.Segments), len(got.Text))
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// WithTimeout gives every call to b its own deadline. A wedged engine does not answer at
|
||||
// all (iguana after a Metal OOM, infra#273), and with only the job's deadline it used up
|
||||
// the whole job before a fallback could run -- and Chain then refuses the fallback because
|
||||
// the job context has expired. With its own deadline the call fails while the job is still
|
||||
// alive, which is exactly when the fallback, or the other engine, can take over.
|
||||
//
|
||||
// A zero or negative d returns b unchanged. The wrapper is invisible: Name is b's.
|
||||
func WithTimeout(b Backend, d time.Duration) Backend {
|
||||
if d <= 0 {
|
||||
return b
|
||||
}
|
||||
return timeoutBackend{Backend: b, d: d}
|
||||
}
|
||||
|
||||
type timeoutBackend struct {
|
||||
Backend
|
||||
d time.Duration
|
||||
}
|
||||
|
||||
func (t timeoutBackend) Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, t.d)
|
||||
defer cancel()
|
||||
return t.Backend.Transcribe(ctx, wavPath)
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// A wedged engine (iguana after a Metal OOM, infra#273) does not answer at all. With only
|
||||
// the job's deadline, the primary used up the whole job before any fallback could run, and
|
||||
// the chain then refused the fallback because the job context had expired. Each engine call
|
||||
// gets its own deadline (adversarial review F4).
|
||||
|
||||
type hangingBackend struct{ name string }
|
||||
|
||||
func (h hangingBackend) Name() string { return h.name }
|
||||
func (hangingBackend) Transcribe(ctx context.Context, _ string) (domain.Transcription, error) {
|
||||
<-ctx.Done()
|
||||
return domain.Transcription{}, ctx.Err()
|
||||
}
|
||||
|
||||
func TestWithTimeout_AHungBackendFailsAtItsOwnDeadlineWhileTheJobIsStillAlive(t *testing.T) {
|
||||
job, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
b := transcribe.WithTimeout(hangingBackend{name: "iguana/kb-whisper"}, 40*time.Millisecond)
|
||||
|
||||
start := time.Now()
|
||||
_, err := b.Transcribe(job, "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "timeout", errkind.Of(err))
|
||||
assert.Less(t, time.Since(start), 2*time.Second, "it must give up at its own deadline, not wait for the job's")
|
||||
assert.NoError(t, job.Err(), "the job context is untouched, which is what lets a fallback run")
|
||||
assert.Equal(t, "iguana/kb-whisper", b.Name(), "the wrapper is invisible in the recorded endpoint")
|
||||
}
|
||||
|
||||
func TestWithTimeout_ASuccessPassesStraightThrough(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
inner := &orderedBackend{name: "iguana/kb-whisper", text: "hej", calls: calls}
|
||||
|
||||
got, err := transcribe.WithTimeout(inner, time.Minute).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hej", got.Text)
|
||||
}
|
||||
|
||||
func TestWithTimeout_ZeroMeansNoDeadlineOfItsOwn(t *testing.T) {
|
||||
inner := &orderedBackend{name: "x", text: "hej", calls: &[]string{}}
|
||||
|
||||
assert.Same(t, inner, transcribe.WithTimeout(inner, 0), "a zero timeout must not wrap or change anything")
|
||||
}
|
||||
|
||||
func TestChain_AHungPrimaryReachesTheFallbackWhenItHasItsOwnDeadline(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
fb := &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
|
||||
c := transcribe.NewChain(transcribe.WithTimeout(hangingBackend{name: "iguana/kb-whisper"}, 40*time.Millisecond), fb)
|
||||
job, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
got, err := c.Transcribe(job, "/a.wav")
|
||||
|
||||
require.NoError(t, err, "a wedged primary is exactly what the fallback exists for")
|
||||
assert.Equal(t, "berget/kb-whisper", got.Endpoint)
|
||||
}
|
||||
|
||||
func TestChain_AJobThatEndedWhileThePrimaryHungStillNeverReachesTheFallback(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
fb := &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
|
||||
c := transcribe.NewChain(transcribe.WithTimeout(hangingBackend{name: "iguana/kb-whisper"}, time.Minute), fb)
|
||||
job, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
_, err := c.Transcribe(job, "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, *calls, "the job's own deadline or a cancel must still stop the audio from reaching a cloud endpoint")
|
||||
}
|
||||
|
||||
func TestEnsemble_AHungEngineIsASoftFailureAndTheOtherEnginesTextIsUsed(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: "text from the other engine", calls: calls}
|
||||
e := transcribe.NewEnsemble(
|
||||
transcribe.WithTimeout(hangingBackend{name: "iguana/pianissimo"}, 40*time.Millisecond), alt, nil,
|
||||
&verdictAdj{v: reconcile.VerdictAlt}, ensembleOpts)
|
||||
job, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
got, err := e.Transcribe(job, "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "text from the other engine", got.Text)
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
assert.Contains(t, got.Reconciliation.Reason, "timeout")
|
||||
assert.False(t, errors.Is(err, context.DeadlineExceeded))
|
||||
}
|
||||
Reference in New Issue
Block a user