Upload a recording (audio, or video with the camera off), get a private Swedish transcript, and chat about it. - ADR-033: RLS-scoped private transcript store; ffprobe/ffmpeg intake; gateway transcription (iguana kb-whisper, berget fallback); in-process single-replica worker; chat over the transcript (local models only). - ADR-034: optional two-engine mode (pianissimo + kb-whisper) reconciled by a LOCAL model; off unless TAPIR_RECONCILE_BASE_MODEL is set. - ADR-035: real progress (phases, running/quiet/lost), phase logging, every judged spot stored, merged segments. - Hardening from four adversarial reviews: errors logged and stored as kinds never text; per-engine deadlines; converted-length check; bounded queue; client hang-ups are not errors; strict local-alias check. - Dockerfile: static ffmpeg/ffprobe 7.0.2 pinned by sha256 (#34). Built and inspected by hand; CI does not build the image on a PR, so its first CI build is this merge. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set in the deployment. Closes #33, #34, #42, #43, #44. Co-authored-by: Mathias <mthbqv@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// ErrUnreadable is what a Prober returns when the FILE is the problem (not media,
|
||||
// truncated, damaged), as opposed to the prober itself failing. Accept turns it
|
||||
// into a user-facing rejection; any other prober error is our fault and surfaces
|
||||
// as an internal error — a missing ffprobe must not read as "your file is bad".
|
||||
var ErrUnreadable = errors.New("uploads: file is not readable media")
|
||||
|
||||
// Prober inspects a file on disk. The ffprobe adapter implements it.
|
||||
type Prober interface {
|
||||
Probe(ctx context.Context, path string) (Probe, error)
|
||||
}
|
||||
|
||||
// Store is the slice of the private transcript store the intake needs.
|
||||
type Store interface {
|
||||
CreateUpload(ctx context.Context, userID, title string, hadVideo bool) (string, error)
|
||||
FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error
|
||||
}
|
||||
|
||||
// Job is one accepted upload awaiting transcription. Path is a temp file the
|
||||
// receiver owns: it must delete it when finished (the audio is never retained).
|
||||
type Job struct {
|
||||
UserID string
|
||||
UploadID string
|
||||
Path string
|
||||
// Duration is what the file CLAIMED when probed. The worker compares it with what
|
||||
// was really converted; zero means unknown (ADR-035 review).
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// Submitter hands accepted jobs to the in-process worker (ADR-033).
|
||||
type Submitter interface {
|
||||
Submit(Job) error
|
||||
}
|
||||
|
||||
// Intake validates an uploaded recording and, if it passes, queues it.
|
||||
type Intake struct {
|
||||
store Store
|
||||
prober Prober
|
||||
sub Submitter
|
||||
limits Limits
|
||||
tmpDir string
|
||||
}
|
||||
|
||||
// NewIntake wires an Intake. tmpDir is where uploads are spooled; it must exist.
|
||||
func NewIntake(s Store, p Prober, sub Submitter, l Limits, tmpDir string) *Intake {
|
||||
return &Intake{store: s, prober: p, sub: sub, limits: l, tmpDir: tmpDir}
|
||||
}
|
||||
|
||||
// ErrClientRead marks an error that came from READING the client's body: a connection that
|
||||
// dropped, a body cut short. It is the sender's to retry, unlike a spool, probe or database
|
||||
// fault, which is ours. It is a marker on the error, not a sentinel that others may wrap:
|
||||
// io.ErrUnexpectedEOF alone is also what a database drop looks like.
|
||||
var ErrClientRead = errors.New("uploads: client body read failed")
|
||||
|
||||
// clientReader remembers the last non-EOF error its source returned, so Accept can tell
|
||||
// a failure reading the client from a failure writing the spool file.
|
||||
type clientReader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *clientReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
c.err = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// busyMessage is what a user is told when the queue is full.
|
||||
const busyMessage = "Too many recordings are waiting to be transcribed. Please try again in a few minutes."
|
||||
|
||||
// Accept spools src to disk (bounded by MaxBytes), probes it, validates it, records
|
||||
// a queued upload and submits the job. It returns the upload id. A file that fails
|
||||
// validation yields a *RejectError and creates NO row — rejection is synchronous
|
||||
// and leaves nothing behind. On success the temp file is handed to the job and the
|
||||
// worker deletes it; on every failure path Accept deletes it itself.
|
||||
func (in *Intake) Accept(ctx context.Context, userID, title string, src io.Reader) (string, error) {
|
||||
// Refuse before spooling when the queue has no room, if the submitter can say so: a full
|
||||
// queue otherwise cost up to the upload cap on disk, an ffprobe run and a failed row for
|
||||
// every refused attempt.
|
||||
if rc, ok := in.sub.(interface{ HasRoom(userID string) bool }); ok && !rc.HasRoom(userID) {
|
||||
return "", &RejectError{Reason: ReasonBusy, Message: busyMessage}
|
||||
}
|
||||
f, err := os.CreateTemp(in.tmpDir, "upload-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: spool: %w", err)
|
||||
}
|
||||
path := f.Name()
|
||||
handedOver := false
|
||||
defer func() {
|
||||
if !handedOver {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
|
||||
// Read one byte past the cap so "exactly at the limit" and "over" are told apart.
|
||||
cr := &clientReader{r: io.LimitReader(src, in.limits.MaxBytes+1)}
|
||||
n, err := io.Copy(f, cr)
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
if cr.err != nil && errors.Is(err, cr.err) {
|
||||
return "", fmt.Errorf("%w: %w", ErrClientRead, err)
|
||||
}
|
||||
return "", fmt.Errorf("uploads: receive: %w", err)
|
||||
}
|
||||
if n > in.limits.MaxBytes {
|
||||
return "", &RejectError{Reason: ReasonTooLarge, Message: fmt.Sprintf(
|
||||
"This file is larger than the %d MB limit.", in.limits.MaxBytes>>20)}
|
||||
}
|
||||
|
||||
probe, err := in.prober.Probe(ctx, path)
|
||||
if errors.Is(err, ErrUnreadable) {
|
||||
return "", &RejectError{
|
||||
Reason: ReasonUnreadable,
|
||||
Message: "This file could not be read as audio; it may be damaged or not a recording.",
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: probe: %w", err)
|
||||
}
|
||||
if err := Validate(probe, in.limits); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
id, err := in.store.CreateUpload(ctx, userID, title, probe.HasVideo)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: create: %w", err)
|
||||
}
|
||||
if err := in.sub.Submit(Job{UserID: userID, UploadID: id, Path: path, Duration: probe.Duration}); err != nil {
|
||||
if errors.Is(err, ErrQueueFull) {
|
||||
// The user's to retry, not a server fault: refuse it as such, and do not leave
|
||||
// the row queued -- that is the stuck spinner.
|
||||
if ferr := in.store.FailUpload(ctx, userID, id, domain.UploadErrBusy); ferr != nil {
|
||||
return "", fmt.Errorf("uploads: submit: %w (and failing the row: %v)", err, ferr)
|
||||
}
|
||||
return "", &RejectError{Reason: ReasonBusy, Message: busyMessage}
|
||||
}
|
||||
// A row that will never run must not sit queued: that is the stuck spinner.
|
||||
if ferr := in.store.FailUpload(ctx, userID, id, domain.UploadErrInterrupted); ferr != nil {
|
||||
return "", fmt.Errorf("uploads: submit: %w (and failing the row: %v)", err, ferr)
|
||||
}
|
||||
return "", fmt.Errorf("uploads: submit: %w", err)
|
||||
}
|
||||
handedOver = true
|
||||
return id, nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// A full queue is the user's to retry, not a server fault: a typed rejection, the row
|
||||
// failed honestly, the audio removed (adversarial review F6).
|
||||
|
||||
func TestAccept_AFullQueueIsABusyRejectionNotAServerErrorAndLeavesNothingBehind(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.sub.err = uploads.ErrQueueFull
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej), "a full queue must be a rejection the web layer can answer with 429: %v", err)
|
||||
assert.Equal(t, uploads.ReasonBusy, rej.Reason)
|
||||
assert.Contains(t, rej.Message, "waiting")
|
||||
assert.Equal(t, domain.UploadErrBusy, r.st.failed["up-1"], "the row must not sit queued: that is the stuck spinner")
|
||||
assert.Empty(t, r.tmpFiles(t), "the audio is deleted")
|
||||
}
|
||||
|
||||
// roomSubmitter is a Submitter that can say, before anything is spooled, that there is no room.
|
||||
type roomSubmitter struct {
|
||||
fakeSubmitter
|
||||
room bool
|
||||
}
|
||||
|
||||
func (r *roomSubmitter) HasRoom(string) bool { return r.room }
|
||||
|
||||
func TestAccept_ANoRoomQueueIsRefusedBeforeAnythingIsSpooledProbedOrStored(t *testing.T) {
|
||||
// Checking only at Submit meant 200 MB was spooled and ffprobe run for every refused
|
||||
// attempt, and a failed row was left behind each time (second adversarial review).
|
||||
r := newRig(t)
|
||||
full := &roomSubmitter{room: false}
|
||||
in := uploads.NewIntake(r.st, r.pr, full, uploads.Limits{MaxBytes: 1024, MaxDuration: 90 * time.Minute}, r.tmp)
|
||||
|
||||
_, err := in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej))
|
||||
assert.Equal(t, uploads.ReasonBusy, rej.Reason)
|
||||
assert.Empty(t, r.st.created, "no row for an upload that was never accepted")
|
||||
assert.Empty(t, r.pr.paths, "ffprobe was never run on it")
|
||||
assert.Empty(t, r.tmpFiles(t), "nothing was spooled")
|
||||
assert.Empty(t, full.jobs)
|
||||
}
|
||||
|
||||
func TestAccept_ASubmitterThatCannotSayStillWorksAsBefore(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.NoError(t, err, "a Submitter without HasRoom is accepted: the check at Submit remains")
|
||||
}
|
||||
|
||||
// brokenBody is a client body that fails mid-read, like a connection that dropped.
|
||||
type brokenBody struct{ n int }
|
||||
|
||||
func (b *brokenBody) Read(p []byte) (int, error) {
|
||||
if b.n > 0 {
|
||||
b.n = 0
|
||||
return copy(p, "partial audio"), nil
|
||||
}
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
func TestAccept_AnErrorReadingTheClientsBodyIsMarkedAsTheClientsAndAWrappedSubmitErrorIsNot(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", &brokenBody{n: 1})
|
||||
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, uploads.ErrClientRead, "a dropped client body is the sender's to retry")
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
|
||||
// A fault that is ours must NOT carry the marker, even if it wraps the same sentinel.
|
||||
r2 := newRig(t)
|
||||
r2.sub.err = fmt.Errorf("db: %w", io.ErrUnexpectedEOF)
|
||||
_, err = r2.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, uploads.ErrClientRead)
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
created []string // titles
|
||||
hadVideo []bool
|
||||
failed map[string]domain.UploadErrorKind
|
||||
nextID string
|
||||
}
|
||||
|
||||
func (f *fakeStore) CreateUpload(_ context.Context, _, title string, hadVideo bool) (string, error) {
|
||||
f.created = append(f.created, title)
|
||||
f.hadVideo = append(f.hadVideo, hadVideo)
|
||||
return f.nextID, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) FailUpload(_ context.Context, _, id string, kind domain.UploadErrorKind) error {
|
||||
if f.failed == nil {
|
||||
f.failed = map[string]domain.UploadErrorKind{}
|
||||
}
|
||||
f.failed[id] = kind
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeProber struct {
|
||||
probe uploads.Probe
|
||||
err error
|
||||
paths []string
|
||||
seen []byte // content of the file at probe time
|
||||
}
|
||||
|
||||
func (f *fakeProber) Probe(_ context.Context, path string) (uploads.Probe, error) {
|
||||
f.paths = append(f.paths, path)
|
||||
f.seen, _ = os.ReadFile(path)
|
||||
return f.probe, f.err
|
||||
}
|
||||
|
||||
type fakeSubmitter struct {
|
||||
jobs []uploads.Job
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeSubmitter) Submit(j uploads.Job) error {
|
||||
f.jobs = append(f.jobs, j)
|
||||
return f.err
|
||||
}
|
||||
|
||||
type rig struct {
|
||||
in *uploads.Intake
|
||||
st *fakeStore
|
||||
pr *fakeProber
|
||||
sub *fakeSubmitter
|
||||
tmp string
|
||||
}
|
||||
|
||||
func newRig(t *testing.T) *rig {
|
||||
t.Helper()
|
||||
r := &rig{
|
||||
st: &fakeStore{nextID: "up-1"},
|
||||
pr: &fakeProber{probe: uploads.Probe{Duration: 10 * time.Minute, HasAudio: true}},
|
||||
sub: &fakeSubmitter{},
|
||||
tmp: t.TempDir(),
|
||||
}
|
||||
r.in = uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: 90 * time.Minute}, r.tmp)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *rig) tmpFiles(t *testing.T) []string {
|
||||
t.Helper()
|
||||
ents, err := os.ReadDir(r.tmp)
|
||||
require.NoError(t, err)
|
||||
var names []string
|
||||
for _, e := range ents {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestAccept_ValidAudioIsStoredProbedQueuedAndSubmitted(t *testing.T) {
|
||||
r := newRig(t)
|
||||
audio := bytes.Repeat([]byte("a"), 500)
|
||||
|
||||
id, err := r.in.Accept(context.Background(), "user-1", "Teams export", bytes.NewReader(audio))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "up-1", id)
|
||||
assert.Equal(t, []string{"Teams export"}, r.st.created)
|
||||
assert.Equal(t, audio, r.pr.seen, "the prober must see the complete uploaded bytes, on disk")
|
||||
require.Len(t, r.sub.jobs, 1)
|
||||
job := r.sub.jobs[0]
|
||||
assert.Equal(t, uploads.Job{UserID: "user-1", UploadID: "up-1", Path: job.Path, Duration: 10 * time.Minute}, job,
|
||||
"the job carries what the file CLAIMED, so the worker can compare it with what was converted")
|
||||
assert.Equal(t, filepath.Dir(job.Path), r.tmp, "the temp file lives in the configured directory")
|
||||
got, err := os.ReadFile(job.Path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, audio, got, "the handed-over file still exists: the worker, not Accept, deletes it")
|
||||
}
|
||||
|
||||
func TestAccept_OversizeBodyIsRejectedAndLeavesNothingBehind(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "big", strings.NewReader(strings.Repeat("a", 1025)))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej))
|
||||
assert.Equal(t, uploads.ReasonTooLarge, rej.Reason)
|
||||
assert.Contains(t, rej.Message, "limit")
|
||||
assert.Empty(t, r.pr.paths, "a file already over the size cap must not be probed")
|
||||
assert.Empty(t, r.st.created, "a rejected upload creates no row")
|
||||
assert.Empty(t, r.sub.jobs)
|
||||
assert.Empty(t, r.tmpFiles(t), "the partial temp file must be removed")
|
||||
}
|
||||
|
||||
func TestAccept_BodyExactlyAtTheLimitIsAccepted(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "edge", strings.NewReader(strings.Repeat("a", 1024)))
|
||||
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestAccept_ProbeVerdictRejectsWithoutRowAndRemovesTheFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
probe uploads.Probe
|
||||
reason uploads.Reason
|
||||
}{
|
||||
{"video with no audio", uploads.Probe{Duration: time.Minute, HasVideo: true}, uploads.ReasonNoAudio},
|
||||
{"too long", uploads.Probe{Duration: 2 * time.Hour, HasAudio: true}, uploads.ReasonTooLong},
|
||||
{"no audio", uploads.Probe{Duration: time.Minute}, uploads.ReasonNoAudio},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.probe = tt.probe
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej))
|
||||
assert.Equal(t, tt.reason, rej.Reason)
|
||||
assert.Empty(t, r.st.created)
|
||||
assert.Empty(t, r.sub.jobs)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccept_UnreadableFileIsARejectionButAProberFaultIsAnError(t *testing.T) {
|
||||
t.Run("ffprobe says the file is not media", func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.err = uploads.ErrUnreadable
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("not media"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej), "the user's file is the problem: tell them")
|
||||
assert.Equal(t, uploads.ReasonUnreadable, rej.Reason)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
})
|
||||
t.Run("the prober itself broke", func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.err = errors.New("exec: ffprobe: executable file not found")
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.Error(t, err)
|
||||
var rej *uploads.RejectError
|
||||
assert.False(t, errors.As(err, &rej), "our fault must not be reported as the user's bad file")
|
||||
assert.Empty(t, r.st.created)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
})
|
||||
}
|
||||
|
||||
func TestAccept_ClientDisconnectMidUploadLeavesNothingBehind(t *testing.T) {
|
||||
r := newRig(t)
|
||||
body := io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(errors.New("client went away")))
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", body)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, r.pr.paths)
|
||||
assert.Empty(t, r.st.created)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
}
|
||||
|
||||
func TestAccept_SubmitFailureFailsTheRowHonestlyAndRemovesTheFile(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.sub.err = errors.New("worker stopped")
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, domain.UploadErrInterrupted, r.st.failed["up-1"],
|
||||
"a row that will never run must not sit queued: that is the stuck spinner ADR-025 forbids")
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
}
|
||||
|
||||
func TestAccept_AFileWithAVideoTrackIsAcceptedAndRecordsThatItHadOne(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
probe uploads.Probe
|
||||
wantVideo bool
|
||||
}{
|
||||
{"audio only", uploads.Probe{Duration: 10 * time.Minute, HasAudio: true}, false},
|
||||
{"audio with a video track (a Teams meeting with the camera off)", uploads.Probe{Duration: 10 * time.Minute, HasAudio: true, HasVideo: true}, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.probe = tt.probe
|
||||
|
||||
id, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "up-1", id)
|
||||
assert.Equal(t, []bool{tt.wantVideo}, r.st.hadVideo,
|
||||
"the row must say whether a video track was present, so the page can say it was ignored")
|
||||
assert.Len(t, r.sub.jobs, 1, "it is queued like any other recording")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Package uploads is the use case behind "upload an audio recording" (ADR-033):
|
||||
// validate the file by PROBING it, keep the transcript private, never retain the
|
||||
// audio. It depends only on ports (a Prober, a store, a job Submitter) — the
|
||||
// concrete ffprobe adapter, the web handler and the transcription worker are wired
|
||||
// from outside.
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Limits bound what an upload may be. Both are rejected synchronously, with a
|
||||
// specific message, rather than by a timeout (ADR-033).
|
||||
type Limits struct {
|
||||
MaxBytes int64
|
||||
MaxDuration time.Duration
|
||||
}
|
||||
|
||||
// Probe is what inspecting an uploaded file revealed. It is the input to Validate
|
||||
// because a name or MIME type cannot say what a container holds: a Teams recording
|
||||
// is an .mp4 whether or not the camera was on.
|
||||
type Probe struct {
|
||||
Duration time.Duration
|
||||
HasAudio bool
|
||||
HasVideo bool
|
||||
}
|
||||
|
||||
// Reason is the machine-checkable class of a rejection.
|
||||
type Reason string
|
||||
|
||||
const (
|
||||
ReasonNoAudio Reason = "no_audio"
|
||||
ReasonTooLong Reason = "too_long"
|
||||
ReasonTooLarge Reason = "too_large"
|
||||
ReasonUnreadable Reason = "unreadable"
|
||||
// ReasonBusy: the queue is full. Not about the file: the user retries later.
|
||||
ReasonBusy Reason = "busy"
|
||||
)
|
||||
|
||||
// RejectError is a synchronous, user-presentable refusal. Message is written for
|
||||
// the person who uploaded the file; Reason is for code and tests.
|
||||
type RejectError struct {
|
||||
Reason Reason
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *RejectError) Error() string { return string(e.Reason) + ": " + e.Message }
|
||||
|
||||
// Validate decides whether a probed file may be transcribed. A video track is NOT
|
||||
// a reason to refuse: a Teams meeting with the camera off is still an .mp4 with a
|
||||
// video track, so rejecting video would reject the primary use case. Only the audio
|
||||
// is used (Probe.HasVideo is recorded so the user can be told the video was
|
||||
// ignored). What IS refused: no audio at all, an unreadable length, too long.
|
||||
func Validate(p Probe, l Limits) error {
|
||||
switch {
|
||||
case !p.HasAudio:
|
||||
return &RejectError{Reason: ReasonNoAudio, Message: "This file has no audio track to transcribe."}
|
||||
case p.Duration <= 0:
|
||||
return &RejectError{Reason: ReasonUnreadable, Message: "Could not read a length from this file; it may be damaged."}
|
||||
case p.Duration > l.MaxDuration:
|
||||
return &RejectError{Reason: ReasonTooLong, Message: fmt.Sprintf(
|
||||
"This recording is %s long; the limit is %s.", FormatDuration(p.Duration), FormatDuration(l.MaxDuration))}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FormatDuration renders a length for people: "1 h 42 min", "45 min", "30 s".
|
||||
func FormatDuration(d time.Duration) string {
|
||||
d = d.Round(time.Second)
|
||||
h := int(d / time.Hour)
|
||||
m := int(d % time.Hour / time.Minute)
|
||||
switch {
|
||||
case h > 0 && m > 0:
|
||||
return fmt.Sprintf("%d h %d min", h, m)
|
||||
case h > 0:
|
||||
return fmt.Sprintf("%d h", h)
|
||||
case m > 0:
|
||||
return fmt.Sprintf("%d min", m)
|
||||
default:
|
||||
return fmt.Sprintf("%d s", int(d/time.Second))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
var testLimits = uploads.Limits{MaxBytes: 200 << 20, MaxDuration: 90 * time.Minute}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
probe uploads.Probe
|
||||
wantReason uploads.Reason // "" = accepted
|
||||
wantInMsg []string
|
||||
}{
|
||||
{
|
||||
name: "audio-only file within the duration limit is accepted",
|
||||
probe: uploads.Probe{Duration: 45 * time.Minute, HasAudio: true},
|
||||
},
|
||||
{
|
||||
name: "a recording exactly at the limit is accepted",
|
||||
probe: uploads.Probe{Duration: 90 * time.Minute, HasAudio: true},
|
||||
},
|
||||
{
|
||||
name: "a recording that also has a video stream is accepted: the audio is what is used",
|
||||
probe: uploads.Probe{Duration: 4 * time.Minute, HasAudio: true, HasVideo: true},
|
||||
},
|
||||
{
|
||||
name: "video with no audio is rejected as having no audio, and says so",
|
||||
probe: uploads.Probe{Duration: 4 * time.Minute, HasVideo: true},
|
||||
wantReason: uploads.ReasonNoAudio,
|
||||
wantInMsg: []string{"no audio"},
|
||||
},
|
||||
{
|
||||
name: "a file with no audio stream is rejected",
|
||||
probe: uploads.Probe{Duration: 4 * time.Minute},
|
||||
wantReason: uploads.ReasonNoAudio,
|
||||
wantInMsg: []string{"no audio"},
|
||||
},
|
||||
{
|
||||
name: "a recording over the limit is rejected, stating both lengths",
|
||||
probe: uploads.Probe{Duration: 102 * time.Minute, HasAudio: true},
|
||||
wantReason: uploads.ReasonTooLong,
|
||||
wantInMsg: []string{"1 h 42 min", "1 h 30 min"},
|
||||
},
|
||||
{
|
||||
name: "zero duration is unreadable, not accepted as an empty recording",
|
||||
probe: uploads.Probe{HasAudio: true},
|
||||
wantReason: uploads.ReasonUnreadable,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := uploads.Validate(tt.probe, testLimits)
|
||||
|
||||
if tt.wantReason == "" {
|
||||
require.NoError(t, err)
|
||||
return
|
||||
}
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej), "want a *RejectError, got %v", err)
|
||||
assert.Equal(t, tt.wantReason, rej.Reason)
|
||||
for _, s := range tt.wantInMsg {
|
||||
assert.Contains(t, rej.Message, s)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
d time.Duration
|
||||
want string
|
||||
}{
|
||||
{90 * time.Minute, "1 h 30 min"},
|
||||
{102 * time.Minute, "1 h 42 min"},
|
||||
{45 * time.Minute, "45 min"},
|
||||
{2 * time.Hour, "2 h"},
|
||||
{30 * time.Second, "30 s"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.want, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, uploads.FormatDuration(tt.d))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
)
|
||||
|
||||
// ErrStopped is returned by Submit once the worker has stopped.
|
||||
var ErrStopped = errors.New("uploads: worker stopped")
|
||||
|
||||
// ErrQueueFull is returned by Submit when the queue, or one user's share of it, is at
|
||||
// its cap. Every queued job holds up to the upload cap on disk until its turn, so an
|
||||
// unbounded queue is a disk-filling lever (ADR-035 review).
|
||||
var ErrQueueFull = errors.New("uploads: queue full")
|
||||
|
||||
// Converter turns the uploaded file into the 16 kHz mono PCM WAV whisper wants.
|
||||
type Converter interface {
|
||||
ToWAV(ctx context.Context, src, dst string) error
|
||||
}
|
||||
|
||||
// Transcriber turns a WAV into a timed transcription. The gateway chain
|
||||
// (iguana primary, berget fallback) implements it.
|
||||
type Transcriber interface {
|
||||
Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error)
|
||||
}
|
||||
|
||||
// Planner is an optional extra on a Transcriber: it declares its own phases (for
|
||||
// example one per engine) so the page can show them. A Transcriber that does not
|
||||
// implement it gets a single "transcribing" phase from the worker. Whatever a Planner
|
||||
// declares it reports itself, through progress.From(ctx) (ADR-035).
|
||||
type Planner interface {
|
||||
Plan() []progress.Planned
|
||||
}
|
||||
|
||||
// WorkerStore is the slice of the private transcript store the worker drives.
|
||||
type WorkerStore interface {
|
||||
StartTranscribing(ctx context.Context, userID, id string) error
|
||||
CompleteUpload(ctx context.Context, userID, id string, t domain.Transcription) error
|
||||
FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error
|
||||
SaveProgress(ctx context.Context, userID, id string, steps []domain.ProgressStep, at time.Time) error
|
||||
Heartbeat(ctx context.Context, userID, id string) error
|
||||
}
|
||||
|
||||
// WorkerOption tunes a Worker.
|
||||
type WorkerOption func(*Worker)
|
||||
|
||||
// WithHeartbeat sets how often a running job stamps its row alive and how often it logs
|
||||
// which phase is still running. The defaults are 10 s and 5 min (ADR-035).
|
||||
func WithHeartbeat(beat, still time.Duration) WorkerOption {
|
||||
return func(w *Worker) { w.beatEvery, w.stillEvery = beat, still }
|
||||
}
|
||||
|
||||
// WithQueueLimits bounds how many jobs may WAIT: in total, and per user. A job being
|
||||
// processed is not waiting. The defaults are 20 and 3 (one instance, one job at a time).
|
||||
func WithQueueLimits(total, perUser int) WorkerOption {
|
||||
return func(w *Worker) { w.maxQueued, w.maxPerUser = total, perUser }
|
||||
}
|
||||
|
||||
// WithMaxDuration makes the worker measure the converted audio and fail a job whose real
|
||||
// length exceeds the limit, or falls far short of what the file claimed. Zero (the
|
||||
// default) measures nothing. The limit is first checked against the file's header, which
|
||||
// can lie (ADR-035 review).
|
||||
func WithMaxDuration(d time.Duration) WorkerOption { return func(w *Worker) { w.maxDuration = d } }
|
||||
|
||||
// WithLogger replaces slog.Default() as the worker's logger.
|
||||
func WithLogger(l *slog.Logger) WorkerOption { return func(w *Worker) { w.log = l } }
|
||||
|
||||
const (
|
||||
defaultMaxQueued = 20
|
||||
defaultMaxPerUser = 3
|
||||
defaultBeatEvery = 10 * time.Second
|
||||
defaultStillEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// jobSink binds the store to one job so a progress.Tracker can persist through it.
|
||||
type jobSink struct {
|
||||
st WorkerStore
|
||||
userID, id string
|
||||
}
|
||||
|
||||
func (s jobSink) SaveProgress(ctx context.Context, steps []domain.ProgressStep, at time.Time) error {
|
||||
return s.st.SaveProgress(ctx, s.userID, s.id, steps, at)
|
||||
}
|
||||
|
||||
func (s jobSink) Beat(ctx context.Context) error { return s.st.Heartbeat(ctx, s.userID, s.id) }
|
||||
|
||||
// Worker transcribes accepted uploads in-process, ONE AT A TIME (ADR-033). It is
|
||||
// not a second deployable: it shares the web process's lifetime, which is why it
|
||||
// inherits ADR-018's single-replica assumption — the queue is process-local. The
|
||||
// concurrency of one is deliberate: two simultaneous 90-minute jobs on iguana were
|
||||
// never measured, and a Metal OOM there latches the whisper backend (infra#273).
|
||||
type Worker struct {
|
||||
store WorkerStore
|
||||
conv Converter
|
||||
tr Transcriber
|
||||
tmpDir string
|
||||
timeout time.Duration
|
||||
|
||||
log *slog.Logger
|
||||
beatEvery, stillEvery time.Duration
|
||||
maxDuration time.Duration
|
||||
maxQueued, maxPerUser int
|
||||
|
||||
mu sync.Mutex
|
||||
queue []Job
|
||||
stopped bool
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
// NewWorker wires a Worker. timeout bounds each job (conversion + transcription);
|
||||
// a job that outlives it is failed rather than left spinning.
|
||||
func NewWorker(s WorkerStore, c Converter, t Transcriber, tmpDir string, timeout time.Duration, opts ...WorkerOption) *Worker {
|
||||
w := &Worker{
|
||||
store: s, conv: c, tr: t, tmpDir: tmpDir, timeout: timeout, wake: make(chan struct{}, 1),
|
||||
beatEvery: defaultBeatEvery, stillEvery: defaultStillEvery,
|
||||
maxQueued: defaultMaxQueued, maxPerUser: defaultMaxPerUser,
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(w)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// HasRoom reports whether a job from this user would be accepted right now. It lets the
|
||||
// intake refuse BEFORE spooling up to the upload cap and running ffprobe; Submit still
|
||||
// enforces the same limits atomically, so a race costs at worst one refused submit.
|
||||
func (w *Worker) HasRoom(userID string) bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped || len(w.queue) >= w.maxQueued {
|
||||
return false
|
||||
}
|
||||
mine := 0
|
||||
for _, q := range w.queue {
|
||||
if q.UserID == userID {
|
||||
mine++
|
||||
}
|
||||
}
|
||||
return mine < w.maxPerUser
|
||||
}
|
||||
|
||||
// Submit queues a job. It never blocks. It satisfies Submitter.
|
||||
func (w *Worker) Submit(j Job) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped {
|
||||
return ErrStopped
|
||||
}
|
||||
if len(w.queue) >= w.maxQueued {
|
||||
return ErrQueueFull
|
||||
}
|
||||
mine := 0
|
||||
for _, q := range w.queue {
|
||||
if q.UserID == j.UserID {
|
||||
mine++
|
||||
}
|
||||
}
|
||||
if mine >= w.maxPerUser {
|
||||
return ErrQueueFull
|
||||
}
|
||||
w.queue = append(w.queue, j)
|
||||
select {
|
||||
case w.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run processes queued jobs until ctx ends, then deletes the audio of anything
|
||||
// still queued and refuses new jobs. The rows of those jobs stay queued or
|
||||
// transcribing and are failed as interrupted by the start-up sweep
|
||||
// (store.FailInterruptedUploads): the audio is not retained, so they cannot resume.
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
defer w.stop()
|
||||
for {
|
||||
job, ok := w.next(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w.Process(ctx, job)
|
||||
}
|
||||
}
|
||||
|
||||
// next blocks for the next job. Cancellation wins over a non-empty queue: a job
|
||||
// still queued at shutdown must not be STARTED (it would begin a long conversion
|
||||
// with an already-cancelled context); stop() deletes its audio instead.
|
||||
func (w *Worker) next(ctx context.Context) (Job, bool) {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return Job{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
if len(w.queue) > 0 {
|
||||
j := w.queue[0]
|
||||
w.queue = w.queue[1:]
|
||||
w.mu.Unlock()
|
||||
return j, true
|
||||
}
|
||||
w.mu.Unlock()
|
||||
select {
|
||||
case <-w.wake:
|
||||
case <-ctx.Done():
|
||||
return Job{}, false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) stop() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.stopped = true
|
||||
for _, j := range w.queue {
|
||||
_ = os.Remove(j.Path)
|
||||
}
|
||||
w.queue = nil
|
||||
}
|
||||
|
||||
// Process runs one job: row -> transcribing, convert, transcribe, store. The
|
||||
// uploaded file and the converted WAV are deleted on every path — the audio is
|
||||
// never retained (ADR-033). A failure is recorded with a TYPED kind; a shutdown
|
||||
// mid-job is not a failure and leaves the row for the start-up sweep.
|
||||
//
|
||||
// Every phase is reported through one progress.Tracker, which saves it to the row for
|
||||
// the page and logs it, and a heartbeat stamps the row on its own while the job runs
|
||||
// (ADR-035). Nothing logged or saved here carries transcript text or the file's name.
|
||||
func (w *Worker) Process(ctx context.Context, job Job) {
|
||||
defer func() { _ = os.Remove(job.Path) }()
|
||||
base := w.log
|
||||
if base == nil {
|
||||
base = slog.Default()
|
||||
}
|
||||
log := base.With("upload", job.UploadID)
|
||||
start := time.Now()
|
||||
|
||||
if err := w.store.StartTranscribing(ctx, job.UserID, job.UploadID); err != nil {
|
||||
log.Error("upload: start transcribing", "kind", errkind.Of(err))
|
||||
w.fail(ctx, job, log, domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
|
||||
plan := []progress.Planned{{Key: "received", Label: "Received"}, {Key: "audio", Label: "Preparing the audio"}}
|
||||
planner, planned := w.tr.(Planner)
|
||||
if planned {
|
||||
plan = append(plan, planner.Plan()...)
|
||||
} else {
|
||||
plan = append(plan, progress.Planned{Key: "transcribe", Label: "Transcribing"})
|
||||
}
|
||||
plan = append(plan, progress.Planned{Key: "save", Label: "Saving the transcript"})
|
||||
tr := progress.NewTracker(ctx, jobSink{st: w.store, userID: job.UserID, id: job.UploadID}, log, plan)
|
||||
tr.Start()
|
||||
tr.Begin("received", 0)
|
||||
tr.Done("received")
|
||||
|
||||
hbCtx, stopBeat := context.WithCancel(ctx)
|
||||
hbDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(hbDone)
|
||||
tr.Heartbeat(hbCtx, w.beatEvery, w.stillEvery)
|
||||
}()
|
||||
defer func() { stopBeat(); <-hbDone }() // no heartbeat outlives its job
|
||||
|
||||
// abort records where the job stopped. A shutdown (the parent context ended) is not
|
||||
// a failure: the row is left for the start-up sweep, so the step stays as it was.
|
||||
abort := func(kind domain.UploadErrorKind) {
|
||||
if ctx.Err() == nil {
|
||||
tr.FailRunning()
|
||||
}
|
||||
w.fail(ctx, job, log, kind)
|
||||
}
|
||||
|
||||
wav, err := w.tempWAV()
|
||||
if err != nil {
|
||||
log.Error("upload: temp wav", "kind", errkind.Of(err))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
defer func() { _ = os.Remove(wav) }()
|
||||
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.timeout)
|
||||
defer cancel()
|
||||
tr.Begin("audio", 0)
|
||||
if err := w.conv.ToWAV(jobCtx, job.Path, wav); err != nil {
|
||||
log.Error("upload: convert", "kind", errkind.Of(err))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
got, tooLong := w.audioLengthProblem(wav)
|
||||
if tooLong {
|
||||
abort(domain.UploadErrTooLong)
|
||||
log.Error("upload: converted audio is over the limit", "audio_s", int(got.Seconds()), "limit_s", int(w.maxDuration.Seconds()))
|
||||
return
|
||||
}
|
||||
if w.maxDuration > 0 && job.Duration > 0 {
|
||||
if note := shortfallNote(got, job.Duration); note != "" {
|
||||
log.Warn("upload: audio is much shorter than the file claimed", "audio_s", int(got.Seconds()), "claimed_s", int(job.Duration.Seconds()))
|
||||
tr.Annotate("audio", note)
|
||||
}
|
||||
}
|
||||
tr.Done("audio")
|
||||
|
||||
if !planned {
|
||||
tr.Begin("transcribe", 0)
|
||||
}
|
||||
t, err := w.tr.Transcribe(progress.With(jobCtx, tr), wav)
|
||||
if err != nil {
|
||||
log.Error("upload: transcribe", "kind", errkind.Of(err), "elapsed", time.Since(start).Round(time.Second))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
if !planned {
|
||||
tr.Done("transcribe")
|
||||
}
|
||||
|
||||
tr.Begin("save", 0)
|
||||
if err := w.store.CompleteUpload(ctx, job.UserID, job.UploadID, t); err != nil {
|
||||
log.Error("upload: store transcript", "kind", errkind.Of(err))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
tr.Done("save")
|
||||
tr.Finish()
|
||||
log.Info("upload transcribed", "endpoint", t.Endpoint, "audio_s", t.AudioSeconds,
|
||||
"segments", len(t.Segments), "elapsed", time.Since(start).Round(time.Second))
|
||||
}
|
||||
|
||||
// fail records a transcription failure — unless the PARENT context ended, which
|
||||
// is a shutdown, not a failure: the row is left for the start-up sweep.
|
||||
func (w *Worker) fail(ctx context.Context, job Job, log *slog.Logger, kind domain.UploadErrorKind) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := w.store.FailUpload(ctx, job.UserID, job.UploadID, kind); err != nil {
|
||||
log.Error("upload: record failure", "kind", errkind.Of(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) tempWAV() (string, error) {
|
||||
f, err := os.CreateTemp(w.tmpDir, "wav-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: temp wav: %w", err)
|
||||
}
|
||||
name := f.Name()
|
||||
if err := f.Close(); err != nil {
|
||||
_ = os.Remove(name)
|
||||
return "", fmt.Errorf("uploads: temp wav: %w", err)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
const (
|
||||
// tooLongSlack is how far over the limit the converted audio may be before the claim
|
||||
// in the file's header is called a lie: ffmpeg rounds.
|
||||
tooLongSlack = 30 * time.Second
|
||||
// A file counts as truncated when it converts to under 80% of the length it claimed
|
||||
// AND more than two minutes short. Probe estimates for variable-bitrate files are
|
||||
// imprecise; refusing real recordings would be worse than the problem.
|
||||
incompleteBelow = 0.8
|
||||
incompleteShortfall = 2 * time.Minute
|
||||
)
|
||||
|
||||
// audioLengthProblem measures the converted WAV (16 kHz mono 16-bit PCM is 32,000 bytes a
|
||||
// second after a 44-byte header) against the limit. It returns the real length and whether
|
||||
// that is over the limit by more than the tolerance.
|
||||
func (w *Worker) audioLengthProblem(wav string) (got time.Duration, tooLong bool) {
|
||||
if w.maxDuration <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
info, err := os.Stat(wav)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
got = time.Duration(float64(max(info.Size()-44, 0)) / 32000 * float64(time.Second))
|
||||
return got, got > w.maxDuration+tooLongSlack
|
||||
}
|
||||
|
||||
// shortfallNote is a remark for the audio step when the converted audio is much shorter
|
||||
// than the file claimed, or "" when it is not. It is a NOTE and never a failure: a
|
||||
// container header can over-estimate a complete file (a variable-bitrate MP3 without a Xing
|
||||
// header claimed 79 minutes for 20 real ones), so refusing would turn away real
|
||||
// recordings, while saying nothing about a truncated one would hide a partial transcript.
|
||||
func shortfallNote(got, claimed time.Duration) string {
|
||||
if float64(got) >= incompleteBelow*float64(claimed) || claimed-got <= incompleteShortfall {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("The audio is %s long; the file claimed %s. If the recording was cut off, upload the complete one.",
|
||||
FormatDuration(got), FormatDuration(claimed))
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// The duration limit is a claim in the file's header. The worker measures what was
|
||||
// really converted and refuses both a recording that is longer than the limit (the
|
||||
// header lied low) and one that ends far earlier than the file claimed (it was truncated
|
||||
// and would otherwise yield a partial transcript with no warning) -- adversarial review F2.
|
||||
|
||||
// sizedConverter writes a (sparse) 16 kHz mono PCM WAV of the given length, without
|
||||
// actually allocating it: 32,000 bytes per second plus a 44-byte header.
|
||||
type sizedConverter struct{ seconds float64 }
|
||||
|
||||
func (c sizedConverter) ToWAV(_ context.Context, _, dst string) error {
|
||||
f, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
return f.Truncate(44 + int64(c.seconds*32000))
|
||||
}
|
||||
|
||||
func durationWorker(t *testing.T, r *wrig, wavSeconds float64, max time.Duration) *uploads.Worker {
|
||||
t.Helper()
|
||||
opts := []uploads.WorkerOption{}
|
||||
if max > 0 {
|
||||
opts = append(opts, uploads.WithMaxDuration(max))
|
||||
}
|
||||
return uploads.NewWorker(r.st, sizedConverter{seconds: wavSeconds}, r.tr, r.tmp, time.Minute, opts...)
|
||||
}
|
||||
|
||||
func jobWith(r *wrig, t *testing.T, claimed time.Duration) uploads.Job {
|
||||
j := r.spool(t, "a")
|
||||
j.Duration = claimed
|
||||
return j
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AConvertedLengthOverTheLimitFailsTheJobBeforeAnyEngineSeesIt(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 100*60, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 10*time.Second))
|
||||
|
||||
_, failed := r.st.snapshot()
|
||||
assert.Equal(t, domain.UploadErrTooLong, failed["a"], "a header that claimed 10 s must not let 100 minutes through")
|
||||
assert.Empty(t, r.tr.paths, "no engine may be sent audio that is over the limit")
|
||||
assert.Empty(t, r.leftovers(t), "the audio is still deleted")
|
||||
}
|
||||
|
||||
func TestWorkerDuration_JustOverTheLimitWithinTheSlackStillTranscribes(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 90*60+20, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 90*time.Minute))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed)
|
||||
assert.Contains(t, completed, "a", "ffmpeg rounds: 20 s over a 90-minute limit is not a lie")
|
||||
}
|
||||
|
||||
func audioStepNote(r *wrig) string {
|
||||
for _, st := range r.st.lastSteps() {
|
||||
if st.Key == "audio" {
|
||||
return st.Note
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AFileThatEndsFarEarlierThanItClaimedIsStillTranscribedAndSaysSo(t *testing.T) {
|
||||
// A container header can over-estimate a complete file (VBR MP3 without a Xing header
|
||||
// claimed 79 minutes for 20 real ones). Refusing it was wrong; staying silent about a
|
||||
// truncated one is wrong too. So it is transcribed, and the audio step carries a note.
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 20*60, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 79*time.Minute))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed, "a complete recording with a bad header estimate must not be refused")
|
||||
assert.Contains(t, completed, "a")
|
||||
note := audioStepNote(r)
|
||||
assert.Contains(t, note, "20 min", "what was really transcribed")
|
||||
assert.Contains(t, note, "1 h 19 min", "what the file claimed")
|
||||
assert.Contains(t, note, "cut off", "and what to do if that is not a surprise")
|
||||
}
|
||||
|
||||
func TestWorkerDuration_TheShortfallNoteIsPinnedAtEightyPercentAndTwoMinutes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
claimed time.Duration
|
||||
wav float64
|
||||
noted bool
|
||||
}{
|
||||
{"79 percent of an hour", 60 * time.Minute, 0.79 * 3600, true},
|
||||
{"exactly 80 percent of an hour", 60 * time.Minute, 2880, false},
|
||||
{"one second under 80 percent of an hour", 60 * time.Minute, 2879, true},
|
||||
{"81 percent of an hour", 60 * time.Minute, 0.81 * 3600, false},
|
||||
{"exactly two minutes short and under 80 percent", 6 * time.Minute, 4 * 60, false},
|
||||
{"two minutes and a second short and under 80 percent", 6 * time.Minute, 4*60 - 1, true},
|
||||
{"unknown claimed length", 0, 20 * 60, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, tt.claimed))
|
||||
|
||||
assert.Equal(t, tt.noted, audioStepNote(r) != "")
|
||||
completed, _ := r.st.snapshot()
|
||||
assert.Contains(t, completed, "a", "a note never stops the job")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_TheTooLongToleranceIsExactlyThirtySeconds(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
wav float64
|
||||
fail bool
|
||||
}{
|
||||
{"at the limit plus 30 s", 90*60 + 30, false},
|
||||
{"at the limit plus 31 s", 90*60 + 31, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 90*time.Minute))
|
||||
|
||||
_, failed := r.st.snapshot()
|
||||
if tt.fail {
|
||||
assert.Equal(t, domain.UploadErrTooLong, failed["a"])
|
||||
} else {
|
||||
assert.Empty(t, failed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AnEstimateThatIsMerelyALittleOffIsNotTruncation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
claimed time.Duration
|
||||
wav float64
|
||||
}{
|
||||
{"five percent short", 60 * time.Minute, 57 * 60},
|
||||
{"under two minutes short of a long file", 60 * time.Minute, 59 * 60},
|
||||
{"unknown claimed length", 0, 20 * 60},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, tt.claimed))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed)
|
||||
assert.Contains(t, completed, "a")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_WithNoMaximumConfiguredNothingIsMeasured(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 100*60, 0)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 0))
|
||||
|
||||
completed, _ := r.st.snapshot()
|
||||
assert.Contains(t, completed, "a")
|
||||
require.NotNil(t, w)
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// Progress, heartbeat and logging of the worker (ADR-035, #42, #43).
|
||||
|
||||
// plannedTr is a transcriber that declares its own phases and reports them, like the
|
||||
// two-engine Ensemble does.
|
||||
type plannedTr struct{ *fakeTranscriber }
|
||||
|
||||
func (plannedTr) Plan() []progress.Planned {
|
||||
return []progress.Planned{{Key: "engine_a", Label: "Transcribing with A"}, {Key: "engine_b", Label: "Transcribing with B"}}
|
||||
}
|
||||
|
||||
func (p plannedTr) Transcribe(ctx context.Context, wav string) (domain.Transcription, error) {
|
||||
r := progress.From(ctx)
|
||||
r.Begin("engine_a", 0)
|
||||
r.Done("engine_a")
|
||||
r.Begin("engine_b", 0)
|
||||
r.Done("engine_b")
|
||||
return p.fakeTranscriber.Transcribe(ctx, wav)
|
||||
}
|
||||
|
||||
func keysAndStates(steps []domain.ProgressStep) []string {
|
||||
var out []string
|
||||
for _, s := range steps {
|
||||
out = append(out, s.Key+":"+string(s.State))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// syncLog is a log sink safe to read while the worker's goroutines write.
|
||||
type syncLog struct {
|
||||
mu sync.Mutex
|
||||
b bytes.Buffer
|
||||
}
|
||||
|
||||
func (l *syncLog) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.b.Write(p)
|
||||
}
|
||||
|
||||
func (l *syncLog) String() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.b.String()
|
||||
}
|
||||
|
||||
func newProgressWorker(t *testing.T, r *wrig, tr uploads.Transcriber, opts ...uploads.WorkerOption) *uploads.Worker {
|
||||
t.Helper()
|
||||
return uploads.NewWorker(r.st, r.conv, tr, r.tmp, time.Minute, opts...)
|
||||
}
|
||||
|
||||
func TestWorkerProgress_ASingleEngineJobShowsEveryPhaseDone(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Equal(t, []string{"received:done", "audio:done", "transcribe:done", "save:done"}, keysAndStates(r.st.lastSteps()))
|
||||
}
|
||||
|
||||
func TestWorkerProgress_ATranscriberThatPlansItsOwnPhasesGetsThemBetweenAudioAndSaving(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := newProgressWorker(t, r, plannedTr{r.tr})
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Equal(t, []string{"received:done", "audio:done", "engine_a:done", "engine_b:done", "save:done"}, keysAndStates(r.st.lastSteps()))
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AFailureMarksWhereItStoppedAndLeavesTheRestPending(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*wrig)
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
"conversion fails", func(r *wrig) { r.conv.err = errors.New("moov atom not found") },
|
||||
[]string{"received:done", "audio:failed", "transcribe:pending", "save:pending"},
|
||||
},
|
||||
{
|
||||
"every engine fails", func(r *wrig) { r.tr.err = errors.New("iguana down") },
|
||||
[]string{"received:done", "audio:done", "transcribe:failed", "save:pending"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
tt.setup(r)
|
||||
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Equal(t, tt.want, keysAndStates(r.st.lastSteps()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AShutdownIsNotRecordedAsAFailedStep(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); r.w.Process(ctx, r.spool(t, "a")) }()
|
||||
require.Eventually(t, func() bool {
|
||||
return strings.Contains(strings.Join(keysAndStates(r.st.lastSteps()), ","), "transcribe:running")
|
||||
}, 2*time.Second, time.Millisecond)
|
||||
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
assert.Contains(t, keysAndStates(r.st.lastSteps()), "transcribe:running",
|
||||
"a shutdown leaves the row for the start-up sweep; it must not pose as a failure of the step")
|
||||
}
|
||||
|
||||
func TestWorkerProgress_TheHeartbeatBeatsWhileAnEngineCallIsStuckAndStopsWhenTheJobEnds(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
r.st.beatDelay = 20 * time.Millisecond
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithHeartbeat(5*time.Millisecond, time.Hour))
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); w.Process(context.Background(), r.spool(t, "a")) }()
|
||||
|
||||
require.Eventually(t, func() bool { return r.st.beatCount() >= 3 }, 2*time.Second, 5*time.Millisecond,
|
||||
"the page tells a slow engine from a dead service by this heartbeat, so it must not wait on the engine")
|
||||
close(r.tr.block)
|
||||
<-done
|
||||
after := r.st.beatCount()
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
|
||||
assert.Equal(t, after, r.st.beatCount(), "no heartbeat may outlive its job")
|
||||
}
|
||||
|
||||
func TestWorkerProgress_OnASuccessfulJobTheLogNamesEveryPhaseInOrderAndNeverTheTranscriptOrThePath(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.result = domain.Transcription{Text: "zebrakaka secret meeting words", Endpoint: "iguana/kb-whisper", AudioSeconds: 3}
|
||||
buf := &syncLog{}
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithLogger(slog.New(slog.NewTextHandler(buf, nil))))
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
out := buf.String()
|
||||
audio := strings.Index(out, "step=audio")
|
||||
transcribe := strings.Index(out, "step=transcribe")
|
||||
save := strings.Index(out, "step=save")
|
||||
require.True(t, audio >= 0 && transcribe > audio && save > transcribe, "phases must be logged in the order they ran:\n%s", out)
|
||||
assert.Contains(t, out, "upload=a", "every line says which upload it is about")
|
||||
assert.NotContains(t, out, "zebrakaka", "no transcript text in the log")
|
||||
assert.NotContains(t, out, r.tmp, "no spool path, which carries the upload's identity, in the log")
|
||||
assert.Contains(t, out, "upload transcribed", "the job's own summary line")
|
||||
assert.Contains(t, out, "endpoint=iguana/kb-whisper", "which endpoint served it is a quality fact")
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AFailureLogsAKindNeverTheErrorTextOrThePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*wrig)
|
||||
line string
|
||||
}{
|
||||
{"conversion fails and names the spool file", func(r *wrig) {
|
||||
r.conv.err = errors.New("Error opening input file " + r.tmp + "/upload-a: Invalid data zebrakaka")
|
||||
}, "upload: convert"},
|
||||
{"every engine fails and the gateway echoed a word", func(r *wrig) {
|
||||
r.tr.err = errors.New("gateway echoed: zebrakaka secret words")
|
||||
}, "upload: transcribe"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
tt.setup(r)
|
||||
buf := &syncLog{}
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithLogger(slog.New(slog.NewTextHandler(buf, nil))))
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
out := buf.String()
|
||||
assert.NotContains(t, out, "zebrakaka", "provider text or file contents in the log")
|
||||
assert.NotContains(t, out, r.tmp, "the spool path in the log")
|
||||
assert.Contains(t, out, tt.line, "the failure must be logged at all, not only scrubbed")
|
||||
assert.Contains(t, out, "kind=error")
|
||||
assert.Contains(t, out, "upload=a")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AStoreFailureLogsItsSQLStateNeverTheValueItQuotes(t *testing.T) {
|
||||
// A Postgres error can quote the offending value; for a transcript row that is the
|
||||
// transcript (fourth adversarial review). The log gets the SQLSTATE.
|
||||
r := newWRig(t, time.Minute)
|
||||
r.st.completeErr = dbErr{}
|
||||
buf := &syncLog{}
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithLogger(slog.New(slog.NewTextHandler(buf, nil))))
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
out := buf.String()
|
||||
assert.NotContains(t, out, "zebrakaka")
|
||||
assert.Contains(t, out, "upload: store transcript")
|
||||
assert.Contains(t, out, "kind=sql_22P02")
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// The queue was append-only: 100,000 submits with the worker idle all succeeded, and each
|
||||
// queued job holds up to the upload cap on disk until its turn (adversarial review F6).
|
||||
|
||||
func queueWorker(t *testing.T, total, perUser int) *uploads.Worker {
|
||||
t.Helper()
|
||||
r := newWRig(t, time.Minute)
|
||||
return uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute, uploads.WithQueueLimits(total, perUser))
|
||||
}
|
||||
|
||||
func submit(w *uploads.Worker, user string, n int) error {
|
||||
return w.Submit(uploads.Job{UserID: user, UploadID: user + "-" + strconv.Itoa(n), Path: "/nonexistent"})
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_RefusesBeyondTheTotalQueueCap(t *testing.T) {
|
||||
w := queueWorker(t, 3, 10)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
require.NoError(t, submit(w, "u"+strconv.Itoa(i), i))
|
||||
}
|
||||
|
||||
err := submit(w, "u9", 9)
|
||||
assert.ErrorIs(t, err, uploads.ErrQueueFull)
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_OneUserCannotFillTheQueueForEveryoneElse(t *testing.T) {
|
||||
w := queueWorker(t, 100, 2)
|
||||
|
||||
require.NoError(t, submit(w, "alice", 1))
|
||||
require.NoError(t, submit(w, "alice", 2))
|
||||
errAlice := submit(w, "alice", 3)
|
||||
errBob := submit(w, "bob", 1)
|
||||
|
||||
assert.True(t, errors.Is(errAlice, uploads.ErrQueueFull), "a third waiting upload from one user is refused")
|
||||
assert.NoError(t, errBob, "another user is not affected by alice's queue")
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_ThereIsADefaultCapSoANeglectedWorkerCannotBeFilledForever(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute)
|
||||
|
||||
refused := 0
|
||||
for i := 0; i < 1000; i++ {
|
||||
if err := submit(w, "u"+strconv.Itoa(i), i); errors.Is(err, uploads.ErrQueueFull) {
|
||||
refused++
|
||||
}
|
||||
}
|
||||
|
||||
assert.Positive(t, refused, "with the defaults, 1,000 submits to an idle worker must not all be accepted")
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_AJobThatHasLeftTheQueueFreesItsPlace(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute, uploads.WithQueueLimits(1, 1))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); w.Run(ctx) }()
|
||||
defer func() { cancel(); <-done }()
|
||||
|
||||
job := r.spool(t, "first")
|
||||
require.NoError(t, w.Submit(job))
|
||||
require.Eventually(t, func() bool { c, _ := r.st.snapshot(); _, ok := c["first"]; return ok },
|
||||
5*time.Second, 5*time.Millisecond, "the first job never completed")
|
||||
|
||||
assert.NoError(t, w.Submit(r.spool(t, "second")), "the cap counts jobs waiting, not jobs ever submitted")
|
||||
}
|
||||
|
||||
func TestWorkerHasRoom_SaysWhetherASubmitWouldBeAccepted(t *testing.T) {
|
||||
w := queueWorker(t, 3, 2)
|
||||
assert.True(t, w.HasRoom("alice"))
|
||||
require.NoError(t, submit(w, "alice", 1))
|
||||
require.NoError(t, submit(w, "alice", 2))
|
||||
|
||||
assert.False(t, w.HasRoom("alice"), "alice is at her share")
|
||||
assert.True(t, w.HasRoom("bob"), "bob is not")
|
||||
require.NoError(t, submit(w, "bob", 1))
|
||||
assert.False(t, w.HasRoom("carol"), "the queue as a whole is at its cap")
|
||||
}
|
||||
|
||||
func TestWorkerHasRoom_AgreesWithSubmitAtEveryStepAndIsFalseOnceStopped(t *testing.T) {
|
||||
w := queueWorker(t, 2, 1)
|
||||
for i, user := range []string{"a", "a", "b", "c"} {
|
||||
room := w.HasRoom(user)
|
||||
err := submit(w, user, i)
|
||||
assert.Equal(t, room, err == nil, "step %d (%s): HasRoom=%v but Submit returned %v", i, user, room, err)
|
||||
}
|
||||
|
||||
r := newWRig(t, time.Minute)
|
||||
stopped := uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
stopped.Run(ctx)
|
||||
assert.False(t, stopped.HasRoom("a"), "a worker that has stopped has no room")
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
type wStore struct {
|
||||
mu sync.Mutex
|
||||
started []string
|
||||
completed map[string]domain.Transcription
|
||||
failed map[string]domain.UploadErrorKind
|
||||
startErr error
|
||||
// completeErr makes CompleteUpload fail, as a database outage would.
|
||||
completeErr error
|
||||
saves [][]domain.ProgressStep
|
||||
beats int
|
||||
// beatDelay makes each heartbeat write slow, so one is still in flight when a job
|
||||
// ends -- which is what proves the worker waits for its heartbeat to finish.
|
||||
beatDelay time.Duration
|
||||
}
|
||||
|
||||
func newWStore() *wStore {
|
||||
return &wStore{completed: map[string]domain.Transcription{}, failed: map[string]domain.UploadErrorKind{}}
|
||||
}
|
||||
|
||||
func (s *wStore) StartTranscribing(_ context.Context, _, id string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.started = append(s.started, id)
|
||||
return s.startErr
|
||||
}
|
||||
|
||||
func (s *wStore) CompleteUpload(_ context.Context, _, id string, t domain.Transcription) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.completeErr != nil {
|
||||
return s.completeErr
|
||||
}
|
||||
s.completed[id] = t
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) FailUpload(_ context.Context, _, id string, k domain.UploadErrorKind) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.failed[id] = k
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) SaveProgress(_ context.Context, _, _ string, steps []domain.ProgressStep, _ time.Time) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.saves = append(s.saves, append([]domain.ProgressStep(nil), steps...))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) Heartbeat(context.Context, string, string) error {
|
||||
time.Sleep(s.beatDelay)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.beats++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) lastSteps() []domain.ProgressStep {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(s.saves) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.saves[len(s.saves)-1]
|
||||
}
|
||||
|
||||
func (s *wStore) beatCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.beats
|
||||
}
|
||||
|
||||
func (s *wStore) snapshot() (map[string]domain.Transcription, map[string]domain.UploadErrorKind) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
c := map[string]domain.Transcription{}
|
||||
f := map[string]domain.UploadErrorKind{}
|
||||
for k, v := range s.completed {
|
||||
c[k] = v
|
||||
}
|
||||
for k, v := range s.failed {
|
||||
f[k] = v
|
||||
}
|
||||
return c, f
|
||||
}
|
||||
|
||||
type fakeConverter struct {
|
||||
err error
|
||||
srcs []string
|
||||
dsts []string
|
||||
}
|
||||
|
||||
func (c *fakeConverter) ToWAV(_ context.Context, src, dst string) error {
|
||||
c.srcs = append(c.srcs, src)
|
||||
c.dsts = append(c.dsts, dst)
|
||||
if c.err != nil {
|
||||
return c.err
|
||||
}
|
||||
return os.WriteFile(dst, []byte("wav"), 0o600)
|
||||
}
|
||||
|
||||
type fakeTranscriber struct {
|
||||
result domain.Transcription
|
||||
err error
|
||||
paths []string
|
||||
// block, when non-nil, holds every call until it is closed or ctx ends.
|
||||
block chan struct{}
|
||||
running atomic.Int32
|
||||
maxSeen atomic.Int32
|
||||
}
|
||||
|
||||
func (f *fakeTranscriber) Transcribe(ctx context.Context, wav string) (domain.Transcription, error) {
|
||||
f.paths = append(f.paths, wav)
|
||||
n := f.running.Add(1)
|
||||
defer f.running.Add(-1)
|
||||
for {
|
||||
m := f.maxSeen.Load()
|
||||
if n <= m || f.maxSeen.CompareAndSwap(m, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if f.block != nil {
|
||||
select {
|
||||
case <-f.block:
|
||||
case <-ctx.Done():
|
||||
return domain.Transcription{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return f.result, f.err
|
||||
}
|
||||
|
||||
type wrig struct {
|
||||
w *uploads.Worker
|
||||
st *wStore
|
||||
conv *fakeConverter
|
||||
tr *fakeTranscriber
|
||||
tmp string
|
||||
}
|
||||
|
||||
func newWRig(t *testing.T, timeout time.Duration) *wrig {
|
||||
t.Helper()
|
||||
r := &wrig{
|
||||
st: newWStore(),
|
||||
conv: &fakeConverter{},
|
||||
tr: &fakeTranscriber{result: domain.Transcription{Text: "synthetic text", Endpoint: "iguana/kb-whisper", AudioSeconds: 3}},
|
||||
tmp: t.TempDir(),
|
||||
}
|
||||
r.w = uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, timeout)
|
||||
return r
|
||||
}
|
||||
|
||||
// spool writes a fake uploaded file (what Intake would hand over) and returns a Job.
|
||||
func (r *wrig) spool(t *testing.T, id string) uploads.Job {
|
||||
t.Helper()
|
||||
p := filepath.Join(r.tmp, "upload-"+id)
|
||||
require.NoError(t, os.WriteFile(p, []byte("audio"), 0o600))
|
||||
return uploads.Job{UserID: "u1", UploadID: id, Path: p}
|
||||
}
|
||||
|
||||
func (r *wrig) leftovers(t *testing.T) []string {
|
||||
t.Helper()
|
||||
ents, err := os.ReadDir(r.tmp)
|
||||
require.NoError(t, err)
|
||||
var n []string
|
||||
for _, e := range ents {
|
||||
n = append(n, e.Name())
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestWorkerProcess_SuccessConvertsTranscribesStoresAndDeletesTheAudio(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
job := r.spool(t, "a")
|
||||
|
||||
r.w.Process(context.Background(), job)
|
||||
|
||||
c, f := r.st.snapshot()
|
||||
assert.Equal(t, []string{"a"}, r.st.started)
|
||||
assert.Equal(t, "synthetic text", c["a"].Text)
|
||||
assert.Equal(t, "iguana/kb-whisper", c["a"].Endpoint)
|
||||
assert.Empty(t, f)
|
||||
require.Len(t, r.conv.srcs, 1)
|
||||
assert.Equal(t, job.Path, r.conv.srcs[0], "the converter reads the uploaded file")
|
||||
assert.Equal(t, r.conv.dsts, r.tr.paths, "whisper is sent the converted WAV, not the original")
|
||||
assert.Equal(t, r.tmp, filepath.Dir(r.conv.dsts[0]))
|
||||
assert.Empty(t, r.leftovers(t), "the audio is never retained: both the upload and the WAV are deleted")
|
||||
}
|
||||
|
||||
func TestWorkerProcess_FailuresAreTypedAndStillDeleteTheAudio(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*wrig)
|
||||
}{
|
||||
{"conversion fails", func(r *wrig) { r.conv.err = errors.New("moov atom not found") }},
|
||||
{"every transcription endpoint fails", func(r *wrig) { r.tr.err = errors.New("iguana down; berget down") }},
|
||||
{"storing the finished transcript fails", func(r *wrig) { r.st.completeErr = errors.New("db down") }},
|
||||
{"the temporary WAV cannot be created", func(r *wrig) {
|
||||
r.w = uploads.NewWorker(r.st, r.conv, r.tr, filepath.Join(r.tmp, "missing"), time.Minute)
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
tt.setup(r)
|
||||
job := r.spool(t, "a")
|
||||
|
||||
r.w.Process(context.Background(), job)
|
||||
|
||||
c, f := r.st.snapshot()
|
||||
assert.Empty(t, c)
|
||||
assert.Equal(t, domain.UploadErrTranscriptionFailed, f["a"])
|
||||
assert.Empty(t, r.leftovers(t))
|
||||
})
|
||||
}
|
||||
t.Run("conversion failure never reaches whisper", func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.conv.err = errors.New("bad")
|
||||
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Empty(t, r.tr.paths)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkerProcess_ShutdownMidJobLeavesTheRowForTheStartupSweep(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
job := r.spool(t, "a")
|
||||
|
||||
go func() { r.w.Process(ctx, job); close(done) }()
|
||||
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
_, f := r.st.snapshot()
|
||||
assert.Empty(t, f, "a shutdown is not a transcription failure: the row stays for FailInterruptedUploads to mark interrupted")
|
||||
assert.Empty(t, r.leftovers(t), "but the audio is still deleted")
|
||||
}
|
||||
|
||||
func TestWorkerProcess_AJobThatOutlivesItsTimeoutIsFailedNotStuck(t *testing.T) {
|
||||
r := newWRig(t, 50*time.Millisecond)
|
||||
r.tr.block = make(chan struct{}) // never released
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() { r.w.Process(context.Background(), r.spool(t, "a")); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Process did not return: a hung endpoint was not bounded by the job timeout")
|
||||
}
|
||||
|
||||
_, f := r.st.snapshot()
|
||||
assert.Equal(t, domain.UploadErrTranscriptionFailed, f["a"], "a hung endpoint must end in failed, not an eternal spinner (ADR-025)")
|
||||
assert.Empty(t, r.leftovers(t))
|
||||
}
|
||||
|
||||
func TestWorkerProcess_StartFailureDropsTheJobAndDeletesTheAudio(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.st.startErr = errors.New("db down")
|
||||
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Empty(t, r.tr.paths, "never transcribe a job whose row could not be moved to transcribing")
|
||||
_, f := r.st.snapshot()
|
||||
assert.Contains(t, f, "a")
|
||||
assert.Empty(t, r.leftovers(t))
|
||||
}
|
||||
|
||||
func TestWorkerRun_ProcessesOneJobAtATime(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go r.w.Run(ctx)
|
||||
|
||||
for _, id := range []string{"a", "b", "c"} {
|
||||
require.NoError(t, r.w.Submit(r.spool(t, id)))
|
||||
}
|
||||
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||
time.Sleep(50 * time.Millisecond) // give a (wrong) second job time to start
|
||||
assert.Equal(t, int32(1), r.tr.running.Load(), "two simultaneous long jobs on iguana were never measured: concurrency is 1")
|
||||
|
||||
close(r.tr.block)
|
||||
require.Eventually(t, func() bool { c, _ := r.st.snapshot(); return len(c) == 3 }, 3*time.Second, 5*time.Millisecond)
|
||||
assert.Equal(t, int32(1), r.tr.maxSeen.Load())
|
||||
}
|
||||
|
||||
func TestWorkerRun_StoppingDeletesQueuedAudioAndRefusesNewJobs(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
stopped := make(chan struct{})
|
||||
go func() { r.w.Run(ctx); close(stopped) }()
|
||||
require.NoError(t, r.w.Submit(r.spool(t, "a")))
|
||||
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||
require.NoError(t, r.w.Submit(r.spool(t, "b"))) // queued behind the running job
|
||||
|
||||
cancel()
|
||||
<-stopped
|
||||
|
||||
assert.Len(t, r.conv.srcs, 1, "a job still queued at shutdown must never be started: only the running job was converted")
|
||||
assert.Empty(t, r.leftovers(t), "audio of queued jobs is deleted on stop: the rows are swept to interrupted at next start")
|
||||
assert.ErrorIs(t, r.w.Submit(r.spool(t, "c")), uploads.ErrStopped)
|
||||
}
|
||||
|
||||
// dbErr is an error shaped like pgx's: a message that quotes a value, and an SQLSTATE.
|
||||
type dbErr struct{}
|
||||
|
||||
func (dbErr) Error() string {
|
||||
return `ERROR: invalid input syntax for type json: "zebrakaka secret words"`
|
||||
}
|
||||
func (dbErr) SQLState() string { return "22P02" }
|
||||
Reference in New Issue
Block a user