mathias/tapir
1
0
Fork 0
Code Issues 35 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(uploads): choose the language of a recording; record it, show it, carry it to the engines (#61)

Browse Source 
Form field (Swedish preselected, Other = auto) read before the file part and refused
before anything is spooled if outside the closed set; stored on the row (migration 021,
CHECK constraint, default sv for history); carried on the Job and the worker's context;
shown on the status page. Feature scenarios and the coverage map are updated, and
ADR-033/034 amended.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
mathias
2026-10-04 22:29:58 +02:00
co-authored by Claude Sonnet 5.5
parent ed9d76b0cf
commit 0338b18a17
26 changed files with 718 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/uploads/intake.go
+7 -4
View File
@@ -25,7 +25,7 @@ type Prober interface {
// Store is the slice of the private transcript store the intake needs.
type Store interface {
CreateUpload(ctx context.Context, userID, title string, hadVideo bool) (string, error)
CreateUpload(ctx context.Context, userID, title string, lang domain.Language, hadVideo bool) (string, error)
FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error
}
@@ -35,6 +35,9 @@ type Job struct {
UserID string
UploadID string
Path string
// Language is what the user chose for this recording; empty means Swedish (a job
// queued before the choice existed).
Language domain.Language
// Duration is what the file CLAIMED when probed. The worker compares it with what
// was really converted; zero means unknown (ADR-035 review).
Duration time.Duration
@@ -135,7 +138,7 @@ const busyMessage = "Too many recordings are waiting to be transcribed. Please t
// validation yields a *RejectError and creates NO row — rejection is synchronous
// and leaves nothing behind. On success the temp file is handed to the job and the
// worker deletes it; on every failure path Accept deletes it itself.
func (in *Intake) Accept(ctx context.Context, userID, title string, src io.Reader) (string, error) {
func (in *Intake) Accept(ctx context.Context, userID, title string, lang domain.Language, src io.Reader) (string, error) {
// Refuse before spooling when the queue has no room, if the submitter can say so: a full
// queue otherwise cost up to the upload cap on disk, an ffprobe run and a failed row for
// every refused attempt.
@@ -190,11 +193,11 @@ func (in *Intake) Accept(ctx context.Context, userID, title string, src io.Reade
return "", err
}
id, err := in.store.CreateUpload(ctx, userID, title, probe.HasVideo)
id, err := in.store.CreateUpload(ctx, userID, title, lang, probe.HasVideo)
if err != nil {
return "", fmt.Errorf("uploads: create: %w", err)
}
if err := in.sub.Submit(Job{UserID: userID, UploadID: id, Path: path, Duration: probe.Duration}); err != nil {
if err := in.sub.Submit(Job{UserID: userID, UploadID: id, Path: path, Language: lang, Duration: probe.Duration}); err != nil {
if errors.Is(err, ErrQueueFull) {
// The user's to retry, not a server fault: refuse it as such, and do not leave
// the row queued -- that is the stuck spinner.
internal/uploads/intake_busy_test.go
+5 -5
View File
@@ -23,7 +23,7 @@ func TestAccept_AFullQueueIsABusyRejectionNotAServerErrorAndLeavesNothingBehind(
r := newRig(t)
r.sub.err = uploads.ErrQueueFull
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
var rej *uploads.RejectError
require.True(t, errors.As(err, &rej), "a full queue must be a rejection the web layer can answer with 429: %v", err)
@@ -48,7 +48,7 @@ func TestAccept_ANoRoomQueueIsRefusedBeforeAnythingIsSpooledProbedOrStored(t *te
full := &roomSubmitter{room: false}
in := uploads.NewIntake(r.st, r.pr, full, uploads.Limits{MaxBytes: 1024, MaxDuration: 90 * time.Minute}, r.tmp)
_, err := in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
var rej *uploads.RejectError
require.True(t, errors.As(err, &rej))
@@ -62,7 +62,7 @@ func TestAccept_ANoRoomQueueIsRefusedBeforeAnythingIsSpooledProbedOrStored(t *te
func TestAccept_ASubmitterThatCannotSayStillWorksAsBefore(t *testing.T) {
r := newRig(t)
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
require.NoError(t, err, "a Submitter without HasRoom is accepted: the check at Submit remains")
}
@@ -81,7 +81,7 @@ func (b *brokenBody) Read(p []byte) (int, error) {
func TestAccept_AnErrorReadingTheClientsBodyIsMarkedAsTheClientsAndAWrappedSubmitErrorIsNot(t *testing.T) {
r := newRig(t)
_, err := r.in.Accept(context.Background(), "user-1", "t", &brokenBody{n: 1})
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, &brokenBody{n: 1})
require.Error(t, err)
assert.ErrorIs(t, err, uploads.ErrClientRead, "a dropped client body is the sender's to retry")
@@ -90,7 +90,7 @@ func TestAccept_AnErrorReadingTheClientsBodyIsMarkedAsTheClientsAndAWrappedSubmi
// A fault that is ours must NOT carry the marker, even if it wraps the same sentinel.
r2 := newRig(t)
r2.sub.err = fmt.Errorf("db: %w", io.ErrUnexpectedEOF)
_, err = r2.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err = r2.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
require.Error(t, err)
assert.NotErrorIs(t, err, uploads.ErrClientRead)
}
internal/uploads/intake_concurrency_test.go
+7 -6
View File
@@ -15,6 +15,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.d-ma.be/mathias/tapir/internal/domain"
"git.d-ma.be/mathias/tapir/internal/uploads"
)
@@ -31,7 +32,7 @@ func startSlowReceive(t *testing.T, r *rig, in *uploads.Intake, user string, wan
pr, pw := io.Pipe()
s := &slowReceive{w: pw, done: make(chan error, 1)}
go func() {
_, err := in.Accept(context.Background(), user, "t", pr)
_, err := in.Accept(context.Background(), user, "t", domain.LangSwedish, pr)
s.done <- err
}()
require.Eventually(t, func() bool { return len(r.tmpFiles(t)) == wantFiles }, 2*time.Second, 5*time.Millisecond,
@@ -80,7 +81,7 @@ func TestAccept_APerUserLimitOnConcurrentReceivesRefusesTheNextBodyBeforeSpoolin
startSlowReceive(t, r, in, "user-1", 1)
startSlowReceive(t, r, in, "user-1", 2)
_, err := in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
requireBusy(t, err)
assert.Len(t, r.tmpFiles(t), 2, "the refused body was never spooled")
@@ -98,7 +99,7 @@ func TestAccept_AnOverallLimitOnConcurrentReceivesRefusesTheNextUser(t *testing.
startSlowReceive(t, r, in, u, i+1)
}
_, err := in.Accept(context.Background(), "d", "t", strings.NewReader("data"))
_, err := in.Accept(context.Background(), "d", "t", domain.LangSwedish, strings.NewReader("data"))
requireBusy(t, err)
assert.Len(t, r.tmpFiles(t), 3)
@@ -112,7 +113,7 @@ func TestAccept_AFinishedReceiveFreesItsSlot(t *testing.T) {
first := startSlowReceive(t, r, in, "user-1", 1)
require.NoError(t, first.finish(t))
_, err := in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
require.NoError(t, err, "the slot is released when Accept returns")
}
@@ -122,10 +123,10 @@ func TestAccept_AFailedReceiveFreesItsSlot(t *testing.T) {
in := uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: time.Hour}, r.tmp)
in.MaxReceivesPerUser = 1
in.MaxReceives = 5
_, err := in.Accept(context.Background(), "user-1", "t", &brokenBody{n: 1})
_, err := in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, &brokenBody{n: 1})
require.ErrorIs(t, err, uploads.ErrClientRead)
_, err = in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err = in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
require.NoError(t, err, "a dropped client must not leak a slot")
}
internal/uploads/intake_defaults_test.go
+3 -2
View File
@@ -12,6 +12,7 @@ import (
"testing"
"time"
"git.d-ma.be/mathias/tapir/internal/domain"
"git.d-ma.be/mathias/tapir/internal/uploads"
)
@@ -21,11 +22,11 @@ func TestNewIntake_DefaultsAllowTwoReceivesPerUserAndFourOverall(t *testing.T) {
startSlowReceive(t, r, in, "a", 1)
startSlowReceive(t, r, in, "a", 2)
_, err := in.Accept(context.Background(), "a", "t", strings.NewReader("x"))
_, err := in.Accept(context.Background(), "a", "t", domain.LangSwedish, strings.NewReader("x"))
requireBusy(t, err) // a third body from one user
startSlowReceive(t, r, in, "b", 3)
startSlowReceive(t, r, in, "b", 4)
_, err = in.Accept(context.Background(), "c", "t", strings.NewReader("x"))
_, err = in.Accept(context.Background(), "c", "t", domain.LangSwedish, strings.NewReader("x"))
requireBusy(t, err) // a fifth body overall
}
internal/uploads/intake_test.go
+24 -11
View File
@@ -21,13 +21,15 @@ import (
type fakeStore struct {
created []string // titles
langs []domain.Language
hadVideo []bool
failed map[string]domain.UploadErrorKind
nextID string
}
func (f *fakeStore) CreateUpload(_ context.Context, _, title string, hadVideo bool) (string, error) {
func (f *fakeStore) CreateUpload(_ context.Context, _, title string, lang domain.Language, hadVideo bool) (string, error) {
f.created = append(f.created, title)
f.langs = append(f.langs, lang)
f.hadVideo = append(f.hadVideo, hadVideo)
return f.nextID, nil
}
@@ -98,7 +100,7 @@ func TestAccept_ValidAudioIsStoredProbedQueuedAndSubmitted(t *testing.T) {
r := newRig(t)
audio := bytes.Repeat([]byte("a"), 500)
id, err := r.in.Accept(context.Background(), "user-1", "Teams export", bytes.NewReader(audio))
id, err := r.in.Accept(context.Background(), "user-1", "Teams export", domain.LangSwedish, bytes.NewReader(audio))
require.NoError(t, err)
assert.Equal(t, "up-1", id)
@@ -106,7 +108,7 @@ func TestAccept_ValidAudioIsStoredProbedQueuedAndSubmitted(t *testing.T) {
assert.Equal(t, audio, r.pr.seen, "the prober must see the complete uploaded bytes, on disk")
require.Len(t, r.sub.jobs, 1)
job := r.sub.jobs[0]
assert.Equal(t, uploads.Job{UserID: "user-1", UploadID: "up-1", Path: job.Path, Duration: 10 * time.Minute}, job,
assert.Equal(t, uploads.Job{UserID: "user-1", UploadID: "up-1", Path: job.Path, Duration: 10 * time.Minute, Language: domain.LangSwedish}, job,
"the job carries what the file CLAIMED, so the worker can compare it with what was converted")
assert.Equal(t, filepath.Dir(job.Path), r.tmp, "the temp file lives in the configured directory")
got, err := os.ReadFile(job.Path)
@@ -117,7 +119,7 @@ func TestAccept_ValidAudioIsStoredProbedQueuedAndSubmitted(t *testing.T) {
func TestAccept_OversizeBodyIsRejectedAndLeavesNothingBehind(t *testing.T) {
r := newRig(t)
_, err := r.in.Accept(context.Background(), "user-1", "big", strings.NewReader(strings.Repeat("a", 1025)))
_, err := r.in.Accept(context.Background(), "user-1", "big", domain.LangSwedish, strings.NewReader(strings.Repeat("a", 1025)))
var rej *uploads.RejectError
require.True(t, errors.As(err, &rej))
@@ -132,7 +134,7 @@ func TestAccept_OversizeBodyIsRejectedAndLeavesNothingBehind(t *testing.T) {
func TestAccept_BodyExactlyAtTheLimitIsAccepted(t *testing.T) {
r := newRig(t)
_, err := r.in.Accept(context.Background(), "user-1", "edge", strings.NewReader(strings.Repeat("a", 1024)))
_, err := r.in.Accept(context.Background(), "user-1", "edge", domain.LangSwedish, strings.NewReader(strings.Repeat("a", 1024)))
require.NoError(t, err)
}
@@ -152,7 +154,7 @@ func TestAccept_ProbeVerdictRejectsWithoutRowAndRemovesTheFile(t *testing.T) {
r := newRig(t)
r.pr.probe = tt.probe
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
var rej *uploads.RejectError
require.True(t, errors.As(err, &rej))
@@ -169,7 +171,7 @@ func TestAccept_UnreadableFileIsARejectionButAProberFaultIsAnError(t *testing.T)
r := newRig(t)
r.pr.err = uploads.ErrUnreadable
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("not media"))
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("not media"))
var rej *uploads.RejectError
require.True(t, errors.As(err, &rej), "the user's file is the problem: tell them")
@@ -180,7 +182,7 @@ func TestAccept_UnreadableFileIsARejectionButAProberFaultIsAnError(t *testing.T)
r := newRig(t)
r.pr.err = errors.New("exec: ffprobe: executable file not found")
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
require.Error(t, err)
var rej *uploads.RejectError
@@ -194,7 +196,7 @@ func TestAccept_ClientDisconnectMidUploadLeavesNothingBehind(t *testing.T) {
r := newRig(t)
body := io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(errors.New("client went away")))
_, err := r.in.Accept(context.Background(), "user-1", "t", body)
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, body)
require.Error(t, err)
assert.Empty(t, r.pr.paths)
@@ -206,7 +208,7 @@ func TestAccept_SubmitFailureFailsTheRowHonestlyAndRemovesTheFile(t *testing.T)
r := newRig(t)
r.sub.err = errors.New("worker stopped")
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
require.Error(t, err)
assert.Equal(t, domain.UploadErrInterrupted, r.st.failed["up-1"],
@@ -228,7 +230,7 @@ func TestAccept_AFileWithAVideoTrackIsAcceptedAndRecordsThatItHadOne(t *testing.
r := newRig(t)
r.pr.probe = tt.probe
id, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
id, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangSwedish, strings.NewReader("data"))
require.NoError(t, err)
assert.Equal(t, "up-1", id)
@@ -238,3 +240,14 @@ func TestAccept_AFileWithAVideoTrackIsAcceptedAndRecordsThatItHadOne(t *testing.
})
}
}
func TestAccept_TheChosenLanguageIsStoredAndHandedToTheJob(t *testing.T) {
r := newRig(t)
_, err := r.in.Accept(context.Background(), "user-1", "t", domain.LangAuto, strings.NewReader("data"))
require.NoError(t, err)
assert.Equal(t, []domain.Language{domain.LangAuto}, r.st.langs, "the row records what the user chose")
require.Len(t, r.sub.jobs, 1)
assert.Equal(t, domain.LangAuto, r.sub.jobs[0].Language, "the worker is told, so the engines are")
}
internal/uploads/worker.go
+1 -1
View File
@@ -313,7 +313,7 @@ func (w *Worker) Process(ctx context.Context, job Job) {
if !planned {
tr.Begin("transcribe", 0)
}
t, err := w.tr.Transcribe(progress.With(jobCtx, tr), wav)
t, err := w.tr.Transcribe(domain.WithLanguage(progress.With(jobCtx, tr), job.Language), wav)
if err != nil {
log.Error("upload: transcribe", "kind", errkind.Of(err), "elapsed", time.Since(start).Round(time.Second))
abort(domain.UploadErrTranscriptionFailed)
internal/uploads/worker_test.go
+23
View File
@@ -126,6 +126,7 @@ type fakeTranscriber struct {
result domain.Transcription
err error
paths []string
langs []domain.Language // what each call's context carried
// block, when non-nil, holds every call until it is closed or ctx ends.
block chan struct{}
running atomic.Int32
@@ -134,6 +135,7 @@ type fakeTranscriber struct {
func (f *fakeTranscriber) Transcribe(ctx context.Context, wav string) (domain.Transcription, error) {
f.paths = append(f.paths, wav)
f.langs = append(f.langs, domain.LanguageFrom(ctx))
n := f.running.Add(1)
defer f.running.Add(-1)
for {
@@ -337,3 +339,24 @@ func (dbErr) Error() string {
return `ERROR: invalid input syntax for type json: "zebrakaka secret words"`
}
func (dbErr) SQLState() string { return "22P02" }
func TestWorker_TheTranscriberSeesTheLanguageOfTheJobOnItsContext(t *testing.T) {
for name, tt := range map[string]struct {
job domain.Language
want domain.Language
}{
"auto": {domain.LangAuto, domain.LangAuto},
"swedish": {domain.LangSwedish, domain.LangSwedish},
"a job with no language": {"", domain.LangSwedish}, // queued before the field existed
} {
t.Run(name, func(t *testing.T) {
r := newWRig(t, time.Minute)
job := r.spool(t, "a")
job.Language = tt.job
r.w.Process(context.Background(), job)
assert.Equal(t, []domain.Language{tt.want}, r.tr.langs)
})
}
}