docs(adr): ADR-033 amended for SIGTERM handling and the recorded deploy limitation (#50, #47)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

DECISIONS.md

@@ -1447,6 +1447,16 @@ an hour of Swedish *was* an unknown here; it is now measured, see decision 8.)
    re-uploads. The single-replica assumption gains a second dependent (the semaphore is
    process-local) and is recorded here.
 
+   *Amended 2026-10-03 (#50, #47).* The process now handles SIGTERM (what Kubernetes sends on
+   every deploy; only SIGINT was handled, so shutdown code never ran). On shutdown the running job
+   is recorded `interrupted` at once and the process waits, bounded to 10 s each, for in-flight
+   requests and for that write; the start-up sweep stays the backstop for a kill. **Known
+   limitation, deliberately not built: a deploy still loses a job in flight** (finishing it needs
+   a drain of up to the job time, a ~330 s termination grace and `strategy: Recreate`, i.e.
+   downtime on every deploy; a 90-minute file takes ~4.5 min and a deploy lands on a running job
+   perhaps one time in five at current volume). Jobs still queued at shutdown stay `queued` until
+   the next start's sweep. The user re-uploads, with an honest message.
+
 8. **Chat reuses `chat.Service`; it is not forked.** The uploaded-transcript chat handlers
    look the row up through the RLS-scoped store, so a non-owner gets 404 by the same mechanism
    as ADR-027's summary gate, **before any model is called**. ADR-027's decisions (ephemeral,

cmd/tapir/main.go

@@ -17,9 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"net"
 	"net/http"
 	"os"
-	"os/signal"
 	"sync"
 	"time"
 
@@ -41,7 +41,7 @@ func main() {
 	}
 
 	log := slog.New(slog.NewTextHandler(os.Stderr, nil))
-	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	ctx, stop := shutdownContext(context.Background())
 	defer stop()
 
 	var err error
@@ -215,9 +215,11 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
 	if err != nil {
 		return err
 	}
+	var workerDone chan struct{} // nil without uploads: nothing to wait for at shutdown
 	wireUploads(app, cfg, intake)
 	if intake != nil {
-		go worker.Run(ctx)
+		workerDone = make(chan struct{})
+		go func() { worker.Run(ctx); close(workerDone) }()
 		log.Info("audio upload enabled", "model", cfg.TranscribeModel, "fallback", cfg.TranscribeFallbackModel,
 			"max_mb", cfg.UploadMaxBytes>>20, "max_minutes", int(cfg.UploadMaxDuration.Minutes()))
 		log.Warn("audio upload assumes a SINGLE replica: the transcription queue is process-local (ADR-033, ADR-018)")
@@ -350,20 +352,19 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
 		}()
 	}
 
-	// Graceful shutdown on signal: stop accepting, drain in-flight requests.
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		_ = srv.Shutdown(shutdownCtx)
-		if metricsSrv != nil {
-			_ = metricsSrv.Shutdown(shutdownCtx)
-		}
-	}()
-
+	// Graceful shutdown on signal: stop accepting, drain in-flight requests, and let the
+	// worker record the job it was running as interrupted (#50).
+	var extra []*http.Server
+	if metricsSrv != nil {
+		extra = append(extra, metricsSrv)
+	}
+	ln, err := net.Listen("tcp", srv.Addr)
+	if err != nil {
+		return fmt.Errorf("serve: listen: %w", err)
+	}
 	log.Info("serving web ui", "addr", cfg.HTTPAddr, "user", cfg.UserID)
-	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
-		return fmt.Errorf("serve: %w", err)
+	if err := serveHTTP(ctx, srv, ln, extra, workerDone, shutdownGrace); err != nil {
+		return err
 	}
 	return nil
 }

cmd/tapir/shutdown.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+// shutdownGrace bounds each wait at shutdown (in-flight requests, then the worker). Two of
+// them stay inside Kubernetes' default 30 s termination grace.
+const shutdownGrace = 10 * time.Second
+
+// shutdownContext is cancelled by SIGINT (a terminal) or SIGTERM (what Kubernetes sends
+// on every deploy). Handling only SIGINT meant a deploy killed the process before any
+// shutdown code ran (#50).
+func shutdownContext(parent context.Context) (context.Context, context.CancelFunc) {
+	return signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM)
+}
+
+// serveHTTP serves srv on ln until it fails or ctx ends, then shuts every server down and
+// waits for in-flight requests and for workerDone (nil: no worker) to finish, each for at
+// most grace. It must not return earlier: Serve returns http.ErrServerClosed as soon as
+// Shutdown STARTS, and returning from main kills the drain and the worker's last write.
+func serveHTTP(ctx context.Context, srv *http.Server, ln net.Listener, extra []*http.Server, workerDone <-chan struct{}, grace time.Duration) error {
+	errc := make(chan error, 1)
+	go func() { errc <- srv.Serve(ln) }()
+
+	select {
+	case err := <-errc:
+		if !errors.Is(err, http.ErrServerClosed) {
+			return fmt.Errorf("serve: %w", err)
+		}
+		return nil
+	case <-ctx.Done():
+	}
+
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), grace)
+	defer cancel()
+	for _, s := range append([]*http.Server{srv}, extra...) {
+		_ = s.Shutdown(shutdownCtx)
+	}
+	if workerDone != nil {
+		select {
+		case <-workerDone:
+		case <-time.After(grace):
+		}
+	}
+	return nil
+}

cmd/tapir/shutdown_test.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// Kubernetes stops a pod with SIGTERM; only SIGINT was handled, so a deploy killed the
+// process without running any shutdown code (#50).
+func TestShutdownContext_IsCancelledBySIGTERMAndSIGINT(t *testing.T) {
+	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
+		t.Run(sig.String(), func(t *testing.T) {
+			ctx, stop := shutdownContext(context.Background())
+			defer stop()
+
+			if err := syscall.Kill(os.Getpid(), sig); err != nil {
+				t.Fatalf("send %v: %v", sig, err)
+			}
+
+			select {
+			case <-ctx.Done():
+			case <-time.After(2 * time.Second):
+				t.Fatalf("context not cancelled by %v", sig)
+			}
+		})
+	}
+}
+
+// serveHTTP must not return while a request is in flight or the worker is still recording
+// its job: ListenAndServe returns ErrServerClosed the moment Shutdown starts, so returning
+// from it let the process exit before either finished (#50).
+func TestServeHTTP_ReturnsOnlyAfterTheInFlightRequestAndTheWorkerHaveFinished(t *testing.T) {
+	release := make(chan struct{})
+	entered := make(chan struct{})
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		close(entered)
+		<-release
+		_, _ = w.Write([]byte("ok"))
+	}))
+	srv.Start()
+	defer srv.Close()
+	ctx, cancel := context.WithCancel(context.Background())
+	workerDone := make(chan struct{})
+	returned := make(chan error, 1)
+	go func() { returned <- serveHTTP(ctx, srv.Config, srv.Listener, nil, workerDone, 5*time.Second) }()
+
+	go func() { _, _ = http.Get(srv.URL) }()
+	<-entered
+	cancel()
+
+	assertPending := func(why string) {
+		t.Helper()
+		select {
+		case err := <-returned:
+			t.Fatalf("serveHTTP returned (%v) %s", err, why)
+		case <-time.After(150 * time.Millisecond):
+		}
+	}
+	assertPending("while a request was still in flight")
+	close(release)
+	assertPending("before the worker finished")
+	close(workerDone)
+	select {
+	case err := <-returned:
+		if err != nil {
+			t.Fatalf("serveHTTP: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("serveHTTP did not return once the request and the worker had finished")
+	}
+}
+
+func TestServeHTTP_GivesUpOnAWorkerThatNeverFinishesAfterTheGrace(t *testing.T) {
+	srv := httptest.NewUnstartedServer(http.NotFoundHandler())
+	srv.Start()
+	defer srv.Close()
+	ctx, cancel := context.WithCancel(context.Background())
+	returned := make(chan error, 1)
+	go func() {
+		returned <- serveHTTP(ctx, srv.Config, srv.Listener, nil, make(chan struct{}), 200*time.Millisecond)
+	}()
+
+	cancel()
+
+	select {
+	case <-returned:
+	case <-time.After(3 * time.Second):
+		t.Fatal("a stuck worker must not hold the process past the grace period")
+	}
+}

internal/adapters/audio/ffprobe.go

@@ -4,6 +4,7 @@
 package audio
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -19,6 +20,51 @@ import (
 type FFprobe struct {
 	// Bin is the ffprobe executable ("ffprobe" resolves via PATH).
 	Bin string
+	// MaxOutput bounds how much of ffprobe's stdout is read; zero means DefaultMaxOutput.
+	MaxOutput int
+	// Timeout bounds one probe; zero means DefaultProbeTimeout. A probe is milliseconds of
+	// work on a real file, so one that runs this long is the FILE's doing (#49).
+	Timeout time.Duration
+}
+
+// DefaultProbeTimeout is the probe deadline when FFprobe.Timeout is unset.
+const DefaultProbeTimeout = 45 * time.Second
+
+// DefaultMaxOutput is far above the narrowed probe output of a legitimate file (a few
+// hundred bytes) and far below what a hostile one can make ffprobe print.
+const DefaultMaxOutput = 1 << 20
+
+// maxStderr is how much of ffprobe's stderr is kept for the error message.
+const maxStderr = 4096
+
+// errOutputCap is the write error that stops ffprobe once it prints past the cap.
+var errOutputCap = errors.New("audio: ffprobe output exceeds the cap")
+
+// capWriter collects up to max bytes and then refuses further writes, which closes the
+// pipe and ends the process instead of buffering whatever it prints.
+type capWriter struct {
+	buf bytes.Buffer
+	max int
+}
+
+func (w *capWriter) Write(p []byte) (int, error) {
+	if w.buf.Len()+len(p) > w.max {
+		return 0, errOutputCap
+	}
+	return w.buf.Write(p)
+}
+
+// headWriter keeps the first max bytes and silently drops the rest.
+type headWriter struct {
+	buf bytes.Buffer
+	max int
+}
+
+func (w *headWriter) Write(p []byte) (int, error) {
+	if room := w.max - w.buf.Len(); room > 0 {
+		w.buf.Write(p[:min(room, len(p))])
+	}
+	return len(p), nil
 }
 
 // AllowedDemuxers are the input formats an upload may be. The input is
@@ -62,20 +108,37 @@ func (f *FFprobe) Probe(ctx context.Context, path string) (uploads.Probe, error)
 	}
 	args := append([]string{"-v", "error"}, inputRestrictions...)
 	args = append(args, "-print_format", "json", "-show_entries", probeEntries, path)
-	cmd := exec.CommandContext(ctx, bin, args...)
+	timeout := f.Timeout
+	if timeout <= 0 {
+		timeout = DefaultProbeTimeout
+	}
+	probeCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	cmd := exec.CommandContext(probeCtx, bin, args...)
 	cmd.WaitDelay = killGrace
-	out, err := cmd.Output()
-	if err != nil {
-		if ctxErr := ctx.Err(); ctxErr != nil {
+	maxOut := f.MaxOutput
+	if maxOut <= 0 {
+		maxOut = DefaultMaxOutput
+	}
+	stdout, stderr := &capWriter{max: maxOut}, &headWriter{max: maxStderr}
+	cmd.Stdout, cmd.Stderr = stdout, stderr
+	if err := cmd.Run(); err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil { // the caller gave up: not a verdict on the file
 			return uploads.Probe{}, fmt.Errorf("audio: ffprobe: %w", ctxErr)
 		}
+		if probeCtx.Err() != nil { // our own deadline passed while the caller was still waiting
+			return uploads.Probe{}, fmt.Errorf("%w: ffprobe did not finish within %s", uploads.ErrUnreadable, timeout)
+		}
+		if errors.Is(err, errOutputCap) {
+			return uploads.Probe{}, fmt.Errorf("%w: ffprobe printed more than %d bytes", uploads.ErrUnreadable, maxOut)
+		}
 		var exit *exec.ExitError
 		if errors.As(err, &exit) {
-			return uploads.Probe{}, fmt.Errorf("%w: ffprobe: %s", uploads.ErrUnreadable, exit.Stderr)
+			return uploads.Probe{}, fmt.Errorf("%w: ffprobe: %s", uploads.ErrUnreadable, stderr.buf.String())
 		}
 		return uploads.Probe{}, fmt.Errorf("audio: run ffprobe: %w", err)
 	}
-	p, err := ParseProbe(out)
+	p, err := ParseProbe(stdout.buf.Bytes())
 	if err != nil {
 		return uploads.Probe{}, fmt.Errorf("audio: ffprobe output: %w", err)
 	}

internal/adapters/audio/ffprobe_test.go

@@ -234,3 +234,28 @@ func TestFFprobeProbe_OutputWithinTheCapIsParsed(t *testing.T) {
 	require.NoError(t, err)
 	assert.True(t, got.HasAudio)
 }
+
+func TestFFprobeProbe_AHungProbeTimesOutOnItsOwnDeadlineAsUnreadableNotAsOurFault(t *testing.T) {
+	// Accept runs the probe on the request context, which has no deadline of its own;
+	// a probe that never returns would pin a request (#49). `exec` so the sleep is the
+	// process itself and not a child holding the pipe open.
+	bin := stub(t, `exec sleep 30`)
+
+	start := time.Now()
+	_, err := (&audio.FFprobe{Bin: bin, Timeout: 200 * time.Millisecond}).Probe(context.Background(), "/x")
+
+	assert.ErrorIs(t, err, uploads.ErrUnreadable)
+	assert.Less(t, time.Since(start), 3*time.Second)
+}
+
+func TestFFprobeProbe_ParentCancellationIsStillNotUnreadableWhenATimeoutIsSet(t *testing.T) {
+	bin := stub(t, `exec sleep 30`)
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	_, err := (&audio.FFprobe{Bin: bin, Timeout: time.Minute}).Probe(ctx, "/x")
+
+	require.Error(t, err)
+	assert.False(t, errors.Is(err, uploads.ErrUnreadable), "the caller giving up is not a verdict on the file")
+	assert.ErrorIs(t, err, context.DeadlineExceeded)
+}

internal/uploads/intake.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sync"
 	"time"
 
 	"git.d-ma.be/mathias/tapir/internal/domain"
@@ -51,6 +52,53 @@ type Intake struct {
 	sub    Submitter
 	limits Limits
 	tmpDir string
+
+	// MaxReceives and MaxReceivesPerUser bound how many request bodies Accept reads at
+	// once; zero means the defaults. Without them N parallel POSTs each spool up to
+	// MaxBytes while the queue still looks empty, and the spool volume fills (#47).
+	MaxReceives        int
+	MaxReceivesPerUser int
+
+	mu       sync.Mutex
+	receives int
+	byUser   map[string]int
+}
+
+const (
+	defaultMaxReceives        = 4
+	defaultMaxReceivesPerUser = 2
+)
+
+// acquireReceive reserves a receive slot for userID, or reports there is none. Every
+// successful call must be paired with releaseReceive.
+func (in *Intake) acquireReceive(userID string) bool {
+	maxAll, maxUser := in.MaxReceives, in.MaxReceivesPerUser
+	if maxAll <= 0 {
+		maxAll = defaultMaxReceives
+	}
+	if maxUser <= 0 {
+		maxUser = defaultMaxReceivesPerUser
+	}
+	in.mu.Lock()
+	defer in.mu.Unlock()
+	if in.receives >= maxAll || in.byUser[userID] >= maxUser {
+		return false
+	}
+	if in.byUser == nil {
+		in.byUser = map[string]int{}
+	}
+	in.receives++
+	in.byUser[userID]++
+	return true
+}
+
+func (in *Intake) releaseReceive(userID string) {
+	in.mu.Lock()
+	defer in.mu.Unlock()
+	in.receives--
+	if in.byUser[userID]--; in.byUser[userID] <= 0 {
+		delete(in.byUser, userID)
+	}
 }
 
 // NewIntake wires an Intake. tmpDir is where uploads are spooled; it must exist.
@@ -94,6 +142,11 @@ func (in *Intake) Accept(ctx context.Context, userID, title string, src io.Reade
 	if rc, ok := in.sub.(interface{ HasRoom(userID string) bool }); ok && !rc.HasRoom(userID) {
 		return "", &RejectError{Reason: ReasonBusy, Message: busyMessage}
 	}
+	// HasRoom cannot see bodies still being received; bound those separately.
+	if !in.acquireReceive(userID) {
+		return "", &RejectError{Reason: ReasonBusy, Message: busyMessage}
+	}
+	defer in.releaseReceive(userID)
 	f, err := os.CreateTemp(in.tmpDir, "upload-*")
 	if err != nil {
 		return "", fmt.Errorf("uploads: spool: %w", err)

internal/uploads/intake_concurrency_test.go

@@ -0,0 +1,131 @@
+package uploads_test
+
+// Observes how many request bodies Accept will read at once (#47). HasRoom and the
+// per-user queue cap act before and after the receive; N parallel POSTs all pass
+// HasRoom while the queue is still empty, so the receive itself is bounded.
+
+import (
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"git.d-ma.be/mathias/tapir/internal/uploads"
+)
+
+// slowReceive starts an Accept whose body stays open until the returned writer is closed,
+// and waits until it holds a spool file, which proves it is inside the receive.
+type slowReceive struct {
+	w        *io.PipeWriter
+	done     chan error
+	finished bool
+}
+
+func startSlowReceive(t *testing.T, r *rig, in *uploads.Intake, user string, wantFiles int) *slowReceive {
+	t.Helper()
+	pr, pw := io.Pipe()
+	s := &slowReceive{w: pw, done: make(chan error, 1)}
+	go func() {
+		_, err := in.Accept(context.Background(), user, "t", pr)
+		s.done <- err
+	}()
+	require.Eventually(t, func() bool { return len(r.tmpFiles(t)) == wantFiles }, 2*time.Second, 5*time.Millisecond,
+		"the receive for %s should have started", user)
+	// An abandoned receive ends in a read error, so it never reaches the rig's fakes, which
+	// are not safe for concurrent use. A receive that finish() already ended is a no-op.
+	t.Cleanup(func() {
+		if s.finished {
+			return
+		}
+		_ = pw.CloseWithError(errors.New("test over"))
+		select {
+		case <-s.done:
+		case <-time.After(2 * time.Second):
+		}
+	})
+	return s
+}
+
+func (s *slowReceive) finish(t *testing.T) error {
+	t.Helper()
+	_, _ = s.w.Write([]byte("data"))
+	require.NoError(t, s.w.Close())
+	s.finished = true
+	select {
+	case err := <-s.done:
+		return err
+	case <-time.After(2 * time.Second):
+		t.Fatal("Accept did not return after its body ended")
+		return nil
+	}
+}
+
+func requireBusy(t *testing.T, err error) {
+	t.Helper()
+	var rej *uploads.RejectError
+	require.True(t, errors.As(err, &rej), "want a busy rejection, got %v", err)
+	assert.Equal(t, uploads.ReasonBusy, rej.Reason)
+}
+
+func TestAccept_APerUserLimitOnConcurrentReceivesRefusesTheNextBodyBeforeSpoolingIt(t *testing.T) {
+	r := newRig(t)
+	in := uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: time.Hour}, r.tmp)
+	in.MaxReceivesPerUser = 2
+	in.MaxReceives = 10
+	startSlowReceive(t, r, in, "user-1", 1)
+	startSlowReceive(t, r, in, "user-1", 2)
+
+	_, err := in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
+
+	requireBusy(t, err)
+	assert.Len(t, r.tmpFiles(t), 2, "the refused body was never spooled")
+	// Another user is not blocked by user-1's two receives.
+	other := startSlowReceive(t, r, in, "user-2", 3)
+	require.NoError(t, other.finish(t))
+}
+
+func TestAccept_AnOverallLimitOnConcurrentReceivesRefusesTheNextUser(t *testing.T) {
+	r := newRig(t)
+	in := uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: time.Hour}, r.tmp)
+	in.MaxReceivesPerUser = 5
+	in.MaxReceives = 3
+	for i, u := range []string{"a", "b", "c"} {
+		startSlowReceive(t, r, in, u, i+1)
+	}
+
+	_, err := in.Accept(context.Background(), "d", "t", strings.NewReader("data"))
+
+	requireBusy(t, err)
+	assert.Len(t, r.tmpFiles(t), 3)
+}
+
+func TestAccept_AFinishedReceiveFreesItsSlot(t *testing.T) {
+	r := newRig(t)
+	in := uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: time.Hour}, r.tmp)
+	in.MaxReceivesPerUser = 1
+	in.MaxReceives = 5
+	first := startSlowReceive(t, r, in, "user-1", 1)
+	require.NoError(t, first.finish(t))
+
+	_, err := in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
+
+	require.NoError(t, err, "the slot is released when Accept returns")
+}
+
+func TestAccept_AFailedReceiveFreesItsSlot(t *testing.T) {
+	r := newRig(t)
+	in := uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: time.Hour}, r.tmp)
+	in.MaxReceivesPerUser = 1
+	in.MaxReceives = 5
+	_, err := in.Accept(context.Background(), "user-1", "t", &brokenBody{n: 1})
+	require.ErrorIs(t, err, uploads.ErrClientRead)
+
+	_, err = in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
+
+	require.NoError(t, err, "a dropped client must not leak a slot")
+}

internal/uploads/worker.go

@@ -176,9 +176,10 @@ func (w *Worker) Submit(j Job) error {
 }
 
 // Run processes queued jobs until ctx ends, then deletes the audio of anything
-// still queued and refuses new jobs. The rows of those jobs stay queued or
-// transcribing and are failed as interrupted by the start-up sweep
-// (store.FailInterruptedUploads): the audio is not retained, so they cannot resume.
+// still queued and refuses new jobs. The job running at shutdown is recorded interrupted
+// by Process; the rows of jobs still queued stay queued and are failed as interrupted by
+// the start-up sweep (store.FailInterruptedUploads): the audio is not retained, so they
+// cannot resume.
 func (w *Worker) Run(ctx context.Context) {
 	defer w.stop()
 	for {
@@ -227,7 +228,7 @@ func (w *Worker) stop() {
 // Process runs one job: row -> transcribing, convert, transcribe, store. The
 // uploaded file and the converted WAV are deleted on every path — the audio is
 // never retained (ADR-033). A failure is recorded with a TYPED kind; a shutdown
-// mid-job is not a failure and leaves the row for the start-up sweep.
+// mid-job is recorded as interrupted at once (see fail).
 //
 // Every phase is reported through one progress.Tracker, which saves it to the row for
 // the page and logs it, and a heartbeat stamps the row on its own while the job runs
@@ -269,7 +270,7 @@ func (w *Worker) Process(ctx context.Context, job Job) {
 	defer func() { stopBeat(); <-hbDone }() // no heartbeat outlives its job
 
 	// abort records where the job stopped. A shutdown (the parent context ended) is not
-	// a failure: the row is left for the start-up sweep, so the step stays as it was.
+	// a failure of the step: it stays as it was, and fail records the row as interrupted.
 	abort := func(kind domain.UploadErrorKind) {
 		if ctx.Err() == nil {
 			tr.FailRunning()
@@ -332,11 +333,19 @@ func (w *Worker) Process(ctx context.Context, job Job) {
 		"segments", len(t.Segments), "elapsed", time.Since(start).Round(time.Second))
 }
 
-// fail records a transcription failure — unless the PARENT context ended, which
-// is a shutdown, not a failure: the row is left for the start-up sweep.
+// interruptedWriteTimeout bounds the one write a dying process makes to record its job.
+const interruptedWriteTimeout = 3 * time.Second
+
+// fail records a transcription failure. When the PARENT context ended the process is
+// shutting down: the job is recorded interrupted at once, on a short context of its own,
+// because the process may exit before any other pod's start-up sweep would do it (#50).
+// The sweep remains the backstop for a process that is killed outright.
 func (w *Worker) fail(ctx context.Context, job Job, log *slog.Logger, kind domain.UploadErrorKind) {
 	if ctx.Err() != nil {
-		return
+		kind = domain.UploadErrInterrupted
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), interruptedWriteTimeout)
+		defer cancel()
 	}
 	if err := w.store.FailUpload(ctx, job.UserID, job.UploadID, kind); err != nil {
 		log.Error("upload: record failure", "kind", errkind.Of(err))

internal/uploads/worker_test.go

@@ -53,7 +53,10 @@ func (s *wStore) CompleteUpload(_ context.Context, _, id string, t domain.Transc
 	return nil
 }
 
-func (s *wStore) FailUpload(_ context.Context, _, id string, k domain.UploadErrorKind) error {
+func (s *wStore) FailUpload(ctx context.Context, _, id string, k domain.UploadErrorKind) error {
+	if err := ctx.Err(); err != nil { // like the real store: a cancelled context cannot write
+		return err
+	}
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.failed[id] = k
@@ -242,7 +245,9 @@ func TestWorkerProcess_FailuresAreTypedAndStillDeleteTheAudio(t *testing.T) {
 	})
 }
 
-func TestWorkerProcess_ShutdownMidJobLeavesTheRowForTheStartupSweep(t *testing.T) {
+func TestWorkerProcess_ShutdownMidJobRecordsTheRowInterruptedAtOnce(t *testing.T) {
+	// SIGTERM on a deploy used to leave the row "transcribing" until the NEXT pod's
+	// start-up sweep; the process may exit before any sweep runs (#50).
 	r := newWRig(t, time.Minute)
 	r.tr.block = make(chan struct{})
 	ctx, cancel := context.WithCancel(context.Background())
@@ -255,8 +260,8 @@ func TestWorkerProcess_ShutdownMidJobLeavesTheRowForTheStartupSweep(t *testing.T
 	<-done
 
 	_, f := r.st.snapshot()
-	assert.Empty(t, f, "a shutdown is not a transcription failure: the row stays for FailInterruptedUploads to mark interrupted")
-	assert.Empty(t, r.leftovers(t), "but the audio is still deleted")
+	assert.Equal(t, domain.UploadErrInterrupted, f["a"], "the row is recorded interrupted by the dying process, not left for a sweep")
+	assert.Empty(t, r.leftovers(t), "and the audio is still deleted")
 }
 
 func TestWorkerProcess_AJobThatOutlivesItsTimeoutIsFailedNotStuck(t *testing.T) {