Compare commits
23
Commits
@@ -207,6 +207,27 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
|
|||||||
log.Info("web chat enabled (stored-transcript only)", "models", chatModels(cfg))
|
log.Info("web chat enabled (stored-transcript only)", "models", chatModels(cfg))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Audio upload -> private transcript (ADR-033). Off unless TAPIR_UPLOADS_ENABLED:
|
||||||
|
// the image must carry ffmpeg first. The worker is IN-PROCESS and one job at a
|
||||||
|
// time, so it inherits the single-replica assumption below (its queue is
|
||||||
|
// process-local); it stops with the same ctx as the server.
|
||||||
|
intake, worker, err := buildUploads(ctx, cfg, st, log)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if intake != nil {
|
||||||
|
app.Uploads = intake
|
||||||
|
app.UploadLimits = uploadLimits(cfg)
|
||||||
|
go worker.Run(ctx)
|
||||||
|
log.Info("audio upload enabled", "model", cfg.TranscribeModel, "fallback", cfg.TranscribeFallbackModel,
|
||||||
|
"max_mb", cfg.UploadMaxBytes>>20, "max_minutes", int(cfg.UploadMaxDuration.Minutes()))
|
||||||
|
log.Warn("audio upload assumes a SINGLE replica: the transcription queue is process-local (ADR-033, ADR-018)")
|
||||||
|
if cfg.TranscribeFallbackModel != "" {
|
||||||
|
log.Warn("audio upload fallback is a CLOUD endpoint: if the primary fails, private audio leaves the homelab",
|
||||||
|
"fallback", cfg.TranscribeFallbackModel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// User onboarding is handled by the IdP (Authentik invite flow), not Tapir —
|
// User onboarding is handled by the IdP (Authentik invite flow), not Tapir —
|
||||||
// the Dex local-password provisioning path was removed (ADR-019). An
|
// the Dex local-password provisioning path was removed (ADR-019). An
|
||||||
// authenticated subject with no Tapir user is routed to /register.
|
// authenticated subject with no Tapir user is routed to /register.
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// uploadsStore is the store surface the upload wiring needs. *store.Store
|
||||||
|
// satisfies it.
|
||||||
|
type uploadsStore interface {
|
||||||
|
uploads.Store
|
||||||
|
uploads.WorkerStore
|
||||||
|
FailInterruptedUploads(ctx context.Context) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildUploads wires audio upload and private transcription (ADR-033). It returns
|
||||||
|
// nils when uploads are disabled — never a typed-nil Intake, which would mount the
|
||||||
|
// /uploads routes over nothing. When enabled it, in order: prepares a private spool
|
||||||
|
// directory, deletes any audio a crashed run left in it, fails the rows such a run
|
||||||
|
// left queued or transcribing (the audio is not retained, so they can never
|
||||||
|
// resume), and only then builds the intake and the worker. The caller runs the
|
||||||
|
// worker. The fallback endpoint is a cloud service; an empty
|
||||||
|
// TAPIR_TRANSCRIBE_FALLBACK_MODEL keeps private audio on the homelab.
|
||||||
|
func buildUploads(ctx context.Context, cfg config.Config, st uploadsStore, log *slog.Logger) (*uploads.Intake, *uploads.Worker, error) {
|
||||||
|
if !cfg.UploadsEnabled {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
if cfg.GatewayURL == "" {
|
||||||
|
return nil, nil, fmt.Errorf("uploads: TAPIR_UPLOADS_ENABLED is set but there is no TAPIR_GATEWAY_URL to transcribe through")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(cfg.UploadTmpDir, 0o700); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("uploads: spool dir: %w", err)
|
||||||
|
}
|
||||||
|
removeSpooledAudio(cfg.UploadTmpDir, log)
|
||||||
|
|
||||||
|
n, err := st.FailInterruptedUploads(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("uploads: fail interrupted rows: %w", err)
|
||||||
|
}
|
||||||
|
if n > 0 {
|
||||||
|
log.Warn("uploads: marked interrupted by a restart (audio is not retained; they must be re-uploaded)", "count", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// No overall client timeout: a 90 minute recording takes minutes. The deadline
|
||||||
|
// is the worker's per-job context.
|
||||||
|
client := &http.Client{}
|
||||||
|
primary := transcribe.NewGateway(cfg.GatewayURL, cfg.GatewayKey, cfg.TranscribeModel, client)
|
||||||
|
var fallback transcribe.Backend
|
||||||
|
if cfg.TranscribeFallbackModel != "" && cfg.TranscribeFallbackModel != cfg.TranscribeModel {
|
||||||
|
fallback = transcribe.NewGateway(cfg.GatewayURL, cfg.GatewayKey, cfg.TranscribeFallbackModel, client)
|
||||||
|
}
|
||||||
|
|
||||||
|
worker := uploads.NewWorker(st, &audio.FFmpeg{Bin: cfg.FFmpegBin}, transcribe.NewChain(primary, fallback),
|
||||||
|
cfg.UploadTmpDir, cfg.TranscribeTimeout)
|
||||||
|
intake := uploads.NewIntake(st, &audio.FFprobe{Bin: cfg.FFprobeBin}, worker,
|
||||||
|
uploadLimits(cfg), cfg.UploadTmpDir)
|
||||||
|
return intake, worker, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadLimits(cfg config.Config) uploads.Limits {
|
||||||
|
return uploads.Limits{MaxBytes: cfg.UploadMaxBytes, MaxDuration: cfg.UploadMaxDuration}
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeSpooledAudio deletes the spool files this process family creates
|
||||||
|
// (upload-*, wav-*) and nothing else.
|
||||||
|
func removeSpooledAudio(dir string, log *slog.Logger) {
|
||||||
|
for _, pat := range []string{"upload-*", "wav-*"} {
|
||||||
|
matches, _ := filepath.Glob(filepath.Join(dir, pat))
|
||||||
|
for _, m := range matches {
|
||||||
|
if err := os.Remove(m); err != nil {
|
||||||
|
log.Warn("uploads: remove leftover audio", "path", m, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeUploadsStore satisfies the store surface buildUploads needs, and counts the
|
||||||
|
// start-up sweep.
|
||||||
|
type fakeUploadsStore struct {
|
||||||
|
sweeps int
|
||||||
|
sweepErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) CreateUpload(context.Context, string, string) (string, error) {
|
||||||
|
return "id", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) FailUpload(context.Context, string, string, domain.UploadErrorKind) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) StartTranscribing(context.Context, string, string) error { return nil }
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) CompleteUpload(context.Context, string, string, domain.Transcription) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) FailInterruptedUploads(context.Context) (int, error) {
|
||||||
|
f.sweeps++
|
||||||
|
return 2, f.sweepErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadCfg(t *testing.T, enabled bool) config.Config {
|
||||||
|
t.Helper()
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.GatewayURL = "http://gateway.invalid/v1"
|
||||||
|
cfg.UploadsEnabled = enabled
|
||||||
|
cfg.UploadMaxBytes = 200 << 20
|
||||||
|
cfg.UploadTmpDir = filepath.Join(t.TempDir(), "spool")
|
||||||
|
cfg.TranscribeModel = "iguana/kb-whisper"
|
||||||
|
cfg.TranscribeFallbackModel = "berget/kb-whisper"
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_DisabledMountsNothingAndSweepsNothing(t *testing.T) {
|
||||||
|
st := &fakeUploadsStore{}
|
||||||
|
|
||||||
|
in, w, err := buildUploads(context.Background(), uploadCfg(t, false), st, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, in, "a typed-nil Intake must never reach web.App.Uploads: it would mount the routes over nothing")
|
||||||
|
assert.Nil(t, w)
|
||||||
|
assert.Zero(t, st.sweeps)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_EnabledFailsInterruptedRowsBeforeAnythingElseStarts(t *testing.T) {
|
||||||
|
st := &fakeUploadsStore{}
|
||||||
|
|
||||||
|
in, w, err := buildUploads(context.Background(), uploadCfg(t, true), st, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, in)
|
||||||
|
require.NotNil(t, w)
|
||||||
|
assert.Equal(t, 1, st.sweeps,
|
||||||
|
"audio is not retained, so rows left queued/transcribing by a restart can never resume: fail them at start-up, not never")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_SweepFailureStopsStartup(t *testing.T) {
|
||||||
|
st := &fakeUploadsStore{sweepErr: errors.New("db down")}
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), uploadCfg(t, true), st, quietLog())
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "db down")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_CreatesTheSpoolDirectoryPrivately(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
info, err := os.Stat(cfg.UploadTmpDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, info.IsDir())
|
||||||
|
assert.Equal(t, os.FileMode(0o700), info.Mode().Perm(), "spooled audio is private third-party content")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_EnabledWithoutAGatewayIsAnError(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.GatewayURL = ""
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
|
||||||
|
assert.Error(t, err, "uploads on with nowhere to transcribe must fail loudly at start-up, not at the first upload")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_RemovesAudioLeftBehindByACrash(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
require.NoError(t, os.MkdirAll(cfg.UploadTmpDir, 0o700))
|
||||||
|
leftovers := []string{"upload-123", "wav-456"}
|
||||||
|
for _, n := range leftovers {
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(cfg.UploadTmpDir, n), []byte("audio"), 0o600))
|
||||||
|
}
|
||||||
|
unrelated := filepath.Join(cfg.UploadTmpDir, "someone-elses-file")
|
||||||
|
require.NoError(t, os.WriteFile(unrelated, []byte("keep"), 0o600))
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, n := range leftovers {
|
||||||
|
_, err := os.Stat(filepath.Join(cfg.UploadTmpDir, n))
|
||||||
|
assert.True(t, os.IsNotExist(err), "%s: audio is never retained, so a spooled file from a crashed run must not survive start-up", n)
|
||||||
|
}
|
||||||
|
_, err = os.Stat(unrelated)
|
||||||
|
assert.NoError(t, err, "only our own spool files are removed")
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FFmpeg converts any audio container to the strict format whisper wants: 16 kHz
|
||||||
|
// mono signed 16-bit PCM WAV (ADR-033). It ALWAYS transcodes, even when the input
|
||||||
|
// already is that format — a Teams export often is, and the transcode is then
|
||||||
|
// nearly free; assuming it would be the bug. This replaces whisper-server's
|
||||||
|
// --convert, whose --tmp-dir is created at process start and rots under macOS
|
||||||
|
// periodic cleanup.
|
||||||
|
type FFmpeg struct {
|
||||||
|
// Bin is the ffmpeg executable ("ffmpeg" resolves via PATH).
|
||||||
|
Bin string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToWAV writes src, converted, to dst. Video streams are dropped (-vn): by now the
|
||||||
|
// file has already been probed as audio-only, but a cover-art stream must not end
|
||||||
|
// up in the WAV.
|
||||||
|
func (f *FFmpeg) ToWAV(ctx context.Context, src, dst string) error {
|
||||||
|
bin := f.Bin
|
||||||
|
if bin == "" {
|
||||||
|
bin = "ffmpeg"
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, bin,
|
||||||
|
"-y", "-v", "error", "-i", src, "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", "-f", "wav", dst)
|
||||||
|
cmd.WaitDelay = killGrace
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return fmt.Errorf("audio: ffmpeg: %w", ctxErr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("audio: ffmpeg: %w: %s", err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
// A zero exit that wrote nothing is still a failure: never hand whisper a
|
||||||
|
// missing file and let the error surface three layers later.
|
||||||
|
if info, err := os.Stat(dst); err != nil || info.Size() == 0 {
|
||||||
|
return fmt.Errorf("audio: ffmpeg wrote no output at %s", dst)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package audio_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_AsksForSixteenKilohertzMonoPCMAlways(t *testing.T) {
|
||||||
|
// Always transcode, even when the input already is 16 kHz mono: a Teams export
|
||||||
|
// often is, and the point is to not ASSUME (ADR-033). Records the argv.
|
||||||
|
rec := filepath.Join(t.TempDir(), "argv")
|
||||||
|
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||||
|
for a; do last="$a"; done
|
||||||
|
printf data > "$last"`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "out.wav"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
raw, err := os.ReadFile(rec)
|
||||||
|
require.NoError(t, err)
|
||||||
|
argv := strings.Split(strings.TrimSpace(string(raw)), "\n")
|
||||||
|
joined := " " + strings.Join(argv, " ") + " "
|
||||||
|
for _, want := range []string{" -ac 1 ", " -ar 16000 ", " -c:a pcm_s16le ", " -vn ", " -i /spool/in "} {
|
||||||
|
assert.Contains(t, joined, want)
|
||||||
|
}
|
||||||
|
assert.True(t, strings.HasSuffix(strings.TrimSpace(joined), "out.wav"), "the output path is the last argument")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_NonzeroExitIsAnErrorCarryingStderr(t *testing.T) {
|
||||||
|
bin := stub(t, `echo "moov atom not found" >&2; exit 1`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/in", filepath.Join(t.TempDir(), "o.wav"))
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "moov atom not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_ZeroExitWithNoOutputIsAnError(t *testing.T) {
|
||||||
|
bin := stub(t, `exit 0`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/in", filepath.Join(t.TempDir(), "never-written.wav"))
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_MissingBinaryIsAnError(t *testing.T) {
|
||||||
|
err := (&audio.FFmpeg{Bin: filepath.Join(t.TempDir(), "no-ffmpeg")}).ToWAV(context.Background(), "/in", "/out.wav")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.False(t, errors.Is(err, context.Canceled))
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
// Package audio holds the adapters that inspect and convert user-supplied audio
|
||||||
|
// with ffmpeg's tools (ADR-033). They sit behind small ports so the use case is
|
||||||
|
// tested with fakes and only this package knows a subprocess is involved.
|
||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FFprobe implements uploads.Prober by running ffprobe on the spooled file.
|
||||||
|
type FFprobe struct {
|
||||||
|
// Bin is the ffprobe executable ("ffprobe" resolves via PATH).
|
||||||
|
Bin string
|
||||||
|
}
|
||||||
|
|
||||||
|
// killGrace bounds how long Probe waits for a cancelled ffprobe's pipes to close.
|
||||||
|
const killGrace = 2 * time.Second
|
||||||
|
|
||||||
|
// Probe runs ffprobe and classifies the outcome. A non-zero exit means ffprobe
|
||||||
|
// looked at the file and could not read it: that is the FILE's fault and returns
|
||||||
|
// uploads.ErrUnreadable. A missing binary, a cancelled context or unparsable
|
||||||
|
// output is OURS and is returned as an ordinary error, so it is never reported to
|
||||||
|
// the user as their file being bad.
|
||||||
|
func (f *FFprobe) Probe(ctx context.Context, path string) (uploads.Probe, error) {
|
||||||
|
bin := f.Bin
|
||||||
|
if bin == "" {
|
||||||
|
bin = "ffprobe"
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, bin,
|
||||||
|
"-v", "error", "-print_format", "json", "-show_format", "-show_streams", path)
|
||||||
|
cmd.WaitDelay = killGrace
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("audio: ffprobe: %w", ctxErr)
|
||||||
|
}
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if errors.As(err, &exit) {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("%w: ffprobe: %s", uploads.ErrUnreadable, exit.Stderr)
|
||||||
|
}
|
||||||
|
return uploads.Probe{}, fmt.Errorf("audio: run ffprobe: %w", err)
|
||||||
|
}
|
||||||
|
p, err := ParseProbe(out)
|
||||||
|
if err != nil {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("audio: ffprobe output: %w", err)
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeJSON is the subset of `ffprobe -print_format json` output that matters.
|
||||||
|
type probeJSON struct {
|
||||||
|
Streams []struct {
|
||||||
|
CodecType string `json:"codec_type"`
|
||||||
|
Disposition struct {
|
||||||
|
AttachedPic int `json:"attached_pic"`
|
||||||
|
} `json:"disposition"`
|
||||||
|
} `json:"streams"`
|
||||||
|
Format struct {
|
||||||
|
Duration string `json:"duration"`
|
||||||
|
} `json:"format"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseProbe turns ffprobe's JSON into a uploads.Probe. A video stream that is
|
||||||
|
// only attached cover art (an mp3/m4a with an album picture) is not video. A
|
||||||
|
// missing or "N/A" duration yields zero, which uploads.Validate rejects as
|
||||||
|
// unreadable rather than accepting an empty recording.
|
||||||
|
func ParseProbe(data []byte) (uploads.Probe, error) {
|
||||||
|
var j probeJSON
|
||||||
|
if err := json.Unmarshal(data, &j); err != nil {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("decode: %w", err)
|
||||||
|
}
|
||||||
|
var p uploads.Probe
|
||||||
|
for _, s := range j.Streams {
|
||||||
|
switch s.CodecType {
|
||||||
|
case "audio":
|
||||||
|
p.HasAudio = true
|
||||||
|
case "video":
|
||||||
|
if s.Disposition.AttachedPic == 0 {
|
||||||
|
p.HasVideo = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if secs, err := strconv.ParseFloat(j.Format.Duration, 64); err == nil && secs > 0 {
|
||||||
|
p.Duration = time.Duration(secs * float64(time.Second))
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package audio_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseProbe(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
json string
|
||||||
|
want uploads.Probe
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "audio-only m4a",
|
||||||
|
json: `{"streams":[{"codec_type":"audio","disposition":{"attached_pic":0}}],"format":{"duration":"2701.5"}}`,
|
||||||
|
want: uploads.Probe{Duration: 2701*time.Second + 500*time.Millisecond, HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mp4 with a real video stream",
|
||||||
|
json: `{"streams":[{"codec_type":"video","disposition":{"attached_pic":0}},{"codec_type":"audio"}],"format":{"duration":"240.0"}}`,
|
||||||
|
want: uploads.Probe{Duration: 240 * time.Second, HasAudio: true, HasVideo: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mp3 whose only video stream is attached cover art is NOT video",
|
||||||
|
json: `{"streams":[{"codec_type":"audio"},{"codec_type":"video","disposition":{"attached_pic":1}}],"format":{"duration":"60"}}`,
|
||||||
|
want: uploads.Probe{Duration: 60 * time.Second, HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no streams at all",
|
||||||
|
json: `{"streams":[],"format":{"duration":"10"}}`,
|
||||||
|
want: uploads.Probe{Duration: 10 * time.Second},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duration absent means zero, which Validate rejects as unreadable",
|
||||||
|
json: `{"streams":[{"codec_type":"audio"}],"format":{}}`,
|
||||||
|
want: uploads.Probe{HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duration reported as N/A means zero",
|
||||||
|
json: `{"streams":[{"codec_type":"audio"}],"format":{"duration":"N/A"}}`,
|
||||||
|
want: uploads.Probe{HasAudio: true},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := audio.ParseProbe([]byte(tt.json))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.want, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseProbe_GarbageIsAnError(t *testing.T) {
|
||||||
|
_, err := audio.ParseProbe([]byte("not json"))
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stub writes an executable shell script standing in for ffprobe. This is a fake
|
||||||
|
// at the process boundary: the real binary is not on every host (it is not on
|
||||||
|
// koala), and what is under test is how Probe drives and interprets a process.
|
||||||
|
func stub(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(t.TempDir(), "ffprobe")
|
||||||
|
require.NoError(t, os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755))
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_ParsesWhatTheProcessPrints(t *testing.T) {
|
||||||
|
bin := stub(t, `echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"90"}}'`)
|
||||||
|
|
||||||
|
got, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/ignored")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, uploads.Probe{Duration: 90 * time.Second, HasAudio: true}, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_PassesTheFilePathAsTheLastArgument(t *testing.T) {
|
||||||
|
out := filepath.Join(t.TempDir(), "args")
|
||||||
|
bin := stub(t, `for a; do last="$a"; done; printf '%s' "$last" > `+out+`
|
||||||
|
echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"1"}}'`)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/spool/upload-123")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
got, err := os.ReadFile(out)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "/spool/upload-123", string(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_NonzeroExitMeansTheFileIsUnreadable(t *testing.T) {
|
||||||
|
bin := stub(t, `echo "Invalid data found when processing input" >&2; exit 1`)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/x")
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, uploads.ErrUnreadable)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_MissingBinaryIsOurFaultNotAnUnreadableFile(t *testing.T) {
|
||||||
|
_, err := (&audio.FFprobe{Bin: filepath.Join(t.TempDir(), "no-such-ffprobe")}).Probe(context.Background(), "/x")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.False(t, errors.Is(err, uploads.ErrUnreadable),
|
||||||
|
"a missing ffprobe must not be reported to the user as their file being bad")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_StopsWhenTheContextIsCancelled(t *testing.T) {
|
||||||
|
bin := stub(t, `sleep 30`)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(ctx, "/x")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Less(t, time.Since(start), 5*time.Second)
|
||||||
|
assert.False(t, errors.Is(err, uploads.ErrUnreadable), "a timeout is not a verdict on the file")
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package transcribe
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Backend is one transcription endpoint. *Gateway satisfies it; Name is the value
|
||||||
|
// recorded as the serving endpoint.
|
||||||
|
type Backend interface {
|
||||||
|
Name() string
|
||||||
|
Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chain tries a primary (local) backend and, if it fails for ANY reason, a
|
||||||
|
// fallback. The fallback exists because iguana can wedge permanently on a Metal
|
||||||
|
// OOM (infra#273) — which is also why a fallback is a data-egress decision: the
|
||||||
|
// berget endpoint is a cloud service, so the result records which one served it.
|
||||||
|
type Chain struct {
|
||||||
|
primary Backend
|
||||||
|
fallback Backend // nil disables the fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChain builds a Chain. A nil fallback means a primary failure is the result.
|
||||||
|
func NewChain(primary, fallback Backend) *Chain {
|
||||||
|
return &Chain{primary: primary, fallback: fallback}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transcribe returns the first backend's success. It falls back on any primary
|
||||||
|
// error — transport, timeout, non-2xx, empty transcript — and never inspects a
|
||||||
|
// response body to decide. A cancelled context stops the chain: a job the user or
|
||||||
|
// shutdown cancelled must not ship its audio to a cloud endpoint as a last act.
|
||||||
|
func (c *Chain) Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error) {
|
||||||
|
t, err := c.primary.Transcribe(ctx, wavPath)
|
||||||
|
if err == nil {
|
||||||
|
t.Endpoint = c.primary.Name()
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
if c.fallback == nil || ctx.Err() != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("%s: %w", c.primary.Name(), err)
|
||||||
|
}
|
||||||
|
t, ferr := c.fallback.Transcribe(ctx, wavPath)
|
||||||
|
if ferr != nil {
|
||||||
|
return domain.Transcription{}, errors.Join(
|
||||||
|
fmt.Errorf("%s: %w", c.primary.Name(), err),
|
||||||
|
fmt.Errorf("%s: %w", c.fallback.Name(), ferr))
|
||||||
|
}
|
||||||
|
t.Endpoint = c.fallback.Name()
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package transcribe_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeBackend struct {
|
||||||
|
name string
|
||||||
|
text string
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeBackend) Name() string { return f.name }
|
||||||
|
func (f *fakeBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||||
|
f.calls++
|
||||||
|
if f.err != nil {
|
||||||
|
return domain.Transcription{}, f.err
|
||||||
|
}
|
||||||
|
return domain.Transcription{Text: f.text, Endpoint: f.name}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_PrimarySuccessNeverTouchesTheFallback(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", text: "primary text"}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||||
|
|
||||||
|
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "primary text", got.Text)
|
||||||
|
assert.Equal(t, "iguana/kb-whisper", got.Endpoint)
|
||||||
|
assert.Zero(t, fallback.calls, "audio must not leave the homelab when the local endpoint worked")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_AnyPrimaryFailureFallsBackAndRecordsTheFallbackEndpoint(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{"server error", &transcribe.StatusError{Code: 500}},
|
||||||
|
{"gateway timeout", &transcribe.StatusError{Code: 504}},
|
||||||
|
{"client error", &transcribe.StatusError{Code: 400}},
|
||||||
|
{"transport failure", errors.New("dial tcp: connection refused")},
|
||||||
|
{"empty transcript", transcribe.ErrEmptyTranscript},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: tt.err}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||||
|
|
||||||
|
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "fallback text", got.Text)
|
||||||
|
assert.Equal(t, "berget/kb-whisper", got.Endpoint,
|
||||||
|
"the row must say berget served it: that means the audio left the homelab")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_BothFailingReportsBothErrors(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", err: errors.New("berget down")}
|
||||||
|
|
||||||
|
_, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "iguana down")
|
||||||
|
assert.Contains(t, err.Error(), "berget down")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_WithoutAFallbackAPrimaryFailureIsTheResult(t *testing.T) {
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||||
|
|
||||||
|
_, err := transcribe.NewChain(primary, nil).Transcribe(context.Background(), "/a.wav")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "iguana down")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChain_ACancelledJobDoesNotFallBack(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
primary := &fakeBackend{name: "iguana/kb-whisper", err: context.Canceled}
|
||||||
|
fallback := &fakeBackend{name: "berget/kb-whisper", text: "x"}
|
||||||
|
|
||||||
|
_, err := transcribe.NewChain(primary, fallback).Transcribe(ctx, "/a.wav")
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, context.Canceled)
|
||||||
|
assert.Zero(t, fallback.calls, "a cancelled job must not ship the audio to a cloud endpoint")
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
// Package transcribe turns a WAV file into a timed Swedish transcript by calling
|
||||||
|
// the LiteLLM gateway's OpenAI-compatible /audio/transcriptions endpoint
|
||||||
|
// (ADR-033). A Gateway is one model alias; a Chain tries a local one and falls
|
||||||
|
// back to a second.
|
||||||
|
package transcribe
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrEmptyTranscript is returned when the endpoint answered 200 but with no text.
|
||||||
|
// An empty transcript is not a success: chat over it could only answer "nothing".
|
||||||
|
var ErrEmptyTranscript = errors.New("transcribe: endpoint returned an empty transcript")
|
||||||
|
|
||||||
|
// StatusError is a non-2xx answer from the gateway. Nothing may branch on the
|
||||||
|
// body — {"error":"failed to process audio"} has had at least five distinct
|
||||||
|
// causes — so only the status code is exposed for logic; Snippet is for logs.
|
||||||
|
type StatusError struct {
|
||||||
|
Code int
|
||||||
|
Snippet string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *StatusError) Error() string {
|
||||||
|
return fmt.Sprintf("transcribe: gateway returned HTTP %d: %s", e.Code, e.Snippet)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gateway transcribes through one model alias on the gateway. baseURL is the
|
||||||
|
// gateway's /v1 root — the in-cluster NodePort, not the public hostname, whose
|
||||||
|
// ingress rejects bodies over ~8-10 MB with 413 (docs/research/audio-length-ceiling).
|
||||||
|
type Gateway struct {
|
||||||
|
baseURL string
|
||||||
|
key string
|
||||||
|
model string
|
||||||
|
client *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGateway builds a Gateway. The client should carry no overall Timeout: a 90
|
||||||
|
// minute recording takes minutes, so the deadline belongs to the caller's context.
|
||||||
|
func NewGateway(baseURL, key, model string, client *http.Client) *Gateway {
|
||||||
|
return &Gateway{baseURL: strings.TrimRight(baseURL, "/"), key: key, model: model, client: client}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name is the model alias; it is what gets recorded as the serving endpoint.
|
||||||
|
func (g *Gateway) Name() string { return g.model }
|
||||||
|
|
||||||
|
// Transcribe uploads the WAV at path and returns the normalised transcription.
|
||||||
|
// The body is streamed from disk with a COMPUTED Content-Length: a 90 minute WAV
|
||||||
|
// is ~170 MB, so it is neither buffered in memory nor sent chunked.
|
||||||
|
func (g *Gateway) Transcribe(ctx context.Context, path string) (domain.Transcription, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("transcribe: open audio: %w", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
info, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("transcribe: stat audio: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var prefix bytes.Buffer
|
||||||
|
mw := multipart.NewWriter(&prefix)
|
||||||
|
for _, kv := range [][2]string{{"model", g.model}, {"response_format", "verbose_json"}, {"language", "sv"}} {
|
||||||
|
if err := mw.WriteField(kv[0], kv[1]); err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The multipart field is `file`; `audio_file` is rejected as "Invalid request".
|
||||||
|
if _, err := mw.CreateFormFile("file", "audio.wav"); err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||||
|
}
|
||||||
|
suffix := []byte("\r\n--" + mw.Boundary() + "--\r\n")
|
||||||
|
|
||||||
|
body := io.MultiReader(bytes.NewReader(prefix.Bytes()), f, bytes.NewReader(suffix))
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.baseURL+"/audio/transcriptions", body)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||||
|
}
|
||||||
|
req.ContentLength = int64(prefix.Len()) + info.Size() + int64(len(suffix))
|
||||||
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||||
|
req.Header.Set("Authorization", "Bearer "+g.key)
|
||||||
|
// No "Expect: 100-continue": it trips the proxy for any body over 1 KB.
|
||||||
|
|
||||||
|
resp, err := g.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||||
|
snip, _ := io.ReadAll(io.LimitReader(resp.Body, 200))
|
||||||
|
return domain.Transcription{}, &StatusError{Code: resp.StatusCode, Snippet: strings.TrimSpace(string(snip))}
|
||||||
|
}
|
||||||
|
|
||||||
|
t, err := parseVerbose(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||||
|
}
|
||||||
|
t.Endpoint = g.model
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// verboseJSON is the subset of the verbose_json response used. Both backends
|
||||||
|
// return it; they differ in segment granularity (iguana: whole seconds, berget:
|
||||||
|
// sub-second) and in the language token, neither of which matters here.
|
||||||
|
type verboseJSON struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
Duration float64 `json:"duration"`
|
||||||
|
Segments []struct {
|
||||||
|
Start float64 `json:"start"`
|
||||||
|
End float64 `json:"end"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
} `json:"segments"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseVerbose normalises a verbose_json body. Whisper's final window can end past
|
||||||
|
// the audio (9.0 s for 4.25 s of audio was observed), so segment ends are clamped
|
||||||
|
// to the reported duration.
|
||||||
|
func parseVerbose(r io.Reader) (domain.Transcription, error) {
|
||||||
|
var v verboseJSON
|
||||||
|
if err := json.NewDecoder(r).Decode(&v); err != nil {
|
||||||
|
return domain.Transcription{}, fmt.Errorf("decode response: %w", err)
|
||||||
|
}
|
||||||
|
text := strings.TrimSpace(v.Text)
|
||||||
|
if text == "" {
|
||||||
|
return domain.Transcription{}, ErrEmptyTranscript
|
||||||
|
}
|
||||||
|
t := domain.Transcription{Text: text, AudioSeconds: v.Duration}
|
||||||
|
for _, s := range v.Segments {
|
||||||
|
seg := domain.Segment{Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text)}
|
||||||
|
if v.Duration > 0 {
|
||||||
|
seg.End = min(seg.End, v.Duration)
|
||||||
|
seg.Start = min(seg.Start, seg.End)
|
||||||
|
}
|
||||||
|
t.Segments = append(t.Segments, seg)
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
package transcribe_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The httptest server stands in for the LiteLLM gateway: it is the process
|
||||||
|
// boundary, so a fake here is legitimate. The JSON it returns mirrors what the
|
||||||
|
// real gateway returned for iguana/kb-whisper and berget/kb-whisper on 2026-10-01
|
||||||
|
// (docs/research/audio-length-ceiling-2026-10-01.md). All text is synthetic.
|
||||||
|
|
||||||
|
const verboseJSON = `{
|
||||||
|
"text": " God morgon allihopa.\n Vi går igenom läget.\n",
|
||||||
|
"language": "swedish",
|
||||||
|
"duration": 4.25,
|
||||||
|
"segments": [
|
||||||
|
{"id": 0, "start": 0.0, "end": 1.0, "text": " God morgon allihopa."},
|
||||||
|
{"id": 1, "start": 1.0, "end": 9.0, "text": " Vi går igenom läget."}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
func wavFile(t *testing.T, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(t.TempDir(), "a.wav")
|
||||||
|
require.NoError(t, os.WriteFile(p, []byte(content), 0o600))
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
type seen struct {
|
||||||
|
method, path, auth, expect, contentType string
|
||||||
|
contentLength int64
|
||||||
|
transferEncoding []string
|
||||||
|
fields map[string]string
|
||||||
|
file string
|
||||||
|
fileField string
|
||||||
|
bodyLen int
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve starts a gateway fake that records the request it received and replies
|
||||||
|
// with status and body.
|
||||||
|
func serve(t *testing.T, status int, reply string, got *seen) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got != nil {
|
||||||
|
got.method, got.path = r.Method, r.URL.Path
|
||||||
|
got.auth = r.Header.Get("Authorization")
|
||||||
|
got.expect = r.Header.Get("Expect")
|
||||||
|
got.contentType = r.Header.Get("Content-Type")
|
||||||
|
got.contentLength = r.ContentLength
|
||||||
|
got.transferEncoding = r.TransferEncoding
|
||||||
|
raw, _ := io.ReadAll(r.Body)
|
||||||
|
got.bodyLen = len(raw)
|
||||||
|
r.Body = io.NopCloser(bytes.NewReader(raw))
|
||||||
|
require.NoError(t, r.ParseMultipartForm(32<<20))
|
||||||
|
got.fields = map[string]string{}
|
||||||
|
for k, v := range r.MultipartForm.Value {
|
||||||
|
got.fields[k] = v[0]
|
||||||
|
}
|
||||||
|
for field, fhs := range r.MultipartForm.File {
|
||||||
|
got.fileField = field
|
||||||
|
f, _ := fhs[0].Open()
|
||||||
|
b, _ := io.ReadAll(f)
|
||||||
|
got.file = string(b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_, _ = io.WriteString(w, reply)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_SendsTheRequestTheGatewayExpects(t *testing.T) {
|
||||||
|
var got seen
|
||||||
|
srv := serve(t, 200, verboseJSON, &got)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "secret-key", "iguana/kb-whisper", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "RIFF-synthetic-pcm"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, http.MethodPost, got.method)
|
||||||
|
assert.Equal(t, "/v1/audio/transcriptions", got.path)
|
||||||
|
assert.Equal(t, "Bearer secret-key", got.auth)
|
||||||
|
assert.Equal(t, "file", got.fileField, "the multipart field is `file`; `audio_file` is rejected as 'Invalid request'")
|
||||||
|
assert.Equal(t, "RIFF-synthetic-pcm", got.file)
|
||||||
|
assert.Equal(t, "iguana/kb-whisper", got.fields["model"])
|
||||||
|
assert.Equal(t, "verbose_json", got.fields["response_format"])
|
||||||
|
assert.Equal(t, "sv", got.fields["language"])
|
||||||
|
assert.Empty(t, got.expect, "no Expect: 100-continue: it trips the proxy for any body over 1 KB")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_SendsAKnownContentLengthNotChunked(t *testing.T) {
|
||||||
|
var got seen
|
||||||
|
srv := serve(t, 200, verboseJSON, &got)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "0123456789"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, got.transferEncoding, "a 90-minute WAV is ~170 MB: stream it with a computed length, not chunked and not buffered")
|
||||||
|
assert.Equal(t, int64(got.bodyLen), got.contentLength)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_NormalisesTheVerboseJSONResponse(t *testing.T) {
|
||||||
|
srv := serve(t, 200, verboseJSON, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
||||||
|
|
||||||
|
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "God morgon allihopa.\n Vi går igenom läget.", got.Text)
|
||||||
|
assert.Equal(t, "iguana/kb-whisper", got.Endpoint, "the serving endpoint is part of the result")
|
||||||
|
assert.InDelta(t, 4.25, got.AudioSeconds, 1e-9)
|
||||||
|
assert.Equal(t, []domain.Segment{
|
||||||
|
{Start: 0, End: 1, Text: "God morgon allihopa."},
|
||||||
|
// The final segment ended at 9.0 s for 4.25 s of audio: whisper's last
|
||||||
|
// window can run past the end. Clamped to the audio duration.
|
||||||
|
{Start: 1, End: 4.25, Text: "Vi går igenom läget."},
|
||||||
|
}, got.Segments)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_ErrorsDoNotDependOnTheBody(t *testing.T) {
|
||||||
|
// {"error":"failed to process audio"} has had at least five distinct causes, so
|
||||||
|
// no code may branch on it. Two different bodies must be treated identically.
|
||||||
|
for _, reply := range []string{`{"error":"failed to process audio"}`, `{"error":"something else entirely"}`} {
|
||||||
|
srv := serve(t, 500, reply, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
var se *transcribe.StatusError
|
||||||
|
require.ErrorAs(t, err, &se)
|
||||||
|
assert.Equal(t, 500, se.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_AnEmptyTranscriptIsAFailureNotASuccess(t *testing.T) {
|
||||||
|
srv := serve(t, 200, `{"text":" ","language":"swedish","duration":3,"segments":[]}`, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, transcribe.ErrEmptyTranscript)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_MalformedJSONIsAnError(t *testing.T) {
|
||||||
|
srv := serve(t, 200, `not json`, nil)
|
||||||
|
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||||
|
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGateway_MissingAudioFileIsAnError(t *testing.T) {
|
||||||
|
g := transcribe.NewGateway("http://127.0.0.1:1/v1", "k", "m", http.DefaultClient)
|
||||||
|
|
||||||
|
_, err := g.Transcribe(context.Background(), filepath.Join(t.TempDir(), "nope.wav"))
|
||||||
|
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package transcribe_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestGateway_LiveGateway is an opt-in check against the REAL gateway, because
|
||||||
|
// the httptest fake above can only prove what we think the gateway accepts. It
|
||||||
|
// is skipped unless TAPIR_LIVE_WAV names a 16 kHz mono WAV and the gateway
|
||||||
|
// coordinates are set:
|
||||||
|
//
|
||||||
|
// TAPIR_LIVE_WAV=/path/a.wav TAPIR_LIVE_GATEWAY_URL=http://koala:30401/v1 \
|
||||||
|
// TAPIR_LIVE_MODEL=iguana/kb-whisper op run --env-file ~/.op-env -- \
|
||||||
|
// go test ./internal/adapters/transcribe -run LiveGateway -v
|
||||||
|
//
|
||||||
|
// Use SYNTHETIC audio only: this repo is public and the gateway may be a cloud
|
||||||
|
// endpoint (#33).
|
||||||
|
func TestGateway_LiveGateway(t *testing.T) {
|
||||||
|
wav := os.Getenv("TAPIR_LIVE_WAV")
|
||||||
|
url := os.Getenv("TAPIR_LIVE_GATEWAY_URL")
|
||||||
|
if wav == "" || url == "" {
|
||||||
|
t.Skip("set TAPIR_LIVE_WAV and TAPIR_LIVE_GATEWAY_URL to run against the real gateway")
|
||||||
|
}
|
||||||
|
model := os.Getenv("TAPIR_LIVE_MODEL")
|
||||||
|
if model == "" {
|
||||||
|
model = "iguana/kb-whisper"
|
||||||
|
}
|
||||||
|
key := os.Getenv("LITELLM_MASTER_KEY")
|
||||||
|
require.NotEmpty(t, key, "LITELLM_MASTER_KEY must be set (via op run)")
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
got, err := transcribe.NewGateway(url, key, model, &http.Client{}).Transcribe(ctx, wav)
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotEmpty(t, got.Text)
|
||||||
|
assert.NotEmpty(t, got.Segments)
|
||||||
|
assert.Equal(t, model, got.Endpoint)
|
||||||
|
t.Logf("endpoint=%s audio_s=%.1f segments=%d chars=%d", got.Endpoint, got.AudioSeconds, len(got.Segments), len(got.Text))
|
||||||
|
}
|
||||||
@@ -20,6 +20,9 @@ import (
|
|||||||
|
|
||||||
// Config is Tapir's fully-resolved runtime configuration.
|
// Config is Tapir's fully-resolved runtime configuration.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
|
// UploadConfig is the audio-upload and private-transcription settings (ADR-033).
|
||||||
|
UploadConfig
|
||||||
|
|
||||||
// UserID is the Tapir user the run operates as. Stage 0 has exactly one.
|
// UserID is the Tapir user the run operates as. Stage 0 has exactly one.
|
||||||
// It must be a UUID: it keys the UUID user_id/video_id columns (data-model).
|
// It must be a UUID: it keys the UUID user_id/video_id columns (data-model).
|
||||||
UserID string
|
UserID string
|
||||||
@@ -320,6 +323,10 @@ func Load() (Config, error) {
|
|||||||
}
|
}
|
||||||
c.OnboardMaxVideoSeconds = onboardMax
|
c.OnboardMaxVideoSeconds = onboardMax
|
||||||
|
|
||||||
|
if c.UploadConfig, err = loadUploadConfig(); err != nil {
|
||||||
|
return Config{}, err
|
||||||
|
}
|
||||||
|
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// UploadConfig holds the settings for audio upload and private transcription
|
||||||
|
// (ADR-033). It is embedded in Config, so its fields read as c.UploadsEnabled etc.
|
||||||
|
type UploadConfig struct {
|
||||||
|
// UploadsEnabled mounts the /uploads routes and starts the worker. Off by
|
||||||
|
// default: the runtime image must contain ffmpeg and ffprobe first (ADR-033).
|
||||||
|
UploadsEnabled bool
|
||||||
|
// UploadMaxBytes caps one upload. UploadMaxDuration caps one recording; 90
|
||||||
|
// minutes is the longest length MEASURED, not a ceiling that was found, so
|
||||||
|
// raising it is a measurement, not an edit.
|
||||||
|
UploadMaxBytes int64
|
||||||
|
UploadMaxDuration time.Duration
|
||||||
|
// TranscribeModel is the primary gateway alias; TranscribeFallbackModel is the
|
||||||
|
// fallback, tried on ANY primary failure. The fallback is a cloud endpoint
|
||||||
|
// (berget): set it EMPTY to keep private audio on the homelab.
|
||||||
|
TranscribeModel string
|
||||||
|
TranscribeFallbackModel string
|
||||||
|
// TranscribeTimeout bounds one job (conversion + transcription); a 90 minute
|
||||||
|
// recording took ~7.5 minutes on iguana.
|
||||||
|
TranscribeTimeout time.Duration
|
||||||
|
// UploadTmpDir is where uploads are spooled and converted. Audio is deleted
|
||||||
|
// after transcription, so this holds at most the in-flight files.
|
||||||
|
UploadTmpDir string
|
||||||
|
FFmpegBin string
|
||||||
|
FFprobeBin string
|
||||||
|
// MaxUploadedTranscriptChars bounds the transcript sent to a chat model for an
|
||||||
|
// UPLOADED recording. The 18,000-char default for captions is ~22 minutes of
|
||||||
|
// speech; an hour of meeting is ~50,000 chars. A guess until measured against
|
||||||
|
// the models chat offers.
|
||||||
|
MaxUploadedTranscriptChars int
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultUploadMaxMB = 200
|
||||||
|
defaultUploadMaxMinutes = 90
|
||||||
|
defaultTranscribeModel = "iguana/kb-whisper"
|
||||||
|
defaultTranscribeFallbackModel = "berget/kb-whisper"
|
||||||
|
defaultTranscribeTimeout = 30 * time.Minute
|
||||||
|
defaultMaxUploadedChars = 120000
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadUploadConfig() (UploadConfig, error) {
|
||||||
|
u := UploadConfig{
|
||||||
|
TranscribeModel: envOr("TAPIR_TRANSCRIBE_MODEL", defaultTranscribeModel),
|
||||||
|
TranscribeFallbackModel: lookupOr("TAPIR_TRANSCRIBE_FALLBACK_MODEL", defaultTranscribeFallbackModel),
|
||||||
|
UploadTmpDir: envOr("TAPIR_UPLOAD_TMP_DIR", os.TempDir()),
|
||||||
|
FFmpegBin: envOr("TAPIR_FFMPEG_BIN", "ffmpeg"),
|
||||||
|
FFprobeBin: envOr("TAPIR_FFPROBE_BIN", "ffprobe"),
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := os.Getenv("TAPIR_UPLOADS_ENABLED"); v != "" {
|
||||||
|
on, err := strconv.ParseBool(v)
|
||||||
|
if err != nil {
|
||||||
|
return UploadConfig{}, fmt.Errorf("config: TAPIR_UPLOADS_ENABLED=%q: %w", v, err)
|
||||||
|
}
|
||||||
|
u.UploadsEnabled = on
|
||||||
|
}
|
||||||
|
|
||||||
|
mb, err := positiveInt("TAPIR_UPLOAD_MAX_MB", defaultUploadMaxMB)
|
||||||
|
if err != nil {
|
||||||
|
return UploadConfig{}, err
|
||||||
|
}
|
||||||
|
u.UploadMaxBytes = int64(mb) << 20
|
||||||
|
|
||||||
|
mins, err := positiveInt("TAPIR_UPLOAD_MAX_MINUTES", defaultUploadMaxMinutes)
|
||||||
|
if err != nil {
|
||||||
|
return UploadConfig{}, err
|
||||||
|
}
|
||||||
|
u.UploadMaxDuration = time.Duration(mins) * time.Minute
|
||||||
|
|
||||||
|
timeout, err := durationOr("TAPIR_TRANSCRIBE_TIMEOUT", defaultTranscribeTimeout)
|
||||||
|
if err != nil {
|
||||||
|
return UploadConfig{}, err
|
||||||
|
}
|
||||||
|
if timeout <= 0 {
|
||||||
|
return UploadConfig{}, fmt.Errorf("config: TAPIR_TRANSCRIBE_TIMEOUT must be positive, got %v", timeout)
|
||||||
|
}
|
||||||
|
u.TranscribeTimeout = timeout
|
||||||
|
|
||||||
|
chars, err := intOr("TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS", defaultMaxUploadedChars)
|
||||||
|
if err != nil {
|
||||||
|
return UploadConfig{}, err
|
||||||
|
}
|
||||||
|
if chars < 0 {
|
||||||
|
chars = 0
|
||||||
|
}
|
||||||
|
u.MaxUploadedTranscriptChars = chars
|
||||||
|
return u, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// positiveInt is intOr that refuses zero and negatives: a zero upload limit would
|
||||||
|
// reject every file, which is a misconfiguration to fail on, not to run with.
|
||||||
|
func positiveInt(key string, fallback int) (int, error) {
|
||||||
|
n, err := intOr(key, fallback)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if n <= 0 {
|
||||||
|
return 0, fmt.Errorf("config: %s must be positive, got %d", key, n)
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// clearUploadEnv unsets every upload knob so a test observes the default, not the
|
||||||
|
// host environment.
|
||||||
|
func clearUploadEnv(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
for _, k := range []string{
|
||||||
|
"TAPIR_UPLOADS_ENABLED", "TAPIR_UPLOAD_MAX_MB", "TAPIR_UPLOAD_MAX_MINUTES",
|
||||||
|
"TAPIR_TRANSCRIBE_MODEL", "TAPIR_TRANSCRIBE_FALLBACK_MODEL", "TAPIR_TRANSCRIBE_TIMEOUT",
|
||||||
|
"TAPIR_UPLOAD_TMP_DIR", "TAPIR_FFMPEG_BIN", "TAPIR_FFPROBE_BIN",
|
||||||
|
"TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS",
|
||||||
|
} {
|
||||||
|
unset(t, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_UploadDefaults(t *testing.T) {
|
||||||
|
clearUploadEnv(t)
|
||||||
|
|
||||||
|
c, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.UploadsEnabled {
|
||||||
|
t.Error("uploads must be OFF by default: the runtime image needs ffmpeg first (ADR-033)")
|
||||||
|
}
|
||||||
|
if c.UploadMaxBytes != 200<<20 {
|
||||||
|
t.Errorf("UploadMaxBytes = %d, want 200 MB", c.UploadMaxBytes)
|
||||||
|
}
|
||||||
|
if c.UploadMaxDuration != 90*time.Minute {
|
||||||
|
t.Errorf("UploadMaxDuration = %v, want 90m (the longest length measured, not a found ceiling)", c.UploadMaxDuration)
|
||||||
|
}
|
||||||
|
if c.TranscribeModel != "iguana/kb-whisper" {
|
||||||
|
t.Errorf("TranscribeModel = %q", c.TranscribeModel)
|
||||||
|
}
|
||||||
|
if c.TranscribeFallbackModel != "berget/kb-whisper" {
|
||||||
|
t.Errorf("TranscribeFallbackModel = %q", c.TranscribeFallbackModel)
|
||||||
|
}
|
||||||
|
if c.TranscribeTimeout != 30*time.Minute {
|
||||||
|
t.Errorf("TranscribeTimeout = %v, want 30m (a 90 min recording took ~7.5 min)", c.TranscribeTimeout)
|
||||||
|
}
|
||||||
|
if c.UploadTmpDir != os.TempDir() {
|
||||||
|
t.Errorf("UploadTmpDir = %q, want os.TempDir()", c.UploadTmpDir)
|
||||||
|
}
|
||||||
|
if c.FFmpegBin != "ffmpeg" || c.FFprobeBin != "ffprobe" {
|
||||||
|
t.Errorf("bins = %q, %q", c.FFmpegBin, c.FFprobeBin)
|
||||||
|
}
|
||||||
|
if c.MaxUploadedTranscriptChars != 120000 {
|
||||||
|
t.Errorf("MaxUploadedTranscriptChars = %d, want 120000", c.MaxUploadedTranscriptChars)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_UploadValuesAreParsed(t *testing.T) {
|
||||||
|
clearUploadEnv(t)
|
||||||
|
setEnv(t, map[string]string{
|
||||||
|
"TAPIR_UPLOADS_ENABLED": "true",
|
||||||
|
"TAPIR_UPLOAD_MAX_MB": "50",
|
||||||
|
"TAPIR_UPLOAD_MAX_MINUTES": "30",
|
||||||
|
"TAPIR_TRANSCRIBE_MODEL": "koala/whisper",
|
||||||
|
"TAPIR_TRANSCRIBE_TIMEOUT": "10m",
|
||||||
|
"TAPIR_UPLOAD_TMP_DIR": "/var/spool/tapir",
|
||||||
|
"TAPIR_FFMPEG_BIN": "/opt/ffmpeg",
|
||||||
|
"TAPIR_FFPROBE_BIN": "/opt/ffprobe",
|
||||||
|
"TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS": "60000",
|
||||||
|
})
|
||||||
|
|
||||||
|
c, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !c.UploadsEnabled || c.UploadMaxBytes != 50<<20 || c.UploadMaxDuration != 30*time.Minute ||
|
||||||
|
c.TranscribeModel != "koala/whisper" || c.TranscribeTimeout != 10*time.Minute ||
|
||||||
|
c.UploadTmpDir != "/var/spool/tapir" || c.FFmpegBin != "/opt/ffmpeg" ||
|
||||||
|
c.FFprobeBin != "/opt/ffprobe" || c.MaxUploadedTranscriptChars != 60000 {
|
||||||
|
t.Errorf("values not applied: %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_EmptyTranscribeFallbackDisablesIt(t *testing.T) {
|
||||||
|
clearUploadEnv(t)
|
||||||
|
t.Setenv("TAPIR_TRANSCRIBE_FALLBACK_MODEL", "")
|
||||||
|
|
||||||
|
c, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.TranscribeFallbackModel != "" {
|
||||||
|
t.Errorf("TranscribeFallbackModel = %q: an explicit empty value must disable the cloud fallback, "+
|
||||||
|
"so private audio can be kept on the homelab", c.TranscribeFallbackModel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_UploadRejectsInvalidValues(t *testing.T) {
|
||||||
|
tests := []struct{ key, val string }{
|
||||||
|
{"TAPIR_UPLOADS_ENABLED", "maybe"},
|
||||||
|
{"TAPIR_UPLOAD_MAX_MB", "lots"},
|
||||||
|
{"TAPIR_UPLOAD_MAX_MB", "0"},
|
||||||
|
{"TAPIR_UPLOAD_MAX_MB", "-5"},
|
||||||
|
{"TAPIR_UPLOAD_MAX_MINUTES", "0"},
|
||||||
|
{"TAPIR_UPLOAD_MAX_MINUTES", "forever"},
|
||||||
|
{"TAPIR_TRANSCRIBE_TIMEOUT", "soon"},
|
||||||
|
{"TAPIR_TRANSCRIBE_TIMEOUT", "0s"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.key+"="+tt.val, func(t *testing.T) {
|
||||||
|
clearUploadEnv(t)
|
||||||
|
t.Setenv(tt.key, tt.val)
|
||||||
|
|
||||||
|
if _, err := Load(); err == nil {
|
||||||
|
t.Errorf("Load accepted %s=%q; a zero or negative limit would reject every upload", tt.key, tt.val)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package uploads
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrUnreadable is what a Prober returns when the FILE is the problem (not media,
|
||||||
|
// truncated, damaged), as opposed to the prober itself failing. Accept turns it
|
||||||
|
// into a user-facing rejection; any other prober error is our fault and surfaces
|
||||||
|
// as an internal error — a missing ffprobe must not read as "your file is bad".
|
||||||
|
var ErrUnreadable = errors.New("uploads: file is not readable media")
|
||||||
|
|
||||||
|
// Prober inspects a file on disk. The ffprobe adapter implements it.
|
||||||
|
type Prober interface {
|
||||||
|
Probe(ctx context.Context, path string) (Probe, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store is the slice of the private transcript store the intake needs.
|
||||||
|
type Store interface {
|
||||||
|
CreateUpload(ctx context.Context, userID, title string) (string, error)
|
||||||
|
FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job is one accepted upload awaiting transcription. Path is a temp file the
|
||||||
|
// receiver owns: it must delete it when finished (the audio is never retained).
|
||||||
|
type Job struct {
|
||||||
|
UserID string
|
||||||
|
UploadID string
|
||||||
|
Path string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Submitter hands accepted jobs to the in-process worker (ADR-033).
|
||||||
|
type Submitter interface {
|
||||||
|
Submit(Job) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Intake validates an uploaded recording and, if it passes, queues it.
|
||||||
|
type Intake struct {
|
||||||
|
store Store
|
||||||
|
prober Prober
|
||||||
|
sub Submitter
|
||||||
|
limits Limits
|
||||||
|
tmpDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewIntake wires an Intake. tmpDir is where uploads are spooled; it must exist.
|
||||||
|
func NewIntake(s Store, p Prober, sub Submitter, l Limits, tmpDir string) *Intake {
|
||||||
|
return &Intake{store: s, prober: p, sub: sub, limits: l, tmpDir: tmpDir}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Accept spools src to disk (bounded by MaxBytes), probes it, validates it, records
|
||||||
|
// a queued upload and submits the job. It returns the upload id. A file that fails
|
||||||
|
// validation yields a *RejectError and creates NO row — rejection is synchronous
|
||||||
|
// and leaves nothing behind. On success the temp file is handed to the job and the
|
||||||
|
// worker deletes it; on every failure path Accept deletes it itself.
|
||||||
|
func (in *Intake) Accept(ctx context.Context, userID, title string, src io.Reader) (string, error) {
|
||||||
|
f, err := os.CreateTemp(in.tmpDir, "upload-*")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("uploads: spool: %w", err)
|
||||||
|
}
|
||||||
|
path := f.Name()
|
||||||
|
handedOver := false
|
||||||
|
defer func() {
|
||||||
|
if !handedOver {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Read one byte past the cap so "exactly at the limit" and "over" are told apart.
|
||||||
|
n, err := io.Copy(f, io.LimitReader(src, in.limits.MaxBytes+1))
|
||||||
|
if cerr := f.Close(); err == nil {
|
||||||
|
err = cerr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("uploads: receive: %w", err)
|
||||||
|
}
|
||||||
|
if n > in.limits.MaxBytes {
|
||||||
|
return "", &RejectError{Reason: ReasonTooLarge, Message: fmt.Sprintf(
|
||||||
|
"This file is larger than the %d MB limit.", in.limits.MaxBytes>>20)}
|
||||||
|
}
|
||||||
|
|
||||||
|
probe, err := in.prober.Probe(ctx, path)
|
||||||
|
if errors.Is(err, ErrUnreadable) {
|
||||||
|
return "", &RejectError{
|
||||||
|
Reason: ReasonUnreadable,
|
||||||
|
Message: "This file could not be read as audio; it may be damaged or not a recording.",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("uploads: probe: %w", err)
|
||||||
|
}
|
||||||
|
if err := Validate(probe, in.limits); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := in.store.CreateUpload(ctx, userID, title)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("uploads: create: %w", err)
|
||||||
|
}
|
||||||
|
if err := in.sub.Submit(Job{UserID: userID, UploadID: id, Path: path}); err != nil {
|
||||||
|
// A row that will never run must not sit queued: that is the stuck spinner.
|
||||||
|
if ferr := in.store.FailUpload(ctx, userID, id, domain.UploadErrInterrupted); ferr != nil {
|
||||||
|
return "", fmt.Errorf("uploads: submit: %w (and failing the row: %v)", err, ferr)
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("uploads: submit: %w", err)
|
||||||
|
}
|
||||||
|
handedOver = true
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
package uploads_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/iotest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeStore struct {
|
||||||
|
created []string // titles
|
||||||
|
failed map[string]domain.UploadErrorKind
|
||||||
|
nextID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStore) CreateUpload(_ context.Context, _, title string) (string, error) {
|
||||||
|
f.created = append(f.created, title)
|
||||||
|
return f.nextID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStore) FailUpload(_ context.Context, _, id string, kind domain.UploadErrorKind) error {
|
||||||
|
if f.failed == nil {
|
||||||
|
f.failed = map[string]domain.UploadErrorKind{}
|
||||||
|
}
|
||||||
|
f.failed[id] = kind
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeProber struct {
|
||||||
|
probe uploads.Probe
|
||||||
|
err error
|
||||||
|
paths []string
|
||||||
|
seen []byte // content of the file at probe time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeProber) Probe(_ context.Context, path string) (uploads.Probe, error) {
|
||||||
|
f.paths = append(f.paths, path)
|
||||||
|
f.seen, _ = os.ReadFile(path)
|
||||||
|
return f.probe, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeSubmitter struct {
|
||||||
|
jobs []uploads.Job
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeSubmitter) Submit(j uploads.Job) error {
|
||||||
|
f.jobs = append(f.jobs, j)
|
||||||
|
return f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
type rig struct {
|
||||||
|
in *uploads.Intake
|
||||||
|
st *fakeStore
|
||||||
|
pr *fakeProber
|
||||||
|
sub *fakeSubmitter
|
||||||
|
tmp string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRig(t *testing.T) *rig {
|
||||||
|
t.Helper()
|
||||||
|
r := &rig{
|
||||||
|
st: &fakeStore{nextID: "up-1"},
|
||||||
|
pr: &fakeProber{probe: uploads.Probe{Duration: 10 * time.Minute, HasAudio: true}},
|
||||||
|
sub: &fakeSubmitter{},
|
||||||
|
tmp: t.TempDir(),
|
||||||
|
}
|
||||||
|
r.in = uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: 90 * time.Minute}, r.tmp)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *rig) tmpFiles(t *testing.T) []string {
|
||||||
|
t.Helper()
|
||||||
|
ents, err := os.ReadDir(r.tmp)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var names []string
|
||||||
|
for _, e := range ents {
|
||||||
|
names = append(names, e.Name())
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccept_ValidAudioIsStoredProbedQueuedAndSubmitted(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
audio := bytes.Repeat([]byte("a"), 500)
|
||||||
|
|
||||||
|
id, err := r.in.Accept(context.Background(), "user-1", "Teams export", bytes.NewReader(audio))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "up-1", id)
|
||||||
|
assert.Equal(t, []string{"Teams export"}, r.st.created)
|
||||||
|
assert.Equal(t, audio, r.pr.seen, "the prober must see the complete uploaded bytes, on disk")
|
||||||
|
require.Len(t, r.sub.jobs, 1)
|
||||||
|
job := r.sub.jobs[0]
|
||||||
|
assert.Equal(t, uploads.Job{UserID: "user-1", UploadID: "up-1", Path: job.Path}, job)
|
||||||
|
assert.Equal(t, filepath.Dir(job.Path), r.tmp, "the temp file lives in the configured directory")
|
||||||
|
got, err := os.ReadFile(job.Path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, audio, got, "the handed-over file still exists: the worker, not Accept, deletes it")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccept_OversizeBodyIsRejectedAndLeavesNothingBehind(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
|
||||||
|
_, err := r.in.Accept(context.Background(), "user-1", "big", strings.NewReader(strings.Repeat("a", 1025)))
|
||||||
|
|
||||||
|
var rej *uploads.RejectError
|
||||||
|
require.True(t, errors.As(err, &rej))
|
||||||
|
assert.Equal(t, uploads.ReasonTooLarge, rej.Reason)
|
||||||
|
assert.Contains(t, rej.Message, "limit")
|
||||||
|
assert.Empty(t, r.pr.paths, "a file already over the size cap must not be probed")
|
||||||
|
assert.Empty(t, r.st.created, "a rejected upload creates no row")
|
||||||
|
assert.Empty(t, r.sub.jobs)
|
||||||
|
assert.Empty(t, r.tmpFiles(t), "the partial temp file must be removed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccept_BodyExactlyAtTheLimitIsAccepted(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
|
||||||
|
_, err := r.in.Accept(context.Background(), "user-1", "edge", strings.NewReader(strings.Repeat("a", 1024)))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccept_ProbeVerdictRejectsWithoutRowAndRemovesTheFile(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
probe uploads.Probe
|
||||||
|
reason uploads.Reason
|
||||||
|
}{
|
||||||
|
{"video", uploads.Probe{Duration: time.Minute, HasAudio: true, HasVideo: true}, uploads.ReasonVideo},
|
||||||
|
{"too long", uploads.Probe{Duration: 2 * time.Hour, HasAudio: true}, uploads.ReasonTooLong},
|
||||||
|
{"no audio", uploads.Probe{Duration: time.Minute}, uploads.ReasonNoAudio},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
r.pr.probe = tt.probe
|
||||||
|
|
||||||
|
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||||
|
|
||||||
|
var rej *uploads.RejectError
|
||||||
|
require.True(t, errors.As(err, &rej))
|
||||||
|
assert.Equal(t, tt.reason, rej.Reason)
|
||||||
|
assert.Empty(t, r.st.created)
|
||||||
|
assert.Empty(t, r.sub.jobs)
|
||||||
|
assert.Empty(t, r.tmpFiles(t))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccept_UnreadableFileIsARejectionButAProberFaultIsAnError(t *testing.T) {
|
||||||
|
t.Run("ffprobe says the file is not media", func(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
r.pr.err = uploads.ErrUnreadable
|
||||||
|
|
||||||
|
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("not media"))
|
||||||
|
|
||||||
|
var rej *uploads.RejectError
|
||||||
|
require.True(t, errors.As(err, &rej), "the user's file is the problem: tell them")
|
||||||
|
assert.Equal(t, uploads.ReasonUnreadable, rej.Reason)
|
||||||
|
assert.Empty(t, r.tmpFiles(t))
|
||||||
|
})
|
||||||
|
t.Run("the prober itself broke", func(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
r.pr.err = errors.New("exec: ffprobe: executable file not found")
|
||||||
|
|
||||||
|
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
var rej *uploads.RejectError
|
||||||
|
assert.False(t, errors.As(err, &rej), "our fault must not be reported as the user's bad file")
|
||||||
|
assert.Empty(t, r.st.created)
|
||||||
|
assert.Empty(t, r.tmpFiles(t))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccept_ClientDisconnectMidUploadLeavesNothingBehind(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
body := io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(errors.New("client went away")))
|
||||||
|
|
||||||
|
_, err := r.in.Accept(context.Background(), "user-1", "t", body)
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Empty(t, r.pr.paths)
|
||||||
|
assert.Empty(t, r.st.created)
|
||||||
|
assert.Empty(t, r.tmpFiles(t))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccept_SubmitFailureFailsTheRowHonestlyAndRemovesTheFile(t *testing.T) {
|
||||||
|
r := newRig(t)
|
||||||
|
r.sub.err = errors.New("worker stopped")
|
||||||
|
|
||||||
|
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, domain.UploadErrInterrupted, r.st.failed["up-1"],
|
||||||
|
"a row that will never run must not sit queued: that is the stuck spinner ADR-025 forbids")
|
||||||
|
assert.Empty(t, r.tmpFiles(t))
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
// Package uploads is the use case behind "upload an audio recording" (ADR-033):
|
||||||
|
// validate the file by PROBING it, keep the transcript private, never retain the
|
||||||
|
// audio. It depends only on ports (a Prober, a store, a job Submitter) — the
|
||||||
|
// concrete ffprobe adapter, the web handler and the transcription worker are wired
|
||||||
|
// from outside.
|
||||||
|
package uploads
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Limits bound what an upload may be. Both are rejected synchronously, with a
|
||||||
|
// specific message, rather than by a timeout (ADR-033).
|
||||||
|
type Limits struct {
|
||||||
|
MaxBytes int64
|
||||||
|
MaxDuration time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Probe is what inspecting an uploaded file revealed. It is the input to Validate
|
||||||
|
// because a name or MIME type cannot say whether a container carries video: a
|
||||||
|
// Teams voice-only export is typically an .mp4 with no video stream.
|
||||||
|
type Probe struct {
|
||||||
|
Duration time.Duration
|
||||||
|
HasAudio bool
|
||||||
|
HasVideo bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reason is the machine-checkable class of a rejection.
|
||||||
|
type Reason string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ReasonVideo Reason = "video"
|
||||||
|
ReasonNoAudio Reason = "no_audio"
|
||||||
|
ReasonTooLong Reason = "too_long"
|
||||||
|
ReasonTooLarge Reason = "too_large"
|
||||||
|
ReasonUnreadable Reason = "unreadable"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RejectError is a synchronous, user-presentable refusal. Message is written for
|
||||||
|
// the person who uploaded the file; Reason is for code and tests.
|
||||||
|
type RejectError struct {
|
||||||
|
Reason Reason
|
||||||
|
Message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *RejectError) Error() string { return string(e.Reason) + ": " + e.Message }
|
||||||
|
|
||||||
|
// Validate decides whether a probed file may be transcribed. Video is checked
|
||||||
|
// first so a video file is always refused as video, whatever its length.
|
||||||
|
func Validate(p Probe, l Limits) error {
|
||||||
|
switch {
|
||||||
|
case p.HasVideo:
|
||||||
|
return &RejectError{Reason: ReasonVideo, Message: "This file contains a video track. " +
|
||||||
|
"Only audio recordings are supported for now; video upload is tracked in issue #28."}
|
||||||
|
case !p.HasAudio:
|
||||||
|
return &RejectError{Reason: ReasonNoAudio, Message: "This file has no audio track to transcribe."}
|
||||||
|
case p.Duration <= 0:
|
||||||
|
return &RejectError{Reason: ReasonUnreadable, Message: "Could not read a length from this file; it may be damaged."}
|
||||||
|
case p.Duration > l.MaxDuration:
|
||||||
|
return &RejectError{Reason: ReasonTooLong, Message: fmt.Sprintf(
|
||||||
|
"This recording is %s long; the limit is %s.", FormatDuration(p.Duration), FormatDuration(l.MaxDuration))}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatDuration renders a length for people: "1 h 42 min", "45 min", "30 s".
|
||||||
|
func FormatDuration(d time.Duration) string {
|
||||||
|
d = d.Round(time.Second)
|
||||||
|
h := int(d / time.Hour)
|
||||||
|
m := int(d % time.Hour / time.Minute)
|
||||||
|
switch {
|
||||||
|
case h > 0 && m > 0:
|
||||||
|
return fmt.Sprintf("%d h %d min", h, m)
|
||||||
|
case h > 0:
|
||||||
|
return fmt.Sprintf("%d h", h)
|
||||||
|
case m > 0:
|
||||||
|
return fmt.Sprintf("%d min", m)
|
||||||
|
default:
|
||||||
|
return fmt.Sprintf("%d s", int(d/time.Second))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package uploads_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
var testLimits = uploads.Limits{MaxBytes: 200 << 20, MaxDuration: 90 * time.Minute}
|
||||||
|
|
||||||
|
func TestValidate(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
probe uploads.Probe
|
||||||
|
wantReason uploads.Reason // "" = accepted
|
||||||
|
wantInMsg []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "audio-only file within the duration limit is accepted",
|
||||||
|
probe: uploads.Probe{Duration: 45 * time.Minute, HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a recording exactly at the limit is accepted",
|
||||||
|
probe: uploads.Probe{Duration: 90 * time.Minute, HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a file with a video stream is rejected and names issue 28",
|
||||||
|
probe: uploads.Probe{Duration: 4 * time.Minute, HasAudio: true, HasVideo: true},
|
||||||
|
wantReason: uploads.ReasonVideo,
|
||||||
|
wantInMsg: []string{"video", "#28"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "video with no audio is still rejected as video, not as missing audio",
|
||||||
|
probe: uploads.Probe{Duration: 4 * time.Minute, HasVideo: true},
|
||||||
|
wantReason: uploads.ReasonVideo,
|
||||||
|
wantInMsg: []string{"#28"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a file with no audio stream is rejected",
|
||||||
|
probe: uploads.Probe{Duration: 4 * time.Minute},
|
||||||
|
wantReason: uploads.ReasonNoAudio,
|
||||||
|
wantInMsg: []string{"no audio"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a recording over the limit is rejected, stating both lengths",
|
||||||
|
probe: uploads.Probe{Duration: 102 * time.Minute, HasAudio: true},
|
||||||
|
wantReason: uploads.ReasonTooLong,
|
||||||
|
wantInMsg: []string{"1 h 42 min", "1 h 30 min"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero duration is unreadable, not accepted as an empty recording",
|
||||||
|
probe: uploads.Probe{HasAudio: true},
|
||||||
|
wantReason: uploads.ReasonUnreadable,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := uploads.Validate(tt.probe, testLimits)
|
||||||
|
|
||||||
|
if tt.wantReason == "" {
|
||||||
|
require.NoError(t, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var rej *uploads.RejectError
|
||||||
|
require.True(t, errors.As(err, &rej), "want a *RejectError, got %v", err)
|
||||||
|
assert.Equal(t, tt.wantReason, rej.Reason)
|
||||||
|
for _, s := range tt.wantInMsg {
|
||||||
|
assert.Contains(t, rej.Message, s)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatDuration(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
d time.Duration
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{90 * time.Minute, "1 h 30 min"},
|
||||||
|
{102 * time.Minute, "1 h 42 min"},
|
||||||
|
{45 * time.Minute, "45 min"},
|
||||||
|
{2 * time.Hour, "2 h"},
|
||||||
|
{30 * time.Second, "30 s"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.want, func(t *testing.T) {
|
||||||
|
assert.Equal(t, tt.want, uploads.FormatDuration(tt.d))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
package uploads
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrStopped is returned by Submit once the worker has stopped.
|
||||||
|
var ErrStopped = errors.New("uploads: worker stopped")
|
||||||
|
|
||||||
|
// Converter turns the uploaded file into the 16 kHz mono PCM WAV whisper wants.
|
||||||
|
type Converter interface {
|
||||||
|
ToWAV(ctx context.Context, src, dst string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transcriber turns a WAV into a timed transcription. The gateway chain
|
||||||
|
// (iguana primary, berget fallback) implements it.
|
||||||
|
type Transcriber interface {
|
||||||
|
Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WorkerStore is the slice of the private transcript store the worker drives.
|
||||||
|
type WorkerStore interface {
|
||||||
|
StartTranscribing(ctx context.Context, userID, id string) error
|
||||||
|
CompleteUpload(ctx context.Context, userID, id string, t domain.Transcription) error
|
||||||
|
FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Worker transcribes accepted uploads in-process, ONE AT A TIME (ADR-033). It is
|
||||||
|
// not a second deployable: it shares the web process's lifetime, which is why it
|
||||||
|
// inherits ADR-018's single-replica assumption — the queue is process-local. The
|
||||||
|
// concurrency of one is deliberate: two simultaneous 90-minute jobs on iguana were
|
||||||
|
// never measured, and a Metal OOM there latches the whisper backend (infra#273).
|
||||||
|
type Worker struct {
|
||||||
|
store WorkerStore
|
||||||
|
conv Converter
|
||||||
|
tr Transcriber
|
||||||
|
tmpDir string
|
||||||
|
timeout time.Duration
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
queue []Job
|
||||||
|
stopped bool
|
||||||
|
wake chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewWorker wires a Worker. timeout bounds each job (conversion + transcription);
|
||||||
|
// a job that outlives it is failed rather than left spinning.
|
||||||
|
func NewWorker(s WorkerStore, c Converter, t Transcriber, tmpDir string, timeout time.Duration) *Worker {
|
||||||
|
return &Worker{store: s, conv: c, tr: t, tmpDir: tmpDir, timeout: timeout, wake: make(chan struct{}, 1)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Submit queues a job. It never blocks. It satisfies Submitter.
|
||||||
|
func (w *Worker) Submit(j Job) error {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
if w.stopped {
|
||||||
|
return ErrStopped
|
||||||
|
}
|
||||||
|
w.queue = append(w.queue, j)
|
||||||
|
select {
|
||||||
|
case w.wake <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run processes queued jobs until ctx ends, then deletes the audio of anything
|
||||||
|
// still queued and refuses new jobs. The rows of those jobs stay queued or
|
||||||
|
// transcribing and are failed as interrupted by the start-up sweep
|
||||||
|
// (store.FailInterruptedUploads): the audio is not retained, so they cannot resume.
|
||||||
|
func (w *Worker) Run(ctx context.Context) {
|
||||||
|
defer w.stop()
|
||||||
|
for {
|
||||||
|
job, ok := w.next(ctx)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Process(ctx, job)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// next blocks for the next job. Cancellation wins over a non-empty queue: a job
|
||||||
|
// still queued at shutdown must not be STARTED (it would begin a long conversion
|
||||||
|
// with an already-cancelled context); stop() deletes its audio instead.
|
||||||
|
func (w *Worker) next(ctx context.Context) (Job, bool) {
|
||||||
|
for {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return Job{}, false
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
if len(w.queue) > 0 {
|
||||||
|
j := w.queue[0]
|
||||||
|
w.queue = w.queue[1:]
|
||||||
|
w.mu.Unlock()
|
||||||
|
return j, true
|
||||||
|
}
|
||||||
|
w.mu.Unlock()
|
||||||
|
select {
|
||||||
|
case <-w.wake:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Job{}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) stop() {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
w.stopped = true
|
||||||
|
for _, j := range w.queue {
|
||||||
|
_ = os.Remove(j.Path)
|
||||||
|
}
|
||||||
|
w.queue = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process runs one job: row -> transcribing, convert, transcribe, store. The
|
||||||
|
// uploaded file and the converted WAV are deleted on every path — the audio is
|
||||||
|
// never retained (ADR-033). A failure is recorded with a TYPED kind; a shutdown
|
||||||
|
// mid-job is not a failure and leaves the row for the start-up sweep.
|
||||||
|
func (w *Worker) Process(ctx context.Context, job Job) {
|
||||||
|
defer func() { _ = os.Remove(job.Path) }()
|
||||||
|
log := slog.Default().With("upload", job.UploadID)
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
if err := w.store.StartTranscribing(ctx, job.UserID, job.UploadID); err != nil {
|
||||||
|
log.Error("upload: start transcribing", "err", err)
|
||||||
|
w.fail(ctx, job, log)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
wav, err := w.tempWAV()
|
||||||
|
if err != nil {
|
||||||
|
log.Error("upload: temp wav", "err", err)
|
||||||
|
w.fail(ctx, job, log)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer func() { _ = os.Remove(wav) }()
|
||||||
|
|
||||||
|
jobCtx, cancel := context.WithTimeout(ctx, w.timeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := w.conv.ToWAV(jobCtx, job.Path, wav); err != nil {
|
||||||
|
log.Error("upload: convert", "err", err)
|
||||||
|
w.fail(ctx, job, log)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t, err := w.tr.Transcribe(jobCtx, wav)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("upload: transcribe", "err", err, "elapsed", time.Since(start).Round(time.Second))
|
||||||
|
w.fail(ctx, job, log)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := w.store.CompleteUpload(ctx, job.UserID, job.UploadID, t); err != nil {
|
||||||
|
log.Error("upload: store transcript", "err", err)
|
||||||
|
w.fail(ctx, job, log)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
log.Info("upload transcribed", "endpoint", t.Endpoint, "audio_s", t.AudioSeconds,
|
||||||
|
"segments", len(t.Segments), "elapsed", time.Since(start).Round(time.Second))
|
||||||
|
}
|
||||||
|
|
||||||
|
// fail records a transcription failure — unless the PARENT context ended, which
|
||||||
|
// is a shutdown, not a failure: the row is left for the start-up sweep.
|
||||||
|
func (w *Worker) fail(ctx context.Context, job Job, log *slog.Logger) {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := w.store.FailUpload(ctx, job.UserID, job.UploadID, domain.UploadErrTranscriptionFailed); err != nil {
|
||||||
|
log.Error("upload: record failure", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) tempWAV() (string, error) {
|
||||||
|
f, err := os.CreateTemp(w.tmpDir, "wav-*")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("uploads: temp wav: %w", err)
|
||||||
|
}
|
||||||
|
name := f.Name()
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
_ = os.Remove(name)
|
||||||
|
return "", fmt.Errorf("uploads: temp wav: %w", err)
|
||||||
|
}
|
||||||
|
return name, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,282 @@
|
|||||||
|
package uploads_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
type wStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
started []string
|
||||||
|
completed map[string]domain.Transcription
|
||||||
|
failed map[string]domain.UploadErrorKind
|
||||||
|
startErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func newWStore() *wStore {
|
||||||
|
return &wStore{completed: map[string]domain.Transcription{}, failed: map[string]domain.UploadErrorKind{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *wStore) StartTranscribing(_ context.Context, _, id string) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.started = append(s.started, id)
|
||||||
|
return s.startErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *wStore) CompleteUpload(_ context.Context, _, id string, t domain.Transcription) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.completed[id] = t
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *wStore) FailUpload(_ context.Context, _, id string, k domain.UploadErrorKind) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.failed[id] = k
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *wStore) snapshot() (map[string]domain.Transcription, map[string]domain.UploadErrorKind) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
c := map[string]domain.Transcription{}
|
||||||
|
f := map[string]domain.UploadErrorKind{}
|
||||||
|
for k, v := range s.completed {
|
||||||
|
c[k] = v
|
||||||
|
}
|
||||||
|
for k, v := range s.failed {
|
||||||
|
f[k] = v
|
||||||
|
}
|
||||||
|
return c, f
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeConverter struct {
|
||||||
|
err error
|
||||||
|
srcs []string
|
||||||
|
dsts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *fakeConverter) ToWAV(_ context.Context, src, dst string) error {
|
||||||
|
c.srcs = append(c.srcs, src)
|
||||||
|
c.dsts = append(c.dsts, dst)
|
||||||
|
if c.err != nil {
|
||||||
|
return c.err
|
||||||
|
}
|
||||||
|
return os.WriteFile(dst, []byte("wav"), 0o600)
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeTranscriber struct {
|
||||||
|
result domain.Transcription
|
||||||
|
err error
|
||||||
|
paths []string
|
||||||
|
// block, when non-nil, holds every call until it is closed or ctx ends.
|
||||||
|
block chan struct{}
|
||||||
|
running atomic.Int32
|
||||||
|
maxSeen atomic.Int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeTranscriber) Transcribe(ctx context.Context, wav string) (domain.Transcription, error) {
|
||||||
|
f.paths = append(f.paths, wav)
|
||||||
|
n := f.running.Add(1)
|
||||||
|
defer f.running.Add(-1)
|
||||||
|
for {
|
||||||
|
m := f.maxSeen.Load()
|
||||||
|
if n <= m || f.maxSeen.CompareAndSwap(m, n) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if f.block != nil {
|
||||||
|
select {
|
||||||
|
case <-f.block:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return domain.Transcription{}, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return f.result, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
type wrig struct {
|
||||||
|
w *uploads.Worker
|
||||||
|
st *wStore
|
||||||
|
conv *fakeConverter
|
||||||
|
tr *fakeTranscriber
|
||||||
|
tmp string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newWRig(t *testing.T, timeout time.Duration) *wrig {
|
||||||
|
t.Helper()
|
||||||
|
r := &wrig{
|
||||||
|
st: newWStore(),
|
||||||
|
conv: &fakeConverter{},
|
||||||
|
tr: &fakeTranscriber{result: domain.Transcription{Text: "synthetic text", Endpoint: "iguana/kb-whisper", AudioSeconds: 3}},
|
||||||
|
tmp: t.TempDir(),
|
||||||
|
}
|
||||||
|
r.w = uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, timeout)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// spool writes a fake uploaded file (what Intake would hand over) and returns a Job.
|
||||||
|
func (r *wrig) spool(t *testing.T, id string) uploads.Job {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(r.tmp, "upload-"+id)
|
||||||
|
require.NoError(t, os.WriteFile(p, []byte("audio"), 0o600))
|
||||||
|
return uploads.Job{UserID: "u1", UploadID: id, Path: p}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *wrig) leftovers(t *testing.T) []string {
|
||||||
|
t.Helper()
|
||||||
|
ents, err := os.ReadDir(r.tmp)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var n []string
|
||||||
|
for _, e := range ents {
|
||||||
|
n = append(n, e.Name())
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerProcess_SuccessConvertsTranscribesStoresAndDeletesTheAudio(t *testing.T) {
|
||||||
|
r := newWRig(t, time.Minute)
|
||||||
|
job := r.spool(t, "a")
|
||||||
|
|
||||||
|
r.w.Process(context.Background(), job)
|
||||||
|
|
||||||
|
c, f := r.st.snapshot()
|
||||||
|
assert.Equal(t, []string{"a"}, r.st.started)
|
||||||
|
assert.Equal(t, "synthetic text", c["a"].Text)
|
||||||
|
assert.Equal(t, "iguana/kb-whisper", c["a"].Endpoint)
|
||||||
|
assert.Empty(t, f)
|
||||||
|
require.Len(t, r.conv.srcs, 1)
|
||||||
|
assert.Equal(t, job.Path, r.conv.srcs[0], "the converter reads the uploaded file")
|
||||||
|
assert.Equal(t, r.conv.dsts, r.tr.paths, "whisper is sent the converted WAV, not the original")
|
||||||
|
assert.Equal(t, r.tmp, filepath.Dir(r.conv.dsts[0]))
|
||||||
|
assert.Empty(t, r.leftovers(t), "the audio is never retained: both the upload and the WAV are deleted")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerProcess_FailuresAreTypedAndStillDeleteTheAudio(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
setup func(*wrig)
|
||||||
|
}{
|
||||||
|
{"conversion fails", func(r *wrig) { r.conv.err = errors.New("moov atom not found") }},
|
||||||
|
{"every transcription endpoint fails", func(r *wrig) { r.tr.err = errors.New("iguana down; berget down") }},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
r := newWRig(t, time.Minute)
|
||||||
|
tt.setup(r)
|
||||||
|
job := r.spool(t, "a")
|
||||||
|
|
||||||
|
r.w.Process(context.Background(), job)
|
||||||
|
|
||||||
|
c, f := r.st.snapshot()
|
||||||
|
assert.Empty(t, c)
|
||||||
|
assert.Equal(t, domain.UploadErrTranscriptionFailed, f["a"])
|
||||||
|
assert.Empty(t, r.leftovers(t))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
t.Run("conversion failure never reaches whisper", func(t *testing.T) {
|
||||||
|
r := newWRig(t, time.Minute)
|
||||||
|
r.conv.err = errors.New("bad")
|
||||||
|
|
||||||
|
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||||
|
|
||||||
|
assert.Empty(t, r.tr.paths)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerProcess_ShutdownMidJobLeavesTheRowForTheStartupSweep(t *testing.T) {
|
||||||
|
r := newWRig(t, time.Minute)
|
||||||
|
r.tr.block = make(chan struct{})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
job := r.spool(t, "a")
|
||||||
|
|
||||||
|
go func() { r.w.Process(ctx, job); close(done) }()
|
||||||
|
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
|
||||||
|
_, f := r.st.snapshot()
|
||||||
|
assert.Empty(t, f, "a shutdown is not a transcription failure: the row stays for FailInterruptedUploads to mark interrupted")
|
||||||
|
assert.Empty(t, r.leftovers(t), "but the audio is still deleted")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerProcess_AJobThatOutlivesItsTimeoutIsFailedNotStuck(t *testing.T) {
|
||||||
|
r := newWRig(t, 50*time.Millisecond)
|
||||||
|
r.tr.block = make(chan struct{}) // never released
|
||||||
|
done := make(chan struct{})
|
||||||
|
|
||||||
|
go func() { r.w.Process(context.Background(), r.spool(t, "a")); close(done) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Process did not return: a hung endpoint was not bounded by the job timeout")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, f := r.st.snapshot()
|
||||||
|
assert.Equal(t, domain.UploadErrTranscriptionFailed, f["a"], "a hung endpoint must end in failed, not an eternal spinner (ADR-025)")
|
||||||
|
assert.Empty(t, r.leftovers(t))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerProcess_StartFailureDropsTheJobAndDeletesTheAudio(t *testing.T) {
|
||||||
|
r := newWRig(t, time.Minute)
|
||||||
|
r.st.startErr = errors.New("db down")
|
||||||
|
|
||||||
|
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||||
|
|
||||||
|
assert.Empty(t, r.tr.paths, "never transcribe a job whose row could not be moved to transcribing")
|
||||||
|
_, f := r.st.snapshot()
|
||||||
|
assert.Contains(t, f, "a")
|
||||||
|
assert.Empty(t, r.leftovers(t))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerRun_ProcessesOneJobAtATime(t *testing.T) {
|
||||||
|
r := newWRig(t, time.Minute)
|
||||||
|
r.tr.block = make(chan struct{})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go r.w.Run(ctx)
|
||||||
|
|
||||||
|
for _, id := range []string{"a", "b", "c"} {
|
||||||
|
require.NoError(t, r.w.Submit(r.spool(t, id)))
|
||||||
|
}
|
||||||
|
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||||
|
time.Sleep(50 * time.Millisecond) // give a (wrong) second job time to start
|
||||||
|
assert.Equal(t, int32(1), r.tr.running.Load(), "two simultaneous long jobs on iguana were never measured: concurrency is 1")
|
||||||
|
|
||||||
|
close(r.tr.block)
|
||||||
|
require.Eventually(t, func() bool { c, _ := r.st.snapshot(); return len(c) == 3 }, 3*time.Second, 5*time.Millisecond)
|
||||||
|
assert.Equal(t, int32(1), r.tr.maxSeen.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerRun_StoppingDeletesQueuedAudioAndRefusesNewJobs(t *testing.T) {
|
||||||
|
r := newWRig(t, time.Minute)
|
||||||
|
r.tr.block = make(chan struct{})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
go func() { r.w.Run(ctx); close(stopped) }()
|
||||||
|
require.NoError(t, r.w.Submit(r.spool(t, "a")))
|
||||||
|
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||||
|
require.NoError(t, r.w.Submit(r.spool(t, "b"))) // queued behind the running job
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
<-stopped
|
||||||
|
|
||||||
|
assert.Len(t, r.conv.srcs, 1, "a job still queued at shutdown must never be started: only the running job was converted")
|
||||||
|
assert.Empty(t, r.leftovers(t), "audio of queued jobs is deleted on stop: the rows are swept to interrupted at next start")
|
||||||
|
assert.ErrorIs(t, r.w.Submit(r.spool(t, "c")), uploads.ErrStopped)
|
||||||
|
}
|
||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Store is the read/write surface the web handlers depend on — a narrow port over
|
// Store is the read/write surface the web handlers depend on — a narrow port over
|
||||||
@@ -53,6 +54,12 @@ type Store interface {
|
|||||||
// the most recent discovery pass, shown on the account page as a warning.
|
// the most recent discovery pass, shown on the account page as a warning.
|
||||||
ListChannelErrors(ctx context.Context, userID string) ([]store.ChannelError, error)
|
ListChannelErrors(ctx context.Context, userID string) ([]store.ChannelError, error)
|
||||||
|
|
||||||
|
// Uploaded-audio transcripts (ADR-033): the caller's own PRIVATE uploads,
|
||||||
|
// stored apart from the shared public captions. Both are user-scoped; a
|
||||||
|
// non-owner's id is ErrUploadNotFound.
|
||||||
|
GetUpload(ctx context.Context, userID, id string) (*domain.UploadedTranscript, error)
|
||||||
|
ListUploads(ctx context.Context, userID string) ([]domain.UploadedTranscript, error)
|
||||||
|
|
||||||
// SetTranscriptStatus clears or updates a video's transcript backoff state.
|
// SetTranscriptStatus clears or updates a video's transcript backoff state.
|
||||||
// Used by handleRetryNow to clear rate_limited_at before immediate processing.
|
// Used by handleRetryNow to clear rate_limited_at before immediate processing.
|
||||||
SetTranscriptStatus(ctx context.Context, userID, videoID, status string) error
|
SetTranscriptStatus(ctx context.Context, userID, videoID, status string) error
|
||||||
@@ -100,6 +107,12 @@ type App struct {
|
|||||||
// summary view shows no "dig deeper" affordance. It holds no caption-fetch
|
// summary view shows no "dig deeper" affordance. It holds no caption-fetch
|
||||||
// dependency, so an enabled chat cannot reach YouTube or the rate gate.
|
// dependency, so an enabled chat cannot reach YouTube or the rate gate.
|
||||||
Chat Chatter
|
Chat Chatter
|
||||||
|
// Uploads, when non-nil, accepts audio recordings for private transcription
|
||||||
|
// (ADR-033). Nil = the /uploads routes are not mounted. UploadLimits are the
|
||||||
|
// same limits the Uploader enforces, shown on the form so they are stated up
|
||||||
|
// front rather than discovered by failing.
|
||||||
|
Uploads Uploader
|
||||||
|
UploadLimits uploads.Limits
|
||||||
// Processing tracks in-flight immediate summarizations so the status endpoint
|
// Processing tracks in-flight immediate summarizations so the status endpoint
|
||||||
// shows the animation until the summary lands. The zero value is ready to use.
|
// shows the animation until the summary lands. The zero value is ready to use.
|
||||||
Processing ProcessingSet
|
Processing ProcessingSet
|
||||||
@@ -166,6 +179,14 @@ func (a *App) Router() http.Handler {
|
|||||||
app.HandleFunc("GET /v/{videoId}/chat", a.handleChat)
|
app.HandleFunc("GET /v/{videoId}/chat", a.handleChat)
|
||||||
app.HandleFunc("POST /v/{videoId}/chat", a.handleChatMessage)
|
app.HandleFunc("POST /v/{videoId}/chat", a.handleChatMessage)
|
||||||
}
|
}
|
||||||
|
// Audio upload → private transcript (ADR-033). Mounted only when an Uploader is
|
||||||
|
// wired; every handler is user-scoped, so another user's upload is a 404.
|
||||||
|
if a.Uploads != nil {
|
||||||
|
app.HandleFunc("GET /uploads", a.handleUploadsPage)
|
||||||
|
app.HandleFunc("POST /uploads", a.handleUploadPost)
|
||||||
|
app.HandleFunc("GET /uploads/{id}", a.handleUploadDetail)
|
||||||
|
app.HandleFunc("GET /uploads/{id}/status", a.handleUploadStatus)
|
||||||
|
}
|
||||||
app.HandleFunc("GET /register", a.handleRegisterForm)
|
app.HandleFunc("GET /register", a.handleRegisterForm)
|
||||||
app.HandleFunc("POST /register", a.handleRegister)
|
app.HandleFunc("POST /register", a.handleRegister)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
package web
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Uploader accepts an uploaded recording for transcription (ADR-033).
|
||||||
|
// *uploads.Intake satisfies it. It is the whole of what the handlers know about
|
||||||
|
// validation, spooling and queueing.
|
||||||
|
type Uploader interface {
|
||||||
|
Accept(ctx context.Context, userID, title string, src io.Reader) (id string, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// uploadOverhead is headroom above the file-size cap for the multipart framing
|
||||||
|
// and the title field; the cap itself is enforced on the file part by Intake.
|
||||||
|
uploadOverhead = 1 << 20
|
||||||
|
maxTitleRunes = 200
|
||||||
|
// uploadPollEvery is how often an unfinished upload re-asks for its status.
|
||||||
|
uploadPollEvery = "5s"
|
||||||
|
)
|
||||||
|
|
||||||
|
// uploadsView is what the /uploads page renders.
|
||||||
|
type uploadsView struct {
|
||||||
|
Limits uploads.Limits
|
||||||
|
Rows []domain.UploadedTranscript
|
||||||
|
Error string // a synchronous rejection message, empty when none
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleUploadsPage renders the upload form and the caller's own uploads.
|
||||||
|
func (a *App) handleUploadsPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID, ok := a.currentUserID(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.renderUploadsPage(w, r, userID, http.StatusOK, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) renderUploadsPage(w http.ResponseWriter, r *http.Request, userID string, status int, msg string) {
|
||||||
|
rows, err := a.Store.ListUploads(r.Context(), userID)
|
||||||
|
if err != nil {
|
||||||
|
a.serverError(w, r, "list uploads", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.renderStatus(w, r, status, UploadsPage(uploadsView{Limits: a.UploadLimits, Rows: rows, Error: msg}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleUploadPost streams the multipart body: the optional title field, then the
|
||||||
|
// file part, which goes straight to Intake without being buffered in memory. A
|
||||||
|
// rejection is synchronous — a status code and a message on the upload page, no
|
||||||
|
// row, no timeout (ADR-033).
|
||||||
|
func (a *App) handleUploadPost(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID, ok := a.currentUserID(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, a.UploadLimits.MaxBytes+uploadOverhead)
|
||||||
|
mr, err := r.MultipartReader()
|
||||||
|
if err != nil {
|
||||||
|
a.renderUploadsPage(w, r, userID, http.StatusBadRequest, "Choose an audio file to upload.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
title := ""
|
||||||
|
for {
|
||||||
|
part, err := mr.NextPart()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
a.uploadFailure(w, r, userID, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch part.FormName() {
|
||||||
|
case "title":
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(part, maxTitleRunes*utf8.UTFMax))
|
||||||
|
title = clipRunes(strings.TrimSpace(string(b)), maxTitleRunes)
|
||||||
|
case "file":
|
||||||
|
if title == "" {
|
||||||
|
title = titleFromFilename(part.FileName())
|
||||||
|
}
|
||||||
|
id, err := a.Uploads.Accept(r.Context(), userID, title, part)
|
||||||
|
if err != nil {
|
||||||
|
a.uploadFailure(w, r, userID, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.Redirect(w, r, "/uploads/"+id, http.StatusSeeOther)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.renderUploadsPage(w, r, userID, http.StatusBadRequest, "Choose an audio file to upload.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadFailure maps an Accept (or multipart) error to a response: a validation
|
||||||
|
// rejection is the user's to fix (422, or 413 for size) and carries its own
|
||||||
|
// message; anything else is ours.
|
||||||
|
func (a *App) uploadFailure(w http.ResponseWriter, r *http.Request, userID string, err error) {
|
||||||
|
var rej *uploads.RejectError
|
||||||
|
var tooBig *http.MaxBytesError
|
||||||
|
switch {
|
||||||
|
case errors.As(err, &rej):
|
||||||
|
status := http.StatusUnprocessableEntity
|
||||||
|
if rej.Reason == uploads.ReasonTooLarge {
|
||||||
|
status = http.StatusRequestEntityTooLarge
|
||||||
|
}
|
||||||
|
a.renderUploadsPage(w, r, userID, status, rej.Message)
|
||||||
|
case errors.As(err, &tooBig):
|
||||||
|
a.renderUploadsPage(w, r, userID, http.StatusRequestEntityTooLarge,
|
||||||
|
"This file is larger than the upload limit.")
|
||||||
|
default:
|
||||||
|
a.serverError(w, r, "upload", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleUploadDetail renders one upload's page (GET /uploads/{id}). The row is
|
||||||
|
// read through the user-scoped store, so another user's id is a 404 — existence is
|
||||||
|
// not disclosed, and the title never appears in the response.
|
||||||
|
func (a *App) handleUploadDetail(w http.ResponseWriter, r *http.Request) {
|
||||||
|
u, ok := a.loadOwnedUpload(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.render(w, r, UploadPage(*u))
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleUploadStatus renders the status fragment HTMX polls. Unfinished states
|
||||||
|
// keep polling; terminal states render without hx-trigger so the page stops
|
||||||
|
// asking (ADR-025: never a stuck spinner).
|
||||||
|
func (a *App) handleUploadStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
|
u, ok := a.loadOwnedUpload(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.render(w, r, uploadStatus(*u))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) loadOwnedUpload(w http.ResponseWriter, r *http.Request) (*domain.UploadedTranscript, bool) {
|
||||||
|
userID, ok := a.currentUserID(w, r)
|
||||||
|
if !ok {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
u, err := a.Store.GetUpload(r.Context(), userID, r.PathValue("id"))
|
||||||
|
if errors.Is(err, store.ErrUploadNotFound) {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
a.serverError(w, r, "get upload", err)
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return u, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// titleFromFilename is the default title: the file's name without its extension.
|
||||||
|
func titleFromFilename(name string) string {
|
||||||
|
base := filepath.Base(name)
|
||||||
|
t := strings.TrimSuffix(base, filepath.Ext(base))
|
||||||
|
if t == "" || t == "." {
|
||||||
|
return "Untitled recording"
|
||||||
|
}
|
||||||
|
return clipRunes(t, maxTitleRunes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func clipRunes(s string, n int) string {
|
||||||
|
if utf8.RuneCountInString(s) <= n {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return string([]rune(s)[:n])
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadFailureMessage turns a typed failure kind into words for the user. It is
|
||||||
|
// keyed on OUR kinds, never on a provider's error string (ADR-033).
|
||||||
|
func uploadFailureMessage(k domain.UploadErrorKind) string {
|
||||||
|
switch k {
|
||||||
|
case domain.UploadErrInterrupted:
|
||||||
|
return "The server restarted before this recording was transcribed. Please upload it again."
|
||||||
|
case domain.UploadErrTranscriptionFailed:
|
||||||
|
return "Transcription failed on every available service. Please upload it again later."
|
||||||
|
default:
|
||||||
|
return "This upload failed. Please upload it again."
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package web
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// UploadsPage is the upload form plus the caller's own uploads (ADR-033).
|
||||||
|
templ UploadsPage(v uploadsView) {
|
||||||
|
@Layout("Tapir — Upload a recording") {
|
||||||
|
<article class="account">
|
||||||
|
<h1>Upload a recording</h1>
|
||||||
|
<p class="muted">
|
||||||
|
Audio only. Up to { fmt.Sprintf("%d MB", v.Limits.MaxBytes>>20) } and
|
||||||
|
{ uploads.FormatDuration(v.Limits.MaxDuration) }. The transcript is private to you;
|
||||||
|
the audio is deleted once it has been transcribed.
|
||||||
|
</p>
|
||||||
|
if v.Error != "" {
|
||||||
|
<div class="flash flash-error" role="alert">{ v.Error }</div>
|
||||||
|
}
|
||||||
|
<form method="post" action="/uploads" enctype="multipart/form-data">
|
||||||
|
<p><label>Title (optional) <input type="text" name="title" maxlength="200"/></label></p>
|
||||||
|
<p><input type="file" name="file" accept="audio/*,.m4a,.mp3,.wav,.opus,.ogg,.mp4" required/></p>
|
||||||
|
<p><button type="submit">Upload and transcribe</button></p>
|
||||||
|
</form>
|
||||||
|
if len(v.Rows) > 0 {
|
||||||
|
<section>
|
||||||
|
<h2>Your recordings</h2>
|
||||||
|
<ul>
|
||||||
|
for _, u := range v.Rows {
|
||||||
|
<li>
|
||||||
|
<a href={ templ.SafeURL("/uploads/" + u.ID) }>{ u.Title }</a>
|
||||||
|
<span class="chip">{ string(u.Status) }</span>
|
||||||
|
</li>
|
||||||
|
}
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
}
|
||||||
|
</article>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadPage is one upload's page: its title and the polling status region.
|
||||||
|
templ UploadPage(u domain.UploadedTranscript) {
|
||||||
|
@Layout("Tapir — " + u.Title) {
|
||||||
|
<article class="account">
|
||||||
|
<h1>{ u.Title }</h1>
|
||||||
|
@uploadStatus(u)
|
||||||
|
<p><a href="/uploads">All recordings</a></p>
|
||||||
|
</article>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadStatus is the region HTMX polls. Unfinished states carry hx-trigger so the
|
||||||
|
// page re-asks; done and failed render without it, so nothing polls forever.
|
||||||
|
templ uploadStatus(u domain.UploadedTranscript) {
|
||||||
|
switch u.Status {
|
||||||
|
case domain.UploadDone:
|
||||||
|
<div class="upload-status upload-done" role="status">
|
||||||
|
<p>
|
||||||
|
Transcribed { uploads.FormatDuration(time.Duration(u.AudioSeconds * float64(time.Second))) }
|
||||||
|
of audio by <span class="chip">{ u.Endpoint }</span>.
|
||||||
|
</p>
|
||||||
|
<details open>
|
||||||
|
<summary>Transcript</summary>
|
||||||
|
<pre class="transcript">{ u.Content }</pre>
|
||||||
|
</details>
|
||||||
|
</div>
|
||||||
|
case domain.UploadFailed:
|
||||||
|
<div class="upload-status upload-failed flash flash-error" role="alert">
|
||||||
|
<p>{ uploadFailureMessage(u.ErrorKind) }</p>
|
||||||
|
</div>
|
||||||
|
default:
|
||||||
|
<div
|
||||||
|
class="upload-status upload-working"
|
||||||
|
role="status"
|
||||||
|
hx-get={ "/uploads/" + u.ID + "/status" }
|
||||||
|
hx-trigger={ "every " + uploadPollEvery }
|
||||||
|
hx-swap="outerHTML"
|
||||||
|
>
|
||||||
|
if u.Status == domain.UploadTranscribing {
|
||||||
|
<p>Transcribing… a long recording takes a few minutes.</p>
|
||||||
|
} else {
|
||||||
|
<p>Queued — waiting for its turn to be transcribed.</p>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,277 @@
|
|||||||
|
package web_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Upload route tests drive the real Router, the real Intake and the real Postgres
|
||||||
|
// store; only the process boundary (ffprobe) and the transcription worker are
|
||||||
|
// faked. All audio bytes and transcript text are synthetic — this repo is public
|
||||||
|
// and no real recording or transcript may appear in it (#33).
|
||||||
|
|
||||||
|
const otherUserID = "22222222-2222-2222-2222-222222222222"
|
||||||
|
|
||||||
|
type uploadProber struct{ probe uploads.Probe }
|
||||||
|
|
||||||
|
func (p uploadProber) Probe(context.Context, string) (uploads.Probe, error) { return p.probe, nil }
|
||||||
|
|
||||||
|
type uploadSubmitter struct{ jobs []uploads.Job }
|
||||||
|
|
||||||
|
func (s *uploadSubmitter) Submit(j uploads.Job) error { s.jobs = append(s.jobs, j); return nil }
|
||||||
|
|
||||||
|
func newUploadApp(t *testing.T, probe uploads.Probe, maxBytes int64) (*web.App, *uploadSubmitter) {
|
||||||
|
t.Helper()
|
||||||
|
app := newApp(t)
|
||||||
|
resetDB(t, rawPool(t))
|
||||||
|
sub := &uploadSubmitter{}
|
||||||
|
limits := uploads.Limits{MaxBytes: maxBytes, MaxDuration: 90 * time.Minute}
|
||||||
|
s := newStore(t)
|
||||||
|
app.Uploads = uploads.NewIntake(s, uploadProber{probe: probe}, sub, limits, t.TempDir())
|
||||||
|
app.UploadLimits = limits
|
||||||
|
return app, sub
|
||||||
|
}
|
||||||
|
|
||||||
|
var audioOK = uploads.Probe{Duration: 45 * time.Minute, HasAudio: true}
|
||||||
|
|
||||||
|
func uploadReq(t *testing.T, title, filename string, content []byte) *http.Request {
|
||||||
|
t.Helper()
|
||||||
|
var buf bytes.Buffer
|
||||||
|
mw := multipart.NewWriter(&buf)
|
||||||
|
if title != "" {
|
||||||
|
require.NoError(t, mw.WriteField("title", title))
|
||||||
|
}
|
||||||
|
if filename != "" {
|
||||||
|
fw, err := mw.CreateFormFile("file", filename)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = fw.Write(content)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
require.NoError(t, mw.Close())
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/uploads", &buf)
|
||||||
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||||
|
return req
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedUpload(t *testing.T, owner, title, status, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
p := rawPool(t)
|
||||||
|
_, err := p.Exec(context.Background(), `INSERT INTO users (id) VALUES ($1) ON CONFLICT DO NOTHING`, owner)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var id string
|
||||||
|
switch status {
|
||||||
|
case "done":
|
||||||
|
require.NoError(t, p.QueryRow(context.Background(),
|
||||||
|
`INSERT INTO uploaded_transcripts (user_id, title, status, content, endpoint, audio_seconds, finished_at)
|
||||||
|
VALUES ($1, $2, 'done', $3, 'berget/kb-whisper', 2700, NOW()) RETURNING id`,
|
||||||
|
owner, title, content).Scan(&id))
|
||||||
|
case "failed":
|
||||||
|
require.NoError(t, p.QueryRow(context.Background(),
|
||||||
|
`INSERT INTO uploaded_transcripts (user_id, title, status, error_kind, finished_at)
|
||||||
|
VALUES ($1, $2, 'failed', $3, NOW()) RETURNING id`,
|
||||||
|
owner, title, content).Scan(&id)) // content carries the error_kind here
|
||||||
|
default:
|
||||||
|
require.NoError(t, p.QueryRow(context.Background(),
|
||||||
|
`INSERT INTO uploaded_transcripts (user_id, title, status) VALUES ($1, $2, $3) RETURNING id`,
|
||||||
|
owner, title, status).Scan(&id))
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
func get(path string) *http.Request { return httptest.NewRequest(http.MethodGet, path, nil) }
|
||||||
|
|
||||||
|
func TestUploadsPage_ShowsAMultipartFormWithAFileField(t *testing.T) {
|
||||||
|
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
|
||||||
|
rec := do(t, app, get("/uploads"))
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusOK, rec.Code)
|
||||||
|
b := body(t, rec)
|
||||||
|
assert.Contains(t, b, `enctype="multipart/form-data"`)
|
||||||
|
assert.Contains(t, b, `name="file"`, "the multipart field is `file`")
|
||||||
|
assert.Contains(t, b, "200 MB", "the size limit is stated up front, not discovered by failing")
|
||||||
|
assert.Contains(t, b, "1 h 30 min", "the length limit is stated up front")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadsPage_ListsOnlyTheCallersUploads(t *testing.T) {
|
||||||
|
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
seedUpload(t, userID, "My weekly sync", "queued", "")
|
||||||
|
seedUpload(t, otherUserID, "Someone else's board meeting", "queued", "")
|
||||||
|
|
||||||
|
b := body(t, do(t, app, get("/uploads")))
|
||||||
|
|
||||||
|
assert.Contains(t, b, "My weekly sync")
|
||||||
|
assert.NotContains(t, b, "Someone else's board meeting")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadPost_AcceptedAudioQueuesARowAndRedirectsToIt(t *testing.T) {
|
||||||
|
app, sub := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
|
||||||
|
rec := do(t, app, uploadReq(t, "Sprint review", "recording.m4a", []byte("synthetic audio bytes")))
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusSeeOther, rec.Code)
|
||||||
|
var id, status, title string
|
||||||
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||||
|
`SELECT id, status, title FROM uploaded_transcripts WHERE user_id = $1`, userID).Scan(&id, &status, &title))
|
||||||
|
assert.Equal(t, "queued", status)
|
||||||
|
assert.Equal(t, "Sprint review", title)
|
||||||
|
assert.Equal(t, "/uploads/"+id, rec.Header().Get("Location"))
|
||||||
|
require.Len(t, sub.jobs, 1)
|
||||||
|
assert.Equal(t, uploads.Job{UserID: userID, UploadID: id, Path: sub.jobs[0].Path}, sub.jobs[0])
|
||||||
|
spooled, err := os.ReadFile(sub.jobs[0].Path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "synthetic audio bytes", string(spooled))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadPost_WithoutATitleUsesTheFilenameMinusItsExtension(t *testing.T) {
|
||||||
|
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
|
||||||
|
rec := do(t, app, uploadReq(t, "", "Teams meeting 2026-09-30.mp4", []byte("x")))
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusSeeOther, rec.Code)
|
||||||
|
var title string
|
||||||
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||||
|
`SELECT title FROM uploaded_transcripts WHERE user_id = $1`, userID).Scan(&title))
|
||||||
|
assert.Equal(t, "Teams meeting 2026-09-30", title)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadPost_RejectionsAreSynchronousWithAMessageAndNoRow(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
probe uploads.Probe
|
||||||
|
maxBytes int64
|
||||||
|
content string
|
||||||
|
wantStatus int
|
||||||
|
wantInBody []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "video names issue 28",
|
||||||
|
probe: uploads.Probe{Duration: time.Minute, HasAudio: true, HasVideo: true},
|
||||||
|
maxBytes: 1 << 20,
|
||||||
|
content: "x",
|
||||||
|
wantStatus: http.StatusUnprocessableEntity,
|
||||||
|
wantInBody: []string{"video", "#28"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "too long states both lengths",
|
||||||
|
probe: uploads.Probe{Duration: 2 * time.Hour, HasAudio: true},
|
||||||
|
maxBytes: 1 << 20,
|
||||||
|
content: "x",
|
||||||
|
wantStatus: http.StatusUnprocessableEntity,
|
||||||
|
wantInBody: []string{"2 h", "1 h 30 min"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "too large is 413",
|
||||||
|
probe: audioOK,
|
||||||
|
maxBytes: 4,
|
||||||
|
content: "more than four bytes",
|
||||||
|
wantStatus: http.StatusRequestEntityTooLarge,
|
||||||
|
wantInBody: []string{"limit"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
app, sub := newUploadApp(t, tt.probe, tt.maxBytes)
|
||||||
|
|
||||||
|
rec := do(t, app, uploadReq(t, "t", "r.m4a", []byte(tt.content)))
|
||||||
|
|
||||||
|
assert.Equal(t, tt.wantStatus, rec.Code)
|
||||||
|
b := body(t, rec)
|
||||||
|
for _, s := range tt.wantInBody {
|
||||||
|
assert.Contains(t, b, s)
|
||||||
|
}
|
||||||
|
var n int
|
||||||
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||||
|
`SELECT count(*) FROM uploaded_transcripts`).Scan(&n))
|
||||||
|
assert.Zero(t, n, "a rejected upload leaves no row")
|
||||||
|
assert.Empty(t, sub.jobs)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadPost_WithNoFilePartIs400(t *testing.T) {
|
||||||
|
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
|
||||||
|
rec := do(t, app, uploadReq(t, "only a title", "", nil))
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusBadRequest, rec.Code)
|
||||||
|
assert.Contains(t, body(t, rec), "Choose an audio file")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadRoutes_AreNotMountedWithoutAnUploader(t *testing.T) {
|
||||||
|
app := newApp(t)
|
||||||
|
resetDB(t, rawPool(t))
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusNotFound, do(t, app, get("/uploads")).Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadPage_OwnerSeesTitleStatusAndNonOwnerGetsNotFound(t *testing.T) {
|
||||||
|
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
mine := seedUpload(t, userID, "Roadmap sync", "queued", "")
|
||||||
|
theirs := seedUpload(t, otherUserID, "Their private meeting", "queued", "")
|
||||||
|
|
||||||
|
own := do(t, app, get("/uploads/"+mine))
|
||||||
|
require.Equal(t, http.StatusOK, own.Code)
|
||||||
|
assert.Contains(t, body(t, own), "Roadmap sync")
|
||||||
|
|
||||||
|
for _, path := range []string{"/uploads/" + theirs, "/uploads/" + theirs + "/status"} {
|
||||||
|
rec := do(t, app, get(path))
|
||||||
|
assert.Equal(t, http.StatusNotFound, rec.Code, path)
|
||||||
|
assert.NotContains(t, body(t, rec), "Their private meeting", "a 404 must not leak the title")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadStatus_UnfinishedStatesKeepPollingTerminalStatesStop(t *testing.T) {
|
||||||
|
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
tests := []struct {
|
||||||
|
name, status, content string
|
||||||
|
polls bool
|
||||||
|
wantInBody string
|
||||||
|
}{
|
||||||
|
{"queued", "queued", "", true, "Queued"},
|
||||||
|
{"transcribing", "transcribing", "", true, "ranscrib"},
|
||||||
|
{"done", "done", "Synthetic transcript text.", false, "Synthetic transcript text."},
|
||||||
|
{"failed interrupted", "failed", "interrupted", false, "upload it again"},
|
||||||
|
{"failed transcription", "failed", "transcription_failed", false, "Transcription failed"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
id := seedUpload(t, userID, "m-"+tt.name, tt.status, tt.content)
|
||||||
|
|
||||||
|
rec := do(t, app, get("/uploads/"+id+"/status"))
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusOK, rec.Code)
|
||||||
|
b := body(t, rec)
|
||||||
|
assert.Contains(t, b, tt.wantInBody)
|
||||||
|
if tt.polls {
|
||||||
|
assert.Contains(t, b, `hx-trigger="every`, "an unfinished upload must keep polling")
|
||||||
|
} else {
|
||||||
|
assert.NotContains(t, b, "hx-trigger", "a finished upload must not poll forever: no stuck spinner")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadStatus_DoneNamesTheEndpointThatServedIt(t *testing.T) {
|
||||||
|
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||||
|
id := seedUpload(t, userID, "m", "done", "Synthetic transcript text.")
|
||||||
|
|
||||||
|
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||||
|
|
||||||
|
assert.Contains(t, b, "berget/kb-whisper",
|
||||||
|
"which backend served a transcript is a quality fact, and berget means the audio left the homelab")
|
||||||
|
assert.True(t, strings.Contains(b, "45 min"), "the audio length is shown")
|
||||||
|
}
|
||||||
@@ -0,0 +1,381 @@
|
|||||||
|
// Code generated by templ - DO NOT EDIT.
|
||||||
|
|
||||||
|
// templ: version: v0.3.1020
|
||||||
|
package web
|
||||||
|
|
||||||
|
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||||
|
|
||||||
|
import "github.com/a-h/templ"
|
||||||
|
import templruntime "github.com/a-h/templ/runtime"
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// UploadsPage is the upload form plus the caller's own uploads (ADR-033).
|
||||||
|
func UploadsPage(v uploadsView) templ.Component {
|
||||||
|
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||||
|
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||||
|
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||||
|
return templ_7745c5c3_CtxErr
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||||
|
if !templ_7745c5c3_IsBuffer {
|
||||||
|
defer func() {
|
||||||
|
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err == nil {
|
||||||
|
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
ctx = templ.InitializeContext(ctx)
|
||||||
|
templ_7745c5c3_Var1 := templ.GetChildren(ctx)
|
||||||
|
if templ_7745c5c3_Var1 == nil {
|
||||||
|
templ_7745c5c3_Var1 = templ.NopComponent
|
||||||
|
}
|
||||||
|
ctx = templ.ClearChildren(ctx)
|
||||||
|
templ_7745c5c3_Var2 := templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||||
|
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||||
|
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||||
|
if !templ_7745c5c3_IsBuffer {
|
||||||
|
defer func() {
|
||||||
|
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err == nil {
|
||||||
|
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
ctx = templ.InitializeContext(ctx)
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<article class=\"account\"><h1>Upload a recording</h1><p class=\"muted\">Audio only. Up to ")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var3 string
|
||||||
|
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d MB", v.Limits.MaxBytes>>20))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 17, Col: 67}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " and ")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var4 string
|
||||||
|
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(uploads.FormatDuration(v.Limits.MaxDuration))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 18, Col: 50}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, ". The transcript is private to you; the audio is deleted once it has been transcribed.</p>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
if v.Error != "" {
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<div class=\"flash flash-error\" role=\"alert\">")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var5 string
|
||||||
|
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(v.Error)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 22, Col: 57}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</div>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<form method=\"post\" action=\"/uploads\" enctype=\"multipart/form-data\"><p><label>Title (optional) <input type=\"text\" name=\"title\" maxlength=\"200\"></label></p><p><input type=\"file\" name=\"file\" accept=\"audio/*,.m4a,.mp3,.wav,.opus,.ogg,.mp4\" required></p><p><button type=\"submit\">Upload and transcribe</button></p></form>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
if len(v.Rows) > 0 {
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<section><h2>Your recordings</h2><ul>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
for _, u := range v.Rows {
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<li><a href=\"")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var6 templ.SafeURL
|
||||||
|
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL("/uploads/" + u.ID))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 35, Col: 51}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\">")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var7 string
|
||||||
|
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(u.Title)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 35, Col: 63}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</a> <span class=\"chip\">")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var8 string
|
||||||
|
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(string(u.Status))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 36, Col: 45}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</span></li>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "</ul></section>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</article>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
templ_7745c5c3_Err = Layout("Tapir — Upload a recording").Render(templ.WithChildren(ctx, templ_7745c5c3_Var2), templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadPage is one upload's page: its title and the polling status region.
|
||||||
|
func UploadPage(u domain.UploadedTranscript) templ.Component {
|
||||||
|
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||||
|
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||||
|
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||||
|
return templ_7745c5c3_CtxErr
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||||
|
if !templ_7745c5c3_IsBuffer {
|
||||||
|
defer func() {
|
||||||
|
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err == nil {
|
||||||
|
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
ctx = templ.InitializeContext(ctx)
|
||||||
|
templ_7745c5c3_Var9 := templ.GetChildren(ctx)
|
||||||
|
if templ_7745c5c3_Var9 == nil {
|
||||||
|
templ_7745c5c3_Var9 = templ.NopComponent
|
||||||
|
}
|
||||||
|
ctx = templ.ClearChildren(ctx)
|
||||||
|
templ_7745c5c3_Var10 := templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||||
|
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||||
|
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||||
|
if !templ_7745c5c3_IsBuffer {
|
||||||
|
defer func() {
|
||||||
|
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err == nil {
|
||||||
|
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
ctx = templ.InitializeContext(ctx)
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<article class=\"account\"><h1>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var11 string
|
||||||
|
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(u.Title)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 50, Col: 16}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</h1>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = uploadStatus(u).Render(ctx, templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<p><a href=\"/uploads\">All recordings</a></p></article>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
templ_7745c5c3_Err = Layout("Tapir — "+u.Title).Render(templ.WithChildren(ctx, templ_7745c5c3_Var10), templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadStatus is the region HTMX polls. Unfinished states carry hx-trigger so the
|
||||||
|
// page re-asks; done and failed render without it, so nothing polls forever.
|
||||||
|
func uploadStatus(u domain.UploadedTranscript) templ.Component {
|
||||||
|
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||||
|
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||||
|
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||||
|
return templ_7745c5c3_CtxErr
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||||
|
if !templ_7745c5c3_IsBuffer {
|
||||||
|
defer func() {
|
||||||
|
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||||
|
if templ_7745c5c3_Err == nil {
|
||||||
|
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
ctx = templ.InitializeContext(ctx)
|
||||||
|
templ_7745c5c3_Var12 := templ.GetChildren(ctx)
|
||||||
|
if templ_7745c5c3_Var12 == nil {
|
||||||
|
templ_7745c5c3_Var12 = templ.NopComponent
|
||||||
|
}
|
||||||
|
ctx = templ.ClearChildren(ctx)
|
||||||
|
switch u.Status {
|
||||||
|
case domain.UploadDone:
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<div class=\"upload-status upload-done\" role=\"status\"><p>Transcribed ")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var13 string
|
||||||
|
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(uploads.FormatDuration(time.Duration(u.AudioSeconds * float64(time.Second))))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 64, Col: 95}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, " of audio by <span class=\"chip\">")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var14 string
|
||||||
|
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(u.Endpoint)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 65, Col: 48}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</span>.</p><details open><summary>Transcript</summary><pre class=\"transcript\">")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var15 string
|
||||||
|
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(u.Content)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 69, Col: 40}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "</pre></details></div>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
case domain.UploadFailed:
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "<div class=\"upload-status upload-failed flash flash-error\" role=\"alert\"><p>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var16 string
|
||||||
|
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(uploadFailureMessage(u.ErrorKind))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 74, Col: 42}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "</p></div>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "<div class=\"upload-status upload-working\" role=\"status\" hx-get=\"")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var17 string
|
||||||
|
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.ResolveAttributeValue("/uploads/" + u.ID + "/status")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 80, Col: 43}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var17)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "\" hx-trigger=\"")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var18 string
|
||||||
|
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.ResolveAttributeValue("every " + uploadPollEvery)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/uploads.templ`, Line: 81, Col: 43}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var18)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "\" hx-swap=\"outerHTML\">")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
if u.Status == domain.UploadTranscribing {
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "<p>Transcribing… a long recording takes a few minutes.</p>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "<p>Queued — waiting for its turn to be transcribed.</p>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</div>")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ = templruntime.GeneratedTemplate
|
||||||
Reference in New Issue
Block a user