Upload a recording (audio, or video with the camera off), get a private Swedish transcript, and chat about it. - ADR-033: RLS-scoped private transcript store; ffprobe/ffmpeg intake; gateway transcription (iguana kb-whisper, berget fallback); in-process single-replica worker; chat over the transcript (local models only). - ADR-034: optional two-engine mode (pianissimo + kb-whisper) reconciled by a LOCAL model; off unless TAPIR_RECONCILE_BASE_MODEL is set. - ADR-035: real progress (phases, running/quiet/lost), phase logging, every judged spot stored, merged segments. - Hardening from four adversarial reviews: errors logged and stored as kinds never text; per-engine deadlines; converted-length check; bounded queue; client hang-ups are not errors; strict local-alias check. - Dockerfile: static ffmpeg/ffprobe 7.0.2 pinned by sha256 (#34). Built and inspected by hand; CI does not build the image on a PR, so its first CI build is this merge. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set in the deployment. Closes #33, #34, #42, #43, #44. Co-authored-by: Mathias <mthbqv@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FFmpeg converts any audio container to the strict format whisper wants: 16 kHz
|
||||
// mono signed 16-bit PCM WAV (ADR-033). It ALWAYS transcodes, even when the input
|
||||
// already is that format — a Teams export often is, and the transcode is then
|
||||
// nearly free; assuming it would be the bug. This replaces whisper-server's
|
||||
// --convert, whose --tmp-dir is created at process start and rots under macOS
|
||||
// periodic cleanup.
|
||||
type FFmpeg struct {
|
||||
// Bin is the ffmpeg executable ("ffmpeg" resolves via PATH).
|
||||
Bin string
|
||||
// MaxSeconds caps the converted length when positive. The duration limit is a claim
|
||||
// in the file's header and can be lied about (a 2 h file claiming 10 s converted to a
|
||||
// 120-minute WAV); this bounds what a lie can cost. A minute of slack is added so a
|
||||
// file at the limit is never cut, and the worker measures the result.
|
||||
MaxSeconds int
|
||||
}
|
||||
|
||||
// lengthSlackSeconds is added to MaxSeconds so ffmpeg's rounding never clips a file that
|
||||
// is really at the limit.
|
||||
const lengthSlackSeconds = 60
|
||||
|
||||
// ToWAV writes src, converted, to dst. Video streams are dropped (-vn): by now the
|
||||
// file has already been probed as audio-only, but a cover-art stream must not end
|
||||
// up in the WAV.
|
||||
func (f *FFmpeg) ToWAV(ctx context.Context, src, dst string) error {
|
||||
bin := f.Bin
|
||||
if bin == "" {
|
||||
bin = "ffmpeg"
|
||||
}
|
||||
// Input restrictions go BEFORE -i: placed after it they would apply to the output.
|
||||
args := append([]string{"-y", "-v", "error"}, inputRestrictions...)
|
||||
args = append(args, "-i", src, "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le")
|
||||
if f.MaxSeconds > 0 {
|
||||
// After -i: an OUTPUT cap. Before it, it would limit what is read, not written.
|
||||
args = append(args, "-t", strconv.Itoa(f.MaxSeconds+lengthSlackSeconds))
|
||||
}
|
||||
args = append(args, "-f", "wav", dst)
|
||||
cmd := exec.CommandContext(ctx, bin, args...)
|
||||
cmd.WaitDelay = killGrace
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return fmt.Errorf("audio: ffmpeg: %w", ctxErr)
|
||||
}
|
||||
return fmt.Errorf("audio: ffmpeg: %w: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
// A zero exit that wrote nothing is still a failure: never hand whisper a
|
||||
// missing file and let the error surface three layers later.
|
||||
if info, err := os.Stat(dst); err != nil || info.Size() == 0 {
|
||||
return fmt.Errorf("audio: ffmpeg wrote no output at %s", dst)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package audio_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||
)
|
||||
|
||||
func TestFFmpegToWAV_AsksForSixteenKilohertzMonoPCMAlways(t *testing.T) {
|
||||
// Always transcode, even when the input already is 16 kHz mono: a Teams export
|
||||
// often is, and the point is to not ASSUME (ADR-033). Records the argv.
|
||||
rec := filepath.Join(t.TempDir(), "argv")
|
||||
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||
for a; do last="$a"; done
|
||||
printf data > "$last"`)
|
||||
|
||||
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "out.wav"))
|
||||
|
||||
require.NoError(t, err)
|
||||
raw, err := os.ReadFile(rec)
|
||||
require.NoError(t, err)
|
||||
argv := strings.Split(strings.TrimSpace(string(raw)), "\n")
|
||||
joined := " " + strings.Join(argv, " ") + " "
|
||||
for _, want := range []string{" -ac 1 ", " -ar 16000 ", " -c:a pcm_s16le ", " -vn ", " -i /spool/in "} {
|
||||
assert.Contains(t, joined, want)
|
||||
}
|
||||
assert.True(t, strings.HasSuffix(strings.TrimSpace(joined), "out.wav"), "the output path is the last argument")
|
||||
}
|
||||
|
||||
func TestFFmpegToWAV_NonzeroExitIsAnErrorCarryingStderr(t *testing.T) {
|
||||
bin := stub(t, `echo "moov atom not found" >&2; exit 1`)
|
||||
|
||||
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/in", filepath.Join(t.TempDir(), "o.wav"))
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "moov atom not found")
|
||||
}
|
||||
|
||||
func TestFFmpegToWAV_ZeroExitWithNoOutputIsAnError(t *testing.T) {
|
||||
bin := stub(t, `exit 0`)
|
||||
|
||||
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/in", filepath.Join(t.TempDir(), "never-written.wav"))
|
||||
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestFFmpegToWAV_MissingBinaryIsAnError(t *testing.T) {
|
||||
err := (&audio.FFmpeg{Bin: filepath.Join(t.TempDir(), "no-ffmpeg")}).ToWAV(context.Background(), "/in", "/out.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.False(t, errors.Is(err, context.Canceled))
|
||||
}
|
||||
|
||||
func TestFFmpegToWAV_RestrictsWhatTheFileMayMakeFFmpegOpen(t *testing.T) {
|
||||
rec := filepath.Join(t.TempDir(), "argv")
|
||||
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||
for a; do last="$a"; done
|
||||
printf data > "$last"`)
|
||||
|
||||
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "o.wav"))
|
||||
|
||||
require.NoError(t, err)
|
||||
args := argvOf(t, rec)
|
||||
pw, fw, in := indexOf(args, "-protocol_whitelist"), indexOf(args, "-format_whitelist"), indexOf(args, "-i")
|
||||
require.GreaterOrEqual(t, pw, 0, "argv: %v", args)
|
||||
require.GreaterOrEqual(t, fw, 0, "argv: %v", args)
|
||||
assert.Equal(t, "file,pipe", args[pw+1])
|
||||
assert.Equal(t, audio.AllowedDemuxers, args[fw+1])
|
||||
assert.Less(t, pw, in, "input options must come BEFORE -i or ffmpeg applies them to the output")
|
||||
assert.Less(t, fw, in)
|
||||
}
|
||||
|
||||
func TestFFmpegToWAV_PassesAnOutputLengthCapAfterTheInputWhenAMaximumIsSet(t *testing.T) {
|
||||
// The duration limit is a CLAIM in the file's header; a file that lies about it can
|
||||
// convert to hours of audio (adversarial review F2: a 2 h file claiming 10 s became a
|
||||
// 120-minute WAV). The cap is an OUTPUT option, after -i, with a minute of slack.
|
||||
rec := filepath.Join(t.TempDir(), "argv")
|
||||
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||
for a; do last="$a"; done
|
||||
printf data > "$last"`)
|
||||
|
||||
err := (&audio.FFmpeg{Bin: bin, MaxSeconds: 5400}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "out.wav"))
|
||||
|
||||
require.NoError(t, err)
|
||||
raw, err := os.ReadFile(rec)
|
||||
require.NoError(t, err)
|
||||
joined := " " + strings.Join(strings.Split(strings.TrimSpace(string(raw)), "\n"), " ") + " "
|
||||
assert.Contains(t, joined, " -t 5460 ", "limit plus a minute of slack")
|
||||
assert.Less(t, strings.Index(joined, " -i /spool/in "), strings.Index(joined, " -t 5460 "), "-t after -i is an output cap; before it, an input one")
|
||||
}
|
||||
|
||||
func TestFFmpegToWAV_PassesNoLengthCapWhenNoMaximumIsSet(t *testing.T) {
|
||||
rec := filepath.Join(t.TempDir(), "argv")
|
||||
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||
for a; do last="$a"; done
|
||||
printf data > "$last"`)
|
||||
|
||||
require.NoError(t, (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "out.wav")))
|
||||
|
||||
raw, err := os.ReadFile(rec)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, " "+strings.ReplaceAll(string(raw), "\n", " ")+" ", " -t ")
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Package audio holds the adapters that inspect and convert user-supplied audio
|
||||
// with ffmpeg's tools (ADR-033). They sit behind small ports so the use case is
|
||||
// tested with fakes and only this package knows a subprocess is involved.
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// FFprobe implements uploads.Prober by running ffprobe on the spooled file.
|
||||
type FFprobe struct {
|
||||
// Bin is the ffprobe executable ("ffprobe" resolves via PATH).
|
||||
Bin string
|
||||
}
|
||||
|
||||
// AllowedDemuxers are the input formats an upload may be. The input is
|
||||
// attacker-supplied, and a file's CONTENT can tell ffmpeg to open other files or
|
||||
// URLs (HLS and concat playlists, image2 sequences, network demuxers). Recent
|
||||
// ffmpeg refuses the ones tried (7.0.2: HLS without a .m3u8 name is not sniffed,
|
||||
// concat rejects unsafe paths), but that is version luck; this list is not. It
|
||||
// holds the containers real recordings use and none of the playlist or network
|
||||
// formats. "mov" covers .mp4/.m4a/.3gp; "matroska" covers .webm.
|
||||
const AllowedDemuxers = "wav,mov,mp4,m4a,3gp,mp3,ogg,matroska,webm,flac,aac,opus,aiff,amr,asf"
|
||||
|
||||
// inputRestrictions are ffmpeg/ffprobe INPUT options and must precede the input.
|
||||
// Local files and pipes only: no network, no other protocols.
|
||||
var inputRestrictions = []string{"-protocol_whitelist", "file,pipe", "-format_whitelist", AllowedDemuxers}
|
||||
|
||||
// killGrace bounds how long Probe waits for a cancelled ffprobe's pipes to close.
|
||||
const killGrace = 2 * time.Second
|
||||
|
||||
// Probe runs ffprobe and classifies the outcome. A non-zero exit means ffprobe
|
||||
// looked at the file and could not read it: that is the FILE's fault and returns
|
||||
// uploads.ErrUnreadable. A missing binary, a cancelled context or unparsable
|
||||
// output is OURS and is returned as an ordinary error, so it is never reported to
|
||||
// the user as their file being bad.
|
||||
func (f *FFprobe) Probe(ctx context.Context, path string) (uploads.Probe, error) {
|
||||
bin := f.Bin
|
||||
if bin == "" {
|
||||
bin = "ffprobe"
|
||||
}
|
||||
args := append([]string{"-v", "error"}, inputRestrictions...)
|
||||
args = append(args, "-print_format", "json", "-show_format", "-show_streams", path)
|
||||
cmd := exec.CommandContext(ctx, bin, args...)
|
||||
cmd.WaitDelay = killGrace
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return uploads.Probe{}, fmt.Errorf("audio: ffprobe: %w", ctxErr)
|
||||
}
|
||||
var exit *exec.ExitError
|
||||
if errors.As(err, &exit) {
|
||||
return uploads.Probe{}, fmt.Errorf("%w: ffprobe: %s", uploads.ErrUnreadable, exit.Stderr)
|
||||
}
|
||||
return uploads.Probe{}, fmt.Errorf("audio: run ffprobe: %w", err)
|
||||
}
|
||||
p, err := ParseProbe(out)
|
||||
if err != nil {
|
||||
return uploads.Probe{}, fmt.Errorf("audio: ffprobe output: %w", err)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// probeJSON is the subset of `ffprobe -print_format json` output that matters.
|
||||
type probeJSON struct {
|
||||
Streams []struct {
|
||||
CodecType string `json:"codec_type"`
|
||||
Disposition struct {
|
||||
AttachedPic int `json:"attached_pic"`
|
||||
} `json:"disposition"`
|
||||
} `json:"streams"`
|
||||
Format struct {
|
||||
Duration string `json:"duration"`
|
||||
} `json:"format"`
|
||||
}
|
||||
|
||||
// ParseProbe turns ffprobe's JSON into a uploads.Probe. A video stream that is
|
||||
// only attached cover art (an mp3/m4a with an album picture) is not video. A
|
||||
// missing or "N/A" duration yields zero, which uploads.Validate rejects as
|
||||
// unreadable rather than accepting an empty recording.
|
||||
func ParseProbe(data []byte) (uploads.Probe, error) {
|
||||
var j probeJSON
|
||||
if err := json.Unmarshal(data, &j); err != nil {
|
||||
return uploads.Probe{}, fmt.Errorf("decode: %w", err)
|
||||
}
|
||||
var p uploads.Probe
|
||||
for _, s := range j.Streams {
|
||||
switch s.CodecType {
|
||||
case "audio":
|
||||
p.HasAudio = true
|
||||
case "video":
|
||||
if s.Disposition.AttachedPic == 0 {
|
||||
p.HasVideo = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if secs, err := strconv.ParseFloat(j.Format.Duration, 64); err == nil && secs > 0 {
|
||||
p.Duration = time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package audio_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
func TestParseProbe(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
json string
|
||||
want uploads.Probe
|
||||
}{
|
||||
{
|
||||
name: "audio-only m4a",
|
||||
json: `{"streams":[{"codec_type":"audio","disposition":{"attached_pic":0}}],"format":{"duration":"2701.5"}}`,
|
||||
want: uploads.Probe{Duration: 2701*time.Second + 500*time.Millisecond, HasAudio: true},
|
||||
},
|
||||
{
|
||||
name: "mp4 with a real video stream",
|
||||
json: `{"streams":[{"codec_type":"video","disposition":{"attached_pic":0}},{"codec_type":"audio"}],"format":{"duration":"240.0"}}`,
|
||||
want: uploads.Probe{Duration: 240 * time.Second, HasAudio: true, HasVideo: true},
|
||||
},
|
||||
{
|
||||
name: "mp3 whose only video stream is attached cover art is NOT video",
|
||||
json: `{"streams":[{"codec_type":"audio"},{"codec_type":"video","disposition":{"attached_pic":1}}],"format":{"duration":"60"}}`,
|
||||
want: uploads.Probe{Duration: 60 * time.Second, HasAudio: true},
|
||||
},
|
||||
{
|
||||
name: "no streams at all",
|
||||
json: `{"streams":[],"format":{"duration":"10"}}`,
|
||||
want: uploads.Probe{Duration: 10 * time.Second},
|
||||
},
|
||||
{
|
||||
name: "duration absent means zero, which Validate rejects as unreadable",
|
||||
json: `{"streams":[{"codec_type":"audio"}],"format":{}}`,
|
||||
want: uploads.Probe{HasAudio: true},
|
||||
},
|
||||
{
|
||||
name: "duration reported as N/A means zero",
|
||||
json: `{"streams":[{"codec_type":"audio"}],"format":{"duration":"N/A"}}`,
|
||||
want: uploads.Probe{HasAudio: true},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := audio.ParseProbe([]byte(tt.json))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProbe_GarbageIsAnError(t *testing.T) {
|
||||
_, err := audio.ParseProbe([]byte("not json"))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// stub writes an executable shell script standing in for ffprobe. This is a fake
|
||||
// at the process boundary: the real binary is not on every host (it is not on
|
||||
// koala), and what is under test is how Probe drives and interprets a process.
|
||||
func stub(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "ffprobe")
|
||||
require.NoError(t, os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755))
|
||||
return p
|
||||
}
|
||||
|
||||
func TestFFprobeProbe_ParsesWhatTheProcessPrints(t *testing.T) {
|
||||
bin := stub(t, `echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"90"}}'`)
|
||||
|
||||
got, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/ignored")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uploads.Probe{Duration: 90 * time.Second, HasAudio: true}, got)
|
||||
}
|
||||
|
||||
func TestFFprobeProbe_PassesTheFilePathAsTheLastArgument(t *testing.T) {
|
||||
out := filepath.Join(t.TempDir(), "args")
|
||||
bin := stub(t, `for a; do last="$a"; done; printf '%s' "$last" > `+out+`
|
||||
echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"1"}}'`)
|
||||
|
||||
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/spool/upload-123")
|
||||
|
||||
require.NoError(t, err)
|
||||
got, err := os.ReadFile(out)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/spool/upload-123", string(got))
|
||||
}
|
||||
|
||||
func TestFFprobeProbe_NonzeroExitMeansTheFileIsUnreadable(t *testing.T) {
|
||||
bin := stub(t, `echo "Invalid data found when processing input" >&2; exit 1`)
|
||||
|
||||
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/x")
|
||||
|
||||
assert.ErrorIs(t, err, uploads.ErrUnreadable)
|
||||
}
|
||||
|
||||
func TestFFprobeProbe_MissingBinaryIsOurFaultNotAnUnreadableFile(t *testing.T) {
|
||||
_, err := (&audio.FFprobe{Bin: filepath.Join(t.TempDir(), "no-such-ffprobe")}).Probe(context.Background(), "/x")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.False(t, errors.Is(err, uploads.ErrUnreadable),
|
||||
"a missing ffprobe must not be reported to the user as their file being bad")
|
||||
}
|
||||
|
||||
func TestFFprobeProbe_StopsWhenTheContextIsCancelled(t *testing.T) {
|
||||
bin := stub(t, `sleep 30`)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
_, err := (&audio.FFprobe{Bin: bin}).Probe(ctx, "/x")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
assert.False(t, errors.Is(err, uploads.ErrUnreadable), "a timeout is not a verdict on the file")
|
||||
}
|
||||
|
||||
// argvStub records the arguments it was called with into a file and then behaves as
|
||||
// a successful ffprobe, so a test can assert exactly what the process was asked.
|
||||
func argvStub(t *testing.T) (bin, argvFile string) {
|
||||
t.Helper()
|
||||
argvFile = filepath.Join(t.TempDir(), "argv")
|
||||
bin = stub(t, `printf '%s\n' "$@" > `+argvFile+`
|
||||
echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"1"}}'`)
|
||||
return bin, argvFile
|
||||
}
|
||||
|
||||
func argvOf(t *testing.T, file string) []string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(file)
|
||||
require.NoError(t, err)
|
||||
return strings.Split(strings.TrimSpace(string(raw)), "\n")
|
||||
}
|
||||
|
||||
func indexOf(args []string, want string) int {
|
||||
for i, a := range args {
|
||||
if a == want {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func TestFFprobeProbe_RestrictsWhatTheFileMayMakeFFmpegOpen(t *testing.T) {
|
||||
// The input is attacker-supplied. ffmpeg can be told by a file's CONTENT to open
|
||||
// other files or URLs (HLS and concat playlists); recent versions refuse by
|
||||
// default, but that is version luck. Pin it: local files and pipes only, and
|
||||
// only the demuxers real audio recordings use.
|
||||
bin, argvFile := argvStub(t)
|
||||
|
||||
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/spool/upload-1")
|
||||
|
||||
require.NoError(t, err)
|
||||
args := argvOf(t, argvFile)
|
||||
pw := indexOf(args, "-protocol_whitelist")
|
||||
require.GreaterOrEqual(t, pw, 0, "argv: %v", args)
|
||||
assert.Equal(t, "file,pipe", args[pw+1])
|
||||
fw := indexOf(args, "-format_whitelist")
|
||||
require.GreaterOrEqual(t, fw, 0, "argv: %v", args)
|
||||
assert.Equal(t, audio.AllowedDemuxers, args[fw+1])
|
||||
assert.Equal(t, "/spool/upload-1", args[len(args)-1], "the path stays the last argument")
|
||||
}
|
||||
|
||||
func TestAllowedDemuxers_ExcludeThePlaylistFormats(t *testing.T) {
|
||||
for _, bad := range []string{"concat", "hls", "applehttp", "dash", "rtsp", "sdp", "image2"} {
|
||||
assert.NotContains(t, strings.Split(audio.AllowedDemuxers, ","), bad,
|
||||
"%s can make ffmpeg open other files or URLs named inside the upload", bad)
|
||||
}
|
||||
for _, good := range []string{"wav", "mov", "mp4", "m4a", "mp3", "ogg", "matroska", "flac"} {
|
||||
assert.Contains(t, strings.Split(audio.AllowedDemuxers, ","), good)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package audio_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// TestFFprobeLive_HostilePlaylistsAreRefusedAndRealAudioIsNot runs the REAL ffprobe
|
||||
// (opt-in: TAPIR_LIVE_FFPROBE) on crafted concat/HLS files and on a real m4a.
|
||||
func TestFFprobeLive_HostilePlaylistsAreRefusedAndRealAudioIsNot(t *testing.T) {
|
||||
bin := os.Getenv("TAPIR_LIVE_FFPROBE")
|
||||
if bin == "" {
|
||||
t.Skip("set TAPIR_LIVE_FFPROBE (and TAPIR_LIVE_HOSTILE_DIR, TAPIR_LIVE_M4A) to run against the real ffprobe")
|
||||
}
|
||||
p := &audio.FFprobe{Bin: bin}
|
||||
for _, name := range []string{"up-concat", "up-hls"} {
|
||||
_, err := p.Probe(context.Background(), os.Getenv("TAPIR_LIVE_HOSTILE_DIR")+"/"+name)
|
||||
assert.ErrorIs(t, err, uploads.ErrUnreadable, name)
|
||||
}
|
||||
got, err := p.Probe(context.Background(), os.Getenv("TAPIR_LIVE_M4A"))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, got.HasAudio)
|
||||
assert.False(t, got.HasVideo)
|
||||
assert.InDelta(t, 82.5, got.Duration.Seconds(), 1)
|
||||
}
|
||||
@@ -41,6 +41,11 @@ type Request struct {
|
||||
Transcript string
|
||||
History []Turn
|
||||
Question string
|
||||
// MaxChars, when positive, overrides the Service's transcript budget for this
|
||||
// request. The Service default is sized for YouTube captions (~22 minutes of
|
||||
// speech); an uploaded recording is much longer, and truncation keeps only the
|
||||
// head (ADR-033). Zero or negative means "use the Service default".
|
||||
MaxChars int
|
||||
}
|
||||
|
||||
// Reply is the model's answer plus whether the transcript was bounded to fit the
|
||||
@@ -113,7 +118,11 @@ func (s *Service) Answer(ctx context.Context, req Request) (Reply, error) {
|
||||
model = s.DefaultModel("")
|
||||
}
|
||||
|
||||
transcript, truncated := truncate(req.Transcript, s.maxChars)
|
||||
budget := s.maxChars
|
||||
if req.MaxChars > 0 {
|
||||
budget = req.MaxChars
|
||||
}
|
||||
transcript, truncated := truncate(req.Transcript, budget)
|
||||
system := buildSystem(transcript, truncated)
|
||||
user := buildUser(req.History, req.Question)
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A Request may carry its own transcript budget. The Service's cap (18,000 chars by
|
||||
// default) is sized for YouTube captions; an uploaded recording is much longer
|
||||
// (~50,000 chars an hour), and keeping only the head of it would answer questions
|
||||
// about the last two-thirds of a meeting from text that does not contain them
|
||||
// (ADR-033). The per-request value overrides; zero means "the service default".
|
||||
|
||||
func TestAnswerMaxCharsOverridesTheServiceCap(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
serviceCap int
|
||||
requestCap int
|
||||
transcriptLen int
|
||||
wantKept int
|
||||
wantTruncated bool
|
||||
}{
|
||||
{"request cap above the service cap keeps the whole transcript", 10, 100, 50, 50, false},
|
||||
{"request cap below the service cap bounds tighter", 100, 10, 50, 10, true},
|
||||
{"zero request cap falls back to the service cap", 20, 0, 50, 20, true},
|
||||
{"negative request cap is treated as unset", 20, -1, 50, 20, true},
|
||||
{"request cap exceeded is reported as truncated", 10, 30, 50, 30, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := &factory{answer: "answer"}
|
||||
s := New(f.make, []string{"koala/phi4-mini"}, tt.serviceCap)
|
||||
|
||||
reply, err := s.Answer(context.Background(), Request{
|
||||
Model: "koala/phi4-mini",
|
||||
Transcript: strings.Repeat("x", tt.transcriptLen),
|
||||
Question: "what was decided?",
|
||||
MaxChars: tt.requestCap,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Answer: %v", err)
|
||||
}
|
||||
if got := strings.Count(f.used.lastSystem, "x"); got != tt.wantKept {
|
||||
t.Errorf("transcript chars sent = %d, want %d", got, tt.wantKept)
|
||||
}
|
||||
if reply.Truncated != tt.wantTruncated {
|
||||
t.Errorf("Truncated = %v, want %v", reply.Truncated, tt.wantTruncated)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -25,12 +25,16 @@ import (
|
||||
// docs/homelab-integration.md.
|
||||
const defaultMaxTokens = 8192
|
||||
|
||||
// defaultTemperature is what every caller got before WithTemperature existed.
|
||||
const defaultTemperature = 0.2
|
||||
|
||||
// Client calls an OpenAI-compatible chat completions endpoint.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
apiKey string
|
||||
model string
|
||||
maxTokens int
|
||||
temp float64
|
||||
httpClient *http.Client
|
||||
usageHook func(model string, prompt, completion int)
|
||||
}
|
||||
@@ -51,6 +55,35 @@ func WithMaxTokens(n int) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// maxErrorBody bounds how much of a failing response ends up in an error message.
|
||||
const maxErrorBody = 512
|
||||
|
||||
// HTTPError is a non-200 answer from the gateway. Its text keeps the form callers that
|
||||
// log it already rely on, with the body bounded; its Code is what code may branch on and
|
||||
// what errkind turns into a kind. The body is provider text and can echo the prompt, so
|
||||
// nothing private may persist or display Error() (ADR-035 review F3).
|
||||
type HTTPError struct {
|
||||
Code int
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e *HTTPError) Error() string {
|
||||
body := e.Body
|
||||
if len(body) > maxErrorBody {
|
||||
body = body[:maxErrorBody] + "…"
|
||||
}
|
||||
return fmt.Sprintf("LLM returned %d: %s", e.Code, body)
|
||||
}
|
||||
|
||||
// HTTPStatus implements errkind.HTTPStatuser.
|
||||
func (e *HTTPError) HTTPStatus() int { return e.Code }
|
||||
|
||||
// WithTemperature overrides the sampling temperature. Zero IS a valid value and is sent
|
||||
// as zero: the reconciler's accuracy was measured at temperature 0 and must run there.
|
||||
func WithTemperature(t float64) Option {
|
||||
return func(c *Client) { c.temp = t }
|
||||
}
|
||||
|
||||
// WithUsageHook registers a callback fired after a successful completion with the
|
||||
// model and the prompt/completion token counts from the response usage block. It
|
||||
// keeps this copied, stdlib-only package (ADR-004) decoupled from metrics: the
|
||||
@@ -66,6 +99,7 @@ func New(baseURL, apiKey, model string, timeout time.Duration, opts ...Option) *
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
maxTokens: defaultMaxTokens,
|
||||
temp: defaultTemperature,
|
||||
httpClient: &http.Client{Timeout: timeout},
|
||||
}
|
||||
for _, opt := range opts {
|
||||
@@ -105,7 +139,7 @@ func (c *Client) Complete(ctx context.Context, system, user string) (string, err
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
},
|
||||
Temperature: 0.2,
|
||||
Temperature: c.temp,
|
||||
MaxTokens: c.maxTokens,
|
||||
}
|
||||
b, err := json.Marshal(body)
|
||||
@@ -155,7 +189,7 @@ func (c *Client) Complete(ctx context.Context, system, user string) (string, err
|
||||
return "", fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("LLM returned %d: %s", resp.StatusCode, out)
|
||||
return "", &HTTPError{Code: resp.StatusCode, Body: string(out)}
|
||||
}
|
||||
|
||||
var cr chatResponse
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
)
|
||||
|
||||
// A non-200 from the gateway is an HTTP status error: its kind can be logged and stored
|
||||
// without its body, which can echo the prompt (adversarial review F3).
|
||||
|
||||
func failingGateway(t *testing.T, status int, body string) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestClient_ANon200IsAnHTTPStatusErrorWhoseKindCarriesNoBody(t *testing.T) {
|
||||
srv := failingGateway(t, http.StatusBadRequest, `{"error":"prompt was: Anders ansvarar för virket"}`)
|
||||
|
||||
_, err := New(srv.URL, "", "m", 5*time.Second).Complete(context.Background(), "s", "u")
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
if got := errkind.Of(err); got != "http_400" {
|
||||
t.Errorf("errkind.Of = %q, want http_400", got)
|
||||
}
|
||||
var he *HTTPError
|
||||
if !errors.As(err, &he) || he.Code != http.StatusBadRequest {
|
||||
t.Errorf("error is not an *HTTPError with the status code: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_TheErrorTextIsUnchangedForCallersThatLogIt(t *testing.T) {
|
||||
srv := failingGateway(t, http.StatusBadGateway, "upstream down")
|
||||
|
||||
_, err := New(srv.URL, "", "m", 5*time.Second).Complete(context.Background(), "s", "u")
|
||||
|
||||
if err == nil || err.Error() != "LLM returned 502: upstream down" {
|
||||
t.Errorf("error text = %v: callers that log it already rely on this form", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_AnEnormousErrorBodyIsBoundedInTheErrorText(t *testing.T) {
|
||||
srv := failingGateway(t, http.StatusInternalServerError, strings.Repeat("x", 100_000))
|
||||
|
||||
_, err := New(srv.URL, "", "m", 5*time.Second).Complete(context.Background(), "s", "u")
|
||||
|
||||
if err == nil || len(err.Error()) > 1024 {
|
||||
t.Errorf("error text is %d bytes: a gateway's whole body must not become an error message", len(err.Error()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// capturedTemperature runs one Complete against a fake gateway and returns the
|
||||
// "temperature" field the request carried, and whether it was present at all.
|
||||
func capturedTemperature(t *testing.T, opts ...Option) (float64, bool) {
|
||||
t.Helper()
|
||||
var body map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(raw, &body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []map[string]any{{"message": map[string]any{"role": "assistant", "content": "ok"}}}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := New(srv.URL, "", "m", 5*time.Second, opts...).Complete(context.Background(), "s", "u"); err != nil {
|
||||
t.Fatalf("Complete: %v", err)
|
||||
}
|
||||
v, ok := body["temperature"].(float64)
|
||||
return v, ok
|
||||
}
|
||||
|
||||
func TestClient_DefaultTemperatureIsUnchanged(t *testing.T) {
|
||||
got, ok := capturedTemperature(t)
|
||||
|
||||
if !ok || got != 0.2 {
|
||||
t.Fatalf("default temperature = %v (present %v), want 0.2: existing callers must not change behaviour", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_WithTemperatureZeroSendsExactlyZero(t *testing.T) {
|
||||
// The reconciler's accuracy (92%/97%) was measured at temperature 0. Zero must be
|
||||
// SENT, not treated as "unset" and replaced by the default.
|
||||
got, ok := capturedTemperature(t, WithTemperature(0))
|
||||
|
||||
if !ok || got != 0 {
|
||||
t.Fatalf("temperature = %v (present %v), want an explicit 0", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_WithTemperatureSendsTheGivenValue(t *testing.T) {
|
||||
got, ok := capturedTemperature(t, WithTemperature(0.7))
|
||||
|
||||
if !ok || got != 0.7 {
|
||||
t.Fatalf("temperature = %v (present %v), want 0.7", got, ok)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Drops the private uploaded-transcript store and every transcript in it. There
|
||||
-- is no revert of the data: the original audio was never kept (ADR-033).
|
||||
DROP TABLE uploaded_transcripts;
|
||||
@@ -0,0 +1,48 @@
|
||||
-- Migration 017: uploaded_transcripts — PRIVATE transcripts of audio a user
|
||||
-- uploaded themselves (ADR-033, tapir#33).
|
||||
--
|
||||
-- This is deliberately NOT the shared `transcripts` table (migration 015,
|
||||
-- ADR-021). That table is non-RLS on the explicit grounds that it holds only
|
||||
-- public caption content and nothing user-identifying. A recorded meeting is the
|
||||
-- opposite: private third-party content owned by exactly one user. Reusing
|
||||
-- `transcripts` — or adding a user_id to it with a partial policy — would put a
|
||||
-- private row in the one table the isolation test asserts is shared, and would
|
||||
-- pass CI green. A separate table keeps ADR-021's exception singular.
|
||||
--
|
||||
-- Row-level security follows migration 003 exactly: ENABLE + FORCE (the app
|
||||
-- connects as the table owner, which bypasses RLS unless forced), and a policy
|
||||
-- keyed off the per-request GUC tapir.current_user_id. With no GUC set the
|
||||
-- predicate is NULL and the table is deny-all.
|
||||
--
|
||||
-- The original audio is never stored (ADR-033): this row is all that is kept.
|
||||
-- segments holds the timestamped cues; re-transcribing 90 minutes costs ~7
|
||||
-- minutes, so discarding the timestamps would be irreversible.
|
||||
CREATE TABLE uploaded_transcripts (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued', 'transcribing', 'done', 'failed')),
|
||||
content TEXT,
|
||||
segments JSONB,
|
||||
endpoint TEXT, -- which backend served it (iguana | berget); a quality fact
|
||||
audio_seconds DOUBLE PRECISION,
|
||||
error_kind TEXT, -- typed, never a provider's error string (ADR-033)
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
finished_at TIMESTAMPTZ,
|
||||
CONSTRAINT uploaded_transcripts_done_has_content
|
||||
CHECK (status <> 'done' OR content IS NOT NULL),
|
||||
CONSTRAINT uploaded_transcripts_failed_has_error_kind
|
||||
CHECK (status <> 'failed' OR error_kind IS NOT NULL)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_uploaded_transcripts_user ON uploaded_transcripts (user_id, created_at DESC);
|
||||
|
||||
ALTER TABLE uploaded_transcripts ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE uploaded_transcripts FORCE ROW LEVEL SECURITY;
|
||||
CREATE POLICY uploaded_transcripts_isolation ON uploaded_transcripts
|
||||
FOR ALL
|
||||
USING (user_id = current_setting('tapir.current_user_id', true)::uuid);
|
||||
|
||||
COMMENT ON TABLE uploaded_transcripts IS
|
||||
'PRIVATE per-user transcripts of user-supplied audio. RLS-scoped (ADR-012). Never merge into the shared transcripts table (ADR-021, ADR-033).';
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE uploaded_transcripts DROP COLUMN had_video;
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration 018: record whether an uploaded file carried a video track (ADR-033,
|
||||
-- amended 2026-10-01).
|
||||
--
|
||||
-- A Teams meeting with the camera off is still an .mp4 with a video track, so video
|
||||
-- files are ACCEPTED: only the audio is extracted and the video is discarded. This
|
||||
-- flag lets the transcript page say so — "this file contained a video track; only its
|
||||
-- audio was used" — so the user is never left to wonder what happened to the picture.
|
||||
-- It outlives the job: completing or failing an upload does not clear it.
|
||||
ALTER TABLE uploaded_transcripts ADD COLUMN had_video BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE uploaded_transcripts DROP COLUMN reconciliation;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Migration 019: how a merged transcript came about (ADR-034).
|
||||
--
|
||||
-- When two engines transcribed the same audio and a local model chose between their
|
||||
-- disagreements, this records the status, the exact counts, and a bounded list of the
|
||||
-- contested spots (changed or unsure, each with a position and both readings) so the
|
||||
-- page can say what was done and the user can check the uncertain spots against the
|
||||
-- audio. NULL means a single engine: no second opinion, and the page must not claim one.
|
||||
--
|
||||
-- It lives on the already-FORCE'd, RLS-scoped uploaded_transcripts, so the exact-set
|
||||
-- isolation test is unaffected. The contested spots hold fragments of the private
|
||||
-- transcript, which is why they are here and nowhere else.
|
||||
ALTER TABLE uploaded_transcripts ADD COLUMN reconciliation JSONB;
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE uploaded_transcripts
|
||||
DROP COLUMN alive_at,
|
||||
DROP COLUMN progress_at,
|
||||
DROP COLUMN progress;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Migration 020: where an upload job is, and whether the service is still reporting (ADR-035).
|
||||
--
|
||||
-- progress the job's ordered phases as JSON (key, label, state, times, done/total).
|
||||
-- progress_at when a phase last REALLY changed or advanced.
|
||||
-- alive_at a heartbeat stamped every few seconds by a goroutine independent of any
|
||||
-- engine call. The two differ on purpose: alive says the service is still
|
||||
-- reporting, progress says work is moving, and a page that cannot tell a
|
||||
-- crashed service from a slow model is the failure this exists to end.
|
||||
--
|
||||
-- All three are NULL until the job starts. They sit on the already-FORCE'd, RLS-scoped
|
||||
-- uploaded_transcripts, so the exact-set isolation test is unaffected.
|
||||
ALTER TABLE uploaded_transcripts
|
||||
ADD COLUMN progress JSONB,
|
||||
ADD COLUMN progress_at TIMESTAMPTZ,
|
||||
ADD COLUMN alive_at TIMESTAMPTZ;
|
||||
@@ -0,0 +1,249 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// ErrUploadNotFound is returned when an uploaded transcript does not exist OR is
|
||||
// not the caller's. The two are deliberately indistinguishable: a non-owner learns
|
||||
// nothing about whether someone else's upload exists (ADR-033).
|
||||
var ErrUploadNotFound = errors.New("store: uploaded transcript not found")
|
||||
|
||||
// Uploaded transcripts are PRIVATE per-user content, stored in their own
|
||||
// RLS-scoped table (migration 017) — never in the shared `transcripts` (ADR-021).
|
||||
// Every method goes through withUser, so row-level security confines it to the
|
||||
// caller's rows; each statement ALSO names user_id in its predicate, so the
|
||||
// isolation does not rest on a single mechanism and stays testable under the
|
||||
// superuser connection the unit tests use.
|
||||
|
||||
// CreateUpload records a new upload in the queued state and returns its id.
|
||||
// hadVideo records that the file carried a video track (only its audio is used);
|
||||
// it is kept for the life of the row so the page can say so.
|
||||
func (s *Store) CreateUpload(ctx context.Context, userID, title string, hadVideo bool) (string, error) {
|
||||
var id string
|
||||
if err := s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||
return tx.QueryRow(ctx,
|
||||
`INSERT INTO uploaded_transcripts (user_id, title, had_video) VALUES ($1, $2, $3) RETURNING id`,
|
||||
userID, title, hadVideo).Scan(&id)
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("store: create upload: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// StartTranscribing moves an upload to transcribing.
|
||||
func (s *Store) StartTranscribing(ctx context.Context, userID, id string) error {
|
||||
return s.updateUpload(ctx, userID, id, "start transcribing",
|
||||
`UPDATE uploaded_transcripts SET status = 'transcribing'
|
||||
WHERE id = $1 AND user_id = $2`)
|
||||
}
|
||||
|
||||
// CompleteUpload stores the finished transcription and moves the upload to done.
|
||||
func (s *Store) CompleteUpload(ctx context.Context, userID, id string, t domain.Transcription) error {
|
||||
segs, err := json.Marshal(t.Segments)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store: complete upload: marshal segments: %w", err)
|
||||
}
|
||||
// A nil Reconciliation must be SQL NULL, not the JSON text "null": the page decides
|
||||
// whether to claim a second opinion by whether this column is NULL.
|
||||
var rec []byte
|
||||
if t.Reconciliation != nil {
|
||||
if rec, err = json.Marshal(t.Reconciliation); err != nil {
|
||||
return fmt.Errorf("store: complete upload: marshal reconciliation: %w", err)
|
||||
}
|
||||
}
|
||||
return s.updateUpload(ctx, userID, id, "complete upload",
|
||||
`UPDATE uploaded_transcripts
|
||||
SET status = 'done', content = $3, segments = $4, endpoint = $5,
|
||||
audio_seconds = $6, reconciliation = $7, error_kind = NULL, finished_at = NOW()
|
||||
WHERE id = $1 AND user_id = $2`,
|
||||
t.Text, segs, t.Endpoint, t.AudioSeconds, rec)
|
||||
}
|
||||
|
||||
// SaveProgress records the job's phases and when something last really changed
|
||||
// (ADR-035). It replaces the previous snapshot: the Tracker holds the whole plan.
|
||||
func (s *Store) SaveProgress(ctx context.Context, userID, id string, steps []domain.ProgressStep, at time.Time) error {
|
||||
b, err := json.Marshal(steps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store: save progress: marshal: %w", err)
|
||||
}
|
||||
return s.updateUpload(ctx, userID, id, "save progress",
|
||||
`UPDATE uploaded_transcripts SET progress = $3, progress_at = $4
|
||||
WHERE id = $1 AND user_id = $2`, b, at)
|
||||
}
|
||||
|
||||
// Heartbeat stamps the upload as alive now. It does not touch progress_at: alive says
|
||||
// the service is still reporting, not that work is moving (ADR-035).
|
||||
func (s *Store) Heartbeat(ctx context.Context, userID, id string) error {
|
||||
return s.updateUpload(ctx, userID, id, "heartbeat",
|
||||
`UPDATE uploaded_transcripts SET alive_at = NOW()
|
||||
WHERE id = $1 AND user_id = $2`)
|
||||
}
|
||||
|
||||
// FailUpload moves an upload to failed with a typed reason.
|
||||
func (s *Store) FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error {
|
||||
return s.updateUpload(ctx, userID, id, "fail upload",
|
||||
`UPDATE uploaded_transcripts
|
||||
SET status = 'failed', error_kind = $3, finished_at = NOW()
|
||||
WHERE id = $1 AND user_id = $2`,
|
||||
string(kind))
|
||||
}
|
||||
|
||||
// updateUpload runs one UPDATE keyed on ($1 = id, $2 = userID) plus extra args,
|
||||
// and maps "no row touched" to ErrUploadNotFound — which is what a write to another
|
||||
// user's upload looks like under RLS (the row is invisible, not forbidden).
|
||||
func (s *Store) updateUpload(ctx context.Context, userID, id, op, sql string, args ...any) error {
|
||||
all := append([]any{id, userID}, args...)
|
||||
return s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||
ct, err := tx.Exec(ctx, sql, all...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store: %s: %w", op, err)
|
||||
}
|
||||
if ct.RowsAffected() == 0 {
|
||||
return ErrUploadNotFound
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// GetUpload returns one of the caller's uploads, including the transcript text.
|
||||
func (s *Store) GetUpload(ctx context.Context, userID, id string) (*domain.UploadedTranscript, error) {
|
||||
var u domain.UploadedTranscript
|
||||
err := s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||
row := tx.QueryRow(ctx, uploadSelect+` WHERE id = $1 AND user_id = $2`, id, userID)
|
||||
return scanUpload(row, &u, true)
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrUploadNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store: get upload: %w", err)
|
||||
}
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// ListUploads returns the caller's uploads, newest first, WITHOUT transcript text
|
||||
// or segments: a list view must not drag every transcript along.
|
||||
func (s *Store) ListUploads(ctx context.Context, userID string) ([]domain.UploadedTranscript, error) {
|
||||
var out []domain.UploadedTranscript
|
||||
err := s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||
rows, err := tx.Query(ctx, uploadSelect+` WHERE user_id = $1 ORDER BY created_at DESC, id`, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var u domain.UploadedTranscript
|
||||
if err := scanUpload(rows, &u, false); err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, u)
|
||||
}
|
||||
return rows.Err()
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store: list uploads: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FailInterruptedUploads is the start-up sweep. The audio of an upload is never
|
||||
// retained (ADR-033), so a job that was queued or transcribing when the process
|
||||
// stopped can never resume; leaving it would be a stuck spinner. Every such row,
|
||||
// for every user, becomes failed/interrupted and the user re-uploads. It
|
||||
// enumerates users from the un-RLS'd identity map and scopes each pass with
|
||||
// withUser, exactly as scheduled discovery does (ADR-018). Returns rows failed.
|
||||
func (s *Store) FailInterruptedUploads(ctx context.Context) (int, error) {
|
||||
users, err := s.ListAllUsers(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("store: fail interrupted uploads: %w", err)
|
||||
}
|
||||
total := 0
|
||||
for _, u := range users {
|
||||
err := s.withUser(ctx, u.UserID, func(tx pgx.Tx) error {
|
||||
ct, err := tx.Exec(ctx,
|
||||
`UPDATE uploaded_transcripts
|
||||
SET status = 'failed', error_kind = $2, finished_at = NOW()
|
||||
WHERE user_id = $1 AND status IN ('queued', 'transcribing')`,
|
||||
u.UserID, string(domain.UploadErrInterrupted))
|
||||
total += int(ct.RowsAffected())
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("store: fail interrupted uploads: user %s: %w", u.UserID, err)
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
const uploadSelect = `SELECT id, user_id, title, status, content, segments, endpoint,
|
||||
audio_seconds, error_kind, had_video, reconciliation, progress, progress_at, alive_at,
|
||||
created_at, finished_at FROM uploaded_transcripts`
|
||||
|
||||
// scanUpload scans one uploadSelect row. withText controls whether the transcript
|
||||
// body and segments are kept (GetUpload) or dropped (ListUploads).
|
||||
func scanUpload(row pgx.Row, u *domain.UploadedTranscript, withText bool) error {
|
||||
var (
|
||||
status, errKind, endpoint, content *string
|
||||
segments, reconciliation, progress []byte
|
||||
seconds *float64
|
||||
finished, progressAt, aliveAt *time.Time
|
||||
)
|
||||
if err := row.Scan(&u.ID, &u.UserID, &u.Title, &status, &content, &segments, &endpoint,
|
||||
&seconds, &errKind, &u.HadVideo, &reconciliation, &progress, &progressAt, &aliveAt,
|
||||
&u.CreatedAt, &finished); err != nil {
|
||||
return err
|
||||
}
|
||||
u.Status = domain.UploadStatus(deref(status))
|
||||
u.Endpoint = deref(endpoint)
|
||||
u.ErrorKind = domain.UploadErrorKind(deref(errKind))
|
||||
if seconds != nil {
|
||||
u.AudioSeconds = *seconds
|
||||
}
|
||||
if finished != nil {
|
||||
u.FinishedAt = *finished
|
||||
}
|
||||
if !withText {
|
||||
return nil
|
||||
}
|
||||
if len(progress) > 0 {
|
||||
if err := json.Unmarshal(progress, &u.Progress); err != nil {
|
||||
return fmt.Errorf("unmarshal progress: %w", err)
|
||||
}
|
||||
}
|
||||
if progressAt != nil {
|
||||
u.ProgressAt = *progressAt
|
||||
}
|
||||
if aliveAt != nil {
|
||||
u.AliveAt = *aliveAt
|
||||
}
|
||||
if len(reconciliation) > 0 {
|
||||
var r domain.Reconciliation
|
||||
if err := json.Unmarshal(reconciliation, &r); err != nil {
|
||||
return fmt.Errorf("unmarshal reconciliation: %w", err)
|
||||
}
|
||||
u.Reconciliation = &r
|
||||
}
|
||||
u.Content = deref(content)
|
||||
if len(segments) > 0 {
|
||||
if err := json.Unmarshal(segments, &u.Segments); err != nil {
|
||||
return fmt.Errorf("unmarshal segments: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deref(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return *s
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Isolation proof for ADR-033: an uploaded transcript is PRIVATE content owned by
|
||||
// one user, so it lives in its own RLS-scoped table — never in the shared,
|
||||
// non-RLS `transcripts` (ADR-021). These tests drive the database directly (raw
|
||||
// SQL as the non-superuser `app` role) so they observe row-level security itself,
|
||||
// not any application-layer WHERE clause. The superuser pool is used only to seed
|
||||
// and to read pg_catalog; it bypasses RLS and so must never be the pool whose
|
||||
// answers the assertions rest on.
|
||||
|
||||
// seedUploadedTranscript inserts one finished upload for userID as the superuser
|
||||
// (RLS bypassed so both users' rows land regardless of any GUC) and returns its id.
|
||||
// The text is synthetic — this repo is public, and a real transcript must never
|
||||
// appear in it (#33).
|
||||
func seedUploadedTranscript(t *testing.T, super *pgxpool.Pool, userID, text string) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
_, err := super.Exec(ctx, `INSERT INTO users (id) VALUES ($1) ON CONFLICT DO NOTHING`, userID)
|
||||
require.NoError(t, err)
|
||||
var id string
|
||||
require.NoError(t, super.QueryRow(ctx,
|
||||
`INSERT INTO uploaded_transcripts (user_id, title, status, content)
|
||||
VALUES ($1, 'synthetic fixture', 'done', $2) RETURNING id`,
|
||||
userID, text).Scan(&id))
|
||||
return id
|
||||
}
|
||||
|
||||
func TestUploadedTranscriptsRLS_ScopedReadsSeeOnlyOwnRows(t *testing.T) {
|
||||
newStore(t)
|
||||
super := rawPool(t)
|
||||
resetDB(t, super)
|
||||
seedUploadedTranscript(t, super, userA, "text owned by A")
|
||||
seedUploadedTranscript(t, super, userB, "text owned by B")
|
||||
app := appPool(t, super)
|
||||
|
||||
require.Equal(t, 0, scopedCount(t, app, "", "uploaded_transcripts"),
|
||||
"no tapir.current_user_id must be deny-all: this proves RLS is live, not bypassed")
|
||||
require.Equal(t, 1, scopedCount(t, app, userA, "uploaded_transcripts"),
|
||||
"user A must see exactly its own upload")
|
||||
require.Equal(t, 1, scopedCount(t, app, userB, "uploaded_transcripts"),
|
||||
"user B must see exactly its own upload")
|
||||
|
||||
// Count alone cannot distinguish "A's row" from "B's row" — read the content.
|
||||
var got string
|
||||
ctx := context.Background()
|
||||
tx, err := app.Begin(ctx)
|
||||
require.NoError(t, err)
|
||||
defer tx.Rollback(ctx) //nolint:errcheck
|
||||
_, err = tx.Exec(ctx, `SELECT set_config('tapir.current_user_id', $1, true)`, userA)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tx.QueryRow(ctx, `SELECT content FROM uploaded_transcripts`).Scan(&got))
|
||||
require.Equal(t, "text owned by A", got, "A's scoped read must return A's text, never B's")
|
||||
}
|
||||
|
||||
func TestUploadedTranscriptsRLS_CrossUserWritesAffectZeroRows(t *testing.T) {
|
||||
newStore(t)
|
||||
super := rawPool(t)
|
||||
resetDB(t, super)
|
||||
seedUploadedTranscript(t, super, userA, "text owned by A")
|
||||
bID := seedUploadedTranscript(t, super, userB, "text owned by B")
|
||||
app := appPool(t, super)
|
||||
|
||||
require.Equal(t, int64(0), scopedRowsAffected(t, app, userA,
|
||||
`UPDATE uploaded_transcripts SET content = 'hacked' WHERE id = $1`, bID),
|
||||
"user A scoped UPDATE must touch zero of B's uploads")
|
||||
require.Equal(t, int64(0), scopedRowsAffected(t, app, userA,
|
||||
`DELETE FROM uploaded_transcripts WHERE id = $1`, bID),
|
||||
"user A scoped DELETE must touch zero of B's uploads")
|
||||
|
||||
var content string
|
||||
require.NoError(t, super.QueryRow(context.Background(),
|
||||
`SELECT content FROM uploaded_transcripts WHERE id = $1`, bID).Scan(&content))
|
||||
require.Equal(t, "text owned by B", content, "B's upload must survive A's writes unchanged")
|
||||
}
|
||||
|
||||
func TestUploadedTranscriptsRLS_InsertForAnotherUserIsRejected(t *testing.T) {
|
||||
newStore(t)
|
||||
super := rawPool(t)
|
||||
resetDB(t, super)
|
||||
seedUploadedTranscript(t, super, userA, "text owned by A")
|
||||
_, err := super.Exec(context.Background(), `INSERT INTO users (id) VALUES ($1)`, userB)
|
||||
require.NoError(t, err)
|
||||
app := appPool(t, super)
|
||||
|
||||
ctx := context.Background()
|
||||
tx, err := app.Begin(ctx)
|
||||
require.NoError(t, err)
|
||||
defer tx.Rollback(ctx) //nolint:errcheck
|
||||
_, err = tx.Exec(ctx, `SELECT set_config('tapir.current_user_id', $1, true)`, userA)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Scoped as A, plant a row that claims to belong to B. The policy's WITH CHECK
|
||||
// (defaulting to USING under FOR ALL) must refuse it.
|
||||
_, err = tx.Exec(ctx,
|
||||
`INSERT INTO uploaded_transcripts (user_id, title, status, content)
|
||||
VALUES ($1, 'planted', 'done', 'planted')`, userB)
|
||||
require.Error(t, err, "a row scoped as A must not be insertable on behalf of B")
|
||||
require.Contains(t, err.Error(), "row-level security")
|
||||
}
|
||||
|
||||
func TestUploadedTranscripts_RowSecurityIsEnabledForcedAndKeyedByUser(t *testing.T) {
|
||||
newStore(t)
|
||||
super := rawPool(t)
|
||||
ctx := context.Background()
|
||||
|
||||
var enabled, forced bool
|
||||
require.NoError(t, super.QueryRow(ctx,
|
||||
`SELECT relrowsecurity, relforcerowsecurity FROM pg_class
|
||||
WHERE oid = 'public.uploaded_transcripts'::regclass`).Scan(&enabled, &forced))
|
||||
require.True(t, enabled, "uploaded_transcripts must have row-level security enabled")
|
||||
require.True(t, forced,
|
||||
"FORCE is what binds the table-owner role tapir; without it the policy is dead in production")
|
||||
|
||||
var userIDNotNull bool
|
||||
require.NoError(t, super.QueryRow(ctx,
|
||||
`SELECT is_nullable = 'NO' FROM information_schema.columns
|
||||
WHERE table_name = 'uploaded_transcripts' AND column_name = 'user_id'`).Scan(&userIDNotNull))
|
||||
require.True(t, userIDNotNull, "an upload with no owner would match no policy and be unreachable")
|
||||
}
|
||||
|
||||
func TestUploadedTranscripts_DeleteUserRemovesTheirUploads(t *testing.T) {
|
||||
s := newStore(t)
|
||||
super := rawPool(t)
|
||||
resetDB(t, super)
|
||||
seedUploadedTranscript(t, super, userA, "text owned by A")
|
||||
bID := seedUploadedTranscript(t, super, userB, "text owned by B")
|
||||
|
||||
require.NoError(t, s.DeleteUser(context.Background(), userA))
|
||||
|
||||
var aRows, bRows int
|
||||
ctx := context.Background()
|
||||
require.NoError(t, super.QueryRow(ctx,
|
||||
`SELECT count(*) FROM uploaded_transcripts WHERE user_id = $1`, userA).Scan(&aRows))
|
||||
require.NoError(t, super.QueryRow(ctx,
|
||||
`SELECT count(*) FROM uploaded_transcripts WHERE id = $1`, bID).Scan(&bRows))
|
||||
require.Equal(t, 0, aRows, "deleting a user must remove their private transcripts (ON DELETE CASCADE)")
|
||||
require.Equal(t, 1, bRows, "deleting A must not touch B's transcripts")
|
||||
}
|
||||
|
||||
// rowSecurityOffTables are the public-schema tables that deliberately have row
|
||||
// security OFF. The list is exact on purpose: a NEW table added without RLS must
|
||||
// fail this test and force a decision, instead of passing CI by omission — the
|
||||
// failure mode ADR-033 names, where a naive reuse "leaks private content across
|
||||
// tenants and passes CI green".
|
||||
//
|
||||
// - transcripts: shared public caption content, the single exception for CONTENT
|
||||
// (ADR-021). It is the one that must never gain a private row.
|
||||
// - user_identities: the Dex-subject -> user_id map, read at login before any
|
||||
// user is known (migration 004). Holds no user content.
|
||||
// - invitations: exists before the user does, so there is no user to scope by;
|
||||
// the single-use token is the capability (migration 009).
|
||||
// - schema_migrations: golang-migrate bookkeeping.
|
||||
//
|
||||
// #33 describes `transcripts` as "the only non-RLS table". That is true of content
|
||||
// and false of tables; this test asserts the real set.
|
||||
var rowSecurityOffTables = []string{
|
||||
"invitations",
|
||||
"schema_migrations",
|
||||
"transcripts",
|
||||
"user_identities",
|
||||
}
|
||||
|
||||
func TestRowSecurityOffTablesAreExactlyTheKnownSet(t *testing.T) {
|
||||
newStore(t)
|
||||
super := rawPool(t)
|
||||
|
||||
rows, err := super.Query(context.Background(),
|
||||
`SELECT c.relname FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p') AND NOT c.relrowsecurity
|
||||
ORDER BY c.relname`)
|
||||
require.NoError(t, err)
|
||||
defer rows.Close()
|
||||
var got []string
|
||||
for rows.Next() {
|
||||
var name string
|
||||
require.NoError(t, rows.Scan(&name))
|
||||
got = append(got, name)
|
||||
}
|
||||
require.NoError(t, rows.Err())
|
||||
|
||||
want := append([]string{}, rowSecurityOffTables...)
|
||||
sort.Strings(want)
|
||||
require.Equal(t, want, got,
|
||||
"every public table must have row security ON unless it is in rowSecurityOffTables; "+
|
||||
"adding a table here is a decision that needs an ADR")
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// Progress and heartbeat of an upload job (ADR-035, migration 020). Like the other
|
||||
// upload store tests these run as the superuser, so the cross-user cases prove the
|
||||
// statement's own user_id predicate; the database-enforced isolation is covered by the
|
||||
// RLS tests, and migration 020 adds columns to the already-FORCE'd table, not a table.
|
||||
|
||||
func startedUpload(t *testing.T, s *store.Store) string {
|
||||
t.Helper()
|
||||
seedUsers(t, userA, userB)
|
||||
id, err := s.CreateUpload(context.Background(), userA, "m", false)
|
||||
require.NoError(t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
func TestUploadProgress_AFreshUploadHasNoProgressAndNoHeartbeat(t *testing.T) {
|
||||
s := newStore(t)
|
||||
id := startedUpload(t, s)
|
||||
|
||||
got, err := s.GetUpload(context.Background(), userA, id)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got.Progress)
|
||||
assert.True(t, got.ProgressAt.IsZero())
|
||||
assert.True(t, got.AliveAt.IsZero())
|
||||
}
|
||||
|
||||
func TestUploadProgress_StepsAndTheirTimesRoundTripInOrder(t *testing.T) {
|
||||
s := newStore(t)
|
||||
id := startedUpload(t, s)
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 10, 2, 8, 30, 0, 0, time.UTC)
|
||||
steps := []domain.ProgressStep{
|
||||
{Key: "audio", Label: "Preparing audio", State: domain.StepDone, StartedAt: at, FinishedAt: at.Add(40 * time.Second)},
|
||||
{Key: "decide", Label: "Deciding", State: domain.StepRunning, StartedAt: at.Add(time.Minute), Done: 12, Total: 282},
|
||||
{Key: "save", Label: "Saving", State: domain.StepPending},
|
||||
}
|
||||
|
||||
require.NoError(t, s.SaveProgress(ctx, userA, id, steps, at.Add(2*time.Minute)))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got.Progress, 3)
|
||||
assert.Equal(t, []string{"audio", "decide", "save"}, []string{got.Progress[0].Key, got.Progress[1].Key, got.Progress[2].Key})
|
||||
assert.Equal(t, domain.StepRunning, got.Progress[1].State)
|
||||
assert.Equal(t, 12, got.Progress[1].Done)
|
||||
assert.Equal(t, 282, got.Progress[1].Total)
|
||||
assert.True(t, got.Progress[0].FinishedAt.Equal(at.Add(40*time.Second)), "a step's finish time must survive the round trip")
|
||||
assert.True(t, got.Progress[2].StartedAt.IsZero(), "a pending step has no start time, not a zero date")
|
||||
assert.True(t, got.ProgressAt.Equal(at.Add(2*time.Minute)))
|
||||
}
|
||||
|
||||
func TestUploadProgress_AHeartbeatMovesAliveAtAndNotProgressAt(t *testing.T) {
|
||||
s := newStore(t)
|
||||
id := startedUpload(t, s)
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 10, 2, 8, 30, 0, 0, time.UTC)
|
||||
require.NoError(t, s.SaveProgress(ctx, userA, id, []domain.ProgressStep{{Key: "audio", Label: "a", State: domain.StepRunning}}, at))
|
||||
before := time.Now().Add(-time.Second)
|
||||
|
||||
require.NoError(t, s.Heartbeat(ctx, userA, id))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, got.AliveAt.After(before), "the heartbeat must stamp the row with now")
|
||||
assert.True(t, got.ProgressAt.Equal(at), "alive is not progress: a heartbeat must not move progress_at")
|
||||
}
|
||||
|
||||
func TestUploadProgress_ASecondSaveReplacesTheFirst(t *testing.T) {
|
||||
s := newStore(t)
|
||||
id := startedUpload(t, s)
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 10, 2, 8, 30, 0, 0, time.UTC)
|
||||
require.NoError(t, s.SaveProgress(ctx, userA, id, []domain.ProgressStep{{Key: "a", Label: "A", State: domain.StepRunning}}, at))
|
||||
|
||||
require.NoError(t, s.SaveProgress(ctx, userA, id, []domain.ProgressStep{{Key: "a", Label: "A", State: domain.StepDone}, {Key: "b", Label: "B", State: domain.StepRunning}}, at.Add(time.Minute)))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got.Progress, 2)
|
||||
assert.Equal(t, domain.StepDone, got.Progress[0].State)
|
||||
}
|
||||
|
||||
func TestUploadProgress_ByAnotherUserChangesNothing(t *testing.T) {
|
||||
s := newStore(t)
|
||||
id := startedUpload(t, s)
|
||||
ctx := context.Background()
|
||||
|
||||
errSave := s.SaveProgress(ctx, userB, id, []domain.ProgressStep{{Key: "x", Label: "X", State: domain.StepDone}}, time.Now())
|
||||
errBeat := s.Heartbeat(ctx, userB, id)
|
||||
|
||||
assert.ErrorIs(t, errSave, store.ErrUploadNotFound)
|
||||
assert.ErrorIs(t, errBeat, store.ErrUploadNotFound)
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got.Progress, "B's write must not have reached A's upload")
|
||||
assert.True(t, got.AliveAt.IsZero())
|
||||
}
|
||||
|
||||
func TestUploadProgress_ListDoesNotDragItAlong(t *testing.T) {
|
||||
s := newStore(t)
|
||||
id := startedUpload(t, s)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, s.SaveProgress(ctx, userA, id, []domain.ProgressStep{{Key: "a", Label: "A", State: domain.StepRunning}}, time.Now()))
|
||||
|
||||
rows, err := s.ListUploads(ctx, userA)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Empty(t, rows[0].Progress, "the list view needs a status, not every row's phases")
|
||||
}
|
||||
|
||||
func TestUploadProgress_SurvivesCompletion(t *testing.T) {
|
||||
s := newStore(t)
|
||||
id := startedUpload(t, s)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, id))
|
||||
require.NoError(t, s.SaveProgress(ctx, userA, id, []domain.ProgressStep{{Key: "a", Label: "A", State: domain.StepDone}}, time.Now()))
|
||||
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, id, domain.Transcription{Text: "x", Endpoint: "e"}))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got.Progress, 1, "a finished upload keeps its record of what was done, and how long each step took")
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// These exercise the Store API over uploaded_transcripts. The test Store connects
|
||||
// as the superuser, which BYPASSES row-level security — so the cross-user cases
|
||||
// here prove the query's own user_id predicate (defence in depth), while the
|
||||
// database-enforced guarantee is proven separately, as the non-superuser app
|
||||
// role, in uploaded_transcripts_isolation_test.go. Text is synthetic; this repo is
|
||||
// public and no real transcript may appear in it (#33).
|
||||
|
||||
func seedUsers(t *testing.T, ids ...string) {
|
||||
t.Helper()
|
||||
super := rawPool(t)
|
||||
resetDB(t, super)
|
||||
for _, id := range ids {
|
||||
_, err := super.Exec(context.Background(), `INSERT INTO users (id) VALUES ($1)`, id)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateUpload_StartsQueuedAndIsOwnedByTheCreator(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
|
||||
id, err := s.CreateUpload(ctx, userA, "Teams export", false)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, id, got.ID)
|
||||
assert.Equal(t, userA, got.UserID)
|
||||
assert.Equal(t, "Teams export", got.Title)
|
||||
assert.Equal(t, domain.UploadQueued, got.Status)
|
||||
assert.Empty(t, got.Content)
|
||||
assert.True(t, got.FinishedAt.IsZero(), "an unfinished upload has no finish time")
|
||||
}
|
||||
|
||||
func TestUploadLifecycle_CompletedKeepsTextSegmentsEndpointAndDuration(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateUpload(ctx, userA, "m", false)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, id))
|
||||
mid, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, domain.UploadTranscribing, mid.Status)
|
||||
|
||||
want := domain.Transcription{
|
||||
Text: "God morgon allihopa. Vi går igenom läget.",
|
||||
Segments: []domain.Segment{
|
||||
{Start: 0.0, End: 1.5, Text: "God morgon allihopa."},
|
||||
{Start: 1.5, End: 4.25, Text: "Vi går igenom läget."},
|
||||
},
|
||||
Endpoint: "iguana/kb-whisper",
|
||||
AudioSeconds: 4.25,
|
||||
}
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, id, want))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, domain.UploadDone, got.Status)
|
||||
assert.Equal(t, want.Text, got.Content)
|
||||
assert.Equal(t, want.Segments, got.Segments, "segment times must survive the JSONB round trip")
|
||||
assert.Equal(t, "iguana/kb-whisper", got.Endpoint, "the serving endpoint is a quality fact and is recorded per row")
|
||||
assert.InDelta(t, 4.25, got.AudioSeconds, 1e-9)
|
||||
assert.False(t, got.FinishedAt.IsZero())
|
||||
assert.Empty(t, got.ErrorKind)
|
||||
}
|
||||
|
||||
func TestUploadLifecycle_FailedRecordsTypedKindAndNoContent(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateUpload(ctx, userA, "m", false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, id))
|
||||
|
||||
require.NoError(t, s.FailUpload(ctx, userA, id, domain.UploadErrTranscriptionFailed))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, domain.UploadFailed, got.Status)
|
||||
assert.Equal(t, domain.UploadErrTranscriptionFailed, got.ErrorKind)
|
||||
assert.Empty(t, got.Content)
|
||||
assert.False(t, got.FinishedAt.IsZero())
|
||||
}
|
||||
|
||||
func TestGetUpload_AnotherUsersIDIsNotFound(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA, userB)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateUpload(ctx, userA, "A's meeting", false)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = s.GetUpload(ctx, userB, id)
|
||||
|
||||
assert.ErrorIs(t, err, store.ErrUploadNotFound,
|
||||
"a non-owner must get not-found, not forbidden: existence is not disclosed")
|
||||
}
|
||||
|
||||
func TestUploadWrites_ByAnotherUserChangeNothing(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA, userB)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateUpload(ctx, userA, "A's meeting", false)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.ErrorIs(t, s.StartTranscribing(ctx, userB, id), store.ErrUploadNotFound)
|
||||
assert.ErrorIs(t, s.CompleteUpload(ctx, userB, id, domain.Transcription{Text: "planted"}), store.ErrUploadNotFound)
|
||||
assert.ErrorIs(t, s.FailUpload(ctx, userB, id, domain.UploadErrTranscriptionFailed), store.ErrUploadNotFound)
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, domain.UploadQueued, got.Status, "B's writes must leave A's upload exactly as it was")
|
||||
assert.Empty(t, got.Content)
|
||||
}
|
||||
|
||||
func TestListUploads_ReturnsOnlyTheCallersNewestFirstWithoutTranscriptText(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA, userB)
|
||||
ctx := context.Background()
|
||||
first, err := s.CreateUpload(ctx, userA, "first", false)
|
||||
require.NoError(t, err)
|
||||
second, err := s.CreateUpload(ctx, userA, "second", false)
|
||||
require.NoError(t, err)
|
||||
_, err = s.CreateUpload(ctx, userB, "B's", false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, first))
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, first, domain.Transcription{Text: "long text body", Endpoint: "e"}))
|
||||
|
||||
got, err := s.ListUploads(ctx, userA)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2, "A must see only A's uploads")
|
||||
assert.Equal(t, second, got[0].ID, "newest first")
|
||||
assert.Equal(t, first, got[1].ID)
|
||||
assert.Empty(t, got[1].Content, "a list row must not drag the whole transcript along")
|
||||
assert.Equal(t, domain.UploadDone, got[1].Status)
|
||||
}
|
||||
|
||||
func TestFailInterruptedUploads_FailsOnlyUnfinishedRowsForEveryUser(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA, userB)
|
||||
super := rawPool(t)
|
||||
ctx := context.Background()
|
||||
for subject, uid := range map[string]string{"sub-a": userA, "sub-b": userB} {
|
||||
_, err := super.Exec(ctx, `INSERT INTO user_identities (dex_subject, user_id) VALUES ($1, $2)`, subject, uid)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
queued, err := s.CreateUpload(ctx, userA, "queued", false)
|
||||
require.NoError(t, err)
|
||||
running, err := s.CreateUpload(ctx, userB, "running", false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userB, running))
|
||||
done, err := s.CreateUpload(ctx, userA, "done", false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, done))
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, done, domain.Transcription{Text: "kept", Endpoint: "e"}))
|
||||
|
||||
n, err := s.FailInterruptedUploads(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, n, "the queued and the transcribing row, across both users")
|
||||
for _, c := range []struct{ user, id string }{{userA, queued}, {userB, running}} {
|
||||
got, err := s.GetUpload(ctx, c.user, c.id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, domain.UploadFailed, got.Status)
|
||||
assert.Equal(t, domain.UploadErrInterrupted, got.ErrorKind,
|
||||
"audio is not retained, so a restart cannot resume: the honest state is failed/interrupted")
|
||||
}
|
||||
kept, err := s.GetUpload(ctx, userA, done)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, domain.UploadDone, kept.Status, "a finished transcript must survive the sweep")
|
||||
assert.Equal(t, "kept", kept.Content)
|
||||
}
|
||||
|
||||
func TestCreateUpload_RecordsWhetherTheFileHadAVideoTrack(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
withVideo, err := s.CreateUpload(ctx, userA, "camera off", true)
|
||||
require.NoError(t, err)
|
||||
audioOnly, err := s.CreateUpload(ctx, userA, "voice only", false)
|
||||
require.NoError(t, err)
|
||||
|
||||
gotVideo, err := s.GetUpload(ctx, userA, withVideo)
|
||||
require.NoError(t, err)
|
||||
gotAudio, err := s.GetUpload(ctx, userA, audioOnly)
|
||||
require.NoError(t, err)
|
||||
listed, err := s.ListUploads(ctx, userA)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, gotVideo.HadVideo, "the page must be able to say the video was ignored")
|
||||
assert.False(t, gotAudio.HadVideo)
|
||||
byID := map[string]bool{}
|
||||
for _, u := range listed {
|
||||
byID[u.ID] = u.HadVideo
|
||||
}
|
||||
assert.Equal(t, map[string]bool{withVideo: true, audioOnly: false}, byID, "the list carries it too")
|
||||
}
|
||||
|
||||
func TestUploadHadVideo_SurvivesCompletionAndFailure(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
done, err := s.CreateUpload(ctx, userA, "d", true)
|
||||
require.NoError(t, err)
|
||||
failed, err := s.CreateUpload(ctx, userA, "f", true)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, done))
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, done, domain.Transcription{Text: "x", Endpoint: "e"}))
|
||||
require.NoError(t, s.FailUpload(ctx, userA, failed, domain.UploadErrTranscriptionFailed))
|
||||
|
||||
for _, id := range []string{done, failed} {
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, got.HadVideo, "completing or failing must not clear it: the notice outlives the job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadLifecycle_ReconciliationRoundTripsWithItsContestedSpots(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateUpload(ctx, userA, "m", false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, id))
|
||||
want := &domain.Reconciliation{
|
||||
Status: domain.ReconcileDone, Base: "iguana/pianissimo", Alt: "iguana/kb-whisper", Reconciler: "iguana/gemma4-26b",
|
||||
Disputes: 380, AutoResolved: 90, Adjudicated: 285, Changed: 120, Unsure: 5, Errors: 0,
|
||||
Contested: []domain.ContestedSpot{
|
||||
{Seconds: 2210.5, Base: "Axel", Alt: "Alex", Used: "alt"},
|
||||
{Seconds: 3400, Base: "A+2", Alt: "en A", Used: "base", Unsure: true},
|
||||
},
|
||||
}
|
||||
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, id, domain.Transcription{Text: "x", Endpoint: "e", Reconciliation: want}))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, got.Reconciliation)
|
||||
}
|
||||
|
||||
func TestUploadLifecycle_ASingleEngineTranscriptHasNoReconciliation(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateUpload(ctx, userA, "m", false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, id))
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, id, domain.Transcription{Text: "x", Endpoint: "e"}))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, got.Reconciliation, "no reconciliation is not an empty one: the page must not claim a second opinion")
|
||||
}
|
||||
|
||||
func TestUploadLifecycle_ASkippedReconciliationKeepsItsReason(t *testing.T) {
|
||||
s := newStore(t)
|
||||
seedUsers(t, userA)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateUpload(ctx, userA, "m", false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.StartTranscribing(ctx, userA, id))
|
||||
rec := &domain.Reconciliation{Status: domain.ReconcileSkipped, Reason: "the model failed 3 times in a row: gateway down", Errors: 3}
|
||||
|
||||
require.NoError(t, s.CompleteUpload(ctx, userA, id, domain.Transcription{Text: "x", Endpoint: "e", Reconciliation: rec}))
|
||||
|
||||
got, err := s.GetUpload(ctx, userA, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rec, got.Reconciliation)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// Backend is one transcription endpoint. *Gateway satisfies it; Name is the value
|
||||
// recorded as the serving endpoint.
|
||||
type Backend interface {
|
||||
Name() string
|
||||
Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error)
|
||||
}
|
||||
|
||||
// Chain tries a primary (local) backend and, if it fails for ANY reason, a
|
||||
// fallback. The fallback exists because iguana can wedge permanently on a Metal
|
||||
// OOM (infra#273) — which is also why a fallback is a data-egress decision: the
|
||||
// berget endpoint is a cloud service, so the result records which one served it.
|
||||
type Chain struct {
|
||||
primary Backend
|
||||
fallback Backend // nil disables the fallback
|
||||
}
|
||||
|
||||
// NewChain builds a Chain. A nil fallback means a primary failure is the result.
|
||||
func NewChain(primary, fallback Backend) *Chain {
|
||||
return &Chain{primary: primary, fallback: fallback}
|
||||
}
|
||||
|
||||
// Transcribe returns the first backend's success. It falls back on any primary
|
||||
// error — transport, timeout, non-2xx, empty transcript — and never inspects a
|
||||
// response body to decide. A cancelled context stops the chain: a job the user or
|
||||
// shutdown cancelled must not ship its audio to a cloud endpoint as a last act.
|
||||
func (c *Chain) Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error) {
|
||||
t, err := c.primary.Transcribe(ctx, wavPath)
|
||||
if err == nil {
|
||||
t.Endpoint = c.primary.Name()
|
||||
return t, nil
|
||||
}
|
||||
if c.fallback == nil || ctx.Err() != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("%s: %w", c.primary.Name(), err)
|
||||
}
|
||||
t, ferr := c.fallback.Transcribe(ctx, wavPath)
|
||||
if ferr != nil {
|
||||
return domain.Transcription{}, errors.Join(
|
||||
fmt.Errorf("%s: %w", c.primary.Name(), err),
|
||||
fmt.Errorf("%s: %w", c.fallback.Name(), ferr))
|
||||
}
|
||||
t.Endpoint = c.fallback.Name()
|
||||
return t, nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
type fakeBackend struct {
|
||||
name string
|
||||
text string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeBackend) Name() string { return f.name }
|
||||
func (f *fakeBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return domain.Transcription{}, f.err
|
||||
}
|
||||
return domain.Transcription{Text: f.text, Endpoint: f.name}, nil
|
||||
}
|
||||
|
||||
func TestChain_PrimarySuccessNeverTouchesTheFallback(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", text: "primary text"}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||
|
||||
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "primary text", got.Text)
|
||||
assert.Equal(t, "iguana/kb-whisper", got.Endpoint)
|
||||
assert.Zero(t, fallback.calls, "audio must not leave the homelab when the local endpoint worked")
|
||||
}
|
||||
|
||||
func TestChain_AnyPrimaryFailureFallsBackAndRecordsTheFallbackEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"server error", &transcribe.StatusError{Code: 500}},
|
||||
{"gateway timeout", &transcribe.StatusError{Code: 504}},
|
||||
{"client error", &transcribe.StatusError{Code: 400}},
|
||||
{"transport failure", errors.New("dial tcp: connection refused")},
|
||||
{"empty transcript", transcribe.ErrEmptyTranscript},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: tt.err}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", text: "fallback text"}
|
||||
|
||||
got, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fallback text", got.Text)
|
||||
assert.Equal(t, "berget/kb-whisper", got.Endpoint,
|
||||
"the row must say berget served it: that means the audio left the homelab")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChain_BothFailingReportsBothErrors(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", err: errors.New("berget down")}
|
||||
|
||||
_, err := transcribe.NewChain(primary, fallback).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "iguana down")
|
||||
assert.Contains(t, err.Error(), "berget down")
|
||||
}
|
||||
|
||||
func TestChain_WithoutAFallbackAPrimaryFailureIsTheResult(t *testing.T) {
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: errors.New("iguana down")}
|
||||
|
||||
_, err := transcribe.NewChain(primary, nil).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "iguana down")
|
||||
}
|
||||
|
||||
func TestChain_ACancelledJobDoesNotFallBack(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
primary := &fakeBackend{name: "iguana/kb-whisper", err: context.Canceled}
|
||||
fallback := &fakeBackend{name: "berget/kb-whisper", text: "x"}
|
||||
|
||||
_, err := transcribe.NewChain(primary, fallback).Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.Zero(t, fallback.calls, "a cancelled job must not ship the audio to a cloud endpoint")
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// Ensemble transcribes the same WAV with two engines, one after the other, and has a
|
||||
// local model reconcile their disagreements (ADR-034). It satisfies uploads.Transcriber,
|
||||
// so the worker neither knows nor cares that two engines were involved.
|
||||
//
|
||||
// It degrades in steps and never turns a recoverable problem into a failed job:
|
||||
// one engine down → the other's text, said so on the result; the model down → the
|
||||
// base text unreconciled; both engines down → the cloud fallback, if configured, and
|
||||
// recorded as such. A cancelled job stops everything and never reaches the cloud.
|
||||
type Ensemble struct {
|
||||
base, alt Backend
|
||||
fallback Backend // nil = no cloud fallback
|
||||
adj reconcile.Adjudicator
|
||||
opts reconcile.Options
|
||||
}
|
||||
|
||||
// NewEnsemble builds an Ensemble. base is the engine whose text stands where the two
|
||||
// disagree and is the faster one, so it runs first; alt supplies the other reading.
|
||||
func NewEnsemble(base, alt, fallback Backend, adj reconcile.Adjudicator, opts reconcile.Options) *Ensemble {
|
||||
return &Ensemble{base: base, alt: alt, fallback: fallback, adj: adj, opts: opts}
|
||||
}
|
||||
|
||||
// Plan implements uploads.Planner: the phases this ensemble reports, so the page can
|
||||
// show them before they run. The fallback is a phase only when one is configured.
|
||||
func (e *Ensemble) Plan() []progress.Planned {
|
||||
plan := []progress.Planned{
|
||||
{Key: "engine_a", Label: "Transcribing with " + e.base.Name()},
|
||||
{Key: "engine_b", Label: "Transcribing with " + e.alt.Name()},
|
||||
{Key: "compare", Label: "Comparing the two transcripts"},
|
||||
{Key: "decide", Label: "Deciding the differences"},
|
||||
}
|
||||
if e.fallback != nil {
|
||||
plan = append(plan, progress.Planned{Key: "fallback", Label: "Cloud fallback (" + e.fallback.Name() + ")"})
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
// Transcribe implements uploads.Transcriber. Each phase is reported through the
|
||||
// progress.Reporter in ctx (a no-op when nobody listens).
|
||||
func (e *Ensemble) Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error) {
|
||||
rep := progress.From(ctx)
|
||||
b, berr := e.run(ctx, rep, "engine_a", e.base, wavPath)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
a, aerr := e.run(ctx, rep, "engine_b", e.alt, wavPath)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
|
||||
switch {
|
||||
case berr == nil && aerr == nil:
|
||||
out, err := reconcile.Reconcile(ctx, b, a, e.adj, e.opts)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
out.Endpoint = fmt.Sprintf("%s + %s, reconciled by %s", e.base.Name(), e.alt.Name(), e.opts.Reconciler)
|
||||
if rc := out.Reconciliation; rc != nil {
|
||||
rep.Note("reconciled", "status", string(rc.Status), "disputes", rc.Disputes, "auto_resolved", rc.AutoResolved,
|
||||
"adjudicated", rc.Adjudicated, "changed", rc.Changed, "unsure", rc.Unsure, "errors", rc.Errors)
|
||||
}
|
||||
return out, nil
|
||||
case berr == nil:
|
||||
return e.single(b, e.base.Name(), fmt.Sprintf("%s failed (%s)", e.alt.Name(), errkind.Of(aerr))), nil
|
||||
case aerr == nil:
|
||||
return e.single(a, e.alt.Name(), fmt.Sprintf("%s failed (%s)", e.base.Name(), errkind.Of(berr))), nil
|
||||
}
|
||||
|
||||
both := errors.Join(fmt.Errorf("%s: %w", e.base.Name(), berr), fmt.Errorf("%s: %w", e.alt.Name(), aerr))
|
||||
if e.fallback == nil {
|
||||
return domain.Transcription{}, both
|
||||
}
|
||||
f, ferr := e.run(ctx, rep, "fallback", e.fallback, wavPath)
|
||||
if ferr != nil {
|
||||
return domain.Transcription{}, errors.Join(both, fmt.Errorf("%s: %w", e.fallback.Name(), ferr))
|
||||
}
|
||||
return e.single(f, e.fallback.Name(), "both local engines failed"), nil
|
||||
}
|
||||
|
||||
// run is one engine call, reported as a phase. A job that was stopped is not a broken
|
||||
// engine, so a cancelled context is never reported as a failed phase.
|
||||
func (e *Ensemble) run(ctx context.Context, rep progress.Reporter, key string, be Backend, wav string) (domain.Transcription, error) {
|
||||
rep.Begin(key, 0)
|
||||
t, err := be.Transcribe(ctx, wav)
|
||||
switch {
|
||||
case err == nil:
|
||||
rep.Done(key)
|
||||
case ctx.Err() == nil:
|
||||
rep.Fail(key, err)
|
||||
}
|
||||
return t, err
|
||||
}
|
||||
|
||||
// single is a transcript from one engine only: nothing to reconcile, and the result
|
||||
// says why, because a reader should know the second opinion was missing.
|
||||
func (e *Ensemble) single(t domain.Transcription, engine, reason string) domain.Transcription {
|
||||
t.Endpoint = engine
|
||||
t.Reconciliation = &domain.Reconciliation{
|
||||
Status: domain.ReconcileSingle, Reason: reason, Base: e.opts.BaseEngine, Alt: e.opts.AltEngine, Reconciler: e.opts.Reconciler,
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// What the Ensemble reports to the page and the log (ADR-035). The reporter here is a
|
||||
// recording double carried in the context, exactly as the worker's Tracker is.
|
||||
|
||||
type recReporter struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
notes []map[string]any
|
||||
}
|
||||
|
||||
func (r *recReporter) add(e string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.events = append(r.events, e)
|
||||
}
|
||||
|
||||
func (r *recReporter) Begin(key string, _ int) { r.add("begin:" + key) }
|
||||
func (r *recReporter) Advance(key string, _ int) { r.add("advance:" + key) }
|
||||
func (r *recReporter) Done(key string) { r.add("done:" + key) }
|
||||
func (r *recReporter) Fail(key string, _ error) { r.add("fail:" + key) }
|
||||
func (r *recReporter) Note(msg string, kv ...any) {
|
||||
m := map[string]any{"msg": msg}
|
||||
for i := 0; i+1 < len(kv); i += 2 {
|
||||
m[kv[i].(string)] = kv[i+1]
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.notes = append(r.notes, m)
|
||||
r.mu.Unlock()
|
||||
r.add("note:" + msg)
|
||||
}
|
||||
|
||||
func (r *recReporter) joined() string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return strings.Join(r.events, ",")
|
||||
}
|
||||
|
||||
func TestEnsemblePlan_DeclaresBothEnginesAndTheReconcileStepsByName(t *testing.T) {
|
||||
r := newEnsemble("a", "a", nil, nil, false, &verdictAdj{})
|
||||
|
||||
plan := r.e.Plan()
|
||||
|
||||
require.Len(t, plan, 4)
|
||||
assert.Equal(t, []string{"engine_a", "engine_b", "compare", "decide"}, []string{plan[0].Key, plan[1].Key, plan[2].Key, plan[3].Key})
|
||||
assert.Contains(t, plan[0].Label, "iguana/pianissimo", "the page says which engine is running")
|
||||
assert.Contains(t, plan[1].Label, "iguana/kb-whisper")
|
||||
}
|
||||
|
||||
func TestEnsemblePlan_TheCloudFallbackIsAStepOnlyWhenOneIsConfigured(t *testing.T) {
|
||||
with := newEnsemble("a", "a", nil, nil, true, &verdictAdj{}).e.Plan()
|
||||
without := newEnsemble("a", "a", nil, nil, false, &verdictAdj{}).e.Plan()
|
||||
|
||||
assert.Equal(t, "fallback", with[len(with)-1].Key)
|
||||
assert.Len(t, without, 4, "no fallback configured: the page must not promise one")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_BothEnginesAreReportedInTheOrderTheyRun(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, false, &verdictAdj{v: reconcile.VerdictAlt})
|
||||
|
||||
_, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
got := rep.joined()
|
||||
a, aDone := strings.Index(got, "begin:engine_a"), strings.Index(got, "done:engine_a")
|
||||
b, bDone := strings.Index(got, "begin:engine_b"), strings.Index(got, "done:engine_b")
|
||||
require.True(t, a >= 0 && aDone > a && b > aDone && bDone > b, "events out of order: %s", got)
|
||||
assert.NotContains(t, got, "fail:")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_AnEngineThatFailsIsShownFailedAndTheOtherStillRuns(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("text", "text", errors.New("pianissimo down"), nil, false, &verdictAdj{})
|
||||
|
||||
got, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err, "one engine down is a degraded result, not a failed job")
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
ev := rep.joined()
|
||||
assert.Contains(t, ev, "fail:engine_a")
|
||||
assert.Contains(t, ev, "done:engine_b")
|
||||
assert.NotContains(t, ev, "begin:compare", "nothing to compare with one engine")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_TheFallbackIsReportedOnlyWhenBothEnginesFailed(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("t", "t", errors.New("a down"), errors.New("b down"), true, &verdictAdj{})
|
||||
|
||||
_, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
ev := rep.joined()
|
||||
assert.Contains(t, ev, "fail:engine_a")
|
||||
assert.Contains(t, ev, "fail:engine_b")
|
||||
assert.Contains(t, ev, "begin:fallback")
|
||||
assert.Contains(t, ev, "done:fallback")
|
||||
|
||||
rep2 := &recReporter{}
|
||||
ok := newEnsemble("x", "x", nil, nil, true, &verdictAdj{})
|
||||
_, err = ok.e.Transcribe(progress.With(context.Background(), rep2), "/a.wav")
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, rep2.joined(), "fallback", "a healthy run never touches the cloud step")
|
||||
}
|
||||
|
||||
// cancellingBackend ends the job in the middle of its own call, like a shutdown.
|
||||
type cancellingBackend struct {
|
||||
name string
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (c cancellingBackend) Name() string { return c.name }
|
||||
func (c cancellingBackend) Transcribe(ctx context.Context, _ string) (domain.Transcription, error) {
|
||||
c.cancel()
|
||||
return domain.Transcription{}, ctx.Err()
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_AShutdownMidEngineIsNotReportedAsAFailedEngine(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
ctx, cancel := context.WithCancel(progress.With(context.Background(), rep))
|
||||
calls := &[]string{}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: "x", calls: calls}
|
||||
e := transcribe.NewEnsemble(cancellingBackend{name: "iguana/pianissimo", cancel: cancel}, alt, nil, &verdictAdj{}, ensembleOpts)
|
||||
|
||||
_, err := e.Transcribe(ctx, "/a.wav")
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.NotContains(t, rep.joined(), "fail:", "a stopped job is not a broken engine: the page must not blame one")
|
||||
assert.Empty(t, *calls, "and the second engine never starts")
|
||||
}
|
||||
|
||||
func TestEnsembleProgress_TheReconcileCountsAreNotedForTheOperator(t *testing.T) {
|
||||
rep := &recReporter{}
|
||||
r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, false, &verdictAdj{v: reconcile.VerdictAlt})
|
||||
|
||||
_, err := r.e.Transcribe(progress.With(context.Background(), rep), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rep.notes, 1)
|
||||
n := rep.notes[0]
|
||||
assert.Equal(t, "reconciled", n["msg"])
|
||||
assert.Equal(t, 1, n["disputes"])
|
||||
assert.Equal(t, 1, n["changed"])
|
||||
assert.Equal(t, 0, n["unsure"])
|
||||
assert.Equal(t, 0, n["errors"])
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// When one engine is down the page says so and why; the "why" is a kind, not the
|
||||
// engine's error text, which can carry provider output (adversarial review F3).
|
||||
|
||||
func TestEnsemble_AFailedEnginesReasonIsAKindNotItsErrorText(t *testing.T) {
|
||||
r := newEnsemble("text", "text", nil, errors.New("kb-whisper said: zebrakaka secret words"), false, &verdictAdj{v: reconcile.VerdictAlt})
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, got.Reconciliation.Reason, "zebrakaka")
|
||||
assert.Contains(t, got.Reconciliation.Reason, "iguana/kb-whisper failed", "which engine failed is still said")
|
||||
}
|
||||
|
||||
func TestEnsemble_AnHTTPFailureShowsItsStatusCodeAsTheKind(t *testing.T) {
|
||||
r := newEnsemble("text", "text", &transcribe.StatusError{Code: 502, Snippet: "zebrakaka"}, nil, false, &verdictAdj{})
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, got.Reconciliation.Reason, "http_502")
|
||||
assert.NotContains(t, got.Reconciliation.Reason, "zebrakaka")
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// orderedBackend records, in a shared slice, when it was called.
|
||||
type orderedBackend struct {
|
||||
name string
|
||||
text string
|
||||
err error
|
||||
calls *[]string
|
||||
}
|
||||
|
||||
func (o *orderedBackend) Name() string { return o.name }
|
||||
func (o *orderedBackend) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||
*o.calls = append(*o.calls, o.name)
|
||||
if o.err != nil {
|
||||
return domain.Transcription{}, o.err
|
||||
}
|
||||
return domain.Transcription{Text: o.text, Endpoint: o.name, AudioSeconds: 10}, nil
|
||||
}
|
||||
|
||||
type verdictAdj struct {
|
||||
v reconcile.Verdict
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (a *verdictAdj) Adjudicate(context.Context, reconcile.Request) (reconcile.Verdict, error) {
|
||||
a.calls++
|
||||
return a.v, a.err
|
||||
}
|
||||
|
||||
var ensembleOpts = reconcile.Options{Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}
|
||||
|
||||
type rig struct {
|
||||
e *transcribe.Ensemble
|
||||
calls *[]string
|
||||
adj *verdictAdj
|
||||
}
|
||||
|
||||
func newEnsemble(baseText, altText string, baseErr, altErr error, withFallback bool, adj *verdictAdj) *rig {
|
||||
calls := &[]string{}
|
||||
base := &orderedBackend{name: "iguana/pianissimo", text: baseText, err: baseErr, calls: calls}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: altText, err: altErr, calls: calls}
|
||||
var fb transcribe.Backend
|
||||
if withFallback {
|
||||
fb = &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
|
||||
}
|
||||
return &rig{e: transcribe.NewEnsemble(base, alt, fb, adj, ensembleOpts), calls: calls, adj: adj}
|
||||
}
|
||||
|
||||
func TestEnsemble_BothEnginesRunInOrderAndTheirDisagreementIsReconciled(t *testing.T) {
|
||||
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
||||
r := newEnsemble("Totalt 1015 kronor idag.", "Totalt 1050 kronor idag.", nil, nil, true, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"iguana/pianissimo", "iguana/kb-whisper"}, *r.calls,
|
||||
"the 40-second engine runs first, so a failure there is known early; the fallback is not touched")
|
||||
assert.Equal(t, "Totalt 1050 kronor idag.", got.Text)
|
||||
assert.Equal(t, "iguana/pianissimo + iguana/kb-whisper, reconciled by iguana/gemma4-26b", got.Endpoint,
|
||||
"every contributor is recorded: which engines touched the audio is a quality and a privacy fact")
|
||||
require.NotNil(t, got.Reconciliation)
|
||||
assert.Equal(t, domain.ReconcileDone, got.Reconciliation.Status)
|
||||
}
|
||||
|
||||
func TestEnsemble_IdenticalTextsNeverCallTheModel(t *testing.T) {
|
||||
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
||||
r := newEnsemble("Vi går igenom läget.", "Vi går igenom läget.", nil, nil, false, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, adj.calls)
|
||||
assert.Equal(t, "Vi går igenom läget.", got.Text)
|
||||
}
|
||||
|
||||
func TestEnsemble_WhenOneEngineFailsTheOtherIsUsedAndItIsSaidSo(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
baseErr error
|
||||
altErr error
|
||||
wantText string
|
||||
wantEndpoint string
|
||||
wantReasonHas string
|
||||
}{
|
||||
{"pianissimo down", errors.New("pianissimo wedged"), nil, "alt text", "iguana/kb-whisper", "iguana/pianissimo failed"},
|
||||
{"kb-whisper down", nil, errors.New("metal oom"), "base text", "iguana/pianissimo", "iguana/kb-whisper failed"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
adj := &verdictAdj{v: reconcile.VerdictAlt}
|
||||
r := newEnsemble("base text", "alt text", tt.baseErr, tt.altErr, true, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.wantText, got.Text)
|
||||
assert.Equal(t, tt.wantEndpoint, got.Endpoint)
|
||||
assert.Zero(t, adj.calls, "there is nothing to reconcile")
|
||||
require.NotNil(t, got.Reconciliation)
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
assert.Contains(t, got.Reconciliation.Reason, tt.wantReasonHas)
|
||||
assert.NotContains(t, *r.calls, "berget/kb-whisper", "one local engine working means audio does not leave the homelab")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsemble_OnlyWhenBothLocalEnginesFailDoesTheCloudFallbackRun(t *testing.T) {
|
||||
r := newEnsemble("", "", errors.New("a"), errors.New("b"), true, &verdictAdj{})
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fallback text", got.Text)
|
||||
assert.Equal(t, "berget/kb-whisper", got.Endpoint, "the row must say berget served it: the audio left the homelab")
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
assert.Contains(t, got.Reconciliation.Reason, "both local engines failed")
|
||||
}
|
||||
|
||||
func TestEnsemble_BothFailingWithNoFallbackReportsBothErrors(t *testing.T) {
|
||||
r := newEnsemble("", "", errors.New("pianissimo down"), errors.New("whisper down"), false, &verdictAdj{})
|
||||
|
||||
_, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "pianissimo down")
|
||||
assert.Contains(t, err.Error(), "whisper down")
|
||||
}
|
||||
|
||||
func TestEnsemble_ACancelledJobNeverReachesTheCloud(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
r := newEnsemble("", "", context.Canceled, context.Canceled, true, &verdictAdj{})
|
||||
|
||||
_, err := r.e.Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.NotContains(t, *r.calls, "berget/kb-whisper", "a cancelled job must not ship its audio to a cloud endpoint as a last act")
|
||||
}
|
||||
|
||||
func TestEnsemble_AModelThatIsDownGivesTheBaseTextUnreconciledNotAFailedJob(t *testing.T) {
|
||||
adj := &verdictAdj{err: errors.New("gateway down")}
|
||||
// Four separate disagreements, each separated by two shared words, so the
|
||||
// breaker trips after three consecutive failures.
|
||||
r := newEnsemble("a1 s t b1 s t c1 s t d1", "a2 s t b2 s t c2 s t d2", nil, nil, false, adj)
|
||||
|
||||
got, err := r.e.Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "a1 s t b1 s t c1 s t d1", got.Text)
|
||||
assert.Equal(t, domain.ReconcileSkipped, got.Reconciliation.Status)
|
||||
assert.Equal(t, 3, adj.calls)
|
||||
}
|
||||
|
||||
// cancelling is a backend that cancels the job's context while it runs, the way a
|
||||
// shutdown arrives in the middle of a long transcription.
|
||||
type cancelling struct {
|
||||
name string
|
||||
cancel context.CancelFunc
|
||||
calls *[]string
|
||||
}
|
||||
|
||||
func (c *cancelling) Name() string { return c.name }
|
||||
func (c *cancelling) Transcribe(_ context.Context, _ string) (domain.Transcription, error) {
|
||||
*c.calls = append(*c.calls, c.name)
|
||||
c.cancel()
|
||||
return domain.Transcription{}, context.Canceled
|
||||
}
|
||||
|
||||
func TestEnsemble_AShutdownDuringTheFirstEngineStopsBeforeTheSecondOrTheCloud(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := &[]string{}
|
||||
base := &cancelling{name: "iguana/pianissimo", cancel: cancel, calls: calls}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: "alt", calls: calls}
|
||||
fb := &orderedBackend{name: "berget/kb-whisper", text: "fb", calls: calls}
|
||||
|
||||
_, err := transcribe.NewEnsemble(base, alt, fb, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, []string{"iguana/pianissimo"}, *calls, "no second engine and no cloud once the job is cancelled")
|
||||
}
|
||||
|
||||
func TestEnsemble_AShutdownDuringTheSecondEngineIsAnErrorNotASingleEngineSuccess(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := &[]string{}
|
||||
base := &orderedBackend{name: "iguana/pianissimo", text: "base", calls: calls}
|
||||
alt := &cancelling{name: "iguana/kb-whisper", cancel: cancel, calls: calls}
|
||||
|
||||
got, err := transcribe.NewEnsemble(base, alt, nil, &verdictAdj{}, ensembleOpts).Transcribe(ctx, "/a.wav")
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled, "a cancelled job must not be recorded as 'kb-whisper failed'")
|
||||
assert.Empty(t, got.Text)
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// Package transcribe turns a WAV file into a timed Swedish transcript by calling
|
||||
// the LiteLLM gateway's OpenAI-compatible /audio/transcriptions endpoint
|
||||
// (ADR-033). A Gateway is one model alias; a Chain tries a local one and falls
|
||||
// back to a second.
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// ErrEmptyTranscript is returned when the endpoint answered 200 but with no text.
|
||||
// An empty transcript is not a success: chat over it could only answer "nothing".
|
||||
var ErrEmptyTranscript = errors.New("transcribe: endpoint returned an empty transcript")
|
||||
|
||||
// StatusError is a non-2xx answer from the gateway. Nothing may branch on the
|
||||
// body — {"error":"failed to process audio"} has had at least five distinct
|
||||
// causes — so only the status code is exposed for logic; Snippet is for logs.
|
||||
type StatusError struct {
|
||||
Code int
|
||||
Snippet string
|
||||
}
|
||||
|
||||
// HTTPStatus implements errkind.HTTPStatuser: the code, never the snippet.
|
||||
func (e *StatusError) HTTPStatus() int { return e.Code }
|
||||
|
||||
func (e *StatusError) Error() string {
|
||||
return fmt.Sprintf("transcribe: gateway returned HTTP %d: %s", e.Code, e.Snippet)
|
||||
}
|
||||
|
||||
// Gateway transcribes through one model alias on the gateway. baseURL is the
|
||||
// gateway's /v1 root — the in-cluster NodePort, not the public hostname, whose
|
||||
// ingress rejects bodies over ~8-10 MB with 413 (docs/research/audio-length-ceiling).
|
||||
type Gateway struct {
|
||||
baseURL string
|
||||
key string
|
||||
model string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewGateway builds a Gateway. The client should carry no overall Timeout: a 90
|
||||
// minute recording takes minutes, so the deadline belongs to the caller's context.
|
||||
func NewGateway(baseURL, key, model string, client *http.Client) *Gateway {
|
||||
return &Gateway{baseURL: strings.TrimRight(baseURL, "/"), key: key, model: model, client: client}
|
||||
}
|
||||
|
||||
// Name is the model alias; it is what gets recorded as the serving endpoint.
|
||||
func (g *Gateway) Name() string { return g.model }
|
||||
|
||||
// Transcribe uploads the WAV at path and returns the normalised transcription.
|
||||
// The body is streamed from disk with a COMPUTED Content-Length: a 90 minute WAV
|
||||
// is ~170 MB, so it is neither buffered in memory nor sent chunked.
|
||||
func (g *Gateway) Transcribe(ctx context.Context, path string) (domain.Transcription, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: open audio: %w", err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: stat audio: %w", err)
|
||||
}
|
||||
|
||||
var prefix bytes.Buffer
|
||||
mw := multipart.NewWriter(&prefix)
|
||||
for _, kv := range [][2]string{{"model", g.model}, {"response_format", "verbose_json"}, {"language", "sv"}} {
|
||||
if err := mw.WriteField(kv[0], kv[1]); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
}
|
||||
// The multipart field is `file`; `audio_file` is rejected as "Invalid request".
|
||||
if _, err := mw.CreateFormFile("file", "audio.wav"); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
suffix := []byte("\r\n--" + mw.Boundary() + "--\r\n")
|
||||
|
||||
body := io.MultiReader(bytes.NewReader(prefix.Bytes()), f, bytes.NewReader(suffix))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.baseURL+"/audio/transcriptions", body)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
req.ContentLength = int64(prefix.Len()) + info.Size() + int64(len(suffix))
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer "+g.key)
|
||||
// No "Expect: 100-continue": it trips the proxy for any body over 1 KB.
|
||||
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
snip, _ := io.ReadAll(io.LimitReader(resp.Body, 200))
|
||||
return domain.Transcription{}, &StatusError{Code: resp.StatusCode, Snippet: strings.TrimSpace(string(snip))}
|
||||
}
|
||||
|
||||
t, err := parseVerbose(resp.Body)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||
}
|
||||
t.Endpoint = g.model
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// verboseJSON is the subset of the verbose_json response used. Both backends
|
||||
// return it; they differ in segment granularity (iguana: whole seconds and
|
||||
// sub-sentence pieces, berget: sub-second sentences), in the language token, and in
|
||||
// whether a segment's text starts with a space.
|
||||
type verboseJSON struct {
|
||||
Text string `json:"text"`
|
||||
Duration float64 `json:"duration"`
|
||||
Segments []struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
|
||||
// parseVerbose normalises a verbose_json body. Whisper's final window can end past
|
||||
// the audio (9.0 s for 4.25 s of audio was observed), so segment ends are clamped
|
||||
// to the reported duration. Text is rebuilt from the segments, not taken from the
|
||||
// response's `text` field: see joinSegments.
|
||||
func parseVerbose(r io.Reader) (domain.Transcription, error) {
|
||||
var v verboseJSON
|
||||
if err := json.NewDecoder(r).Decode(&v); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
raw := make([]string, 0, len(v.Segments))
|
||||
for _, s := range v.Segments {
|
||||
raw = append(raw, s.Text)
|
||||
}
|
||||
text := joinSegments(raw)
|
||||
if text == "" {
|
||||
text = collapseSpace(v.Text)
|
||||
}
|
||||
if text == "" {
|
||||
return domain.Transcription{}, ErrEmptyTranscript
|
||||
}
|
||||
t := domain.Transcription{Text: text, AudioSeconds: v.Duration}
|
||||
for _, s := range v.Segments {
|
||||
seg := domain.Segment{Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text)}
|
||||
if v.Duration > 0 {
|
||||
seg.End = min(seg.End, v.Duration)
|
||||
seg.Start = min(seg.Start, seg.End)
|
||||
}
|
||||
t.Segments = append(t.Segments, seg)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// joinSegments rebuilds the running text from raw (untrimmed) segment texts.
|
||||
//
|
||||
// whisper.cpp wraps its output at ~60 characters and a wrap can land INSIDE a word
|
||||
// ("nyt" | "t godkännande"): a segment that begins with a space starts a new word,
|
||||
// one that does not continues the previous one, and the response's own `text`
|
||||
// field joins them with newlines, which breaks the word. So when any segment
|
||||
// carries a leading space the raw texts are concatenated as they are. Backends
|
||||
// that return whole sentences with no leading space (berget) are joined with a
|
||||
// space instead — concatenating those would glue sentences together.
|
||||
func joinSegments(raw []string) string {
|
||||
leadingSpace := false
|
||||
for _, s := range raw {
|
||||
if strings.HasPrefix(s, " ") {
|
||||
leadingSpace = true
|
||||
break
|
||||
}
|
||||
}
|
||||
sep := " "
|
||||
if leadingSpace {
|
||||
sep = ""
|
||||
}
|
||||
return collapseSpace(strings.Join(raw, sep))
|
||||
}
|
||||
|
||||
// collapseSpace turns every run of whitespace (including newlines) into one space.
|
||||
func collapseSpace(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
)
|
||||
|
||||
// The httptest server stands in for the LiteLLM gateway: it is the process
|
||||
// boundary, so a fake here is legitimate. The JSON it returns mirrors what the
|
||||
// real gateway returned for iguana/kb-whisper and berget/kb-whisper on 2026-10-01
|
||||
// (docs/research/audio-length-ceiling-2026-10-01.md). All text is synthetic.
|
||||
|
||||
const verboseJSON = `{
|
||||
"text": " God morgon allihopa.\n Vi går igenom läget.\n",
|
||||
"language": "swedish",
|
||||
"duration": 4.25,
|
||||
"segments": [
|
||||
{"id": 0, "start": 0.0, "end": 1.0, "text": " God morgon allihopa."},
|
||||
{"id": 1, "start": 1.0, "end": 9.0, "text": " Vi går igenom läget."}
|
||||
]
|
||||
}`
|
||||
|
||||
func wavFile(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "a.wav")
|
||||
require.NoError(t, os.WriteFile(p, []byte(content), 0o600))
|
||||
return p
|
||||
}
|
||||
|
||||
type seen struct {
|
||||
method, path, auth, expect, contentType string
|
||||
contentLength int64
|
||||
transferEncoding []string
|
||||
fields map[string]string
|
||||
file string
|
||||
fileField string
|
||||
bodyLen int
|
||||
}
|
||||
|
||||
// serve starts a gateway fake that records the request it received and replies
|
||||
// with status and body.
|
||||
func serve(t *testing.T, status int, reply string, got *seen) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got != nil {
|
||||
got.method, got.path = r.Method, r.URL.Path
|
||||
got.auth = r.Header.Get("Authorization")
|
||||
got.expect = r.Header.Get("Expect")
|
||||
got.contentType = r.Header.Get("Content-Type")
|
||||
got.contentLength = r.ContentLength
|
||||
got.transferEncoding = r.TransferEncoding
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
got.bodyLen = len(raw)
|
||||
r.Body = io.NopCloser(bytes.NewReader(raw))
|
||||
require.NoError(t, r.ParseMultipartForm(32<<20))
|
||||
got.fields = map[string]string{}
|
||||
for k, v := range r.MultipartForm.Value {
|
||||
got.fields[k] = v[0]
|
||||
}
|
||||
for field, fhs := range r.MultipartForm.File {
|
||||
got.fileField = field
|
||||
f, _ := fhs[0].Open()
|
||||
b, _ := io.ReadAll(f)
|
||||
got.file = string(b)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, reply)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestGateway_SendsTheRequestTheGatewayExpects(t *testing.T) {
|
||||
var got seen
|
||||
srv := serve(t, 200, verboseJSON, &got)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "secret-key", "iguana/kb-whisper", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "RIFF-synthetic-pcm"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.MethodPost, got.method)
|
||||
assert.Equal(t, "/v1/audio/transcriptions", got.path)
|
||||
assert.Equal(t, "Bearer secret-key", got.auth)
|
||||
assert.Equal(t, "file", got.fileField, "the multipart field is `file`; `audio_file` is rejected as 'Invalid request'")
|
||||
assert.Equal(t, "RIFF-synthetic-pcm", got.file)
|
||||
assert.Equal(t, "iguana/kb-whisper", got.fields["model"])
|
||||
assert.Equal(t, "verbose_json", got.fields["response_format"])
|
||||
assert.Equal(t, "sv", got.fields["language"])
|
||||
assert.Empty(t, got.expect, "no Expect: 100-continue: it trips the proxy for any body over 1 KB")
|
||||
}
|
||||
|
||||
func TestGateway_SendsAKnownContentLengthNotChunked(t *testing.T) {
|
||||
var got seen
|
||||
srv := serve(t, 200, verboseJSON, &got)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "0123456789"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got.transferEncoding, "a 90-minute WAV is ~170 MB: stream it with a computed length, not chunked and not buffered")
|
||||
assert.Equal(t, int64(got.bodyLen), got.contentLength)
|
||||
}
|
||||
|
||||
func TestGateway_NormalisesTheVerboseJSONResponse(t *testing.T) {
|
||||
srv := serve(t, 200, verboseJSON, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "God morgon allihopa. Vi går igenom läget.", got.Text)
|
||||
assert.Equal(t, "iguana/kb-whisper", got.Endpoint, "the serving endpoint is part of the result")
|
||||
assert.InDelta(t, 4.25, got.AudioSeconds, 1e-9)
|
||||
assert.Equal(t, []domain.Segment{
|
||||
{Start: 0, End: 1, Text: "God morgon allihopa."},
|
||||
// The final segment ended at 9.0 s for 4.25 s of audio: whisper's last
|
||||
// window can run past the end. Clamped to the audio duration.
|
||||
{Start: 1, End: 4.25, Text: "Vi går igenom läget."},
|
||||
}, got.Segments)
|
||||
}
|
||||
|
||||
func TestGateway_ErrorsDoNotDependOnTheBody(t *testing.T) {
|
||||
// {"error":"failed to process audio"} has had at least five distinct causes, so
|
||||
// no code may branch on it. Two different bodies must be treated identically.
|
||||
for _, reply := range []string{`{"error":"failed to process audio"}`, `{"error":"something else entirely"}`} {
|
||||
srv := serve(t, 500, reply, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
var se *transcribe.StatusError
|
||||
require.ErrorAs(t, err, &se)
|
||||
assert.Equal(t, 500, se.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGateway_AnEmptyTranscriptIsAFailureNotASuccess(t *testing.T) {
|
||||
srv := serve(t, 200, `{"text":" ","language":"swedish","duration":3,"segments":[]}`, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
assert.ErrorIs(t, err, transcribe.ErrEmptyTranscript)
|
||||
}
|
||||
|
||||
func TestGateway_MalformedJSONIsAnError(t *testing.T) {
|
||||
srv := serve(t, 200, `not json`, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
_, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestGateway_MissingAudioFileIsAnError(t *testing.T) {
|
||||
g := transcribe.NewGateway("http://127.0.0.1:1/v1", "k", "m", http.DefaultClient)
|
||||
|
||||
_, err := g.Transcribe(context.Background(), filepath.Join(t.TempDir(), "nope.wav"))
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// whisper.cpp wraps its output at ~60 characters, and a wrap can land INSIDE a word:
|
||||
// the continuation segment has no leading space, the `text` field joins segments
|
||||
// with newlines, and a segment can be a lone ".". Measured on real gateway output
|
||||
// 2026-10-01: 7% of a one-hour recording's segments were such fragments. Joining by
|
||||
// newline hands the chat model broken words ("nyt\nt"), so Text is rebuilt from the
|
||||
// segments: a segment that begins with a space starts a new word, one that does not
|
||||
// continues the previous one.
|
||||
func TestGateway_RejoinsWordsWhisperCppSplitAcrossSegments(t *testing.T) {
|
||||
reply := `{
|
||||
"text": " Vi ligger just nu 12 procent över den ursprungliga kalkylen\n.\n Det kräver ett nyt\nt godkännande från beställaren.\n Framförallt på arbetspl\natsen.\n",
|
||||
"language": "swedish", "duration": 20,
|
||||
"segments": [
|
||||
{"start": 0, "end": 4, "text": " Vi ligger just nu 12 procent över den ursprungliga kalkylen"},
|
||||
{"start": 4, "end": 5, "text": "."},
|
||||
{"start": 5, "end": 8, "text": " Det kräver ett nyt"},
|
||||
{"start": 8, "end": 11, "text": "t godkännande från beställaren."},
|
||||
{"start": 11, "end": 14, "text": " Framförallt på arbetspl"},
|
||||
{"start": 14, "end": 16, "text": "atsen."}
|
||||
]}`
|
||||
srv := serve(t, 200, reply, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "iguana/kb-whisper", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Vi ligger just nu 12 procent över den ursprungliga kalkylen. "+
|
||||
"Det kräver ett nytt godkännande från beställaren. Framförallt på arbetsplatsen.", got.Text)
|
||||
assert.Len(t, got.Segments, 6, "the timed segments are kept as the backend gave them (trimmed)")
|
||||
assert.Equal(t, ".", got.Segments[1].Text)
|
||||
}
|
||||
|
||||
func TestGateway_SentenceLevelSegmentsWithNoLeadingSpaceAreJoinedWithASpace(t *testing.T) {
|
||||
// berget/kb-whisper returns whole sentences with no leading space; rebuilding by
|
||||
// concatenation would glue them ("allihopa!Vi").
|
||||
reply := `{"text": "God morgon allihopa!\nVi går igenom läget.", "language": "sv", "duration": 5,
|
||||
"segments": [
|
||||
{"start": 0.05, "end": 1.2, "text": "God morgon allihopa!"},
|
||||
{"start": 1.6, "end": 4.2, "text": "Vi går igenom läget."}
|
||||
]}`
|
||||
srv := serve(t, 200, reply, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "berget/kb-whisper", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "God morgon allihopa! Vi går igenom läget.", got.Text)
|
||||
}
|
||||
|
||||
func TestGateway_WithNoSegmentsTheTextFieldIsUsedWithWhitespaceCollapsed(t *testing.T) {
|
||||
srv := serve(t, 200, `{"text": " Hej allihopa.\n Vi börjar. ", "duration": 3, "segments": []}`, nil)
|
||||
g := transcribe.NewGateway(srv.URL+"/v1", "k", "m", srv.Client())
|
||||
|
||||
got, err := g.Transcribe(context.Background(), wavFile(t, "x"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Hej allihopa. Vi börjar.", got.Text)
|
||||
}
|
||||
|
||||
func TestStatusError_ExposesItsCodeAsAKindAndNotItsSnippet(t *testing.T) {
|
||||
err := fmt.Errorf("transcribe: %w", &transcribe.StatusError{Code: 502, Snippet: "echoed: Anders ansvarar"})
|
||||
|
||||
assert.Equal(t, "http_502", errkind.Of(err))
|
||||
}
|
||||
|
||||
func TestGateway_AnErrorBodyIsBoundedToTwoHundredBytesInTheStatusError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte(strings.Repeat("x", 100_000)))
|
||||
}))
|
||||
defer srv.Close()
|
||||
wav := filepath.Join(t.TempDir(), "a.wav")
|
||||
require.NoError(t, os.WriteFile(wav, []byte("RIFF....WAVEfmt "), 0o600))
|
||||
|
||||
_, err := transcribe.NewGateway(srv.URL, "", "iguana/kb-whisper", srv.Client()).Transcribe(context.Background(), wav)
|
||||
|
||||
var se *transcribe.StatusError
|
||||
require.ErrorAs(t, err, &se)
|
||||
assert.LessOrEqual(t, len(se.Snippet), 200, "a gateway's whole body must not become part of an error")
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
)
|
||||
|
||||
// TestGateway_LiveGateway is an opt-in check against the REAL gateway, because
|
||||
// the httptest fake above can only prove what we think the gateway accepts. It
|
||||
// is skipped unless TAPIR_LIVE_WAV names a 16 kHz mono WAV and the gateway
|
||||
// coordinates are set:
|
||||
//
|
||||
// TAPIR_LIVE_WAV=/path/a.wav TAPIR_LIVE_GATEWAY_URL=http://koala:30401/v1 \
|
||||
// TAPIR_LIVE_MODEL=iguana/kb-whisper op run --env-file ~/.op-env -- \
|
||||
// go test ./internal/adapters/transcribe -run LiveGateway -v
|
||||
//
|
||||
// Use SYNTHETIC audio only: this repo is public and the gateway may be a cloud
|
||||
// endpoint (#33).
|
||||
func TestGateway_LiveGateway(t *testing.T) {
|
||||
wav := os.Getenv("TAPIR_LIVE_WAV")
|
||||
url := os.Getenv("TAPIR_LIVE_GATEWAY_URL")
|
||||
if wav == "" || url == "" {
|
||||
t.Skip("set TAPIR_LIVE_WAV and TAPIR_LIVE_GATEWAY_URL to run against the real gateway")
|
||||
}
|
||||
model := os.Getenv("TAPIR_LIVE_MODEL")
|
||||
if model == "" {
|
||||
model = "iguana/kb-whisper"
|
||||
}
|
||||
key := os.Getenv("LITELLM_MASTER_KEY")
|
||||
require.NotEmpty(t, key, "LITELLM_MASTER_KEY must be set (via op run)")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute)
|
||||
defer cancel()
|
||||
got, err := transcribe.NewGateway(url, key, model, &http.Client{}).Transcribe(ctx, wav)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, got.Text)
|
||||
assert.NotEmpty(t, got.Segments)
|
||||
assert.Equal(t, model, got.Endpoint)
|
||||
t.Logf("endpoint=%s audio_s=%.1f segments=%d chars=%d", got.Endpoint, got.AudioSeconds, len(got.Segments), len(got.Text))
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// WithTimeout gives every call to b its own deadline. A wedged engine does not answer at
|
||||
// all (iguana after a Metal OOM, infra#273), and with only the job's deadline it used up
|
||||
// the whole job before a fallback could run -- and Chain then refuses the fallback because
|
||||
// the job context has expired. With its own deadline the call fails while the job is still
|
||||
// alive, which is exactly when the fallback, or the other engine, can take over.
|
||||
//
|
||||
// A zero or negative d returns b unchanged. The wrapper is invisible: Name is b's.
|
||||
func WithTimeout(b Backend, d time.Duration) Backend {
|
||||
if d <= 0 {
|
||||
return b
|
||||
}
|
||||
return timeoutBackend{Backend: b, d: d}
|
||||
}
|
||||
|
||||
type timeoutBackend struct {
|
||||
Backend
|
||||
d time.Duration
|
||||
}
|
||||
|
||||
func (t timeoutBackend) Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, t.d)
|
||||
defer cancel()
|
||||
return t.Backend.Transcribe(ctx, wavPath)
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package transcribe_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// A wedged engine (iguana after a Metal OOM, infra#273) does not answer at all. With only
|
||||
// the job's deadline, the primary used up the whole job before any fallback could run, and
|
||||
// the chain then refused the fallback because the job context had expired. Each engine call
|
||||
// gets its own deadline (adversarial review F4).
|
||||
|
||||
type hangingBackend struct{ name string }
|
||||
|
||||
func (h hangingBackend) Name() string { return h.name }
|
||||
func (hangingBackend) Transcribe(ctx context.Context, _ string) (domain.Transcription, error) {
|
||||
<-ctx.Done()
|
||||
return domain.Transcription{}, ctx.Err()
|
||||
}
|
||||
|
||||
func TestWithTimeout_AHungBackendFailsAtItsOwnDeadlineWhileTheJobIsStillAlive(t *testing.T) {
|
||||
job, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
b := transcribe.WithTimeout(hangingBackend{name: "iguana/kb-whisper"}, 40*time.Millisecond)
|
||||
|
||||
start := time.Now()
|
||||
_, err := b.Transcribe(job, "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "timeout", errkind.Of(err))
|
||||
assert.Less(t, time.Since(start), 2*time.Second, "it must give up at its own deadline, not wait for the job's")
|
||||
assert.NoError(t, job.Err(), "the job context is untouched, which is what lets a fallback run")
|
||||
assert.Equal(t, "iguana/kb-whisper", b.Name(), "the wrapper is invisible in the recorded endpoint")
|
||||
}
|
||||
|
||||
func TestWithTimeout_ASuccessPassesStraightThrough(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
inner := &orderedBackend{name: "iguana/kb-whisper", text: "hej", calls: calls}
|
||||
|
||||
got, err := transcribe.WithTimeout(inner, time.Minute).Transcribe(context.Background(), "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hej", got.Text)
|
||||
}
|
||||
|
||||
func TestWithTimeout_ZeroMeansNoDeadlineOfItsOwn(t *testing.T) {
|
||||
inner := &orderedBackend{name: "x", text: "hej", calls: &[]string{}}
|
||||
|
||||
assert.Same(t, inner, transcribe.WithTimeout(inner, 0), "a zero timeout must not wrap or change anything")
|
||||
}
|
||||
|
||||
func TestChain_AHungPrimaryReachesTheFallbackWhenItHasItsOwnDeadline(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
fb := &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
|
||||
c := transcribe.NewChain(transcribe.WithTimeout(hangingBackend{name: "iguana/kb-whisper"}, 40*time.Millisecond), fb)
|
||||
job, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
got, err := c.Transcribe(job, "/a.wav")
|
||||
|
||||
require.NoError(t, err, "a wedged primary is exactly what the fallback exists for")
|
||||
assert.Equal(t, "berget/kb-whisper", got.Endpoint)
|
||||
}
|
||||
|
||||
func TestChain_AJobThatEndedWhileThePrimaryHungStillNeverReachesTheFallback(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
fb := &orderedBackend{name: "berget/kb-whisper", text: "fallback text", calls: calls}
|
||||
c := transcribe.NewChain(transcribe.WithTimeout(hangingBackend{name: "iguana/kb-whisper"}, time.Minute), fb)
|
||||
job, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
_, err := c.Transcribe(job, "/a.wav")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, *calls, "the job's own deadline or a cancel must still stop the audio from reaching a cloud endpoint")
|
||||
}
|
||||
|
||||
func TestEnsemble_AHungEngineIsASoftFailureAndTheOtherEnginesTextIsUsed(t *testing.T) {
|
||||
calls := &[]string{}
|
||||
alt := &orderedBackend{name: "iguana/kb-whisper", text: "text from the other engine", calls: calls}
|
||||
e := transcribe.NewEnsemble(
|
||||
transcribe.WithTimeout(hangingBackend{name: "iguana/pianissimo"}, 40*time.Millisecond), alt, nil,
|
||||
&verdictAdj{v: reconcile.VerdictAlt}, ensembleOpts)
|
||||
job, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
got, err := e.Transcribe(job, "/a.wav")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "text from the other engine", got.Text)
|
||||
assert.Equal(t, domain.ReconcileSingle, got.Reconciliation.Status)
|
||||
assert.Contains(t, got.Reconciliation.Reason, "timeout")
|
||||
assert.False(t, errors.Is(err, context.DeadlineExceeded))
|
||||
}
|
||||
@@ -20,6 +20,9 @@ import (
|
||||
|
||||
// Config is Tapir's fully-resolved runtime configuration.
|
||||
type Config struct {
|
||||
// UploadConfig is the audio-upload and private-transcription settings (ADR-033).
|
||||
UploadConfig
|
||||
|
||||
// UserID is the Tapir user the run operates as. Stage 0 has exactly one.
|
||||
// It must be a UUID: it keys the UUID user_id/video_id columns (data-model).
|
||||
UserID string
|
||||
@@ -320,6 +323,10 @@ func Load() (Config, error) {
|
||||
}
|
||||
c.OnboardMaxVideoSeconds = onboardMax
|
||||
|
||||
if c.UploadConfig, err = loadUploadConfig(); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// UploadConfig holds the settings for audio upload and private transcription
|
||||
// (ADR-033). It is embedded in Config, so its fields read as c.UploadsEnabled etc.
|
||||
type UploadConfig struct {
|
||||
// UploadsEnabled mounts the /uploads routes and starts the worker. Off by
|
||||
// default: the runtime image must contain ffmpeg and ffprobe first (ADR-033).
|
||||
UploadsEnabled bool
|
||||
// UploadMaxBytes caps one upload. UploadMaxDuration caps one recording; 90
|
||||
// minutes is the longest length MEASURED, not a ceiling that was found, so
|
||||
// raising it is a measurement, not an edit.
|
||||
UploadMaxBytes int64
|
||||
UploadMaxDuration time.Duration
|
||||
// TranscribeModel is the primary gateway alias; TranscribeFallbackModel is the
|
||||
// fallback, tried on ANY primary failure. The fallback is a cloud endpoint
|
||||
// (berget): set it EMPTY to keep private audio on the homelab.
|
||||
TranscribeModel string
|
||||
TranscribeFallbackModel string
|
||||
// TranscribeTimeout bounds one job (conversion + transcription); a 90 minute
|
||||
// recording took ~7.5 minutes on iguana.
|
||||
TranscribeTimeout time.Duration
|
||||
// TranscribeEngineTimeout bounds ONE engine call, so a wedged engine fails while the
|
||||
// job is still alive and the fallback (or the other engine) can take over. The job
|
||||
// timeout alone cannot do that (ADR-035 review).
|
||||
TranscribeEngineTimeout time.Duration
|
||||
// UploadTmpDir is where uploads are spooled and converted. Audio is deleted
|
||||
// after transcription, so this holds at most the in-flight files.
|
||||
UploadTmpDir string
|
||||
FFmpegBin string
|
||||
FFprobeBin string
|
||||
// MaxUploadedTranscriptChars bounds the transcript sent to a chat model for an
|
||||
// UPLOADED recording. The 18,000-char default for captions is ~22 minutes of
|
||||
// speech; an hour of meeting is ~50,000 chars. MEASURED 2026-10-01 against the
|
||||
// real models: iguana/gemma4-26b answers correctly to 90,000 chars but silently
|
||||
// drops the head of a 120,000-char transcript, so 90,000 is the ceiling of the
|
||||
// best model, not a round number.
|
||||
MaxUploadedTranscriptChars int
|
||||
// UploadChatModel is the model chat over an uploaded recording opens with. It is
|
||||
// NOT the summarizer default: koala/phi4-mini has an 8,192-token context and
|
||||
// returns HTTP 400 on an hour of speech (~13,000 tokens).
|
||||
UploadChatModel string
|
||||
// ReconcileBaseModel turns on two-engine transcription (ADR-034): the engine whose
|
||||
// TEXT is the base of the merged transcript (e.g. iguana/pianissimo). TranscribeModel
|
||||
// stays the other engine (e.g. kb-whisper). Empty = off.
|
||||
ReconcileBaseModel string
|
||||
// ReconcileModel is the LOCAL chat model that chooses between the two engines'
|
||||
// disagreements. Required when ReconcileBaseModel is set.
|
||||
ReconcileModel string
|
||||
// ReconcileCheckOrder asks the model twice per dispute with the reading order
|
||||
// swapped and keeps only agreement. It doubles the time; off by default.
|
||||
ReconcileCheckOrder bool
|
||||
}
|
||||
|
||||
const (
|
||||
defaultUploadMaxMB = 200
|
||||
defaultUploadMaxMinutes = 90
|
||||
defaultTranscribeModel = "iguana/kb-whisper"
|
||||
defaultTranscribeFallbackModel = "berget/kb-whisper"
|
||||
defaultTranscribeTimeout = 60 * time.Minute
|
||||
defaultTranscribeEngineTimeout = 20 * time.Minute
|
||||
defaultMaxUploadedChars = 90000
|
||||
defaultUploadChatModel = "iguana/gemma4-26b"
|
||||
)
|
||||
|
||||
func loadUploadConfig() (UploadConfig, error) {
|
||||
u := UploadConfig{
|
||||
TranscribeModel: envOr("TAPIR_TRANSCRIBE_MODEL", defaultTranscribeModel),
|
||||
TranscribeFallbackModel: lookupOr("TAPIR_TRANSCRIBE_FALLBACK_MODEL", defaultTranscribeFallbackModel),
|
||||
UploadTmpDir: envOr("TAPIR_UPLOAD_TMP_DIR", os.TempDir()),
|
||||
FFmpegBin: envOr("TAPIR_FFMPEG_BIN", "ffmpeg"),
|
||||
FFprobeBin: envOr("TAPIR_FFPROBE_BIN", "ffprobe"),
|
||||
UploadChatModel: envOr("TAPIR_UPLOAD_CHAT_MODEL", defaultUploadChatModel),
|
||||
ReconcileBaseModel: os.Getenv("TAPIR_RECONCILE_BASE_MODEL"),
|
||||
ReconcileModel: os.Getenv("TAPIR_RECONCILE_MODEL"),
|
||||
}
|
||||
|
||||
if v := os.Getenv("TAPIR_UPLOADS_ENABLED"); v != "" {
|
||||
on, err := strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return UploadConfig{}, fmt.Errorf("config: TAPIR_UPLOADS_ENABLED=%q: %w", v, err)
|
||||
}
|
||||
u.UploadsEnabled = on
|
||||
}
|
||||
|
||||
mb, err := positiveInt("TAPIR_UPLOAD_MAX_MB", defaultUploadMaxMB)
|
||||
if err != nil {
|
||||
return UploadConfig{}, err
|
||||
}
|
||||
u.UploadMaxBytes = int64(mb) << 20
|
||||
|
||||
mins, err := positiveInt("TAPIR_UPLOAD_MAX_MINUTES", defaultUploadMaxMinutes)
|
||||
if err != nil {
|
||||
return UploadConfig{}, err
|
||||
}
|
||||
u.UploadMaxDuration = time.Duration(mins) * time.Minute
|
||||
|
||||
timeout, err := durationOr("TAPIR_TRANSCRIBE_TIMEOUT", defaultTranscribeTimeout)
|
||||
if err != nil {
|
||||
return UploadConfig{}, err
|
||||
}
|
||||
if timeout <= 0 {
|
||||
return UploadConfig{}, fmt.Errorf("config: TAPIR_TRANSCRIBE_TIMEOUT must be positive, got %v", timeout)
|
||||
}
|
||||
u.TranscribeTimeout = timeout
|
||||
|
||||
engineTimeout, err := durationOr("TAPIR_TRANSCRIBE_ENGINE_TIMEOUT", defaultTranscribeEngineTimeout)
|
||||
if err != nil {
|
||||
return UploadConfig{}, err
|
||||
}
|
||||
if engineTimeout <= 0 {
|
||||
return UploadConfig{}, fmt.Errorf("config: TAPIR_TRANSCRIBE_ENGINE_TIMEOUT must be positive, got %v", engineTimeout)
|
||||
}
|
||||
u.TranscribeEngineTimeout = engineTimeout
|
||||
|
||||
if v := os.Getenv("TAPIR_RECONCILE_CHECK_ORDER"); v != "" {
|
||||
on, err := strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return UploadConfig{}, fmt.Errorf("config: TAPIR_RECONCILE_CHECK_ORDER=%q: %w", v, err)
|
||||
}
|
||||
u.ReconcileCheckOrder = on
|
||||
}
|
||||
if err := u.validateReconcile(); err != nil {
|
||||
return UploadConfig{}, err
|
||||
}
|
||||
if !IsLocalModel(u.UploadChatModel) {
|
||||
return UploadConfig{}, fmt.Errorf("config: TAPIR_UPLOAD_CHAT_MODEL=%q is not a local model (koala/ or iguana/): the whole private transcript is sent to it", u.UploadChatModel)
|
||||
}
|
||||
|
||||
chars, err := intOr("TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS", defaultMaxUploadedChars)
|
||||
if err != nil {
|
||||
return UploadConfig{}, err
|
||||
}
|
||||
if chars < 0 {
|
||||
chars = 0
|
||||
}
|
||||
u.MaxUploadedTranscriptChars = chars
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// positiveInt is intOr that refuses zero and negatives: a zero upload limit would
|
||||
// reject every file, which is a misconfiguration to fail on, not to run with.
|
||||
func positiveInt(key string, fallback int) (int, error) {
|
||||
n, err := intOr(key, fallback)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if n <= 0 {
|
||||
return 0, fmt.Errorf("config: %s must be positive, got %d", key, n)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// localAlias is the shape of a model alias that runs on the homelab: a local host
|
||||
// prefix, then one plain name. A bare prefix check let through aliases that are shaped
|
||||
// like an escape (koala/../berget/x, koala//x, koala/x?host=berget): whether the
|
||||
// gateway would route them elsewhere is unknown, so they are refused (ADR-035 review).
|
||||
var localAlias = regexp.MustCompile(`^(koala|iguana)/[A-Za-z0-9][A-Za-z0-9._:-]*$`)
|
||||
|
||||
// IsLocalModel reports whether alias is a local model. Private meeting content may be
|
||||
// sent to these and to nothing else: reconciliation (ADR-034) and chat over an uploaded
|
||||
// recording (ADR-035) are both held to it, and a cloud alias is refused at start-up
|
||||
// rather than discovered later.
|
||||
func IsLocalModel(alias string) bool {
|
||||
return localAlias.MatchString(alias) && !strings.Contains(alias, "..")
|
||||
}
|
||||
|
||||
// validateReconcile checks the two-engine settings. They are inert while no base
|
||||
// engine is set; once one is, the setup must be complete and entirely local.
|
||||
func (u UploadConfig) validateReconcile() error {
|
||||
if u.ReconcileBaseModel == "" {
|
||||
return nil
|
||||
}
|
||||
if u.ReconcileModel == "" {
|
||||
return fmt.Errorf("config: TAPIR_RECONCILE_BASE_MODEL is set but TAPIR_RECONCILE_MODEL (the adjudicator) is not")
|
||||
}
|
||||
for _, m := range []struct{ name, alias string }{
|
||||
{"TAPIR_RECONCILE_BASE_MODEL", u.ReconcileBaseModel},
|
||||
{"TAPIR_RECONCILE_MODEL", u.ReconcileModel},
|
||||
} {
|
||||
if !IsLocalModel(m.alias) {
|
||||
return fmt.Errorf("config: %s=%q is not a local model (koala/ or iguana/): private meeting content must not leave the homelab for reconciliation", m.name, m.alias)
|
||||
}
|
||||
}
|
||||
if u.ReconcileBaseModel == u.TranscribeModel {
|
||||
return fmt.Errorf("config: TAPIR_RECONCILE_BASE_MODEL and TAPIR_TRANSCRIBE_MODEL are both %q: two engines must differ", u.TranscribeModel)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// What counts as a LOCAL model alias, and where that is demanded (ADR-034, ADR-035 review).
|
||||
// A private transcript may be sent to a local model and to nothing else; the check is a
|
||||
// guard against a misconfiguration or a crafted alias, so it must be strict about shape,
|
||||
// not just about a prefix.
|
||||
|
||||
func TestIsLocalModel_AcceptsRealLocalAliasesAndNothingShapedLikeAnEscape(t *testing.T) {
|
||||
tests := []struct {
|
||||
alias string
|
||||
want bool
|
||||
}{
|
||||
{"koala/phi4-mini", true},
|
||||
{"iguana/gemma4-26b", true},
|
||||
{"iguana/gemma4-31b-think", true},
|
||||
{"koala/qwen36-35b-a3b", true},
|
||||
{"iguana/kb-whisper", true},
|
||||
{"iguana/hf.co-model_v1:q4", true},
|
||||
{"", false},
|
||||
{"koala/", false},
|
||||
{"iguana/", false},
|
||||
{"gemma4-26b", false},
|
||||
{"berget/mistral-small", false},
|
||||
{"berget/koala/x", false},
|
||||
{"xkoala/x", false},
|
||||
{"Koala/x", false},
|
||||
{" koala/x", false},
|
||||
{"koala/x ", false},
|
||||
{"koala/x y", false},
|
||||
{"koala/x\n", false},
|
||||
{"koala/../berget/kb", false},
|
||||
{"koala//berget/mistral-small", false},
|
||||
{"koala/x/y", false},
|
||||
{"koala/x?host=berget", false},
|
||||
{"koala/x#frag", false},
|
||||
{"koala/x@evil", false},
|
||||
{"koalа/x", false}, // Cyrillic а
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.alias, func(t *testing.T) {
|
||||
if got := IsLocalModel(tt.alias); got != tt.want {
|
||||
t.Errorf("IsLocalModel(%q) = %v, want %v", tt.alias, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_UploadChatModelMustBeLocalBecauseTheWholeTranscriptIsSentToIt(t *testing.T) {
|
||||
for _, alias := range []string{"berget/mistral-small", "gemini/gemini-2.5-pro", "gemma4-26b", "koala/../berget/kb"} {
|
||||
t.Run(alias, func(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
setEnv(t, map[string]string{"TAPIR_UPLOAD_CHAT_MODEL": alias})
|
||||
|
||||
if _, err := Load(); err == nil {
|
||||
t.Errorf("Load accepted TAPIR_UPLOAD_CHAT_MODEL=%q: a private transcript of up to 90,000 characters would be sent to it", alias)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_ALocalUploadChatModelIsAccepted(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
setEnv(t, map[string]string{"TAPIR_UPLOAD_CHAT_MODEL": "iguana/gemma4-31b"})
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.UploadChatModel != "iguana/gemma4-31b" {
|
||||
t.Errorf("UploadChatModel = %q", cfg.UploadChatModel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_TheEngineTimeoutDefaultsToTwentyMinutesAndMustBePositive(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.TranscribeEngineTimeout != 20*time.Minute {
|
||||
t.Errorf("TranscribeEngineTimeout = %v, want 20m: a 90-minute recording takes about 8 minutes on kb-whisper, and two wedged engines in sequence must leave the job time to reach a fallback", cfg.TranscribeEngineTimeout)
|
||||
}
|
||||
|
||||
setEnv(t, map[string]string{"TAPIR_TRANSCRIBE_ENGINE_TIMEOUT": "7m"})
|
||||
cfg, err = Load()
|
||||
if err != nil || cfg.TranscribeEngineTimeout != 7*time.Minute {
|
||||
t.Errorf("override: %v, %v", cfg.TranscribeEngineTimeout, err)
|
||||
}
|
||||
|
||||
for _, bad := range []string{"0", "-1m", "soon"} {
|
||||
setEnv(t, map[string]string{"TAPIR_TRANSCRIBE_ENGINE_TIMEOUT": bad})
|
||||
if _, err := Load(); err == nil {
|
||||
t.Errorf("Load accepted TAPIR_TRANSCRIBE_ENGINE_TIMEOUT=%q", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// clearUploadEnv unsets every upload knob so a test observes the default, not the
|
||||
// host environment.
|
||||
func clearUploadEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, k := range []string{
|
||||
"TAPIR_UPLOADS_ENABLED", "TAPIR_UPLOAD_MAX_MB", "TAPIR_UPLOAD_MAX_MINUTES",
|
||||
"TAPIR_TRANSCRIBE_MODEL", "TAPIR_TRANSCRIBE_FALLBACK_MODEL", "TAPIR_TRANSCRIBE_TIMEOUT", "TAPIR_TRANSCRIBE_ENGINE_TIMEOUT",
|
||||
"TAPIR_UPLOAD_TMP_DIR", "TAPIR_FFMPEG_BIN", "TAPIR_FFPROBE_BIN",
|
||||
"TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS", "TAPIR_UPLOAD_CHAT_MODEL",
|
||||
"TAPIR_RECONCILE_BASE_MODEL", "TAPIR_RECONCILE_MODEL", "TAPIR_RECONCILE_CHECK_ORDER",
|
||||
} {
|
||||
unset(t, k)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_UploadDefaults(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
if c.UploadsEnabled {
|
||||
t.Error("uploads must be OFF by default: the runtime image needs ffmpeg first (ADR-033)")
|
||||
}
|
||||
if c.UploadMaxBytes != 200<<20 {
|
||||
t.Errorf("UploadMaxBytes = %d, want 200 MB", c.UploadMaxBytes)
|
||||
}
|
||||
if c.UploadMaxDuration != 90*time.Minute {
|
||||
t.Errorf("UploadMaxDuration = %v, want 90m (the longest length measured, not a found ceiling)", c.UploadMaxDuration)
|
||||
}
|
||||
if c.TranscribeModel != "iguana/kb-whisper" {
|
||||
t.Errorf("TranscribeModel = %q", c.TranscribeModel)
|
||||
}
|
||||
if c.TranscribeFallbackModel != "berget/kb-whisper" {
|
||||
t.Errorf("TranscribeFallbackModel = %q", c.TranscribeFallbackModel)
|
||||
}
|
||||
if c.TranscribeTimeout != 60*time.Minute {
|
||||
t.Errorf("TranscribeTimeout = %v, want 60m (ADR-034: two engines plus ~300 adjudications)", c.TranscribeTimeout)
|
||||
}
|
||||
if c.ReconcileBaseModel != "" || c.ReconcileModel != "" || c.ReconcileCheckOrder {
|
||||
t.Errorf("reconciliation must be OFF by default: %q %q %v", c.ReconcileBaseModel, c.ReconcileModel, c.ReconcileCheckOrder)
|
||||
}
|
||||
if c.UploadTmpDir != os.TempDir() {
|
||||
t.Errorf("UploadTmpDir = %q, want os.TempDir()", c.UploadTmpDir)
|
||||
}
|
||||
if c.FFmpegBin != "ffmpeg" || c.FFprobeBin != "ffprobe" {
|
||||
t.Errorf("bins = %q, %q", c.FFmpegBin, c.FFprobeBin)
|
||||
}
|
||||
if c.MaxUploadedTranscriptChars != 90000 {
|
||||
t.Errorf("MaxUploadedTranscriptChars = %d, want 90000: iguana/gemma4-26b answered correctly at 90,000 chars "+
|
||||
"but silently dropped the head of a 120,000-char transcript (docs/research/chat-context)", c.MaxUploadedTranscriptChars)
|
||||
}
|
||||
if c.UploadChatModel != "iguana/gemma4-26b" {
|
||||
t.Errorf("UploadChatModel = %q: koala/phi4-mini has an 8,192-token context and answers HTTP 400 on an hour of speech", c.UploadChatModel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_UploadChatModelOverride(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
t.Setenv("TAPIR_UPLOAD_CHAT_MODEL", "koala/qwen36-35b-a3b")
|
||||
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
if c.UploadChatModel != "koala/qwen36-35b-a3b" {
|
||||
t.Errorf("UploadChatModel = %q", c.UploadChatModel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_UploadValuesAreParsed(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
setEnv(t, map[string]string{
|
||||
"TAPIR_UPLOADS_ENABLED": "true",
|
||||
"TAPIR_UPLOAD_MAX_MB": "50",
|
||||
"TAPIR_UPLOAD_MAX_MINUTES": "30",
|
||||
"TAPIR_TRANSCRIBE_MODEL": "koala/whisper",
|
||||
"TAPIR_TRANSCRIBE_TIMEOUT": "10m",
|
||||
"TAPIR_UPLOAD_TMP_DIR": "/var/spool/tapir",
|
||||
"TAPIR_FFMPEG_BIN": "/opt/ffmpeg",
|
||||
"TAPIR_FFPROBE_BIN": "/opt/ffprobe",
|
||||
"TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS": "60000",
|
||||
})
|
||||
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
if !c.UploadsEnabled || c.UploadMaxBytes != 50<<20 || c.UploadMaxDuration != 30*time.Minute ||
|
||||
c.TranscribeModel != "koala/whisper" || c.TranscribeTimeout != 10*time.Minute ||
|
||||
c.UploadTmpDir != "/var/spool/tapir" || c.FFmpegBin != "/opt/ffmpeg" ||
|
||||
c.FFprobeBin != "/opt/ffprobe" || c.MaxUploadedTranscriptChars != 60000 {
|
||||
t.Errorf("values not applied: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_EmptyTranscribeFallbackDisablesIt(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
t.Setenv("TAPIR_TRANSCRIBE_FALLBACK_MODEL", "")
|
||||
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
if c.TranscribeFallbackModel != "" {
|
||||
t.Errorf("TranscribeFallbackModel = %q: an explicit empty value must disable the cloud fallback, "+
|
||||
"so private audio can be kept on the homelab", c.TranscribeFallbackModel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_UploadRejectsInvalidValues(t *testing.T) {
|
||||
tests := []struct{ key, val string }{
|
||||
{"TAPIR_UPLOADS_ENABLED", "maybe"},
|
||||
{"TAPIR_UPLOAD_MAX_MB", "lots"},
|
||||
{"TAPIR_UPLOAD_MAX_MB", "0"},
|
||||
{"TAPIR_UPLOAD_MAX_MB", "-5"},
|
||||
{"TAPIR_UPLOAD_MAX_MINUTES", "0"},
|
||||
{"TAPIR_UPLOAD_MAX_MINUTES", "forever"},
|
||||
{"TAPIR_TRANSCRIBE_TIMEOUT", "soon"},
|
||||
{"TAPIR_TRANSCRIBE_TIMEOUT", "0s"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.key+"="+tt.val, func(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
t.Setenv(tt.key, tt.val)
|
||||
|
||||
if _, err := Load(); err == nil {
|
||||
t.Errorf("Load accepted %s=%q; a zero or negative limit would reject every upload", tt.key, tt.val)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_ReconcileSettingsAreParsed(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
setEnv(t, map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "iguana/pianissimo",
|
||||
"TAPIR_RECONCILE_MODEL": "iguana/gemma4-31b",
|
||||
"TAPIR_RECONCILE_CHECK_ORDER": "true",
|
||||
})
|
||||
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
if c.ReconcileBaseModel != "iguana/pianissimo" || c.ReconcileModel != "iguana/gemma4-31b" || !c.ReconcileCheckOrder {
|
||||
t.Errorf("not applied: %+v", c.UploadConfig)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_ReconcileRefusesAnUnsafeOrIncompleteSetup(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
}{
|
||||
{"a base engine with no adjudicator", map[string]string{"TAPIR_RECONCILE_BASE_MODEL": "iguana/pianissimo"}},
|
||||
{"a cloud adjudicator: private meeting content must not reach it", map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "iguana/pianissimo", "TAPIR_RECONCILE_MODEL": "berget/mistral-small",
|
||||
}},
|
||||
{"a cloud adjudicator from another provider", map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "iguana/pianissimo", "TAPIR_RECONCILE_MODEL": "gemini/gemini-2.5-pro",
|
||||
}},
|
||||
{"an adjudicator alias with no host prefix", map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "iguana/pianissimo", "TAPIR_RECONCILE_MODEL": "gemma4-26b",
|
||||
}},
|
||||
{"a cloud alias that merely contains a local host name", map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "iguana/pianissimo", "TAPIR_RECONCILE_MODEL": "berget/iguana/gemma4-26b",
|
||||
}},
|
||||
{"a cloud base engine", map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "berget/kb-whisper", "TAPIR_RECONCILE_MODEL": "iguana/gemma4-26b",
|
||||
}},
|
||||
{"the same engine twice", map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "iguana/kb-whisper", "TAPIR_RECONCILE_MODEL": "iguana/gemma4-26b",
|
||||
}},
|
||||
{"a bad order-check flag", map[string]string{
|
||||
"TAPIR_RECONCILE_BASE_MODEL": "iguana/pianissimo", "TAPIR_RECONCILE_MODEL": "iguana/gemma4-26b", "TAPIR_RECONCILE_CHECK_ORDER": "perhaps",
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
setEnv(t, tt.env)
|
||||
|
||||
if _, err := Load(); err == nil {
|
||||
t.Errorf("Load accepted %v", tt.env)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_AnAdjudicatorWithoutABaseEngineIsIgnoredNotAnError(t *testing.T) {
|
||||
clearUploadEnv(t)
|
||||
t.Setenv("TAPIR_RECONCILE_MODEL", "iguana/gemma4-26b")
|
||||
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
if c.ReconcileBaseModel != "" {
|
||||
t.Errorf("reconciliation must stay off without a base engine")
|
||||
}
|
||||
}
|
||||
@@ -111,3 +111,161 @@ type Summary struct {
|
||||
FallbackUsed bool
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// Segment is one timed span of a transcription. Times are seconds from the start
|
||||
// of the audio.
|
||||
type Segment struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// Transcription is the result of transcribing user-supplied audio (ADR-033):
|
||||
// the full text, its timed segments, which backend served it, and how long the
|
||||
// audio was. Endpoint is recorded because the backends differ in timestamp
|
||||
// granularity — which one served a transcript is a quality fact.
|
||||
type Transcription struct {
|
||||
Text string
|
||||
Segments []Segment
|
||||
Endpoint string
|
||||
AudioSeconds float64
|
||||
// Reconciliation is set when two engines transcribed the same audio and a local
|
||||
// model chose between their disagreements (ADR-034). Nil for a single engine.
|
||||
Reconciliation *Reconciliation
|
||||
}
|
||||
|
||||
// ReconcileStatus says what became of an attempt to merge two transcripts.
|
||||
type ReconcileStatus string
|
||||
|
||||
const (
|
||||
// ReconcileDone: both engines produced text and the disagreements were resolved.
|
||||
ReconcileDone ReconcileStatus = "reconciled"
|
||||
// ReconcileSingle: only one engine produced text, so there was nothing to merge.
|
||||
ReconcileSingle ReconcileStatus = "single_engine"
|
||||
// ReconcileSkipped: both engines produced text but the model could not be used;
|
||||
// the base text stands, unreconciled.
|
||||
ReconcileSkipped ReconcileStatus = "skipped"
|
||||
)
|
||||
|
||||
// ContestedSpot is a place the engines disagreed that the model judged: it changed it,
|
||||
// kept the base's reading, or could not decide. Every judged spot is stored, not only
|
||||
// the ones a reader should check, because the audio is not retained and the kept ones
|
||||
// are what a later evaluation needs (ADR-035). The page lists only the changed and
|
||||
// uncertain ones.
|
||||
type ContestedSpot struct {
|
||||
Seconds float64 `json:"seconds"` // roughly where in the recording
|
||||
Base string `json:"base"` // what the base engine heard
|
||||
Alt string `json:"alt"` // what the other engine heard
|
||||
Used string `json:"used"` // "base" or "alt": the reading now in the text
|
||||
Unsure bool `json:"unsure"` // the model could not decide; the base was kept
|
||||
Before string `json:"before,omitempty"` // the few words just before the spot, to find it by
|
||||
}
|
||||
|
||||
// Reconciliation records how a merged transcript came about, so the page can say so
|
||||
// plainly and the user can check the uncertain spots. The counts are exact; Contested
|
||||
// is bounded.
|
||||
type Reconciliation struct {
|
||||
Status ReconcileStatus `json:"status"`
|
||||
Reason string `json:"reason,omitempty"` // why single_engine or skipped
|
||||
Base string `json:"base"` // engine whose text is the base
|
||||
Alt string `json:"alt"`
|
||||
Reconciler string `json:"reconciler"`
|
||||
Disputes int `json:"disputes"` // places the engines disagreed
|
||||
AutoResolved int `json:"auto_resolved"` // trivial: compounds, filler words
|
||||
Adjudicated int `json:"adjudicated"` // decided by the model
|
||||
Changed int `json:"changed"` // the alternative's reading was used
|
||||
Unsure int `json:"unsure"` // base kept, flagged
|
||||
Errors int `json:"errors"` // model calls that failed
|
||||
Contested []ContestedSpot `json:"contested,omitempty"`
|
||||
}
|
||||
|
||||
// StepState is where one phase of an upload job stands (ADR-035).
|
||||
type StepState string
|
||||
|
||||
const (
|
||||
StepPending StepState = "pending"
|
||||
StepRunning StepState = "running"
|
||||
StepDone StepState = "done"
|
||||
StepFailed StepState = "failed"
|
||||
// StepSkipped is a step that never ran because the job finished without it, for
|
||||
// example the second engine's step when the first engine's text was used alone.
|
||||
StepSkipped StepState = "skipped"
|
||||
)
|
||||
|
||||
// ProgressStep is one named phase of an upload job. Total is zero for a phase with no
|
||||
// count; only a phase whose work is really countable carries one (ADR-035: no
|
||||
// percentage that does not come from real counts).
|
||||
type ProgressStep struct {
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
State StepState `json:"state"`
|
||||
StartedAt time.Time `json:"started_at,omitzero"`
|
||||
FinishedAt time.Time `json:"finished_at,omitzero"`
|
||||
Done int `json:"done,omitempty"`
|
||||
Total int `json:"total,omitempty"`
|
||||
// Note is a short plain remark about the phase that the reader should see even though
|
||||
// it did not fail, for example that the audio is much shorter than the file claimed.
|
||||
// Durations and counts only: never transcript text, a name or a path.
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// UploadStatus is the state of an uploaded-audio transcription job.
|
||||
type UploadStatus string
|
||||
|
||||
const (
|
||||
UploadQueued UploadStatus = "queued"
|
||||
UploadTranscribing UploadStatus = "transcribing"
|
||||
UploadDone UploadStatus = "done"
|
||||
UploadFailed UploadStatus = "failed"
|
||||
)
|
||||
|
||||
// UploadErrorKind is the typed reason an upload failed. It is deliberately NOT a
|
||||
// provider's error string: the whisper server's "failed to process audio" has had
|
||||
// at least five distinct causes, so nothing may branch on it (ADR-033).
|
||||
type UploadErrorKind string
|
||||
|
||||
const (
|
||||
// UploadErrInterrupted: the process restarted mid-job. The audio is not
|
||||
// retained, so the job cannot resume; the user re-uploads.
|
||||
UploadErrInterrupted UploadErrorKind = "interrupted"
|
||||
// UploadErrTranscriptionFailed: every configured transcription endpoint failed.
|
||||
UploadErrTranscriptionFailed UploadErrorKind = "transcription_failed"
|
||||
// UploadErrTooLong: the audio, once converted, is longer than the limit. The limit is
|
||||
// first checked against the file's header, which a hostile or damaged file can lie
|
||||
// about (ADR-035 review).
|
||||
UploadErrTooLong UploadErrorKind = "audio_too_long"
|
||||
// UploadErrBusy: too many recordings were already waiting, so this one was not
|
||||
// queued. The user's to retry later (ADR-035 review).
|
||||
UploadErrBusy UploadErrorKind = "queue_full"
|
||||
)
|
||||
|
||||
// UploadedTranscript is a user's PRIVATE transcript of audio they uploaded. It is
|
||||
// never stored in, or merged with, the shared public-caption Transcript (ADR-021).
|
||||
// Content and Segments are empty until Status is UploadDone.
|
||||
type UploadedTranscript struct {
|
||||
ID string
|
||||
UserID string
|
||||
Title string
|
||||
Status UploadStatus
|
||||
Content string
|
||||
Segments []Segment
|
||||
Endpoint string
|
||||
AudioSeconds float64
|
||||
ErrorKind UploadErrorKind
|
||||
// HadVideo records that the uploaded file carried a video track. Only its audio
|
||||
// was used and the video was discarded; the page says so. It is set at upload
|
||||
// and never cleared (ADR-033).
|
||||
HadVideo bool
|
||||
// Reconciliation is how a merged transcript came about, or nil for a single
|
||||
// engine (ADR-034). Set by Get, not by List.
|
||||
Reconciliation *Reconciliation
|
||||
// Progress is the job's phases, in order. Set by Get, not by List (ADR-035).
|
||||
Progress []ProgressStep
|
||||
// ProgressAt is when a phase last really changed or advanced; AliveAt is the
|
||||
// heartbeat. They differ on purpose: alive says the service is still reporting,
|
||||
// progress says work is moving (ADR-035). Both are zero until the job starts.
|
||||
ProgressAt time.Time
|
||||
AliveAt time.Time
|
||||
CreatedAt time.Time
|
||||
FinishedAt time.Time // zero until the job reaches done or failed
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// Package errkind reduces an error to a short kind that is safe to log, store and show.
|
||||
//
|
||||
// An error's own text can hold a provider's response body (a gateway that echoes the
|
||||
// prompt), a file path (ffmpeg names the spool file), or a word from a private
|
||||
// transcript. None of that may reach a log line, a database column or a page about a
|
||||
// private recording (ADR-033: "typed, never a provider's string"; ADR-035). Of returns
|
||||
// only a fixed vocabulary, so it can never carry any.
|
||||
package errkind
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
)
|
||||
|
||||
// HTTPStatuser is implemented by errors that are a non-2xx answer from a service.
|
||||
type HTTPStatuser interface{ HTTPStatus() int }
|
||||
|
||||
// SQLStater is implemented by database errors that carry a SQLSTATE (pgx's PgError does).
|
||||
// The five-character code is a fixed vocabulary; the error's message is not: Postgres quotes
|
||||
// the offending value, which for a transcript row is transcript text.
|
||||
type SQLStater interface{ SQLState() string }
|
||||
|
||||
// Of is the kind of err: "timeout", "canceled", "http_<code>", "sql_<state>", "network",
|
||||
// or "error".
|
||||
// It is empty for nil.
|
||||
func Of(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
var hs HTTPStatuser
|
||||
var ss SQLStater
|
||||
var ne net.Error
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
return "timeout"
|
||||
case errors.Is(err, context.Canceled):
|
||||
return "canceled"
|
||||
case errors.As(err, &hs):
|
||||
return fmt.Sprintf("http_%d", hs.HTTPStatus())
|
||||
case errors.As(err, &ss) && ss.SQLState() != "":
|
||||
return "sql_" + ss.SQLState()
|
||||
case errors.As(err, &ne):
|
||||
if ne.Timeout() {
|
||||
return "timeout"
|
||||
}
|
||||
return "network"
|
||||
}
|
||||
return "error"
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package errkind_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
)
|
||||
|
||||
// Of reduces an error to a short kind that is safe to log, store and show: it never
|
||||
// carries the error's own text, because that text can hold a provider's response body, a
|
||||
// file path or a word from a private transcript (ADR-035 review F3, F5).
|
||||
|
||||
type statusErr struct{ code int }
|
||||
|
||||
func (e statusErr) Error() string {
|
||||
return fmt.Sprintf("gateway said: %d and echoed the transcript", e.code)
|
||||
}
|
||||
func (e statusErr) HTTPStatus() int { return e.code }
|
||||
|
||||
type timeoutNet struct{}
|
||||
|
||||
func (timeoutNet) Error() string { return "i/o timeout to 10.0.0.1" }
|
||||
func (timeoutNet) Timeout() bool { return true }
|
||||
func (timeoutNet) Temporary() bool { return true }
|
||||
|
||||
func TestOf_ReducesAnErrorToAKindAndNeverItsText(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
{"nothing", nil, ""},
|
||||
{"a deadline", context.DeadlineExceeded, "timeout"},
|
||||
{"a wrapped deadline", fmt.Errorf("transcribe: %w", context.DeadlineExceeded), "timeout"},
|
||||
{"a cancellation", fmt.Errorf("job: %w", context.Canceled), "canceled"},
|
||||
{"an HTTP status", statusErr{code: 502}, "http_502"},
|
||||
{"a wrapped HTTP status", fmt.Errorf("model: %w", statusErr{code: 400}), "http_400"},
|
||||
{"a network timeout", timeoutNet{}, "timeout"},
|
||||
{"a network failure", &net.OpError{Op: "dial", Err: errors.New("connection refused")}, "network"},
|
||||
{"a url error wrapping a refusal", &url.Error{Op: "Post", URL: "http://gateway/secret-path", Err: errors.New("refused")}, "network"},
|
||||
{"anything else", errors.New("moov atom not found in /spool/upload-555"), "error"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, errkind.Of(tt.err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type sqlErr struct{}
|
||||
|
||||
func (sqlErr) Error() string {
|
||||
return `ERROR: invalid input syntax for type json: "Anders ansvarar för virket"`
|
||||
}
|
||||
func (sqlErr) SQLState() string { return "22P02" }
|
||||
|
||||
func TestOf_ADatabaseErrorIsReducedToItsSQLStateNeverItsMessage(t *testing.T) {
|
||||
// A Postgres error can quote the offending value, which for a transcript row is
|
||||
// transcript text. The five-character SQLSTATE is a fixed vocabulary (fourth review).
|
||||
err := fmt.Errorf("store: complete upload: %w", sqlErr{})
|
||||
|
||||
got := errkind.Of(err)
|
||||
|
||||
assert.Equal(t, "sql_22P02", got)
|
||||
assert.NotContains(t, got, "Anders")
|
||||
}
|
||||
|
||||
func TestOf_AKindCarriesNoWordFromTheError(t *testing.T) {
|
||||
err := fmt.Errorf("convert /spool/upload-555: %w", errors.New("Anders ansvarar för virket"))
|
||||
|
||||
got := errkind.Of(err)
|
||||
|
||||
assert.NotContains(t, got, "Anders")
|
||||
assert.NotContains(t, got, "spool")
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
// Package progress tracks the phases of one upload job (ADR-035). A Tracker is the
|
||||
// only writer of a job's progress: every transition is saved to the job's row and
|
||||
// logged by the same call, so what the page shows and what the log says cannot
|
||||
// disagree. The engines and the reconciler report through a Reporter carried in the
|
||||
// context, so the Transcriber interface stays what it was.
|
||||
package progress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
)
|
||||
|
||||
// Planned is one phase of a job, declared up front so the page can show what is coming.
|
||||
type Planned struct {
|
||||
Key, Label string
|
||||
}
|
||||
|
||||
// Reporter is what a long-running component reports to. Begin starts a phase (total
|
||||
// is the number of countable units, 0 when there is no honest count), Advance says how
|
||||
// many are done, Done closes the phase. Fail closes a phase that did not work while
|
||||
// the job goes on (an engine that was down, the other one's text used instead) and
|
||||
// says why in the log. Note is a log line for the operator that costs no write.
|
||||
type Reporter interface {
|
||||
Begin(key string, total int)
|
||||
Advance(key string, done int)
|
||||
Done(key string)
|
||||
Fail(key string, err error)
|
||||
Note(msg string, kv ...any)
|
||||
}
|
||||
|
||||
// Sink is where a job's progress is persisted. Both calls are best effort: progress
|
||||
// is for people, so a failed write is logged and never stops the job.
|
||||
type Sink interface {
|
||||
SaveProgress(ctx context.Context, steps []domain.ProgressStep, at time.Time) error
|
||||
Beat(ctx context.Context) error
|
||||
}
|
||||
|
||||
const (
|
||||
defaultMinSave = 2 * time.Second
|
||||
logAdvanceEach = time.Minute
|
||||
)
|
||||
|
||||
// Option tunes a Tracker.
|
||||
type Option func(*Tracker)
|
||||
|
||||
// WithClock fixes the clock, for tests.
|
||||
func WithClock(now func() time.Time) Option { return func(t *Tracker) { t.now = now } }
|
||||
|
||||
// WithMinSave sets the least time between two saves caused by Advance alone. Begin,
|
||||
// Done, Fail and Finish always save.
|
||||
func WithMinSave(d time.Duration) Option { return func(t *Tracker) { t.minSave = d } }
|
||||
|
||||
// Tracker implements Reporter for one job.
|
||||
type Tracker struct {
|
||||
ctx context.Context
|
||||
sink Sink
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
minSave time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
steps []domain.ProgressStep
|
||||
lastAdvanceSave time.Time
|
||||
lastAdvanceLog time.Time
|
||||
}
|
||||
|
||||
// NewTracker plans a job's phases. ctx is used for the saves, so it should outlive the
|
||||
// job's own timeout: a job that timed out still gets its failure recorded.
|
||||
func NewTracker(ctx context.Context, sink Sink, log *slog.Logger, plan []Planned, opts ...Option) *Tracker {
|
||||
t := &Tracker{ctx: ctx, sink: sink, log: log, now: time.Now, minSave: defaultMinSave}
|
||||
for _, o := range opts {
|
||||
o(t)
|
||||
}
|
||||
for _, p := range plan {
|
||||
t.steps = append(t.steps, domain.ProgressStep{Key: p.Key, Label: p.Label, State: domain.StepPending})
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// Start saves the plan, every phase pending.
|
||||
func (t *Tracker) Start() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.log.Info("job started", "steps", len(t.steps))
|
||||
t.save()
|
||||
}
|
||||
|
||||
// Begin implements Reporter.
|
||||
func (t *Tracker) Begin(key string, total int) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
s := t.find(key)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.State, s.StartedAt, s.FinishedAt, s.Done, s.Total = domain.StepRunning, t.now(), time.Time{}, 0, total
|
||||
t.log.Info("step started", "step", key, "total", total)
|
||||
t.save()
|
||||
}
|
||||
|
||||
// Advance implements Reporter. The count is always kept; it is saved at most once per
|
||||
// minimum interval and logged at most once a minute, so a counted phase does not turn
|
||||
// into a write and a log line per unit.
|
||||
func (t *Tracker) Advance(key string, done int) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
s := t.find(key)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.Done = done
|
||||
now := t.now()
|
||||
if t.lastAdvanceLog.IsZero() || now.Sub(t.lastAdvanceLog) >= logAdvanceEach {
|
||||
t.lastAdvanceLog = now
|
||||
t.log.Info("step progress", "step", key, "done", done, "total", s.Total)
|
||||
}
|
||||
if t.lastAdvanceSave.IsZero() || now.Sub(t.lastAdvanceSave) >= t.minSave {
|
||||
t.lastAdvanceSave = now
|
||||
t.save()
|
||||
}
|
||||
}
|
||||
|
||||
// Done implements Reporter.
|
||||
func (t *Tracker) Done(key string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
s := t.find(key)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
t.close(s, domain.StepDone)
|
||||
t.log.Info("step finished", "step", key, "elapsed", s.FinishedAt.Sub(s.StartedAt))
|
||||
t.save()
|
||||
}
|
||||
|
||||
// Fail implements Reporter: one phase failed and the job carries on.
|
||||
func (t *Tracker) Fail(key string, err error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
s := t.find(key)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
t.close(s, domain.StepFailed)
|
||||
t.log.Warn("step failed", "step", key, "elapsed", s.FinishedAt.Sub(s.StartedAt), "kind", errkind.Of(err))
|
||||
t.save()
|
||||
}
|
||||
|
||||
// Note implements Reporter.
|
||||
func (t *Tracker) Note(msg string, kv ...any) { t.log.Info(msg, kv...) }
|
||||
|
||||
// FailRunning marks the phase that was running as failed, because the whole job
|
||||
// failed. What never started stays pending, which is what shows the user where the
|
||||
// job stopped.
|
||||
func (t *Tracker) FailRunning() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
for i := range t.steps {
|
||||
if t.steps[i].State == domain.StepRunning {
|
||||
t.close(&t.steps[i], domain.StepFailed)
|
||||
t.log.Warn("step failed", "step", t.steps[i].Key, "elapsed", t.steps[i].FinishedAt.Sub(t.steps[i].StartedAt))
|
||||
}
|
||||
}
|
||||
t.save()
|
||||
}
|
||||
|
||||
// Finish closes the job: a phase still running is done, a phase that never ran is
|
||||
// skipped.
|
||||
func (t *Tracker) Finish() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
for i := range t.steps {
|
||||
switch t.steps[i].State {
|
||||
case domain.StepRunning:
|
||||
t.close(&t.steps[i], domain.StepDone)
|
||||
case domain.StepPending:
|
||||
t.steps[i].State = domain.StepSkipped
|
||||
}
|
||||
}
|
||||
t.log.Info("job finished")
|
||||
t.save()
|
||||
}
|
||||
|
||||
// Annotate attaches a short plain note to a phase and saves it at once. It is for a
|
||||
// remark the reader should see although nothing failed. Durations and counts only: a note
|
||||
// is shown on a page about a private recording.
|
||||
func (t *Tracker) Annotate(key, note string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
s := t.find(key)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.Note = note
|
||||
t.save()
|
||||
}
|
||||
|
||||
// Heartbeat stamps the job's row every beatEvery and logs which phase is still running
|
||||
// every stillEvery, until ctx ends. It runs on its own, independent of any engine call:
|
||||
// a heartbeat that stopped means the process is gone, not that a request is slow.
|
||||
func (t *Tracker) Heartbeat(ctx context.Context, beatEvery, stillEvery time.Duration) {
|
||||
beat, still := time.NewTicker(beatEvery), time.NewTicker(stillEvery)
|
||||
defer beat.Stop()
|
||||
defer still.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-beat.C:
|
||||
if err := t.sink.Beat(ctx); err != nil && ctx.Err() == nil {
|
||||
t.log.Warn("progress: heartbeat not saved", "err", err)
|
||||
}
|
||||
case <-still.C:
|
||||
t.logStill()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) logStill() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
for _, s := range t.steps {
|
||||
if s.State == domain.StepRunning {
|
||||
t.log.Info("still running", "step", s.Key, "elapsed", t.now().Sub(s.StartedAt), "done", s.Done, "total", s.Total)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// close ends a phase. Callers hold t.mu.
|
||||
func (t *Tracker) close(s *domain.ProgressStep, state domain.StepState) {
|
||||
s.State, s.FinishedAt = state, t.now()
|
||||
if s.StartedAt.IsZero() {
|
||||
s.StartedAt = s.FinishedAt
|
||||
}
|
||||
}
|
||||
|
||||
// find returns the phase with this key, or nil. Callers hold t.mu.
|
||||
func (t *Tracker) find(key string) *domain.ProgressStep {
|
||||
for i := range t.steps {
|
||||
if t.steps[i].Key == key {
|
||||
return &t.steps[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// save persists a copy of the phases. Callers hold t.mu. A failed write is logged and
|
||||
// dropped: progress is cosmetic and must never fail the job it describes.
|
||||
func (t *Tracker) save() {
|
||||
snapshot := append([]domain.ProgressStep(nil), t.steps...)
|
||||
if err := t.sink.SaveProgress(t.ctx, snapshot, t.now()); err != nil && t.ctx.Err() == nil {
|
||||
t.log.Warn("progress: not saved", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
type ctxKey struct{}
|
||||
|
||||
// With returns a context that carries r.
|
||||
func With(ctx context.Context, r Reporter) context.Context {
|
||||
return context.WithValue(ctx, ctxKey{}, r)
|
||||
}
|
||||
|
||||
// From returns the Reporter carried by ctx, or one that does nothing, so a component
|
||||
// can always report without checking whether anyone is listening.
|
||||
func From(ctx context.Context) Reporter {
|
||||
if r, ok := ctx.Value(ctxKey{}).(Reporter); ok {
|
||||
return r
|
||||
}
|
||||
return noop{}
|
||||
}
|
||||
|
||||
type noop struct{}
|
||||
|
||||
func (noop) Begin(string, int) {}
|
||||
func (noop) Advance(string, int) {}
|
||||
func (noop) Done(string) {}
|
||||
func (noop) Fail(string, error) {}
|
||||
func (noop) Note(string, ...any) {}
|
||||
@@ -0,0 +1,402 @@
|
||||
package progress_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
)
|
||||
|
||||
// fakeSink records every saved snapshot and every heartbeat.
|
||||
type fakeSink struct {
|
||||
mu sync.Mutex
|
||||
saves [][]domain.ProgressStep
|
||||
at []time.Time
|
||||
beats int
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeSink) SaveProgress(_ context.Context, steps []domain.ProgressStep, at time.Time) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.saves = append(f.saves, append([]domain.ProgressStep(nil), steps...))
|
||||
f.at = append(f.at, at)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeSink) Beat(context.Context) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.beats++
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeSink) last(t *testing.T) []domain.ProgressStep {
|
||||
t.Helper()
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
require.NotEmpty(t, f.saves, "nothing was saved")
|
||||
return f.saves[len(f.saves)-1]
|
||||
}
|
||||
|
||||
func (f *fakeSink) saveCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.saves)
|
||||
}
|
||||
|
||||
func (f *fakeSink) beatCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.beats
|
||||
}
|
||||
|
||||
// syncBuf is a log sink safe to read while the heartbeat goroutine is still writing.
|
||||
type syncBuf struct {
|
||||
mu sync.Mutex
|
||||
b strings.Builder
|
||||
}
|
||||
|
||||
func (s *syncBuf) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.Write(p)
|
||||
}
|
||||
|
||||
func (s *syncBuf) String() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.String()
|
||||
}
|
||||
|
||||
// clock is a hand-wound clock so durations and throttling are exact.
|
||||
type clock struct{ t time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.t }
|
||||
func (c *clock) tick(d time.Duration) { c.t = c.t.Add(d) }
|
||||
func newClock() *clock { return &clock{t: time.Date(2026, 10, 2, 8, 0, 0, 0, time.UTC)} }
|
||||
|
||||
func plan() []progress.Planned {
|
||||
return []progress.Planned{{Key: "audio", Label: "Preparing audio"}, {Key: "decide", Label: "Deciding"}, {Key: "save", Label: "Saving"}}
|
||||
}
|
||||
|
||||
func newTracker(t *testing.T) (*progress.Tracker, *fakeSink, *clock, *syncBuf) {
|
||||
t.Helper()
|
||||
sink, clk, buf := &fakeSink{}, newClock(), &syncBuf{}
|
||||
log := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
||||
return progress.NewTracker(context.Background(), sink, log, plan(), progress.WithClock(clk.now), progress.WithMinSave(2*time.Second)), sink, clk, buf
|
||||
}
|
||||
|
||||
func stateOf(steps []domain.ProgressStep, key string) domain.StepState {
|
||||
for _, s := range steps {
|
||||
if s.Key == key {
|
||||
return s.State
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestTracker_PlanIsShownUpFrontInOrderAllPending(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
|
||||
tr.Start()
|
||||
|
||||
steps := sink.last(t)
|
||||
require.Len(t, steps, 3)
|
||||
require.Equal(t, []string{"audio", "decide", "save"}, []string{steps[0].Key, steps[1].Key, steps[2].Key})
|
||||
for _, s := range steps {
|
||||
require.Equal(t, domain.StepPending, s.State, s.Key)
|
||||
}
|
||||
require.Equal(t, "Preparing audio", steps[0].Label)
|
||||
}
|
||||
|
||||
func TestTracker_BeginRunsAStepAndDoneRecordsItsDuration(t *testing.T) {
|
||||
tr, sink, clk, _ := newTracker(t)
|
||||
tr.Start()
|
||||
|
||||
tr.Begin("audio", 0)
|
||||
require.Equal(t, domain.StepRunning, stateOf(sink.last(t), "audio"))
|
||||
clk.tick(90 * time.Second)
|
||||
tr.Done("audio")
|
||||
|
||||
s := sink.last(t)[0]
|
||||
require.Equal(t, domain.StepDone, s.State)
|
||||
require.Equal(t, 90*time.Second, s.FinishedAt.Sub(s.StartedAt), "the finished step must carry how long it took")
|
||||
}
|
||||
|
||||
func TestTracker_AdvanceCountsAreSavedAndThrottled(t *testing.T) {
|
||||
tr, sink, clk, _ := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("decide", 282)
|
||||
base := sink.saveCount()
|
||||
|
||||
tr.Advance("decide", 1)
|
||||
clk.tick(500 * time.Millisecond)
|
||||
tr.Advance("decide", 2)
|
||||
clk.tick(500 * time.Millisecond)
|
||||
tr.Advance("decide", 3)
|
||||
|
||||
require.Equal(t, base+1, sink.saveCount(), "advances inside the minimum interval must not each hit the database")
|
||||
clk.tick(2 * time.Second)
|
||||
tr.Advance("decide", 4)
|
||||
require.Equal(t, base+2, sink.saveCount())
|
||||
got := sink.last(t)[1]
|
||||
require.Equal(t, 4, got.Done)
|
||||
require.Equal(t, 282, got.Total)
|
||||
}
|
||||
|
||||
func TestTracker_DoneFlushesTheFinalCountEvenInsideTheThrottle(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("decide", 10)
|
||||
tr.Advance("decide", 9)
|
||||
|
||||
tr.Advance("decide", 10)
|
||||
tr.Done("decide")
|
||||
|
||||
got := sink.last(t)[1]
|
||||
require.Equal(t, domain.StepDone, got.State)
|
||||
require.Equal(t, 10, got.Done, "a finished step must show its full count, not the last throttled one")
|
||||
}
|
||||
|
||||
func TestTracker_FinishSkipsWhatNeverRanAndClosesWhatDid(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("audio", 0)
|
||||
tr.Done("audio")
|
||||
tr.Begin("decide", 5)
|
||||
|
||||
tr.Finish()
|
||||
|
||||
steps := sink.last(t)
|
||||
require.Equal(t, domain.StepDone, stateOf(steps, "audio"))
|
||||
require.Equal(t, domain.StepDone, stateOf(steps, "decide"), "a step still running when the job completes is closed")
|
||||
require.Equal(t, domain.StepSkipped, stateOf(steps, "save"), "a step that never ran is skipped, not left pending")
|
||||
}
|
||||
|
||||
func TestTracker_FailRunningMarksTheRunningStepFailedAndLeavesTheRestPending(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("audio", 0)
|
||||
tr.Done("audio")
|
||||
tr.Begin("decide", 5)
|
||||
|
||||
tr.FailRunning()
|
||||
|
||||
steps := sink.last(t)
|
||||
require.Equal(t, domain.StepDone, stateOf(steps, "audio"))
|
||||
require.Equal(t, domain.StepFailed, stateOf(steps, "decide"))
|
||||
require.Equal(t, domain.StepPending, stateOf(steps, "save"), "what never started stays pending: it shows where the job stopped")
|
||||
}
|
||||
|
||||
func TestTracker_AKeyedFailMarksOneStepFailedLogsItsKindAndTheJobGoesOn(t *testing.T) {
|
||||
tr, sink, _, buf := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("audio", 0)
|
||||
|
||||
tr.Fail("audio", errors.New("gateway said no"))
|
||||
tr.Begin("decide", 5)
|
||||
|
||||
steps := sink.last(t)
|
||||
require.Equal(t, domain.StepFailed, stateOf(steps, "audio"))
|
||||
require.Equal(t, domain.StepRunning, stateOf(steps, "decide"), "a soft failure must not stop the next step from running")
|
||||
require.Contains(t, buf.String(), "level=WARN")
|
||||
require.NotContains(t, buf.String(), "gateway said no", "an error's text can carry provider output or a path: the log gets its kind")
|
||||
require.Contains(t, buf.String(), "kind=error")
|
||||
require.Contains(t, buf.String(), "step=audio")
|
||||
}
|
||||
|
||||
func TestTracker_NoteIsALogLineAndNeverASave(t *testing.T) {
|
||||
tr, sink, _, buf := newTracker(t)
|
||||
tr.Start()
|
||||
before := sink.saveCount()
|
||||
|
||||
tr.Note("reconciled", "disputes", 362, "auto_resolved", 80)
|
||||
|
||||
require.Equal(t, before, sink.saveCount(), "a note is for the operator; it must not cost a database write")
|
||||
require.Contains(t, buf.String(), "reconciled")
|
||||
require.Contains(t, buf.String(), "disputes=362")
|
||||
require.Contains(t, buf.String(), "auto_resolved=80")
|
||||
}
|
||||
|
||||
func TestTracker_AnUnknownKeyIsIgnored(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
tr.Start()
|
||||
before := sink.saveCount()
|
||||
|
||||
require.NotPanics(t, func() {
|
||||
tr.Begin("no-such-step", 0)
|
||||
tr.Advance("no-such-step", 1)
|
||||
tr.Done("no-such-step")
|
||||
tr.Fail("no-such-step", errors.New("x"))
|
||||
})
|
||||
require.Equal(t, before, sink.saveCount())
|
||||
}
|
||||
|
||||
func TestTracker_ASinkFailureIsLoggedAndNeverStopsTheJob(t *testing.T) {
|
||||
tr, sink, _, buf := newTracker(t)
|
||||
sink.err = errors.New("db down")
|
||||
|
||||
require.NotPanics(t, func() {
|
||||
tr.Start()
|
||||
tr.Begin("audio", 0)
|
||||
tr.Done("audio")
|
||||
})
|
||||
require.Contains(t, buf.String(), "level=WARN", "a lost progress write is worth a WARN")
|
||||
require.Contains(t, buf.String(), "db down")
|
||||
}
|
||||
|
||||
func TestTracker_EveryTransitionIsLoggedByTheSameCall(t *testing.T) {
|
||||
tr, _, clk, buf := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("audio", 0)
|
||||
clk.tick(3 * time.Second)
|
||||
tr.Done("audio")
|
||||
tr.Begin("decide", 5)
|
||||
clk.tick(2 * time.Second)
|
||||
tr.FailRunning()
|
||||
tr.Finish()
|
||||
|
||||
out := buf.String()
|
||||
for _, want := range []string{
|
||||
`msg="job started"`,
|
||||
`msg="step started" step=audio`,
|
||||
`msg="step finished" step=audio elapsed=3s`,
|
||||
`msg="step started" step=decide total=5`,
|
||||
`msg="step failed" step=decide elapsed=2s`,
|
||||
`msg="job finished"`,
|
||||
} {
|
||||
require.Contains(t, out, want, "a transition with no log line is invisible to the operator")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracker_AKeyedFailIsSavedAtOnceNotOnTheNextTransition(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("audio", 0)
|
||||
before := sink.saveCount()
|
||||
|
||||
tr.Fail("audio", errors.New("down"))
|
||||
|
||||
require.Equal(t, before+1, sink.saveCount(), "a soft-failed phase must show as failed on the page now")
|
||||
require.Equal(t, domain.StepFailed, stateOf(sink.last(t), "audio"))
|
||||
}
|
||||
|
||||
// retainingSink keeps the very slice it is handed, like a sink that queues it for later.
|
||||
type retainingSink struct{ got [][]domain.ProgressStep }
|
||||
|
||||
func (r *retainingSink) SaveProgress(_ context.Context, steps []domain.ProgressStep, _ time.Time) error {
|
||||
r.got = append(r.got, steps)
|
||||
return nil
|
||||
}
|
||||
func (r *retainingSink) Beat(context.Context) error { return nil }
|
||||
|
||||
func TestTracker_ASnapshotHandedToTheSinkIsNotChangedByLaterTransitions(t *testing.T) {
|
||||
sink := &retainingSink{}
|
||||
tr := progress.NewTracker(context.Background(), sink, slog.New(slog.NewTextHandler(&syncBuf{}, nil)), plan(),
|
||||
progress.WithClock(newClock().now))
|
||||
tr.Start()
|
||||
first := sink.got[0]
|
||||
|
||||
tr.Begin("audio", 0)
|
||||
tr.Done("audio")
|
||||
|
||||
require.Equal(t, domain.StepPending, first[0].State, "a sink that keeps a snapshot must not see it change under it")
|
||||
}
|
||||
|
||||
func TestTracker_AdvanceIsLoggedAtMostOncePerMinute(t *testing.T) {
|
||||
tr, _, clk, buf := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("decide", 100)
|
||||
|
||||
for i := 1; i <= 30; i++ {
|
||||
clk.tick(1 * time.Second)
|
||||
tr.Advance("decide", i)
|
||||
}
|
||||
require.LessOrEqual(t, strings.Count(buf.String(), "step progress"), 1,
|
||||
"30 seconds of progress is at most one log line, not thirty")
|
||||
clk.tick(61 * time.Second)
|
||||
tr.Advance("decide", 31)
|
||||
require.Equal(t, 2, strings.Count(buf.String(), "step progress"))
|
||||
require.Contains(t, buf.String(), "done=31")
|
||||
require.Contains(t, buf.String(), "total=100")
|
||||
}
|
||||
|
||||
func TestTracker_HeartbeatBeatsOnItsOwnUntilStopped(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
tr.Heartbeat(ctx, 5*time.Millisecond, time.Hour)
|
||||
}()
|
||||
|
||||
require.Eventually(t, func() bool { return sink.beatCount() >= 3 }, 2*time.Second, 5*time.Millisecond,
|
||||
"a heartbeat that does not depend on any engine call is the point")
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Heartbeat did not stop when its context ended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracker_HeartbeatSaysWhichStepIsStillRunningAndForHowLong(t *testing.T) {
|
||||
tr, _, clk, buf := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("decide", 100)
|
||||
clk.tick(7 * time.Minute)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
go tr.Heartbeat(ctx, time.Hour, 5*time.Millisecond)
|
||||
|
||||
require.Eventually(t, func() bool { return strings.Contains(buf.String(), "still running") }, 2*time.Second, 5*time.Millisecond)
|
||||
require.Contains(t, buf.String(), "step=decide")
|
||||
require.Contains(t, buf.String(), "elapsed=7m0s")
|
||||
}
|
||||
|
||||
func TestFrom_WithoutAReporterReturnsOneThatDoesNothing(t *testing.T) {
|
||||
r := progress.From(context.Background())
|
||||
|
||||
require.NotNil(t, r)
|
||||
require.NotPanics(t, func() {
|
||||
r.Begin("x", 3)
|
||||
r.Advance("x", 1)
|
||||
r.Done("x")
|
||||
r.Fail("x", errors.New("e"))
|
||||
r.Note("n", "k", 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithAndFrom_ACarriedReporterIsTheOneReturned(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
tr.Start()
|
||||
ctx := progress.With(context.Background(), tr)
|
||||
|
||||
progress.From(ctx).Begin("audio", 0)
|
||||
|
||||
require.Equal(t, domain.StepRunning, stateOf(sink.last(t), "audio"))
|
||||
}
|
||||
|
||||
func TestTracker_AnnotateAttachesANoteToAStepAndSavesItAtOnce(t *testing.T) {
|
||||
tr, sink, _, _ := newTracker(t)
|
||||
tr.Start()
|
||||
tr.Begin("audio", 0)
|
||||
before := sink.saveCount()
|
||||
|
||||
tr.Annotate("audio", "20 min of the 79 the file claimed")
|
||||
|
||||
require.Equal(t, before+1, sink.saveCount())
|
||||
require.Equal(t, "20 min of the 79 the file claimed", sink.last(t)[0].Note)
|
||||
require.NotPanics(t, func() { tr.Annotate("no-such-step", "x") })
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verdict is an adjudicator's decision about one dispute. The zero value is
|
||||
// VerdictUnsure: with no decision the base reading stands and the spot is flagged.
|
||||
type Verdict int
|
||||
|
||||
const (
|
||||
VerdictUnsure Verdict = iota
|
||||
VerdictBase
|
||||
VerdictAlt
|
||||
)
|
||||
|
||||
// Request is one dispute as the adjudicator sees it: the surrounding words, the two
|
||||
// readings, and which of them is shown as "Reading 1". Alternating AltFirst between
|
||||
// disputes averages out position bias.
|
||||
type Request struct {
|
||||
Before, After string
|
||||
Base, Alt string
|
||||
AltFirst bool
|
||||
}
|
||||
|
||||
// Adjudicator decides which reading of a dispute the speaker most likely said.
|
||||
type Adjudicator interface {
|
||||
Adjudicate(ctx context.Context, r Request) (Verdict, error)
|
||||
}
|
||||
|
||||
// Completer is the chat surface the LLM adjudicator needs. *llm.Client satisfies it.
|
||||
type Completer interface {
|
||||
Complete(ctx context.Context, system, user string) (string, error)
|
||||
}
|
||||
|
||||
// systemPrompt is the wording that was MEASURED (docs/research/
|
||||
// local-model-adjudicator-sweep-2026-10-01.md): gemma4-26b 92% and gemma4-31b 97% against
|
||||
// the speaker's blind answers. Changing it invalidates those figures.
|
||||
const systemPrompt = "You compare two automatic transcriptions of the same Swedish speech. A single person is dictating project notes " +
|
||||
"(finance and payments jargon, with English terms mixed into Swedish). The two transcriptions differ at ONE marked spot. " +
|
||||
"Decide which reading is what the speaker most likely actually said, using the surrounding text and ordinary Swedish. " +
|
||||
"Reply with exactly one line, either 'ANSWER: 1', 'ANSWER: 2' or 'ANSWER: UNSURE', then one short sentence of reasoning."
|
||||
|
||||
var answerRe = regexp.MustCompile(`(?i)ANSWER:\s*(1|2|UNSURE)\b`)
|
||||
|
||||
// LLMAdjudicator asks a chat model. It never invents an answer: a reply it cannot read
|
||||
// is VerdictUnsure, and a transport error is returned as an error so the caller can tell
|
||||
// "the model is down" from "the model is unsure".
|
||||
type LLMAdjudicator struct{ c Completer }
|
||||
|
||||
// NewLLMAdjudicator wraps a Completer.
|
||||
func NewLLMAdjudicator(c Completer) *LLMAdjudicator { return &LLMAdjudicator{c: c} }
|
||||
|
||||
// Adjudicate implements Adjudicator.
|
||||
func (a *LLMAdjudicator) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
|
||||
first, second := r.Base, r.Alt
|
||||
if r.AltFirst {
|
||||
first, second = r.Alt, r.Base
|
||||
}
|
||||
user := fmt.Sprintf("Text before the spot:\n%s\n\n[[ THE SPOT ]]\n\nText after the spot:\n%s\n\nReading 1: %s\nReading 2: %s\n",
|
||||
r.Before, r.After, describe(first), describe(second))
|
||||
reply, err := a.c.Complete(ctx, systemPrompt, user)
|
||||
if err != nil {
|
||||
return VerdictUnsure, fmt.Errorf("reconcile: adjudicate: %w", err)
|
||||
}
|
||||
m := answerRe.FindStringSubmatch(reply)
|
||||
if m == nil {
|
||||
return VerdictUnsure, nil
|
||||
}
|
||||
switch strings.ToUpper(m[1]) {
|
||||
case "1":
|
||||
if r.AltFirst {
|
||||
return VerdictAlt, nil
|
||||
}
|
||||
return VerdictBase, nil
|
||||
case "2":
|
||||
if r.AltFirst {
|
||||
return VerdictBase, nil
|
||||
}
|
||||
return VerdictAlt, nil
|
||||
}
|
||||
return VerdictUnsure, nil
|
||||
}
|
||||
|
||||
func describe(reading string) string {
|
||||
if strings.TrimSpace(reading) == "" {
|
||||
return "(nothing: the words are simply absent)"
|
||||
}
|
||||
return reading
|
||||
}
|
||||
|
||||
// OrderChecked asks the inner adjudicator twice with the reading order swapped and
|
||||
// keeps a verdict only when both agree. A model that flips with the order is showing
|
||||
// position bias, not judgement, so disagreement is VerdictUnsure. It doubles the time.
|
||||
type OrderChecked struct{ inner Adjudicator }
|
||||
|
||||
// NewOrderChecked wraps an adjudicator.
|
||||
func NewOrderChecked(inner Adjudicator) *OrderChecked { return &OrderChecked{inner: inner} }
|
||||
|
||||
// Adjudicate implements Adjudicator.
|
||||
func (o *OrderChecked) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
|
||||
r.AltFirst = false
|
||||
v1, err := o.inner.Adjudicate(ctx, r)
|
||||
if err != nil {
|
||||
return VerdictUnsure, err
|
||||
}
|
||||
r.AltFirst = true
|
||||
v2, err := o.inner.Adjudicate(ctx, r)
|
||||
if err != nil {
|
||||
return VerdictUnsure, err
|
||||
}
|
||||
if v1 == v2 {
|
||||
return v1, nil
|
||||
}
|
||||
return VerdictUnsure, nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
type fakeCompleter struct {
|
||||
replies []string
|
||||
err error
|
||||
systems []string
|
||||
users []string
|
||||
}
|
||||
|
||||
func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) {
|
||||
f.systems = append(f.systems, system)
|
||||
f.users = append(f.users, user)
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
i := len(f.users) - 1
|
||||
if i >= len(f.replies) {
|
||||
i = len(f.replies) - 1
|
||||
}
|
||||
return f.replies[i], nil
|
||||
}
|
||||
|
||||
var req = reconcile.Request{Before: "grannarna ska då", After: "De ska kunna måla", Base: "målade", Alt: "mölade"}
|
||||
|
||||
func TestLLMAdjudicator_SendsTheMeasuredPromptWithTheReadingsInTheRequestedOrder(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
a := reconcile.NewLLMAdjudicator(fc)
|
||||
|
||||
_, err := a.Adjudicate(context.Background(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fc.users, 1)
|
||||
assert.Contains(t, fc.systems[0], "two automatic transcriptions of the same Swedish speech")
|
||||
assert.Contains(t, fc.systems[0], "ANSWER: 1")
|
||||
u := fc.users[0]
|
||||
assert.Contains(t, u, "Text before the spot:\ngrannarna ska då")
|
||||
assert.Contains(t, u, "[[ THE SPOT ]]")
|
||||
assert.Contains(t, u, "Text after the spot:\nDe ska kunna måla")
|
||||
assert.Contains(t, u, "Reading 1: målade\nReading 2: mölade", "base first when AltFirst is false")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_AltFirstSwapsWhichReadingIsNumberOne(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
r := req
|
||||
r.AltFirst = true
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, fc.users[0], "Reading 1: mölade\nReading 2: målade")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
altFirst bool
|
||||
reply string
|
||||
want reconcile.Verdict
|
||||
}{
|
||||
{"1 with base first is base", false, "ANSWER: 1", reconcile.VerdictBase},
|
||||
{"2 with base first is alt", false, "ANSWER: 2", reconcile.VerdictAlt},
|
||||
{"1 with alt first is alt", true, "ANSWER: 1", reconcile.VerdictAlt},
|
||||
{"2 with alt first is base", true, "ANSWER: 2", reconcile.VerdictBase},
|
||||
{"unsure is unsure", false, "ANSWER: UNSURE", reconcile.VerdictUnsure},
|
||||
{"case and trailing reasoning are tolerated", false, "answer: 2\nThe second reading fits the context.", reconcile.VerdictAlt},
|
||||
{"reasoning before the answer line is tolerated", true, "Hmm.\nANSWER: 2", reconcile.VerdictBase},
|
||||
{"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure},
|
||||
{"an empty reply is unsure", false, "", reconcile.VerdictUnsure},
|
||||
{"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure},
|
||||
{"a number that merely starts with 1 is not 1", false, "ANSWER: 12", reconcile.VerdictUnsure},
|
||||
{"a word that merely starts with 2 is not 2", false, "ANSWER: 2nd option", reconcile.VerdictUnsure},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := req
|
||||
r.AltFirst = tt.altFirst
|
||||
|
||||
got, err := reconcile.NewLLMAdjudicator(&fakeCompleter{replies: []string{tt.reply}}).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_AnEmptyReadingIsDescribedNotLeftBlank(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
r := req
|
||||
r.Base = ""
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, fc.users[0], "Reading 1: (nothing: the words are simply absent)")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_ACompleterErrorIsReturnedNotTurnedIntoAVerdict(t *testing.T) {
|
||||
boom := errors.New("gateway down")
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(&fakeCompleter{err: boom}).Adjudicate(context.Background(), req)
|
||||
|
||||
assert.ErrorIs(t, err, boom, "the caller must be able to tell 'the model is down' from 'the model is unsure'")
|
||||
}
|
||||
|
||||
func TestOrderChecked_AsksTwiceWithTheOrderSwappedAndKeepsOnlyAgreement(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
replies []string // for [base-first call, alt-first call]
|
||||
want reconcile.Verdict
|
||||
}{
|
||||
{"both orders pick the base", []string{"ANSWER: 1", "ANSWER: 2"}, reconcile.VerdictBase},
|
||||
{"both orders pick the alternative", []string{"ANSWER: 2", "ANSWER: 1"}, reconcile.VerdictAlt},
|
||||
{"it flips with the order, so it is position bias, not a judgement", []string{"ANSWER: 1", "ANSWER: 1"}, reconcile.VerdictUnsure},
|
||||
{"one order unsure is unsure", []string{"ANSWER: 1", "ANSWER: UNSURE"}, reconcile.VerdictUnsure},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: tt.replies}
|
||||
|
||||
got, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(fc)).Adjudicate(context.Background(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
require.Len(t, fc.users, 2)
|
||||
assert.True(t, strings.Contains(fc.users[0], "Reading 1: målade"), "first call shows the base first")
|
||||
assert.True(t, strings.Contains(fc.users[1], "Reading 1: mölade"), "second call shows the alternative first")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrderChecked_AnErrorOnEitherCallIsAnError(t *testing.T) {
|
||||
_, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(&fakeCompleter{err: errors.New("down")})).Adjudicate(context.Background(), req)
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package reconcile
|
||||
|
||||
// OpKind is the kind of one run in an edit script.
|
||||
type OpKind int
|
||||
|
||||
const (
|
||||
// Equal: A[AStart:AEnd] and B[BStart:BEnd] are the same words.
|
||||
Equal OpKind = iota
|
||||
// Replace: different words on both sides.
|
||||
Replace
|
||||
// Delete: words only in A.
|
||||
Delete
|
||||
// Insert: words only in B.
|
||||
Insert
|
||||
)
|
||||
|
||||
// Op is one run of the edit script. The ops of a Diff tile both inputs: each op starts
|
||||
// where the previous one ended, in A and in B.
|
||||
type Op struct {
|
||||
Kind OpKind
|
||||
AStart, AEnd int
|
||||
BStart, BEnd int
|
||||
}
|
||||
|
||||
// maxEdits bounds the Myers search. Memory grows with the square of the edit
|
||||
// distance, so two texts that share almost nothing (one engine produced garbage)
|
||||
// are reported as one Replace instead of spending gigabytes proving it. A real
|
||||
// hour disagrees in roughly 400 places, i.e. about 800 edits, far below this.
|
||||
const maxEdits = 3000
|
||||
|
||||
// Diff returns a minimal edit script from a to b (Myers, O(ND)). Adjacent deletions
|
||||
// and insertions are reported as one Replace.
|
||||
func Diff(a, b []string) []Op {
|
||||
pre := 0
|
||||
for pre < len(a) && pre < len(b) && a[pre] == b[pre] {
|
||||
pre++
|
||||
}
|
||||
suf := 0
|
||||
for suf < len(a)-pre && suf < len(b)-pre && a[len(a)-1-suf] == b[len(b)-1-suf] {
|
||||
suf++
|
||||
}
|
||||
midA, midB := a[pre:len(a)-suf], b[pre:len(b)-suf]
|
||||
|
||||
steps, ok := myers(midA, midB)
|
||||
if !ok {
|
||||
steps = steps[:0]
|
||||
for range midA {
|
||||
steps = append(steps, stepDel)
|
||||
}
|
||||
for range midB {
|
||||
steps = append(steps, stepIns)
|
||||
}
|
||||
}
|
||||
|
||||
var ops []Op
|
||||
ai, bi := 0, 0
|
||||
emit := func(k OpKind, da, db int) {
|
||||
if da == 0 && db == 0 {
|
||||
return
|
||||
}
|
||||
if k == Equal && len(ops) > 0 && ops[len(ops)-1].Kind == Equal {
|
||||
ops[len(ops)-1].AEnd += da
|
||||
ops[len(ops)-1].BEnd += db
|
||||
} else {
|
||||
ops = append(ops, Op{Kind: k, AStart: ai, AEnd: ai + da, BStart: bi, BEnd: bi + db})
|
||||
}
|
||||
ai += da
|
||||
bi += db
|
||||
}
|
||||
emit(Equal, pre, pre)
|
||||
for i := 0; i < len(steps); {
|
||||
if steps[i] == stepEq {
|
||||
j := i
|
||||
for j < len(steps) && steps[j] == stepEq {
|
||||
j++
|
||||
}
|
||||
emit(Equal, j-i, j-i)
|
||||
i = j
|
||||
continue
|
||||
}
|
||||
dels, inss := 0, 0
|
||||
for i < len(steps) && steps[i] != stepEq {
|
||||
if steps[i] == stepDel {
|
||||
dels++
|
||||
} else {
|
||||
inss++
|
||||
}
|
||||
i++
|
||||
}
|
||||
switch {
|
||||
case dels > 0 && inss > 0:
|
||||
emit(Replace, dels, inss)
|
||||
case dels > 0:
|
||||
emit(Delete, dels, 0)
|
||||
default:
|
||||
emit(Insert, 0, inss)
|
||||
}
|
||||
}
|
||||
emit(Equal, suf, suf)
|
||||
return ops
|
||||
}
|
||||
|
||||
type step int8
|
||||
|
||||
const (
|
||||
stepEq step = iota
|
||||
stepDel
|
||||
stepIns
|
||||
)
|
||||
|
||||
// myers returns one step per element of the edit script, in order. ok is false when
|
||||
// the edit distance exceeds maxEdits.
|
||||
func myers(a, b []string) ([]step, bool) {
|
||||
n, m := len(a), len(b)
|
||||
if n == 0 && m == 0 {
|
||||
return nil, true
|
||||
}
|
||||
limit := min(n+m, maxEdits)
|
||||
off := limit + 1
|
||||
v := make([]int32, 2*limit+3)
|
||||
var trace [][]int32 // trace[d] is v as it stood at the start of round d, for k in [-d, d]
|
||||
|
||||
for d := 0; d <= limit; d++ {
|
||||
snap := make([]int32, 2*d+1)
|
||||
for k := -d; k <= d; k++ {
|
||||
snap[k+d] = v[off+k]
|
||||
}
|
||||
trace = append(trace, snap)
|
||||
for k := -d; k <= d; k += 2 {
|
||||
var x int
|
||||
if k == -d || (k != d && v[off+k-1] < v[off+k+1]) {
|
||||
x = int(v[off+k+1])
|
||||
} else {
|
||||
x = int(v[off+k-1]) + 1
|
||||
}
|
||||
y := x - k
|
||||
for x < n && y < m && a[x] == b[y] {
|
||||
x++
|
||||
y++
|
||||
}
|
||||
v[off+k] = int32(x)
|
||||
if x >= n && y >= m {
|
||||
return backtrack(trace, n, m, d), true
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func backtrack(trace [][]int32, n, m, d int) []step {
|
||||
var rev []step
|
||||
x, y := n, m
|
||||
for ; d >= 0; d-- {
|
||||
snap := trace[d]
|
||||
at := func(k int) int { return int(snap[k+d]) }
|
||||
k := x - y
|
||||
var prevK int
|
||||
if k == -d || (k != d && at(k-1) < at(k+1)) {
|
||||
prevK = k + 1
|
||||
} else {
|
||||
prevK = k - 1
|
||||
}
|
||||
prevX := 0
|
||||
if d > 0 {
|
||||
prevX = at(prevK)
|
||||
}
|
||||
prevY := prevX - prevK
|
||||
for x > prevX && y > prevY {
|
||||
rev = append(rev, stepEq)
|
||||
x--
|
||||
y--
|
||||
}
|
||||
if d > 0 {
|
||||
if x == prevX {
|
||||
rev = append(rev, stepIns)
|
||||
} else {
|
||||
rev = append(rev, stepDel)
|
||||
}
|
||||
}
|
||||
x, y = prevX, prevY
|
||||
}
|
||||
out := make([]step, len(rev))
|
||||
for i, s := range rev {
|
||||
out[len(rev)-1-i] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
func words(s string) []string { return strings.Fields(s) }
|
||||
|
||||
// apply rebuilds b from a using the ops, copying equal runs from a and the
|
||||
// replacement/insertion runs from b. If Diff is right this must equal b.
|
||||
func apply(a, b []string, ops []reconcile.Op) []string {
|
||||
var out []string
|
||||
for _, o := range ops {
|
||||
switch o.Kind {
|
||||
case reconcile.Equal:
|
||||
out = append(out, a[o.AStart:o.AEnd]...)
|
||||
default:
|
||||
out = append(out, b[o.BStart:o.BEnd]...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDiff_Cases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a, b string
|
||||
want []reconcile.Op
|
||||
}{
|
||||
{"identical", "a b c", "a b c", []reconcile.Op{{Kind: reconcile.Equal, AStart: 0, AEnd: 3, BStart: 0, BEnd: 3}}},
|
||||
{"both empty", "", "", nil},
|
||||
{"one word replaced", "a b c", "a x c", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Replace, AStart: 1, AEnd: 2, BStart: 1, BEnd: 2},
|
||||
{Kind: reconcile.Equal, AStart: 2, AEnd: 3, BStart: 2, BEnd: 3},
|
||||
}},
|
||||
{"word only in b", "a c", "a b c", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Insert, AStart: 1, AEnd: 1, BStart: 1, BEnd: 2},
|
||||
{Kind: reconcile.Equal, AStart: 1, AEnd: 2, BStart: 2, BEnd: 3},
|
||||
}},
|
||||
{"word only in a", "a b c", "a c", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Delete, AStart: 1, AEnd: 2, BStart: 1, BEnd: 1},
|
||||
{Kind: reconcile.Equal, AStart: 2, AEnd: 3, BStart: 1, BEnd: 2},
|
||||
}},
|
||||
{"nothing in common", "a b", "x y z", []reconcile.Op{
|
||||
{Kind: reconcile.Replace, AStart: 0, AEnd: 2, BStart: 0, BEnd: 3},
|
||||
}},
|
||||
{"a is empty", "", "x y", []reconcile.Op{{Kind: reconcile.Insert, AStart: 0, AEnd: 0, BStart: 0, BEnd: 2}}},
|
||||
{"b is empty", "x y", "", []reconcile.Op{{Kind: reconcile.Delete, AStart: 0, AEnd: 2, BStart: 0, BEnd: 0}}},
|
||||
{"adjacent delete and insert read as one replace", "a b c d", "a x y d", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Replace, AStart: 1, AEnd: 3, BStart: 1, BEnd: 3},
|
||||
{Kind: reconcile.Equal, AStart: 3, AEnd: 4, BStart: 3, BEnd: 4},
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, reconcile.Diff(words(tt.a), words(tt.b)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// lcsLen is a plain dynamic-programming reference. Myers must find an edit script
|
||||
// whose equal runs add up to exactly this: a diff that is merely valid but not
|
||||
// minimal would turn real agreement into disputes the adjudicator is then asked about.
|
||||
func lcsLen(a, b []string) int {
|
||||
prev := make([]int, len(b)+1)
|
||||
for i := 1; i <= len(a); i++ {
|
||||
cur := make([]int, len(b)+1)
|
||||
for j := 1; j <= len(b); j++ {
|
||||
if a[i-1] == b[j-1] {
|
||||
cur[j] = prev[j-1] + 1
|
||||
} else {
|
||||
cur[j] = max(prev[j], cur[j-1])
|
||||
}
|
||||
}
|
||||
prev = cur
|
||||
}
|
||||
return prev[len(b)]
|
||||
}
|
||||
|
||||
func TestDiff_RebuildsBAndKeepsEveryCommonWordOnRandomInputs(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(7))
|
||||
alphabet := []string{"a", "b", "c", "d", "e", "f"}
|
||||
gen := func(n int) []string {
|
||||
out := make([]string, n)
|
||||
for i := range out {
|
||||
out[i] = alphabet[rng.Intn(len(alphabet))]
|
||||
}
|
||||
return out
|
||||
}
|
||||
for i := 0; i < 400; i++ {
|
||||
a, b := gen(rng.Intn(25)), gen(rng.Intn(25))
|
||||
|
||||
ops := reconcile.Diff(a, b)
|
||||
|
||||
require.Equal(t, append([]string{}, b...), append([]string{}, apply(a, b, ops)...), "a=%v b=%v ops=%v", a, b, ops)
|
||||
common := 0
|
||||
prevA, prevB := 0, 0
|
||||
for _, o := range ops {
|
||||
require.Equal(t, prevA, o.AStart, "ops must tile a without gaps: %v", ops)
|
||||
require.Equal(t, prevB, o.BStart, "ops must tile b without gaps: %v", ops)
|
||||
prevA, prevB = o.AEnd, o.BEnd
|
||||
if o.Kind == reconcile.Equal {
|
||||
common += o.AEnd - o.AStart
|
||||
require.Equal(t, a[o.AStart:o.AEnd], b[o.BStart:o.BEnd], "an Equal op must really be equal")
|
||||
}
|
||||
}
|
||||
require.Equal(t, len(a), prevA)
|
||||
require.Equal(t, len(b), prevB)
|
||||
require.Equal(t, lcsLen(a, b), common, "the diff must be minimal: a=%v b=%v", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiff_HandlesAnHoursWorthOfWordsFast(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(3))
|
||||
a := make([]string, 6500)
|
||||
for i := range a {
|
||||
a[i] = string(rune('a'+rng.Intn(26))) + string(rune('a'+rng.Intn(26)))
|
||||
}
|
||||
b := append([]string{}, a...)
|
||||
for i := 0; i < 380; i++ { // ~380 disputes, as measured on a real hour
|
||||
b[rng.Intn(len(b))] = "zz" + string(rune('a'+rng.Intn(26)))
|
||||
}
|
||||
|
||||
ops := reconcile.Diff(a, b)
|
||||
|
||||
assert.NotEmpty(t, ops)
|
||||
assert.Equal(t, append([]string{}, b...), append([]string{}, apply(a, b, ops)...))
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package reconcile
|
||||
|
||||
import "strings"
|
||||
|
||||
// Dispute is one place where the two transcripts disagree. BaseStart:BaseEnd and
|
||||
// AltStart:AltEnd are token ranges into the two Tokenize results; an empty range
|
||||
// means that engine heard nothing there. Base and Alt are the two readings as each
|
||||
// engine wrote them, so a replaced span takes the alternative's own spelling.
|
||||
type Dispute struct {
|
||||
BaseStart, BaseEnd int
|
||||
AltStart, AltEnd int
|
||||
Base, Alt string
|
||||
}
|
||||
|
||||
// FindDisputes aligns base against alt word by word and returns the disagreements.
|
||||
// Differences separated by a single shared word are merged into one dispute: for a
|
||||
// reader that is one decision, and asking about "b" and "d" separately in "b c d"
|
||||
// hides that "c" sits between them. Two shared words keep disputes apart.
|
||||
func FindDisputes(base, alt []Token) []Dispute {
|
||||
a, b := keysOf(base), keysOf(alt)
|
||||
ops := Diff(a, b)
|
||||
var out []Dispute
|
||||
for i := 0; i < len(ops); i++ {
|
||||
if ops[i].Kind == Equal {
|
||||
continue
|
||||
}
|
||||
cur := ops[i]
|
||||
for i+2 < len(ops) && ops[i+1].Kind == Equal && ops[i+1].AEnd-ops[i+1].AStart == 1 && ops[i+2].Kind != Equal {
|
||||
cur.AEnd, cur.BEnd = ops[i+2].AEnd, ops[i+2].BEnd
|
||||
i += 2
|
||||
}
|
||||
out = append(out, Dispute{
|
||||
BaseStart: cur.AStart, BaseEnd: cur.AEnd,
|
||||
AltStart: cur.BStart, AltEnd: cur.BEnd,
|
||||
Base: join(base[cur.AStart:cur.AEnd]),
|
||||
Alt: join(alt[cur.BStart:cur.BEnd]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fillers are Swedish discourse particles. A dispute that is only one or two of them
|
||||
// on one side changes no meaning, so it is resolved without asking a model. The list
|
||||
// is deliberately short and leaves out words that can be an answer ("nej") or that
|
||||
// carry content.
|
||||
var fillers = map[string]bool{
|
||||
"ju": true, "då": true, "liksom": true, "alltså": true, "så": true, "ja": true,
|
||||
"eh": true, "ehm": true, "öh": true, "mm": true, "hm": true, "typ": true, "va": true, "asså": true,
|
||||
}
|
||||
|
||||
// maxFillerRun is how many filler words may be waved through together.
|
||||
const maxFillerRun = 2
|
||||
|
||||
// AutoResolve reports whether a dispute needs no model, in which case the base
|
||||
// reading is kept: either the two sides are the same letters once spaces are removed
|
||||
// (a compound written as one word or two), or one side is empty and the other is a
|
||||
// short run of filler words.
|
||||
func AutoResolve(d Dispute) bool {
|
||||
baseKeys, altKeys := keysOfText(d.Base), keysOfText(d.Alt)
|
||||
if strings.Join(baseKeys, "") == strings.Join(altKeys, "") {
|
||||
return true
|
||||
}
|
||||
return (len(baseKeys) == 0 && onlyFillers(altKeys)) || (len(altKeys) == 0 && onlyFillers(baseKeys))
|
||||
}
|
||||
|
||||
func onlyFillers(keys []string) bool {
|
||||
if len(keys) == 0 || len(keys) > maxFillerRun {
|
||||
return false
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !fillers[k] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ContextAround returns up to window words of the base before and after the dispute,
|
||||
// as the base wrote them.
|
||||
func ContextAround(base []Token, d Dispute, window int) (before, after string) {
|
||||
return join(base[max(0, d.BaseStart-window):d.BaseStart]), join(base[d.BaseEnd:min(len(base), d.BaseEnd+window)])
|
||||
}
|
||||
|
||||
func keysOf(ts []Token) []string {
|
||||
out := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = t.Key
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysOfText(s string) []string { return keysOf(Tokenize(s)) }
|
||||
|
||||
func join(ts []Token) string {
|
||||
parts := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
parts[i] = t.Text
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
func disputes(base, alt string) []reconcile.Dispute {
|
||||
return reconcile.FindDisputes(reconcile.Tokenize(base), reconcile.Tokenize(alt))
|
||||
}
|
||||
|
||||
func TestFindDisputes_NoneWhenTheTextsAgreeWordForWord(t *testing.T) {
|
||||
assert.Empty(t, disputes("Vi går igenom läget.", "vi går igenom läget"),
|
||||
"casing and punctuation are not disagreements")
|
||||
}
|
||||
|
||||
func TestFindDisputes_OneReplacedWordIsOneDisputeCarryingBothReadings(t *testing.T) {
|
||||
got := disputes("Vi ska målade grannarna.", "Vi ska mölade grannarna.")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
d := got[0]
|
||||
assert.Equal(t, "målade", d.Base)
|
||||
assert.Equal(t, "mölade", d.Alt)
|
||||
assert.Equal(t, 2, d.BaseStart)
|
||||
assert.Equal(t, 3, d.BaseEnd)
|
||||
assert.Equal(t, 2, d.AltStart)
|
||||
assert.Equal(t, 3, d.AltEnd)
|
||||
}
|
||||
|
||||
func TestFindDisputes_ReadingsKeepTheirOwnSpellingAndPunctuation(t *testing.T) {
|
||||
got := disputes("Det blir A+2, alltså.", "Det blir A 2, alltså.")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "A+2,", got[0].Base)
|
||||
assert.Equal(t, "A 2,", got[0].Alt, "the alternative's words replace the span as the engine wrote them")
|
||||
}
|
||||
|
||||
func TestFindDisputes_AWordOnlyOneEngineHeardHasAnEmptyOtherSide(t *testing.T) {
|
||||
got := disputes("Man får tillbaka ett förslag.", "Man får tillbaka ett coolt förslag.")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "", got[0].Base)
|
||||
assert.Equal(t, "coolt", got[0].Alt)
|
||||
assert.Equal(t, got[0].BaseStart, got[0].BaseEnd, "an insertion has an empty base range at its position")
|
||||
assert.Equal(t, 4, got[0].BaseStart)
|
||||
}
|
||||
|
||||
func TestFindDisputes_TwoDifferencesSeparatedByOneSharedWordAreOneDispute(t *testing.T) {
|
||||
// "x ... y" around a single agreed word is one decision for the reader, not two.
|
||||
got := disputes("a b c d e", "a x c y e")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "b c d", got[0].Base)
|
||||
assert.Equal(t, "x c y", got[0].Alt)
|
||||
}
|
||||
|
||||
func TestFindDisputes_TwoSharedWordsKeepTheDisputesApart(t *testing.T) {
|
||||
got := disputes("a b c d e f", "a x c d y f")
|
||||
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, "b", got[0].Base)
|
||||
assert.Equal(t, "e", got[1].Base)
|
||||
}
|
||||
|
||||
func TestFindDisputes_ChainsOfCloseDifferencesMergeIntoOne(t *testing.T) {
|
||||
got := disputes("a b c d e f g", "a x c y e z g")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "b c d e f", got[0].Base)
|
||||
}
|
||||
|
||||
func TestFindDisputes_LeavesIndicesThatAddressBothTokenSlices(t *testing.T) {
|
||||
base := reconcile.Tokenize("Hej Anders och Maria")
|
||||
alt := reconcile.Tokenize("Hej Sanders och Maria")
|
||||
|
||||
got := reconcile.FindDisputes(base, alt)
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "Anders", base[got[0].BaseStart].Text)
|
||||
assert.Equal(t, "Sanders", alt[got[0].AltStart].Text)
|
||||
}
|
||||
|
||||
func TestAutoResolve(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
base, alt string
|
||||
resolved bool
|
||||
}{
|
||||
{"a compound written as two words or one", "i takrenns delen", "i takrennsdelen", true},
|
||||
{"a compound with a hyphen", "takrenns-delen", "takrenns delen", true},
|
||||
{"a filler only the alternative heard", "vi ska ju betala", "vi ska betala", true},
|
||||
{"a filler only the base heard", "vi ska betala", "vi ska ju betala", true},
|
||||
{"two fillers in a row", "vi ska ja ju betala", "vi ska betala", true},
|
||||
{"a different word is for the model", "vi ska målade", "vi ska mölade", false},
|
||||
{"a different number is for the model", "totalt 1050 kronor", "totalt 2050 kronor", false},
|
||||
{"a content word only one side heard is for the model", "ett förslag", "ett coolt förslag", false},
|
||||
{"three filler words are too many to wave through", "vi ska ju då liksom betala", "vi ska betala", false},
|
||||
{"a filler swapped for a content word is for the model", "vi ska ju betala", "vi ska nu betala", false},
|
||||
{"nej is an answer, not a filler", "men nej det går inte", "men det går inte", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := disputes(tt.base, tt.alt)
|
||||
require.Len(t, got, 1, "the fixture must be exactly one dispute")
|
||||
|
||||
assert.Equal(t, tt.resolved, reconcile.AutoResolve(got[0]))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextAround_TakesWordsFromTheBaseOnBothSidesAndStopsAtTheEdges(t *testing.T) {
|
||||
base := reconcile.Tokenize("ett två tre fyra fem sex sju åtta nio tio")
|
||||
d := reconcile.Dispute{BaseStart: 4, BaseEnd: 6} // "fem sex"
|
||||
|
||||
before, after := reconcile.ContextAround(base, d, 3)
|
||||
|
||||
assert.Equal(t, "två tre fyra", before)
|
||||
assert.Equal(t, "sju åtta nio", after)
|
||||
|
||||
b2, a2 := reconcile.ContextAround(base, reconcile.Dispute{BaseStart: 0, BaseEnd: 1}, 3)
|
||||
assert.Equal(t, "", b2)
|
||||
assert.Equal(t, "två tre fyra", a2)
|
||||
|
||||
b3, a3 := reconcile.ContextAround(base, reconcile.Dispute{BaseStart: 9, BaseEnd: 10}, 3)
|
||||
assert.Equal(t, "sju åtta nio", b3)
|
||||
assert.Equal(t, "", a3)
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
)
|
||||
|
||||
// MaxContested is the default bound on the stored list of judged spots. A 90-minute
|
||||
// recording has a few hundred, so this is a guard against a runaway, not a budget. The
|
||||
// counts on the Reconciliation are always exact; only the list is capped.
|
||||
const MaxContested = 2000
|
||||
|
||||
// beforeWords is how many words of left context a stored spot carries: enough to find
|
||||
// the place again, small enough to stay out of the way.
|
||||
const beforeWords = 8
|
||||
|
||||
// Options configure Reconcile. Zero Window and BreakAfter take their defaults.
|
||||
type Options struct {
|
||||
// Window is how many words of the base to show on each side of a dispute. 100 is
|
||||
// what was measured (docs/research/local-model-adjudicator-sweep-2026-10-01.md).
|
||||
Window int
|
||||
// BreakAfter is how many consecutive model errors stop the reconciliation, so a
|
||||
// model that is down costs three timeouts, not one per dispute.
|
||||
BreakAfter int
|
||||
// MaxContested caps the stored judged spots; zero takes the package default.
|
||||
MaxContested int
|
||||
// BaseEngine, AltEngine and Reconciler are recorded on the result.
|
||||
BaseEngine, AltEngine, Reconciler string
|
||||
}
|
||||
|
||||
// Reconcile merges two transcripts of the same speech. The base text stands except
|
||||
// where the model chooses the alternative's reading of a dispute; trivial disputes
|
||||
// (a compound split, a filler word) are resolved without the model. It never fails a
|
||||
// transcription: if the model cannot be used the base text comes back unreconciled
|
||||
// and the result says why. The only error it returns is a cancelled context, which
|
||||
// is the job stopping, not a verdict on the model.
|
||||
func Reconcile(ctx context.Context, base, alt domain.Transcription, adj Adjudicator, o Options) (domain.Transcription, error) {
|
||||
window, breakAfter, maxSpots := o.Window, o.BreakAfter, o.MaxContested
|
||||
if window <= 0 {
|
||||
window = 100
|
||||
}
|
||||
if breakAfter <= 0 {
|
||||
breakAfter = 3
|
||||
}
|
||||
if maxSpots <= 0 {
|
||||
maxSpots = MaxContested
|
||||
}
|
||||
rep := progress.From(ctx)
|
||||
|
||||
rep.Begin("compare", 0)
|
||||
bt, at := Tokenize(base.Text), Tokenize(alt.Text)
|
||||
ds := FindDisputes(bt, at)
|
||||
rep.Done("compare")
|
||||
|
||||
// The deciding phase counts what the MODEL decides: the trivial disputes are settled
|
||||
// without it, so counting them would leave the bar short of its end.
|
||||
deciding := 0
|
||||
for _, d := range ds {
|
||||
if !AutoResolve(d) {
|
||||
deciding++
|
||||
}
|
||||
}
|
||||
rec := &domain.Reconciliation{
|
||||
Status: domain.ReconcileDone, Base: o.BaseEngine, Alt: o.AltEngine, Reconciler: o.Reconciler, Disputes: len(ds),
|
||||
}
|
||||
seconds := positioner(base, len(bt))
|
||||
spot := func(d Dispute, used string, unsure bool) {
|
||||
before, _ := ContextAround(bt, d, window)
|
||||
if w := strings.Fields(before); len(w) > beforeWords {
|
||||
before = strings.Join(w[len(w)-beforeWords:], " ")
|
||||
}
|
||||
if len(rec.Contested) < maxSpots {
|
||||
rec.Contested = append(rec.Contested, domain.ContestedSpot{
|
||||
Seconds: seconds(d.BaseStart), Base: d.Base, Alt: d.Alt, Used: used, Unsure: unsure, Before: before,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
rep.Begin("decide", deciding)
|
||||
verdicts := make([]Verdict, len(ds))
|
||||
consecutive, handled := 0, 0
|
||||
for i, d := range ds {
|
||||
if AutoResolve(d) {
|
||||
rec.AutoResolved++
|
||||
verdicts[i] = VerdictBase
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
before, after := ContextAround(bt, d, window)
|
||||
v, err := adj.Adjudicate(ctx, Request{Before: before, After: after, Base: d.Base, Alt: d.Alt, AltFirst: i%2 == 1})
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return domain.Transcription{}, ctxErr
|
||||
}
|
||||
rec.Errors++
|
||||
consecutive++
|
||||
if consecutive >= breakAfter {
|
||||
rep.Fail("decide", err)
|
||||
return skipped(base, rec, fmt.Sprintf("the model failed %d times in a row (%s)", consecutive, errkind.Of(err))), nil
|
||||
}
|
||||
v = VerdictUnsure
|
||||
} else {
|
||||
consecutive = 0
|
||||
rec.Adjudicated++
|
||||
}
|
||||
verdicts[i] = v
|
||||
switch v {
|
||||
case VerdictAlt:
|
||||
rec.Changed++
|
||||
spot(d, "alt", false)
|
||||
case VerdictUnsure:
|
||||
rec.Unsure++
|
||||
spot(d, "base", true)
|
||||
default:
|
||||
spot(d, "base", false)
|
||||
}
|
||||
handled++
|
||||
rep.Advance("decide", handled)
|
||||
}
|
||||
rep.Done("decide")
|
||||
|
||||
out := base
|
||||
out.Text, out.Segments = merge(base, bt, at, ds, verdicts)
|
||||
out.Reconciliation = rec
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// merge applies the verdicts: the base's words stand except where the alternative's
|
||||
// reading was chosen. It returns the merged text and the base's segments carrying the
|
||||
// merged words (timing is the base's). Segments are rewritten only when their words line
|
||||
// up with the text; a word wrapped across a segment edge makes them differ, and guessing
|
||||
// would put words in the wrong place, so then the base's segments are returned as they
|
||||
// were (ADR-035). The base's own segments are never edited in place.
|
||||
func merge(base domain.Transcription, bt, at []Token, ds []Dispute, verdicts []Verdict) (string, []domain.Segment) {
|
||||
segOf, aligned := segmentIndex(base.Segments, len(bt))
|
||||
var perSeg [][]string
|
||||
var changed map[int]bool
|
||||
if aligned {
|
||||
perSeg, changed = make([][]string, len(base.Segments)), map[int]bool{}
|
||||
}
|
||||
|
||||
var parts []string
|
||||
keep := func(from, to int) {
|
||||
for i := from; i < to; i++ {
|
||||
parts = append(parts, bt[i].Text)
|
||||
if aligned {
|
||||
perSeg[segOf[i]] = append(perSeg[segOf[i]], bt[i].Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
pos := 0
|
||||
for i, d := range ds {
|
||||
keep(pos, d.BaseStart)
|
||||
if verdicts[i] == VerdictAlt {
|
||||
for _, t := range at[d.AltStart:d.AltEnd] {
|
||||
parts = append(parts, t.Text)
|
||||
}
|
||||
if aligned {
|
||||
anchor := min(d.BaseStart, len(bt)-1)
|
||||
for _, t := range at[d.AltStart:d.AltEnd] {
|
||||
perSeg[segOf[anchor]] = append(perSeg[segOf[anchor]], t.Text)
|
||||
}
|
||||
changed[segOf[anchor]] = true
|
||||
for k := d.BaseStart; k < d.BaseEnd; k++ {
|
||||
changed[segOf[k]] = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
keep(d.BaseStart, d.BaseEnd)
|
||||
}
|
||||
pos = d.BaseEnd
|
||||
}
|
||||
keep(pos, len(bt))
|
||||
|
||||
if !aligned {
|
||||
return strings.Join(parts, " "), base.Segments
|
||||
}
|
||||
segs := append([]domain.Segment(nil), base.Segments...)
|
||||
for k := range changed {
|
||||
segs[k].Text = strings.Join(perSeg[k], " ")
|
||||
}
|
||||
return strings.Join(parts, " "), segs
|
||||
}
|
||||
|
||||
// segmentIndex maps each base token to the segment it came from. aligned is false when
|
||||
// there are no segments or their word count is not the text's.
|
||||
func segmentIndex(segs []domain.Segment, tokens int) (segOf []int, aligned bool) {
|
||||
if len(segs) == 0 || tokens == 0 {
|
||||
return nil, false
|
||||
}
|
||||
segOf = make([]int, 0, tokens)
|
||||
for k, s := range segs {
|
||||
for range Tokenize(s.Text) {
|
||||
segOf = append(segOf, k)
|
||||
}
|
||||
}
|
||||
return segOf, len(segOf) == tokens
|
||||
}
|
||||
|
||||
// skipped is the base text, none of the partial changes, and the reason.
|
||||
func skipped(base domain.Transcription, rec *domain.Reconciliation, reason string) domain.Transcription {
|
||||
out := base
|
||||
out.Reconciliation = &domain.Reconciliation{
|
||||
Status: domain.ReconcileSkipped, Reason: reason, Base: rec.Base, Alt: rec.Alt, Reconciler: rec.Reconciler,
|
||||
Disputes: rec.Disputes, AutoResolved: rec.AutoResolved, Errors: rec.Errors,
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// positioner maps a token index of the base to the second it starts at: through the
|
||||
// segments when there are any, proportionally otherwise.
|
||||
func positioner(base domain.Transcription, tokens int) func(int) float64 {
|
||||
if len(base.Segments) == 0 {
|
||||
return func(i int) float64 {
|
||||
if tokens == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(i) / float64(tokens) * base.AudioSeconds
|
||||
}
|
||||
}
|
||||
starts := make([]int, len(base.Segments))
|
||||
n := 0
|
||||
for i, s := range base.Segments {
|
||||
starts[i] = n
|
||||
n += len(Tokenize(s.Text))
|
||||
}
|
||||
return func(i int) float64 {
|
||||
k := 0
|
||||
for k+1 < len(starts) && starts[k+1] <= i {
|
||||
k++
|
||||
}
|
||||
return base.Segments[k].Start
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// What Reconcile reports while it works (ADR-035): the comparison, then the deciding
|
||||
// phase with the real number of model decisions as its total.
|
||||
|
||||
type repRec struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
}
|
||||
|
||||
func (r *repRec) add(e string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.events = append(r.events, e)
|
||||
}
|
||||
|
||||
func (r *repRec) Begin(key string, total int) { r.add(fmt.Sprintf("begin:%s:%d", key, total)) }
|
||||
func (r *repRec) Advance(key string, done int) { r.add(fmt.Sprintf("advance:%s:%d", key, done)) }
|
||||
func (r *repRec) Done(key string) { r.add("done:" + key) }
|
||||
func (r *repRec) Fail(key string, _ error) { r.add("fail:" + key) }
|
||||
func (r *repRec) Note(string, ...any) {}
|
||||
func (r *repRec) joined() string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return strings.Join(r.events, ",")
|
||||
}
|
||||
func withRep(r *repRec) context.Context { return progress.With(context.Background(), r) }
|
||||
func reconcileWith(t *testing.T, ctx context.Context, base, alt string, adj reconcile.Adjudicator) error {
|
||||
t.Helper()
|
||||
_, err := reconcile.Reconcile(ctx, tr(base), tr(alt), adj, opts)
|
||||
return err
|
||||
}
|
||||
|
||||
// two compound/filler disputes the model is never asked about, then two it is.
|
||||
const (
|
||||
progBase = "i takrenns delen ska vi ju betala totalt 1015 kronor till Axel idag"
|
||||
progAlt = "i takrennsdelen ska vi betala totalt 1050 kronor till Alex idag"
|
||||
)
|
||||
|
||||
func TestReconcileProgress_TheDecidingPhaseTotalIsTheModelDecisionsNotTheDisputes(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
|
||||
require.NoError(t, reconcileWith(t, withRep(rep), progBase, progAlt, always(reconcile.VerdictAlt)))
|
||||
|
||||
assert.Equal(t,
|
||||
"begin:compare:0,done:compare,begin:decide:2,advance:decide:1,advance:decide:2,done:decide",
|
||||
rep.joined(),
|
||||
"4 disputes, 2 settled without the model: the bar counts the 2 the model decides, so it can reach its end")
|
||||
}
|
||||
|
||||
func TestReconcileProgress_TextsThatAgreeStillClosePhasesSoNothingIsLeftRunning(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
|
||||
require.NoError(t, reconcileWith(t, withRep(rep), "samma text hela vägen", "samma text hela vägen", always(reconcile.VerdictAlt)))
|
||||
|
||||
assert.Equal(t, "begin:compare:0,done:compare,begin:decide:0,done:decide", rep.joined())
|
||||
}
|
||||
|
||||
func TestReconcileProgress_AModelErrorStillCountsAsADecisionTaken(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
adj := &scripted{fn: func(i int, _ reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i == 0 {
|
||||
return reconcile.VerdictBase, errors.New("one timeout")
|
||||
}
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
require.NoError(t, reconcileWith(t, withRep(rep), progBase, progAlt, adj))
|
||||
|
||||
assert.Contains(t, rep.joined(), "advance:decide:1,advance:decide:2", "a spot that errored was still handled; the bar must move past it")
|
||||
}
|
||||
|
||||
func TestReconcileProgress_TheBreakerTrippingShowsTheDecidingPhaseFailedNotDone(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
return reconcile.VerdictBase, errors.New("model down")
|
||||
}}
|
||||
small := opts
|
||||
small.BreakAfter = 2
|
||||
|
||||
_, err := reconcile.Reconcile(withRep(rep), tr(progBase), tr(progAlt), adj, small)
|
||||
|
||||
require.NoError(t, err, "a model that is down degrades the result; it does not fail the job")
|
||||
assert.Contains(t, rep.joined(), "fail:decide")
|
||||
assert.NotContains(t, rep.joined(), "done:decide")
|
||||
}
|
||||
|
||||
func TestReconcileProgress_AShutdownIsNeitherDoneNorFailed(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
ctx, cancel := context.WithCancel(withRep(rep))
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
cancel()
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
err := reconcileWith(t, ctx, progBase, progAlt, adj)
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.NotContains(t, rep.joined(), "fail:decide")
|
||||
assert.NotContains(t, rep.joined(), "done:decide")
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// The reason a reconciliation was skipped is stored and shown on a page about a private
|
||||
// recording, so it carries a kind, never the model's error text, which can echo the
|
||||
// prompt (adversarial review F3).
|
||||
|
||||
func TestReconcile_TheBreakerReasonCarriesAKindNotTheModelsText(t *testing.T) {
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
return reconcile.VerdictBase, errors.New("gateway echoed the prompt: zebrakaka secret words")
|
||||
}}
|
||||
small := opts
|
||||
small.BreakAfter = 2
|
||||
|
||||
out, err := reconcile.Reconcile(context.Background(), tr(progBase), tr(progAlt), adj, small)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, domain.ReconcileSkipped, out.Reconciliation.Status)
|
||||
assert.NotContains(t, out.Reconciliation.Reason, "zebrakaka", "provider text must not be stored or shown")
|
||||
assert.Contains(t, out.Reconciliation.Reason, "2 times in a row", "the reason still says what happened")
|
||||
assert.Contains(t, out.Reconciliation.Reason, "error", "and carries a kind")
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// Every judged spot is stored with a few words of left context, and the saved segments
|
||||
// carry the merged words while keeping the base's timing (ADR-035 decisions 6 and 7).
|
||||
|
||||
func TestReconcileSpots_AKeptSpotAndAChangedSpotAreBothStoredWithTheirVerdict(t *testing.T) {
|
||||
adj := &scripted{fn: func(i int, _ reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i == 0 {
|
||||
return reconcile.VerdictBase, nil
|
||||
}
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
out := merge(t, tr(progBase), tr(progAlt), adj)
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 2)
|
||||
assert.Equal(t, "base", out.Reconciliation.Contested[0].Used)
|
||||
assert.Equal(t, "1015", out.Reconciliation.Contested[0].Base)
|
||||
assert.Equal(t, "alt", out.Reconciliation.Contested[1].Used)
|
||||
assert.Equal(t, 1, out.Reconciliation.Changed, "the exact counts are unchanged by storing more")
|
||||
}
|
||||
|
||||
func TestReconcileSpots_AutoResolvedDisputesAreNotStored(t *testing.T) {
|
||||
out := merge(t, tr("i takrenns delen ska vi ju betala"), tr("i takrennsdelen ska vi betala"), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Empty(t, out.Reconciliation.Contested, "compounds and fillers were never judged; they are counted, not listed")
|
||||
}
|
||||
|
||||
func TestReconcileSpots_ASpotCarriesTheWordsJustBeforeIt(t *testing.T) {
|
||||
out := merge(t, tr("grannen målar ett vanligt staket mot gatan"), tr("grannen målar ett KORT staket mot gatan"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.Equal(t, "grannen målar ett", out.Reconciliation.Contested[0].Before)
|
||||
}
|
||||
|
||||
func TestReconcileSpots_TheContextIsAFewWordsNotTheWholeMeeting(t *testing.T) {
|
||||
long := "ett två tre fyra fem sex sju åtta nio tio elva tolv"
|
||||
out := merge(t, tr(long+" X slut"), tr(long+" Y slut"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.Equal(t, "fem sex sju åtta nio tio elva tolv", out.Reconciliation.Contested[0].Before,
|
||||
"eight words: enough to find the place, small enough to stay out of the way")
|
||||
}
|
||||
|
||||
func segmented(text string, segs ...domain.Segment) domain.Transcription {
|
||||
t := tr(text)
|
||||
t.Segments = segs
|
||||
return t
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AChangedWordReplacesTheBaseWordInItsOwnSegmentOnly(t *testing.T) {
|
||||
base := segmented("ett två tre. fyra fem sex. sju åtta nio.",
|
||||
domain.Segment{Start: 0, End: 10, Text: "ett två tre."},
|
||||
domain.Segment{Start: 10, End: 20, Text: "fyra fem sex."},
|
||||
domain.Segment{Start: 20, End: 30, Text: "sju åtta nio."})
|
||||
|
||||
out := merge(t, base, tr("ett två tre. fyra femm sex. sju åtta nio."), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Segments, 3)
|
||||
assert.Equal(t, "ett två tre.", out.Segments[0].Text, "an untouched segment keeps its exact text")
|
||||
assert.Equal(t, "fyra femm sex.", out.Segments[1].Text)
|
||||
assert.Equal(t, "sju åtta nio.", out.Segments[2].Text)
|
||||
assert.InDelta(t, 10.0, out.Segments[1].Start, 1e-9, "timing stays the base's")
|
||||
assert.InDelta(t, 20.0, out.Segments[1].End, 1e-9)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AWordOnlyTheAlternativeHeardLandsInTheSegmentItBelongsTo(t *testing.T) {
|
||||
base := segmented("a b c. d e f.", domain.Segment{Start: 0, End: 5, Text: "a b c."}, domain.Segment{Start: 5, End: 10, Text: "d e f."})
|
||||
|
||||
out := merge(t, base, tr("a b c. d NY e f."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "a b c.", out.Segments[0].Text)
|
||||
assert.Equal(t, "d NY e f.", out.Segments[1].Text)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AWordOnlyTheBaseHeardIsRemovedFromItsSegment(t *testing.T) {
|
||||
base := segmented("a b c. d X e f.", domain.Segment{Start: 0, End: 5, Text: "a b c."}, domain.Segment{Start: 5, End: 10, Text: "d X e f."})
|
||||
|
||||
out := merge(t, base, tr("a b c. d e f."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "d e f.", out.Segments[1].Text)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AKeptBaseLeavesEverySegmentAsItWas(t *testing.T) {
|
||||
base := segmented("a x b", domain.Segment{Start: 0, End: 4, Text: "a x b"})
|
||||
|
||||
out := merge(t, base, tr("a y b"), always(reconcile.VerdictBase))
|
||||
|
||||
assert.Equal(t, base.Segments, out.Segments)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_TheCallersSegmentsAreNeverChangedInPlace(t *testing.T) {
|
||||
segs := []domain.Segment{{Start: 0, End: 4, Text: "a x b"}}
|
||||
base := segmented("a x b", segs...)
|
||||
|
||||
_ = merge(t, base, tr("a y b"), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "a x b", segs[0].Text, "Reconcile builds new segments; it does not edit the base transcription's")
|
||||
}
|
||||
|
||||
func TestReconcileSegments_SegmentsThatDoNotLineUpWithTheTextAreLeftAsTheBases(t *testing.T) {
|
||||
// A word wrapped across a segment edge makes the segments' word count differ from
|
||||
// the joined text's. Guessing would put words in the wrong place, so leave them.
|
||||
base := segmented("a x b c", domain.Segment{Start: 0, End: 4, Text: "a x"}, domain.Segment{Start: 4, End: 8, Text: "b c d"})
|
||||
|
||||
out := merge(t, base, tr("a y b c"), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, base.Segments, out.Segments)
|
||||
assert.Equal(t, "a y b c", out.Text, "the merged text is unaffected")
|
||||
}
|
||||
|
||||
func TestReconcileSegments_ASpanThatCrossesASegmentEdgeIsRemovedFromBothSegments(t *testing.T) {
|
||||
base := segmented("a b c d", domain.Segment{Start: 0, End: 5, Text: "a b"}, domain.Segment{Start: 5, End: 10, Text: "c d"})
|
||||
|
||||
out := merge(t, base, tr("a X d"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Equal(t, "a X d", out.Text)
|
||||
assert.Equal(t, "a X", out.Segments[0].Text)
|
||||
assert.Equal(t, "d", out.Segments[1].Text, "'c' was replaced too: it must not linger in the second segment")
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// scripted is an Adjudicator that answers from a function and records every request.
|
||||
type scripted struct {
|
||||
fn func(i int, r reconcile.Request) (reconcile.Verdict, error)
|
||||
calls []reconcile.Request
|
||||
}
|
||||
|
||||
func (s *scripted) Adjudicate(_ context.Context, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
i := len(s.calls)
|
||||
s.calls = append(s.calls, r)
|
||||
return s.fn(i, r)
|
||||
}
|
||||
|
||||
func always(v reconcile.Verdict) *scripted {
|
||||
return &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return v, nil }}
|
||||
}
|
||||
|
||||
func tr(text string) domain.Transcription {
|
||||
return domain.Transcription{Text: text, AudioSeconds: 100, Endpoint: "e"}
|
||||
}
|
||||
|
||||
var opts = reconcile.Options{Window: 100, BreakAfter: 3, Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}
|
||||
|
||||
func merge(t *testing.T, base, alt domain.Transcription, adj reconcile.Adjudicator) domain.Transcription {
|
||||
t.Helper()
|
||||
out, err := reconcile.Reconcile(context.Background(), base, alt, adj, opts)
|
||||
require.NoError(t, err)
|
||||
return out
|
||||
}
|
||||
|
||||
func TestReconcile_IdenticalTextsNeedNoModelAndComeBackUnchanged(t *testing.T) {
|
||||
adj := always(reconcile.VerdictAlt)
|
||||
|
||||
out := merge(t, tr("Vi går igenom läget."), tr("vi går igenom läget"), adj)
|
||||
|
||||
assert.Equal(t, "Vi går igenom läget.", out.Text, "the base's own spelling and punctuation stand")
|
||||
assert.Empty(t, adj.calls)
|
||||
require.NotNil(t, out.Reconciliation)
|
||||
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
||||
assert.Zero(t, out.Reconciliation.Disputes)
|
||||
}
|
||||
|
||||
func TestReconcile_TrivialDisputesAreResolvedWithoutAskingTheModel(t *testing.T) {
|
||||
adj := always(reconcile.VerdictAlt)
|
||||
|
||||
out := merge(t, tr("i takrenns delen ska vi ju betala"), tr("i takrennsdelen ska vi betala"), adj)
|
||||
|
||||
assert.Empty(t, adj.calls, "a compound split and a filler particle are not worth a model call")
|
||||
assert.Equal(t, "i takrenns delen ska vi ju betala", out.Text, "the base reading is kept")
|
||||
assert.Equal(t, 2, out.Reconciliation.Disputes)
|
||||
assert.Equal(t, 2, out.Reconciliation.AutoResolved)
|
||||
assert.Zero(t, out.Reconciliation.Adjudicated)
|
||||
}
|
||||
|
||||
func TestReconcile_AltVerdictReplacesTheSpanWithTheAlternativesOwnWords(t *testing.T) {
|
||||
out := merge(t, tr("Totalt 1015 kronor. Det ska betalas."), tr("Totalt 1050 kronor. Det ska betalas."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "Totalt 1050 kronor. Det ska betalas.", out.Text)
|
||||
assert.Equal(t, 1, out.Reconciliation.Changed)
|
||||
assert.Equal(t, 1, out.Reconciliation.Adjudicated)
|
||||
}
|
||||
|
||||
func TestReconcile_BaseAndUnsureVerdictsKeepTheBaseButOnlyUnsureIsFlagged(t *testing.T) {
|
||||
base := tr("Totalt 1015 kronor till Axel.")
|
||||
alt := tr("Totalt 1050 kronor till Alex.")
|
||||
|
||||
keptBase := merge(t, base, alt, always(reconcile.VerdictBase))
|
||||
unsure := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
||||
|
||||
assert.Equal(t, "Totalt 1015 kronor till Axel.", keptBase.Text)
|
||||
assert.Zero(t, keptBase.Reconciliation.Unsure)
|
||||
require.Len(t, keptBase.Reconciliation.Contested, 2,
|
||||
"every judged spot is stored, kept ones too: without them the reconciler cannot be compared with either engine alone (ADR-035)")
|
||||
for _, c := range keptBase.Reconciliation.Contested {
|
||||
assert.Equal(t, "base", c.Used)
|
||||
assert.False(t, c.Unsure, "a confident 'the base is right' is stored but is not an uncertain spot")
|
||||
}
|
||||
assert.Equal(t, "Totalt 1015 kronor till Axel.", unsure.Text, "unsure keeps the base")
|
||||
assert.Equal(t, 2, unsure.Reconciliation.Unsure)
|
||||
require.Len(t, unsure.Reconciliation.Contested, 2)
|
||||
assert.True(t, unsure.Reconciliation.Contested[0].Unsure)
|
||||
assert.Equal(t, "base", unsure.Reconciliation.Contested[0].Used)
|
||||
}
|
||||
|
||||
func TestReconcile_ChangedSpotsAreRecordedWithBothReadings(t *testing.T) {
|
||||
out := merge(t, tr("det blir A+2 imorgon"), tr("det blir en A imorgon"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
c := out.Reconciliation.Contested[0]
|
||||
assert.Equal(t, "A+2", c.Base)
|
||||
assert.Equal(t, "en A", c.Alt)
|
||||
assert.Equal(t, "alt", c.Used)
|
||||
assert.False(t, c.Unsure)
|
||||
}
|
||||
|
||||
func TestReconcile_AWordOnlyTheAlternativeHeardIsInsertedWhereItBelongs(t *testing.T) {
|
||||
out := merge(t, tr("Man får tillbaka ett förslag."), tr("Man får tillbaka ett coolt förslag."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "Man får tillbaka ett coolt förslag.", out.Text)
|
||||
}
|
||||
|
||||
func TestReconcile_AWordOnlyTheBaseHeardCanBeRemoved(t *testing.T) {
|
||||
out := merge(t, tr("Man får tillbaka ett coolt förslag."), tr("Man får tillbaka ett förslag."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "Man får tillbaka ett förslag.", out.Text)
|
||||
}
|
||||
|
||||
func TestReconcile_SeveralDisputesAreAppliedIndependentlyAndInOrder(t *testing.T) {
|
||||
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
if r.Alt == "mölade" || r.Alt == "Alex" {
|
||||
return reconcile.VerdictAlt, nil
|
||||
}
|
||||
return reconcile.VerdictBase, nil
|
||||
}}
|
||||
|
||||
out := merge(t, tr("kunderna målade nu och sen betalas till Axel via ombud med 1050 kronor"),
|
||||
tr("kunderna mölade nu och sen betalas till Alex via ombud med 2050 kronor"), adj)
|
||||
|
||||
assert.Equal(t, "kunderna mölade nu och sen betalas till Alex via ombud med 1050 kronor", out.Text)
|
||||
assert.Equal(t, 3, out.Reconciliation.Disputes)
|
||||
assert.Equal(t, 2, out.Reconciliation.Changed)
|
||||
}
|
||||
|
||||
func TestReconcile_TheReadingOrderAlternatesFromOneDisputeToTheNext(t *testing.T) {
|
||||
adj := always(reconcile.VerdictBase)
|
||||
|
||||
merge(t, tr("a x s1 s2 y t1 t2 z d"), tr("a p s1 s2 q t1 t2 r d"), adj)
|
||||
|
||||
require.Len(t, adj.calls, 3)
|
||||
assert.False(t, adj.calls[0].AltFirst)
|
||||
assert.True(t, adj.calls[1].AltFirst)
|
||||
assert.False(t, adj.calls[2].AltFirst, "alternating averages out position bias across a transcript")
|
||||
}
|
||||
|
||||
func TestReconcile_TheModelGetsTheContextWindowFromTheBase(t *testing.T) {
|
||||
adj := always(reconcile.VerdictBase)
|
||||
o := opts
|
||||
o.Window = 2
|
||||
|
||||
_, err := reconcile.Reconcile(context.Background(), tr("ett två tre fyra fem sex sju åtta nio"), tr("ett två tre fyra FEMM sex sju åtta nio"), adj, o)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, adj.calls, 1)
|
||||
assert.Equal(t, "tre fyra", adj.calls[0].Before)
|
||||
assert.Equal(t, "sex sju", adj.calls[0].After)
|
||||
assert.Equal(t, "fem", adj.calls[0].Base)
|
||||
assert.Equal(t, "FEMM", adj.calls[0].Alt)
|
||||
}
|
||||
|
||||
func TestReconcile_OneModelErrorLeavesThatSpotUnsureAndCarriesOn(t *testing.T) {
|
||||
boom := errors.New("timeout")
|
||||
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i == 0 {
|
||||
return reconcile.VerdictUnsure, boom
|
||||
}
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
out := merge(t, tr("a x s1 s2 y c"), tr("a p s1 s2 q c"), adj)
|
||||
|
||||
assert.Equal(t, "a x s1 s2 q c", out.Text, "the first spot stayed as the base, the second was decided")
|
||||
assert.Equal(t, 1, out.Reconciliation.Errors)
|
||||
assert.Equal(t, 1, out.Reconciliation.Unsure)
|
||||
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
||||
}
|
||||
|
||||
func TestReconcile_ConsecutiveModelErrorsStopTheWholeReconciliationInsteadOfWaitingOutEveryTimeout(t *testing.T) {
|
||||
boom := errors.New("gateway down")
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return reconcile.VerdictUnsure, boom }}
|
||||
a, b := make([]string, 0, 20), make([]string, 0, 20)
|
||||
for i := 0; i < 10; i++ {
|
||||
a = append(a, fmt.Sprintf("w%d", i), "s", "t")
|
||||
b = append(b, fmt.Sprintf("v%d", i), "s", "t")
|
||||
}
|
||||
|
||||
out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)
|
||||
|
||||
assert.Len(t, adj.calls, 3, "three consecutive failures and it stops asking")
|
||||
assert.Equal(t, strings.Join(a, " "), out.Text, "the base text, with none of the partial changes")
|
||||
assert.Equal(t, domain.ReconcileSkipped, out.Reconciliation.Status)
|
||||
assert.Contains(t, out.Reconciliation.Reason, "3 times in a row")
|
||||
assert.NotContains(t, out.Reconciliation.Reason, "gateway down", "the reason is shown on the page: a kind, not the model's own text")
|
||||
}
|
||||
|
||||
func TestReconcile_ASuccessBetweenFailuresResetsTheBreaker(t *testing.T) {
|
||||
boom := errors.New("blip")
|
||||
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i%3 == 2 {
|
||||
return reconcile.VerdictBase, nil
|
||||
}
|
||||
return reconcile.VerdictUnsure, boom
|
||||
}}
|
||||
a, b := make([]string, 0, 20), make([]string, 0, 20)
|
||||
for i := 0; i < 9; i++ {
|
||||
a = append(a, fmt.Sprintf("w%d", i), "s", "t")
|
||||
b = append(b, fmt.Sprintf("v%d", i), "s", "t")
|
||||
}
|
||||
|
||||
out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)
|
||||
|
||||
assert.Len(t, adj.calls, 9, "two failures then a success never reach three in a row")
|
||||
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
||||
assert.Equal(t, 6, out.Reconciliation.Errors)
|
||||
}
|
||||
|
||||
func TestReconcile_ACancelledJobIsAnErrorNotASkippedReconciliation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
cancel()
|
||||
return reconcile.VerdictUnsure, ctx.Err()
|
||||
}}
|
||||
|
||||
_, err := reconcile.Reconcile(ctx, tr("a x b"), tr("a y b"), adj, opts)
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled, "a shutdown must not be recorded as 'the model was unavailable'")
|
||||
}
|
||||
|
||||
func TestReconcile_KeepsTheBaseTimingAndDurationAndNamesTheEngines(t *testing.T) {
|
||||
base := tr("a x b")
|
||||
base.Segments = []domain.Segment{{Start: 0, End: 4, Text: "a x b"}}
|
||||
base.AudioSeconds = 3821.5
|
||||
|
||||
out := merge(t, base, tr("a y b"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Segments, 1)
|
||||
assert.InDelta(t, 0.0, out.Segments[0].Start, 1e-9, "timing is the base engine's")
|
||||
assert.InDelta(t, 4.0, out.Segments[0].End, 1e-9)
|
||||
assert.InDelta(t, 3821.5, out.AudioSeconds, 1e-9)
|
||||
assert.Equal(t, "iguana/pianissimo", out.Reconciliation.Base)
|
||||
assert.Equal(t, "iguana/kb-whisper", out.Reconciliation.Alt)
|
||||
assert.Equal(t, "iguana/gemma4-26b", out.Reconciliation.Reconciler)
|
||||
}
|
||||
|
||||
func TestReconcile_ASpotCarriesTheSecondItStartsAt(t *testing.T) {
|
||||
base := tr("ett två tre. fyra fem sex. sju åtta nio.")
|
||||
base.Segments = []domain.Segment{
|
||||
{Start: 0, End: 10, Text: "ett två tre."},
|
||||
{Start: 10, End: 20, Text: "fyra fem sex."},
|
||||
{Start: 20, End: 30, Text: "sju åtta nio."},
|
||||
}
|
||||
alt := tr("ett två tre. fyra femm sex. sju åtta nio.")
|
||||
|
||||
out := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.InDelta(t, 10.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "'fem' is in the second segment, which starts at 10 s")
|
||||
}
|
||||
|
||||
func TestReconcile_WithNoSegmentsASpotIsPlacedProportionally(t *testing.T) {
|
||||
base := tr("a b c d e f g h i j")
|
||||
base.AudioSeconds = 100
|
||||
alt := tr("a b c d e f g h i J2")
|
||||
|
||||
out := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.InDelta(t, 90.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "token 9 of 10 in 100 s")
|
||||
}
|
||||
|
||||
func TestReconcile_JudgedSpotsAreCappedButTheCountsAreExact(t *testing.T) {
|
||||
a, b := make([]string, 0, 600), make([]string, 0, 600)
|
||||
for i := 0; i < 300; i++ {
|
||||
a = append(a, fmt.Sprintf("w%d", i), "same", "same2")
|
||||
b = append(b, fmt.Sprintf("v%d", i), "same", "same2")
|
||||
}
|
||||
capped := opts
|
||||
capped.MaxContested = 50
|
||||
|
||||
out, err := reconcile.Reconcile(context.Background(), tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), always(reconcile.VerdictUnsure), capped)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 300, out.Reconciliation.Unsure)
|
||||
assert.Equal(t, 300, out.Reconciliation.Disputes)
|
||||
assert.Len(t, out.Reconciliation.Contested, 50, "the stored list is bounded; the counts are not")
|
||||
}
|
||||
|
||||
func TestReconcile_TheDefaultCapIsAFewThousandSoAWholeMeetingIsKept(t *testing.T) {
|
||||
assert.Equal(t, 2000, reconcile.MaxContested, "a 90-minute recording has a few hundred judged spots; the cap is a guard, not a budget")
|
||||
}
|
||||
|
||||
func TestReconcile_AVerdictReturnedAlongsideAnErrorIsIgnored(t *testing.T) {
|
||||
// An adjudicator that returns a verdict AND an error must not have the verdict used:
|
||||
// the contract is that an error means no decision.
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
return reconcile.VerdictAlt, errors.New("half a reply")
|
||||
}}
|
||||
|
||||
out := merge(t, tr("a x b"), tr("a y b"), adj)
|
||||
|
||||
assert.Equal(t, "a x b", out.Text, "no decision means the base stands")
|
||||
assert.Zero(t, out.Reconciliation.Changed)
|
||||
assert.Equal(t, 1, out.Reconciliation.Unsure)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// Package reconcile merges two transcripts of the same speech: it finds where they
|
||||
// disagree word by word, resolves the trivial disagreements itself, and asks a LOCAL
|
||||
// model to choose between the two readings of the rest (ADR-034). It depends on
|
||||
// nothing outside the standard library and the domain; the model, the engines and
|
||||
// the store are wired in from outside.
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Token is one word of a transcript. Text is the word as written, with its casing
|
||||
// and punctuation, so the merged transcript reads naturally; Key is what two engines
|
||||
// are compared on: lower case, letters and digits only.
|
||||
type Token struct {
|
||||
Text string
|
||||
Key string
|
||||
}
|
||||
|
||||
// Tokenize splits s on whitespace. A "word" that has no letter or digit (a dash, a
|
||||
// lone full stop) is not something two engines can disagree about, so it is joined
|
||||
// to the neighbouring word instead of becoming a token of its own.
|
||||
func Tokenize(s string) []Token {
|
||||
var out []Token
|
||||
pending := ""
|
||||
for _, w := range strings.Fields(s) {
|
||||
k := key(w)
|
||||
if k == "" {
|
||||
if len(out) > 0 {
|
||||
out[len(out)-1].Text += " " + w
|
||||
} else if pending == "" {
|
||||
pending = w
|
||||
} else {
|
||||
pending += " " + w
|
||||
}
|
||||
continue
|
||||
}
|
||||
text := w
|
||||
if pending != "" {
|
||||
text = pending + " " + w
|
||||
pending = ""
|
||||
}
|
||||
out = append(out, Token{Text: text, Key: k})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func key(w string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range w {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(unicode.ToLower(r))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
func texts(ts []reconcile.Token) []string {
|
||||
out := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = t.Text
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keys(ts []reconcile.Token) []string {
|
||||
out := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = t.Key
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTokenize_KeepsTheWordAsWrittenAndKeysItWithoutCaseOrPunctuation(t *testing.T) {
|
||||
got := reconcile.Tokenize("Hej, allihopa! Vi går igenom ÅÄÖ.")
|
||||
|
||||
assert.Equal(t, []string{"Hej,", "allihopa!", "Vi", "går", "igenom", "ÅÄÖ."}, texts(got),
|
||||
"the original spelling and punctuation are kept for the merged text")
|
||||
assert.Equal(t, []string{"hej", "allihopa", "vi", "går", "igenom", "åäö"}, keys(got),
|
||||
"the key is lower case with punctuation removed, Swedish letters lowered correctly")
|
||||
}
|
||||
|
||||
func TestTokenize_KeysIgnoreHyphensAndPlusSoOnlyWordsDecideEquality(t *testing.T) {
|
||||
got := reconcile.Tokenize("end-to-end A+2 5 000 kr")
|
||||
|
||||
assert.Equal(t, []string{"endtoend", "a2", "5", "000", "kr"}, keys(got))
|
||||
}
|
||||
|
||||
func TestTokenize_APunctuationOnlyWordJoinsThePreviousWordInsteadOfBecomingOne(t *testing.T) {
|
||||
got := reconcile.Tokenize("Vi går – sen tillbaka . Slut")
|
||||
|
||||
assert.Equal(t, []string{"Vi", "går –", "sen", "tillbaka .", "Slut"}, texts(got),
|
||||
"a dash or a lone full stop is not a word the engines can disagree about")
|
||||
assert.Equal(t, []string{"vi", "går", "sen", "tillbaka", "slut"}, keys(got))
|
||||
}
|
||||
|
||||
func TestTokenize_LeadingPunctuationOnlyWordJoinsTheNextWord(t *testing.T) {
|
||||
got := reconcile.Tokenize("— Hej då")
|
||||
|
||||
assert.Equal(t, []string{"— Hej", "då"}, texts(got))
|
||||
}
|
||||
|
||||
func TestTokenize_NewlinesAndRunsOfSpaceAreJustSeparators(t *testing.T) {
|
||||
got := reconcile.Tokenize(" Hej\n allihopa \t Vi ")
|
||||
|
||||
assert.Equal(t, []string{"Hej", "allihopa", "Vi"}, texts(got))
|
||||
}
|
||||
|
||||
func TestTokenize_EmptyAndPunctuationOnlyInputGiveNoTokens(t *testing.T) {
|
||||
assert.Empty(t, reconcile.Tokenize(""))
|
||||
assert.Empty(t, reconcile.Tokenize(" \n "))
|
||||
assert.Empty(t, reconcile.Tokenize("— . !"))
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// ErrUnreadable is what a Prober returns when the FILE is the problem (not media,
|
||||
// truncated, damaged), as opposed to the prober itself failing. Accept turns it
|
||||
// into a user-facing rejection; any other prober error is our fault and surfaces
|
||||
// as an internal error — a missing ffprobe must not read as "your file is bad".
|
||||
var ErrUnreadable = errors.New("uploads: file is not readable media")
|
||||
|
||||
// Prober inspects a file on disk. The ffprobe adapter implements it.
|
||||
type Prober interface {
|
||||
Probe(ctx context.Context, path string) (Probe, error)
|
||||
}
|
||||
|
||||
// Store is the slice of the private transcript store the intake needs.
|
||||
type Store interface {
|
||||
CreateUpload(ctx context.Context, userID, title string, hadVideo bool) (string, error)
|
||||
FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error
|
||||
}
|
||||
|
||||
// Job is one accepted upload awaiting transcription. Path is a temp file the
|
||||
// receiver owns: it must delete it when finished (the audio is never retained).
|
||||
type Job struct {
|
||||
UserID string
|
||||
UploadID string
|
||||
Path string
|
||||
// Duration is what the file CLAIMED when probed. The worker compares it with what
|
||||
// was really converted; zero means unknown (ADR-035 review).
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// Submitter hands accepted jobs to the in-process worker (ADR-033).
|
||||
type Submitter interface {
|
||||
Submit(Job) error
|
||||
}
|
||||
|
||||
// Intake validates an uploaded recording and, if it passes, queues it.
|
||||
type Intake struct {
|
||||
store Store
|
||||
prober Prober
|
||||
sub Submitter
|
||||
limits Limits
|
||||
tmpDir string
|
||||
}
|
||||
|
||||
// NewIntake wires an Intake. tmpDir is where uploads are spooled; it must exist.
|
||||
func NewIntake(s Store, p Prober, sub Submitter, l Limits, tmpDir string) *Intake {
|
||||
return &Intake{store: s, prober: p, sub: sub, limits: l, tmpDir: tmpDir}
|
||||
}
|
||||
|
||||
// ErrClientRead marks an error that came from READING the client's body: a connection that
|
||||
// dropped, a body cut short. It is the sender's to retry, unlike a spool, probe or database
|
||||
// fault, which is ours. It is a marker on the error, not a sentinel that others may wrap:
|
||||
// io.ErrUnexpectedEOF alone is also what a database drop looks like.
|
||||
var ErrClientRead = errors.New("uploads: client body read failed")
|
||||
|
||||
// clientReader remembers the last non-EOF error its source returned, so Accept can tell
|
||||
// a failure reading the client from a failure writing the spool file.
|
||||
type clientReader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *clientReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
c.err = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// busyMessage is what a user is told when the queue is full.
|
||||
const busyMessage = "Too many recordings are waiting to be transcribed. Please try again in a few minutes."
|
||||
|
||||
// Accept spools src to disk (bounded by MaxBytes), probes it, validates it, records
|
||||
// a queued upload and submits the job. It returns the upload id. A file that fails
|
||||
// validation yields a *RejectError and creates NO row — rejection is synchronous
|
||||
// and leaves nothing behind. On success the temp file is handed to the job and the
|
||||
// worker deletes it; on every failure path Accept deletes it itself.
|
||||
func (in *Intake) Accept(ctx context.Context, userID, title string, src io.Reader) (string, error) {
|
||||
// Refuse before spooling when the queue has no room, if the submitter can say so: a full
|
||||
// queue otherwise cost up to the upload cap on disk, an ffprobe run and a failed row for
|
||||
// every refused attempt.
|
||||
if rc, ok := in.sub.(interface{ HasRoom(userID string) bool }); ok && !rc.HasRoom(userID) {
|
||||
return "", &RejectError{Reason: ReasonBusy, Message: busyMessage}
|
||||
}
|
||||
f, err := os.CreateTemp(in.tmpDir, "upload-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: spool: %w", err)
|
||||
}
|
||||
path := f.Name()
|
||||
handedOver := false
|
||||
defer func() {
|
||||
if !handedOver {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
|
||||
// Read one byte past the cap so "exactly at the limit" and "over" are told apart.
|
||||
cr := &clientReader{r: io.LimitReader(src, in.limits.MaxBytes+1)}
|
||||
n, err := io.Copy(f, cr)
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
if cr.err != nil && errors.Is(err, cr.err) {
|
||||
return "", fmt.Errorf("%w: %w", ErrClientRead, err)
|
||||
}
|
||||
return "", fmt.Errorf("uploads: receive: %w", err)
|
||||
}
|
||||
if n > in.limits.MaxBytes {
|
||||
return "", &RejectError{Reason: ReasonTooLarge, Message: fmt.Sprintf(
|
||||
"This file is larger than the %d MB limit.", in.limits.MaxBytes>>20)}
|
||||
}
|
||||
|
||||
probe, err := in.prober.Probe(ctx, path)
|
||||
if errors.Is(err, ErrUnreadable) {
|
||||
return "", &RejectError{
|
||||
Reason: ReasonUnreadable,
|
||||
Message: "This file could not be read as audio; it may be damaged or not a recording.",
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: probe: %w", err)
|
||||
}
|
||||
if err := Validate(probe, in.limits); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
id, err := in.store.CreateUpload(ctx, userID, title, probe.HasVideo)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: create: %w", err)
|
||||
}
|
||||
if err := in.sub.Submit(Job{UserID: userID, UploadID: id, Path: path, Duration: probe.Duration}); err != nil {
|
||||
if errors.Is(err, ErrQueueFull) {
|
||||
// The user's to retry, not a server fault: refuse it as such, and do not leave
|
||||
// the row queued -- that is the stuck spinner.
|
||||
if ferr := in.store.FailUpload(ctx, userID, id, domain.UploadErrBusy); ferr != nil {
|
||||
return "", fmt.Errorf("uploads: submit: %w (and failing the row: %v)", err, ferr)
|
||||
}
|
||||
return "", &RejectError{Reason: ReasonBusy, Message: busyMessage}
|
||||
}
|
||||
// A row that will never run must not sit queued: that is the stuck spinner.
|
||||
if ferr := in.store.FailUpload(ctx, userID, id, domain.UploadErrInterrupted); ferr != nil {
|
||||
return "", fmt.Errorf("uploads: submit: %w (and failing the row: %v)", err, ferr)
|
||||
}
|
||||
return "", fmt.Errorf("uploads: submit: %w", err)
|
||||
}
|
||||
handedOver = true
|
||||
return id, nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// A full queue is the user's to retry, not a server fault: a typed rejection, the row
|
||||
// failed honestly, the audio removed (adversarial review F6).
|
||||
|
||||
func TestAccept_AFullQueueIsABusyRejectionNotAServerErrorAndLeavesNothingBehind(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.sub.err = uploads.ErrQueueFull
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej), "a full queue must be a rejection the web layer can answer with 429: %v", err)
|
||||
assert.Equal(t, uploads.ReasonBusy, rej.Reason)
|
||||
assert.Contains(t, rej.Message, "waiting")
|
||||
assert.Equal(t, domain.UploadErrBusy, r.st.failed["up-1"], "the row must not sit queued: that is the stuck spinner")
|
||||
assert.Empty(t, r.tmpFiles(t), "the audio is deleted")
|
||||
}
|
||||
|
||||
// roomSubmitter is a Submitter that can say, before anything is spooled, that there is no room.
|
||||
type roomSubmitter struct {
|
||||
fakeSubmitter
|
||||
room bool
|
||||
}
|
||||
|
||||
func (r *roomSubmitter) HasRoom(string) bool { return r.room }
|
||||
|
||||
func TestAccept_ANoRoomQueueIsRefusedBeforeAnythingIsSpooledProbedOrStored(t *testing.T) {
|
||||
// Checking only at Submit meant 200 MB was spooled and ffprobe run for every refused
|
||||
// attempt, and a failed row was left behind each time (second adversarial review).
|
||||
r := newRig(t)
|
||||
full := &roomSubmitter{room: false}
|
||||
in := uploads.NewIntake(r.st, r.pr, full, uploads.Limits{MaxBytes: 1024, MaxDuration: 90 * time.Minute}, r.tmp)
|
||||
|
||||
_, err := in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej))
|
||||
assert.Equal(t, uploads.ReasonBusy, rej.Reason)
|
||||
assert.Empty(t, r.st.created, "no row for an upload that was never accepted")
|
||||
assert.Empty(t, r.pr.paths, "ffprobe was never run on it")
|
||||
assert.Empty(t, r.tmpFiles(t), "nothing was spooled")
|
||||
assert.Empty(t, full.jobs)
|
||||
}
|
||||
|
||||
func TestAccept_ASubmitterThatCannotSayStillWorksAsBefore(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.NoError(t, err, "a Submitter without HasRoom is accepted: the check at Submit remains")
|
||||
}
|
||||
|
||||
// brokenBody is a client body that fails mid-read, like a connection that dropped.
|
||||
type brokenBody struct{ n int }
|
||||
|
||||
func (b *brokenBody) Read(p []byte) (int, error) {
|
||||
if b.n > 0 {
|
||||
b.n = 0
|
||||
return copy(p, "partial audio"), nil
|
||||
}
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
func TestAccept_AnErrorReadingTheClientsBodyIsMarkedAsTheClientsAndAWrappedSubmitErrorIsNot(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", &brokenBody{n: 1})
|
||||
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, uploads.ErrClientRead, "a dropped client body is the sender's to retry")
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
|
||||
// A fault that is ours must NOT carry the marker, even if it wraps the same sentinel.
|
||||
r2 := newRig(t)
|
||||
r2.sub.err = fmt.Errorf("db: %w", io.ErrUnexpectedEOF)
|
||||
_, err = r2.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, uploads.ErrClientRead)
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
created []string // titles
|
||||
hadVideo []bool
|
||||
failed map[string]domain.UploadErrorKind
|
||||
nextID string
|
||||
}
|
||||
|
||||
func (f *fakeStore) CreateUpload(_ context.Context, _, title string, hadVideo bool) (string, error) {
|
||||
f.created = append(f.created, title)
|
||||
f.hadVideo = append(f.hadVideo, hadVideo)
|
||||
return f.nextID, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) FailUpload(_ context.Context, _, id string, kind domain.UploadErrorKind) error {
|
||||
if f.failed == nil {
|
||||
f.failed = map[string]domain.UploadErrorKind{}
|
||||
}
|
||||
f.failed[id] = kind
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeProber struct {
|
||||
probe uploads.Probe
|
||||
err error
|
||||
paths []string
|
||||
seen []byte // content of the file at probe time
|
||||
}
|
||||
|
||||
func (f *fakeProber) Probe(_ context.Context, path string) (uploads.Probe, error) {
|
||||
f.paths = append(f.paths, path)
|
||||
f.seen, _ = os.ReadFile(path)
|
||||
return f.probe, f.err
|
||||
}
|
||||
|
||||
type fakeSubmitter struct {
|
||||
jobs []uploads.Job
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeSubmitter) Submit(j uploads.Job) error {
|
||||
f.jobs = append(f.jobs, j)
|
||||
return f.err
|
||||
}
|
||||
|
||||
type rig struct {
|
||||
in *uploads.Intake
|
||||
st *fakeStore
|
||||
pr *fakeProber
|
||||
sub *fakeSubmitter
|
||||
tmp string
|
||||
}
|
||||
|
||||
func newRig(t *testing.T) *rig {
|
||||
t.Helper()
|
||||
r := &rig{
|
||||
st: &fakeStore{nextID: "up-1"},
|
||||
pr: &fakeProber{probe: uploads.Probe{Duration: 10 * time.Minute, HasAudio: true}},
|
||||
sub: &fakeSubmitter{},
|
||||
tmp: t.TempDir(),
|
||||
}
|
||||
r.in = uploads.NewIntake(r.st, r.pr, r.sub, uploads.Limits{MaxBytes: 1024, MaxDuration: 90 * time.Minute}, r.tmp)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *rig) tmpFiles(t *testing.T) []string {
|
||||
t.Helper()
|
||||
ents, err := os.ReadDir(r.tmp)
|
||||
require.NoError(t, err)
|
||||
var names []string
|
||||
for _, e := range ents {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestAccept_ValidAudioIsStoredProbedQueuedAndSubmitted(t *testing.T) {
|
||||
r := newRig(t)
|
||||
audio := bytes.Repeat([]byte("a"), 500)
|
||||
|
||||
id, err := r.in.Accept(context.Background(), "user-1", "Teams export", bytes.NewReader(audio))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "up-1", id)
|
||||
assert.Equal(t, []string{"Teams export"}, r.st.created)
|
||||
assert.Equal(t, audio, r.pr.seen, "the prober must see the complete uploaded bytes, on disk")
|
||||
require.Len(t, r.sub.jobs, 1)
|
||||
job := r.sub.jobs[0]
|
||||
assert.Equal(t, uploads.Job{UserID: "user-1", UploadID: "up-1", Path: job.Path, Duration: 10 * time.Minute}, job,
|
||||
"the job carries what the file CLAIMED, so the worker can compare it with what was converted")
|
||||
assert.Equal(t, filepath.Dir(job.Path), r.tmp, "the temp file lives in the configured directory")
|
||||
got, err := os.ReadFile(job.Path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, audio, got, "the handed-over file still exists: the worker, not Accept, deletes it")
|
||||
}
|
||||
|
||||
func TestAccept_OversizeBodyIsRejectedAndLeavesNothingBehind(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "big", strings.NewReader(strings.Repeat("a", 1025)))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej))
|
||||
assert.Equal(t, uploads.ReasonTooLarge, rej.Reason)
|
||||
assert.Contains(t, rej.Message, "limit")
|
||||
assert.Empty(t, r.pr.paths, "a file already over the size cap must not be probed")
|
||||
assert.Empty(t, r.st.created, "a rejected upload creates no row")
|
||||
assert.Empty(t, r.sub.jobs)
|
||||
assert.Empty(t, r.tmpFiles(t), "the partial temp file must be removed")
|
||||
}
|
||||
|
||||
func TestAccept_BodyExactlyAtTheLimitIsAccepted(t *testing.T) {
|
||||
r := newRig(t)
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "edge", strings.NewReader(strings.Repeat("a", 1024)))
|
||||
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestAccept_ProbeVerdictRejectsWithoutRowAndRemovesTheFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
probe uploads.Probe
|
||||
reason uploads.Reason
|
||||
}{
|
||||
{"video with no audio", uploads.Probe{Duration: time.Minute, HasVideo: true}, uploads.ReasonNoAudio},
|
||||
{"too long", uploads.Probe{Duration: 2 * time.Hour, HasAudio: true}, uploads.ReasonTooLong},
|
||||
{"no audio", uploads.Probe{Duration: time.Minute}, uploads.ReasonNoAudio},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.probe = tt.probe
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej))
|
||||
assert.Equal(t, tt.reason, rej.Reason)
|
||||
assert.Empty(t, r.st.created)
|
||||
assert.Empty(t, r.sub.jobs)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccept_UnreadableFileIsARejectionButAProberFaultIsAnError(t *testing.T) {
|
||||
t.Run("ffprobe says the file is not media", func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.err = uploads.ErrUnreadable
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("not media"))
|
||||
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej), "the user's file is the problem: tell them")
|
||||
assert.Equal(t, uploads.ReasonUnreadable, rej.Reason)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
})
|
||||
t.Run("the prober itself broke", func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.err = errors.New("exec: ffprobe: executable file not found")
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.Error(t, err)
|
||||
var rej *uploads.RejectError
|
||||
assert.False(t, errors.As(err, &rej), "our fault must not be reported as the user's bad file")
|
||||
assert.Empty(t, r.st.created)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
})
|
||||
}
|
||||
|
||||
func TestAccept_ClientDisconnectMidUploadLeavesNothingBehind(t *testing.T) {
|
||||
r := newRig(t)
|
||||
body := io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(errors.New("client went away")))
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", body)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, r.pr.paths)
|
||||
assert.Empty(t, r.st.created)
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
}
|
||||
|
||||
func TestAccept_SubmitFailureFailsTheRowHonestlyAndRemovesTheFile(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.sub.err = errors.New("worker stopped")
|
||||
|
||||
_, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, domain.UploadErrInterrupted, r.st.failed["up-1"],
|
||||
"a row that will never run must not sit queued: that is the stuck spinner ADR-025 forbids")
|
||||
assert.Empty(t, r.tmpFiles(t))
|
||||
}
|
||||
|
||||
func TestAccept_AFileWithAVideoTrackIsAcceptedAndRecordsThatItHadOne(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
probe uploads.Probe
|
||||
wantVideo bool
|
||||
}{
|
||||
{"audio only", uploads.Probe{Duration: 10 * time.Minute, HasAudio: true}, false},
|
||||
{"audio with a video track (a Teams meeting with the camera off)", uploads.Probe{Duration: 10 * time.Minute, HasAudio: true, HasVideo: true}, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.pr.probe = tt.probe
|
||||
|
||||
id, err := r.in.Accept(context.Background(), "user-1", "t", strings.NewReader("data"))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "up-1", id)
|
||||
assert.Equal(t, []bool{tt.wantVideo}, r.st.hadVideo,
|
||||
"the row must say whether a video track was present, so the page can say it was ignored")
|
||||
assert.Len(t, r.sub.jobs, 1, "it is queued like any other recording")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Package uploads is the use case behind "upload an audio recording" (ADR-033):
|
||||
// validate the file by PROBING it, keep the transcript private, never retain the
|
||||
// audio. It depends only on ports (a Prober, a store, a job Submitter) — the
|
||||
// concrete ffprobe adapter, the web handler and the transcription worker are wired
|
||||
// from outside.
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Limits bound what an upload may be. Both are rejected synchronously, with a
|
||||
// specific message, rather than by a timeout (ADR-033).
|
||||
type Limits struct {
|
||||
MaxBytes int64
|
||||
MaxDuration time.Duration
|
||||
}
|
||||
|
||||
// Probe is what inspecting an uploaded file revealed. It is the input to Validate
|
||||
// because a name or MIME type cannot say what a container holds: a Teams recording
|
||||
// is an .mp4 whether or not the camera was on.
|
||||
type Probe struct {
|
||||
Duration time.Duration
|
||||
HasAudio bool
|
||||
HasVideo bool
|
||||
}
|
||||
|
||||
// Reason is the machine-checkable class of a rejection.
|
||||
type Reason string
|
||||
|
||||
const (
|
||||
ReasonNoAudio Reason = "no_audio"
|
||||
ReasonTooLong Reason = "too_long"
|
||||
ReasonTooLarge Reason = "too_large"
|
||||
ReasonUnreadable Reason = "unreadable"
|
||||
// ReasonBusy: the queue is full. Not about the file: the user retries later.
|
||||
ReasonBusy Reason = "busy"
|
||||
)
|
||||
|
||||
// RejectError is a synchronous, user-presentable refusal. Message is written for
|
||||
// the person who uploaded the file; Reason is for code and tests.
|
||||
type RejectError struct {
|
||||
Reason Reason
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *RejectError) Error() string { return string(e.Reason) + ": " + e.Message }
|
||||
|
||||
// Validate decides whether a probed file may be transcribed. A video track is NOT
|
||||
// a reason to refuse: a Teams meeting with the camera off is still an .mp4 with a
|
||||
// video track, so rejecting video would reject the primary use case. Only the audio
|
||||
// is used (Probe.HasVideo is recorded so the user can be told the video was
|
||||
// ignored). What IS refused: no audio at all, an unreadable length, too long.
|
||||
func Validate(p Probe, l Limits) error {
|
||||
switch {
|
||||
case !p.HasAudio:
|
||||
return &RejectError{Reason: ReasonNoAudio, Message: "This file has no audio track to transcribe."}
|
||||
case p.Duration <= 0:
|
||||
return &RejectError{Reason: ReasonUnreadable, Message: "Could not read a length from this file; it may be damaged."}
|
||||
case p.Duration > l.MaxDuration:
|
||||
return &RejectError{Reason: ReasonTooLong, Message: fmt.Sprintf(
|
||||
"This recording is %s long; the limit is %s.", FormatDuration(p.Duration), FormatDuration(l.MaxDuration))}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FormatDuration renders a length for people: "1 h 42 min", "45 min", "30 s".
|
||||
func FormatDuration(d time.Duration) string {
|
||||
d = d.Round(time.Second)
|
||||
h := int(d / time.Hour)
|
||||
m := int(d % time.Hour / time.Minute)
|
||||
switch {
|
||||
case h > 0 && m > 0:
|
||||
return fmt.Sprintf("%d h %d min", h, m)
|
||||
case h > 0:
|
||||
return fmt.Sprintf("%d h", h)
|
||||
case m > 0:
|
||||
return fmt.Sprintf("%d min", m)
|
||||
default:
|
||||
return fmt.Sprintf("%d s", int(d/time.Second))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
var testLimits = uploads.Limits{MaxBytes: 200 << 20, MaxDuration: 90 * time.Minute}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
probe uploads.Probe
|
||||
wantReason uploads.Reason // "" = accepted
|
||||
wantInMsg []string
|
||||
}{
|
||||
{
|
||||
name: "audio-only file within the duration limit is accepted",
|
||||
probe: uploads.Probe{Duration: 45 * time.Minute, HasAudio: true},
|
||||
},
|
||||
{
|
||||
name: "a recording exactly at the limit is accepted",
|
||||
probe: uploads.Probe{Duration: 90 * time.Minute, HasAudio: true},
|
||||
},
|
||||
{
|
||||
name: "a recording that also has a video stream is accepted: the audio is what is used",
|
||||
probe: uploads.Probe{Duration: 4 * time.Minute, HasAudio: true, HasVideo: true},
|
||||
},
|
||||
{
|
||||
name: "video with no audio is rejected as having no audio, and says so",
|
||||
probe: uploads.Probe{Duration: 4 * time.Minute, HasVideo: true},
|
||||
wantReason: uploads.ReasonNoAudio,
|
||||
wantInMsg: []string{"no audio"},
|
||||
},
|
||||
{
|
||||
name: "a file with no audio stream is rejected",
|
||||
probe: uploads.Probe{Duration: 4 * time.Minute},
|
||||
wantReason: uploads.ReasonNoAudio,
|
||||
wantInMsg: []string{"no audio"},
|
||||
},
|
||||
{
|
||||
name: "a recording over the limit is rejected, stating both lengths",
|
||||
probe: uploads.Probe{Duration: 102 * time.Minute, HasAudio: true},
|
||||
wantReason: uploads.ReasonTooLong,
|
||||
wantInMsg: []string{"1 h 42 min", "1 h 30 min"},
|
||||
},
|
||||
{
|
||||
name: "zero duration is unreadable, not accepted as an empty recording",
|
||||
probe: uploads.Probe{HasAudio: true},
|
||||
wantReason: uploads.ReasonUnreadable,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := uploads.Validate(tt.probe, testLimits)
|
||||
|
||||
if tt.wantReason == "" {
|
||||
require.NoError(t, err)
|
||||
return
|
||||
}
|
||||
var rej *uploads.RejectError
|
||||
require.True(t, errors.As(err, &rej), "want a *RejectError, got %v", err)
|
||||
assert.Equal(t, tt.wantReason, rej.Reason)
|
||||
for _, s := range tt.wantInMsg {
|
||||
assert.Contains(t, rej.Message, s)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
d time.Duration
|
||||
want string
|
||||
}{
|
||||
{90 * time.Minute, "1 h 30 min"},
|
||||
{102 * time.Minute, "1 h 42 min"},
|
||||
{45 * time.Minute, "45 min"},
|
||||
{2 * time.Hour, "2 h"},
|
||||
{30 * time.Second, "30 s"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.want, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, uploads.FormatDuration(tt.d))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
)
|
||||
|
||||
// ErrStopped is returned by Submit once the worker has stopped.
|
||||
var ErrStopped = errors.New("uploads: worker stopped")
|
||||
|
||||
// ErrQueueFull is returned by Submit when the queue, or one user's share of it, is at
|
||||
// its cap. Every queued job holds up to the upload cap on disk until its turn, so an
|
||||
// unbounded queue is a disk-filling lever (ADR-035 review).
|
||||
var ErrQueueFull = errors.New("uploads: queue full")
|
||||
|
||||
// Converter turns the uploaded file into the 16 kHz mono PCM WAV whisper wants.
|
||||
type Converter interface {
|
||||
ToWAV(ctx context.Context, src, dst string) error
|
||||
}
|
||||
|
||||
// Transcriber turns a WAV into a timed transcription. The gateway chain
|
||||
// (iguana primary, berget fallback) implements it.
|
||||
type Transcriber interface {
|
||||
Transcribe(ctx context.Context, wavPath string) (domain.Transcription, error)
|
||||
}
|
||||
|
||||
// Planner is an optional extra on a Transcriber: it declares its own phases (for
|
||||
// example one per engine) so the page can show them. A Transcriber that does not
|
||||
// implement it gets a single "transcribing" phase from the worker. Whatever a Planner
|
||||
// declares it reports itself, through progress.From(ctx) (ADR-035).
|
||||
type Planner interface {
|
||||
Plan() []progress.Planned
|
||||
}
|
||||
|
||||
// WorkerStore is the slice of the private transcript store the worker drives.
|
||||
type WorkerStore interface {
|
||||
StartTranscribing(ctx context.Context, userID, id string) error
|
||||
CompleteUpload(ctx context.Context, userID, id string, t domain.Transcription) error
|
||||
FailUpload(ctx context.Context, userID, id string, kind domain.UploadErrorKind) error
|
||||
SaveProgress(ctx context.Context, userID, id string, steps []domain.ProgressStep, at time.Time) error
|
||||
Heartbeat(ctx context.Context, userID, id string) error
|
||||
}
|
||||
|
||||
// WorkerOption tunes a Worker.
|
||||
type WorkerOption func(*Worker)
|
||||
|
||||
// WithHeartbeat sets how often a running job stamps its row alive and how often it logs
|
||||
// which phase is still running. The defaults are 10 s and 5 min (ADR-035).
|
||||
func WithHeartbeat(beat, still time.Duration) WorkerOption {
|
||||
return func(w *Worker) { w.beatEvery, w.stillEvery = beat, still }
|
||||
}
|
||||
|
||||
// WithQueueLimits bounds how many jobs may WAIT: in total, and per user. A job being
|
||||
// processed is not waiting. The defaults are 20 and 3 (one instance, one job at a time).
|
||||
func WithQueueLimits(total, perUser int) WorkerOption {
|
||||
return func(w *Worker) { w.maxQueued, w.maxPerUser = total, perUser }
|
||||
}
|
||||
|
||||
// WithMaxDuration makes the worker measure the converted audio and fail a job whose real
|
||||
// length exceeds the limit, or falls far short of what the file claimed. Zero (the
|
||||
// default) measures nothing. The limit is first checked against the file's header, which
|
||||
// can lie (ADR-035 review).
|
||||
func WithMaxDuration(d time.Duration) WorkerOption { return func(w *Worker) { w.maxDuration = d } }
|
||||
|
||||
// WithLogger replaces slog.Default() as the worker's logger.
|
||||
func WithLogger(l *slog.Logger) WorkerOption { return func(w *Worker) { w.log = l } }
|
||||
|
||||
const (
|
||||
defaultMaxQueued = 20
|
||||
defaultMaxPerUser = 3
|
||||
defaultBeatEvery = 10 * time.Second
|
||||
defaultStillEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// jobSink binds the store to one job so a progress.Tracker can persist through it.
|
||||
type jobSink struct {
|
||||
st WorkerStore
|
||||
userID, id string
|
||||
}
|
||||
|
||||
func (s jobSink) SaveProgress(ctx context.Context, steps []domain.ProgressStep, at time.Time) error {
|
||||
return s.st.SaveProgress(ctx, s.userID, s.id, steps, at)
|
||||
}
|
||||
|
||||
func (s jobSink) Beat(ctx context.Context) error { return s.st.Heartbeat(ctx, s.userID, s.id) }
|
||||
|
||||
// Worker transcribes accepted uploads in-process, ONE AT A TIME (ADR-033). It is
|
||||
// not a second deployable: it shares the web process's lifetime, which is why it
|
||||
// inherits ADR-018's single-replica assumption — the queue is process-local. The
|
||||
// concurrency of one is deliberate: two simultaneous 90-minute jobs on iguana were
|
||||
// never measured, and a Metal OOM there latches the whisper backend (infra#273).
|
||||
type Worker struct {
|
||||
store WorkerStore
|
||||
conv Converter
|
||||
tr Transcriber
|
||||
tmpDir string
|
||||
timeout time.Duration
|
||||
|
||||
log *slog.Logger
|
||||
beatEvery, stillEvery time.Duration
|
||||
maxDuration time.Duration
|
||||
maxQueued, maxPerUser int
|
||||
|
||||
mu sync.Mutex
|
||||
queue []Job
|
||||
stopped bool
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
// NewWorker wires a Worker. timeout bounds each job (conversion + transcription);
|
||||
// a job that outlives it is failed rather than left spinning.
|
||||
func NewWorker(s WorkerStore, c Converter, t Transcriber, tmpDir string, timeout time.Duration, opts ...WorkerOption) *Worker {
|
||||
w := &Worker{
|
||||
store: s, conv: c, tr: t, tmpDir: tmpDir, timeout: timeout, wake: make(chan struct{}, 1),
|
||||
beatEvery: defaultBeatEvery, stillEvery: defaultStillEvery,
|
||||
maxQueued: defaultMaxQueued, maxPerUser: defaultMaxPerUser,
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(w)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// HasRoom reports whether a job from this user would be accepted right now. It lets the
|
||||
// intake refuse BEFORE spooling up to the upload cap and running ffprobe; Submit still
|
||||
// enforces the same limits atomically, so a race costs at worst one refused submit.
|
||||
func (w *Worker) HasRoom(userID string) bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped || len(w.queue) >= w.maxQueued {
|
||||
return false
|
||||
}
|
||||
mine := 0
|
||||
for _, q := range w.queue {
|
||||
if q.UserID == userID {
|
||||
mine++
|
||||
}
|
||||
}
|
||||
return mine < w.maxPerUser
|
||||
}
|
||||
|
||||
// Submit queues a job. It never blocks. It satisfies Submitter.
|
||||
func (w *Worker) Submit(j Job) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped {
|
||||
return ErrStopped
|
||||
}
|
||||
if len(w.queue) >= w.maxQueued {
|
||||
return ErrQueueFull
|
||||
}
|
||||
mine := 0
|
||||
for _, q := range w.queue {
|
||||
if q.UserID == j.UserID {
|
||||
mine++
|
||||
}
|
||||
}
|
||||
if mine >= w.maxPerUser {
|
||||
return ErrQueueFull
|
||||
}
|
||||
w.queue = append(w.queue, j)
|
||||
select {
|
||||
case w.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run processes queued jobs until ctx ends, then deletes the audio of anything
|
||||
// still queued and refuses new jobs. The rows of those jobs stay queued or
|
||||
// transcribing and are failed as interrupted by the start-up sweep
|
||||
// (store.FailInterruptedUploads): the audio is not retained, so they cannot resume.
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
defer w.stop()
|
||||
for {
|
||||
job, ok := w.next(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w.Process(ctx, job)
|
||||
}
|
||||
}
|
||||
|
||||
// next blocks for the next job. Cancellation wins over a non-empty queue: a job
|
||||
// still queued at shutdown must not be STARTED (it would begin a long conversion
|
||||
// with an already-cancelled context); stop() deletes its audio instead.
|
||||
func (w *Worker) next(ctx context.Context) (Job, bool) {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return Job{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
if len(w.queue) > 0 {
|
||||
j := w.queue[0]
|
||||
w.queue = w.queue[1:]
|
||||
w.mu.Unlock()
|
||||
return j, true
|
||||
}
|
||||
w.mu.Unlock()
|
||||
select {
|
||||
case <-w.wake:
|
||||
case <-ctx.Done():
|
||||
return Job{}, false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) stop() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.stopped = true
|
||||
for _, j := range w.queue {
|
||||
_ = os.Remove(j.Path)
|
||||
}
|
||||
w.queue = nil
|
||||
}
|
||||
|
||||
// Process runs one job: row -> transcribing, convert, transcribe, store. The
|
||||
// uploaded file and the converted WAV are deleted on every path — the audio is
|
||||
// never retained (ADR-033). A failure is recorded with a TYPED kind; a shutdown
|
||||
// mid-job is not a failure and leaves the row for the start-up sweep.
|
||||
//
|
||||
// Every phase is reported through one progress.Tracker, which saves it to the row for
|
||||
// the page and logs it, and a heartbeat stamps the row on its own while the job runs
|
||||
// (ADR-035). Nothing logged or saved here carries transcript text or the file's name.
|
||||
func (w *Worker) Process(ctx context.Context, job Job) {
|
||||
defer func() { _ = os.Remove(job.Path) }()
|
||||
base := w.log
|
||||
if base == nil {
|
||||
base = slog.Default()
|
||||
}
|
||||
log := base.With("upload", job.UploadID)
|
||||
start := time.Now()
|
||||
|
||||
if err := w.store.StartTranscribing(ctx, job.UserID, job.UploadID); err != nil {
|
||||
log.Error("upload: start transcribing", "kind", errkind.Of(err))
|
||||
w.fail(ctx, job, log, domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
|
||||
plan := []progress.Planned{{Key: "received", Label: "Received"}, {Key: "audio", Label: "Preparing the audio"}}
|
||||
planner, planned := w.tr.(Planner)
|
||||
if planned {
|
||||
plan = append(plan, planner.Plan()...)
|
||||
} else {
|
||||
plan = append(plan, progress.Planned{Key: "transcribe", Label: "Transcribing"})
|
||||
}
|
||||
plan = append(plan, progress.Planned{Key: "save", Label: "Saving the transcript"})
|
||||
tr := progress.NewTracker(ctx, jobSink{st: w.store, userID: job.UserID, id: job.UploadID}, log, plan)
|
||||
tr.Start()
|
||||
tr.Begin("received", 0)
|
||||
tr.Done("received")
|
||||
|
||||
hbCtx, stopBeat := context.WithCancel(ctx)
|
||||
hbDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(hbDone)
|
||||
tr.Heartbeat(hbCtx, w.beatEvery, w.stillEvery)
|
||||
}()
|
||||
defer func() { stopBeat(); <-hbDone }() // no heartbeat outlives its job
|
||||
|
||||
// abort records where the job stopped. A shutdown (the parent context ended) is not
|
||||
// a failure: the row is left for the start-up sweep, so the step stays as it was.
|
||||
abort := func(kind domain.UploadErrorKind) {
|
||||
if ctx.Err() == nil {
|
||||
tr.FailRunning()
|
||||
}
|
||||
w.fail(ctx, job, log, kind)
|
||||
}
|
||||
|
||||
wav, err := w.tempWAV()
|
||||
if err != nil {
|
||||
log.Error("upload: temp wav", "kind", errkind.Of(err))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
defer func() { _ = os.Remove(wav) }()
|
||||
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.timeout)
|
||||
defer cancel()
|
||||
tr.Begin("audio", 0)
|
||||
if err := w.conv.ToWAV(jobCtx, job.Path, wav); err != nil {
|
||||
log.Error("upload: convert", "kind", errkind.Of(err))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
got, tooLong := w.audioLengthProblem(wav)
|
||||
if tooLong {
|
||||
abort(domain.UploadErrTooLong)
|
||||
log.Error("upload: converted audio is over the limit", "audio_s", int(got.Seconds()), "limit_s", int(w.maxDuration.Seconds()))
|
||||
return
|
||||
}
|
||||
if w.maxDuration > 0 && job.Duration > 0 {
|
||||
if note := shortfallNote(got, job.Duration); note != "" {
|
||||
log.Warn("upload: audio is much shorter than the file claimed", "audio_s", int(got.Seconds()), "claimed_s", int(job.Duration.Seconds()))
|
||||
tr.Annotate("audio", note)
|
||||
}
|
||||
}
|
||||
tr.Done("audio")
|
||||
|
||||
if !planned {
|
||||
tr.Begin("transcribe", 0)
|
||||
}
|
||||
t, err := w.tr.Transcribe(progress.With(jobCtx, tr), wav)
|
||||
if err != nil {
|
||||
log.Error("upload: transcribe", "kind", errkind.Of(err), "elapsed", time.Since(start).Round(time.Second))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
if !planned {
|
||||
tr.Done("transcribe")
|
||||
}
|
||||
|
||||
tr.Begin("save", 0)
|
||||
if err := w.store.CompleteUpload(ctx, job.UserID, job.UploadID, t); err != nil {
|
||||
log.Error("upload: store transcript", "kind", errkind.Of(err))
|
||||
abort(domain.UploadErrTranscriptionFailed)
|
||||
return
|
||||
}
|
||||
tr.Done("save")
|
||||
tr.Finish()
|
||||
log.Info("upload transcribed", "endpoint", t.Endpoint, "audio_s", t.AudioSeconds,
|
||||
"segments", len(t.Segments), "elapsed", time.Since(start).Round(time.Second))
|
||||
}
|
||||
|
||||
// fail records a transcription failure — unless the PARENT context ended, which
|
||||
// is a shutdown, not a failure: the row is left for the start-up sweep.
|
||||
func (w *Worker) fail(ctx context.Context, job Job, log *slog.Logger, kind domain.UploadErrorKind) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := w.store.FailUpload(ctx, job.UserID, job.UploadID, kind); err != nil {
|
||||
log.Error("upload: record failure", "kind", errkind.Of(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) tempWAV() (string, error) {
|
||||
f, err := os.CreateTemp(w.tmpDir, "wav-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("uploads: temp wav: %w", err)
|
||||
}
|
||||
name := f.Name()
|
||||
if err := f.Close(); err != nil {
|
||||
_ = os.Remove(name)
|
||||
return "", fmt.Errorf("uploads: temp wav: %w", err)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
const (
|
||||
// tooLongSlack is how far over the limit the converted audio may be before the claim
|
||||
// in the file's header is called a lie: ffmpeg rounds.
|
||||
tooLongSlack = 30 * time.Second
|
||||
// A file counts as truncated when it converts to under 80% of the length it claimed
|
||||
// AND more than two minutes short. Probe estimates for variable-bitrate files are
|
||||
// imprecise; refusing real recordings would be worse than the problem.
|
||||
incompleteBelow = 0.8
|
||||
incompleteShortfall = 2 * time.Minute
|
||||
)
|
||||
|
||||
// audioLengthProblem measures the converted WAV (16 kHz mono 16-bit PCM is 32,000 bytes a
|
||||
// second after a 44-byte header) against the limit. It returns the real length and whether
|
||||
// that is over the limit by more than the tolerance.
|
||||
func (w *Worker) audioLengthProblem(wav string) (got time.Duration, tooLong bool) {
|
||||
if w.maxDuration <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
info, err := os.Stat(wav)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
got = time.Duration(float64(max(info.Size()-44, 0)) / 32000 * float64(time.Second))
|
||||
return got, got > w.maxDuration+tooLongSlack
|
||||
}
|
||||
|
||||
// shortfallNote is a remark for the audio step when the converted audio is much shorter
|
||||
// than the file claimed, or "" when it is not. It is a NOTE and never a failure: a
|
||||
// container header can over-estimate a complete file (a variable-bitrate MP3 without a Xing
|
||||
// header claimed 79 minutes for 20 real ones), so refusing would turn away real
|
||||
// recordings, while saying nothing about a truncated one would hide a partial transcript.
|
||||
func shortfallNote(got, claimed time.Duration) string {
|
||||
if float64(got) >= incompleteBelow*float64(claimed) || claimed-got <= incompleteShortfall {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("The audio is %s long; the file claimed %s. If the recording was cut off, upload the complete one.",
|
||||
FormatDuration(got), FormatDuration(claimed))
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// The duration limit is a claim in the file's header. The worker measures what was
|
||||
// really converted and refuses both a recording that is longer than the limit (the
|
||||
// header lied low) and one that ends far earlier than the file claimed (it was truncated
|
||||
// and would otherwise yield a partial transcript with no warning) -- adversarial review F2.
|
||||
|
||||
// sizedConverter writes a (sparse) 16 kHz mono PCM WAV of the given length, without
|
||||
// actually allocating it: 32,000 bytes per second plus a 44-byte header.
|
||||
type sizedConverter struct{ seconds float64 }
|
||||
|
||||
func (c sizedConverter) ToWAV(_ context.Context, _, dst string) error {
|
||||
f, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
return f.Truncate(44 + int64(c.seconds*32000))
|
||||
}
|
||||
|
||||
func durationWorker(t *testing.T, r *wrig, wavSeconds float64, max time.Duration) *uploads.Worker {
|
||||
t.Helper()
|
||||
opts := []uploads.WorkerOption{}
|
||||
if max > 0 {
|
||||
opts = append(opts, uploads.WithMaxDuration(max))
|
||||
}
|
||||
return uploads.NewWorker(r.st, sizedConverter{seconds: wavSeconds}, r.tr, r.tmp, time.Minute, opts...)
|
||||
}
|
||||
|
||||
func jobWith(r *wrig, t *testing.T, claimed time.Duration) uploads.Job {
|
||||
j := r.spool(t, "a")
|
||||
j.Duration = claimed
|
||||
return j
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AConvertedLengthOverTheLimitFailsTheJobBeforeAnyEngineSeesIt(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 100*60, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 10*time.Second))
|
||||
|
||||
_, failed := r.st.snapshot()
|
||||
assert.Equal(t, domain.UploadErrTooLong, failed["a"], "a header that claimed 10 s must not let 100 minutes through")
|
||||
assert.Empty(t, r.tr.paths, "no engine may be sent audio that is over the limit")
|
||||
assert.Empty(t, r.leftovers(t), "the audio is still deleted")
|
||||
}
|
||||
|
||||
func TestWorkerDuration_JustOverTheLimitWithinTheSlackStillTranscribes(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 90*60+20, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 90*time.Minute))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed)
|
||||
assert.Contains(t, completed, "a", "ffmpeg rounds: 20 s over a 90-minute limit is not a lie")
|
||||
}
|
||||
|
||||
func audioStepNote(r *wrig) string {
|
||||
for _, st := range r.st.lastSteps() {
|
||||
if st.Key == "audio" {
|
||||
return st.Note
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AFileThatEndsFarEarlierThanItClaimedIsStillTranscribedAndSaysSo(t *testing.T) {
|
||||
// A container header can over-estimate a complete file (VBR MP3 without a Xing header
|
||||
// claimed 79 minutes for 20 real ones). Refusing it was wrong; staying silent about a
|
||||
// truncated one is wrong too. So it is transcribed, and the audio step carries a note.
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 20*60, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 79*time.Minute))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed, "a complete recording with a bad header estimate must not be refused")
|
||||
assert.Contains(t, completed, "a")
|
||||
note := audioStepNote(r)
|
||||
assert.Contains(t, note, "20 min", "what was really transcribed")
|
||||
assert.Contains(t, note, "1 h 19 min", "what the file claimed")
|
||||
assert.Contains(t, note, "cut off", "and what to do if that is not a surprise")
|
||||
}
|
||||
|
||||
func TestWorkerDuration_TheShortfallNoteIsPinnedAtEightyPercentAndTwoMinutes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
claimed time.Duration
|
||||
wav float64
|
||||
noted bool
|
||||
}{
|
||||
{"79 percent of an hour", 60 * time.Minute, 0.79 * 3600, true},
|
||||
{"exactly 80 percent of an hour", 60 * time.Minute, 2880, false},
|
||||
{"one second under 80 percent of an hour", 60 * time.Minute, 2879, true},
|
||||
{"81 percent of an hour", 60 * time.Minute, 0.81 * 3600, false},
|
||||
{"exactly two minutes short and under 80 percent", 6 * time.Minute, 4 * 60, false},
|
||||
{"two minutes and a second short and under 80 percent", 6 * time.Minute, 4*60 - 1, true},
|
||||
{"unknown claimed length", 0, 20 * 60, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, tt.claimed))
|
||||
|
||||
assert.Equal(t, tt.noted, audioStepNote(r) != "")
|
||||
completed, _ := r.st.snapshot()
|
||||
assert.Contains(t, completed, "a", "a note never stops the job")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_TheTooLongToleranceIsExactlyThirtySeconds(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
wav float64
|
||||
fail bool
|
||||
}{
|
||||
{"at the limit plus 30 s", 90*60 + 30, false},
|
||||
{"at the limit plus 31 s", 90*60 + 31, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 90*time.Minute))
|
||||
|
||||
_, failed := r.st.snapshot()
|
||||
if tt.fail {
|
||||
assert.Equal(t, domain.UploadErrTooLong, failed["a"])
|
||||
} else {
|
||||
assert.Empty(t, failed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AnEstimateThatIsMerelyALittleOffIsNotTruncation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
claimed time.Duration
|
||||
wav float64
|
||||
}{
|
||||
{"five percent short", 60 * time.Minute, 57 * 60},
|
||||
{"under two minutes short of a long file", 60 * time.Minute, 59 * 60},
|
||||
{"unknown claimed length", 0, 20 * 60},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, tt.claimed))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed)
|
||||
assert.Contains(t, completed, "a")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_WithNoMaximumConfiguredNothingIsMeasured(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 100*60, 0)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 0))
|
||||
|
||||
completed, _ := r.st.snapshot()
|
||||
assert.Contains(t, completed, "a")
|
||||
require.NotNil(t, w)
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// Progress, heartbeat and logging of the worker (ADR-035, #42, #43).
|
||||
|
||||
// plannedTr is a transcriber that declares its own phases and reports them, like the
|
||||
// two-engine Ensemble does.
|
||||
type plannedTr struct{ *fakeTranscriber }
|
||||
|
||||
func (plannedTr) Plan() []progress.Planned {
|
||||
return []progress.Planned{{Key: "engine_a", Label: "Transcribing with A"}, {Key: "engine_b", Label: "Transcribing with B"}}
|
||||
}
|
||||
|
||||
func (p plannedTr) Transcribe(ctx context.Context, wav string) (domain.Transcription, error) {
|
||||
r := progress.From(ctx)
|
||||
r.Begin("engine_a", 0)
|
||||
r.Done("engine_a")
|
||||
r.Begin("engine_b", 0)
|
||||
r.Done("engine_b")
|
||||
return p.fakeTranscriber.Transcribe(ctx, wav)
|
||||
}
|
||||
|
||||
func keysAndStates(steps []domain.ProgressStep) []string {
|
||||
var out []string
|
||||
for _, s := range steps {
|
||||
out = append(out, s.Key+":"+string(s.State))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// syncLog is a log sink safe to read while the worker's goroutines write.
|
||||
type syncLog struct {
|
||||
mu sync.Mutex
|
||||
b bytes.Buffer
|
||||
}
|
||||
|
||||
func (l *syncLog) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.b.Write(p)
|
||||
}
|
||||
|
||||
func (l *syncLog) String() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.b.String()
|
||||
}
|
||||
|
||||
func newProgressWorker(t *testing.T, r *wrig, tr uploads.Transcriber, opts ...uploads.WorkerOption) *uploads.Worker {
|
||||
t.Helper()
|
||||
return uploads.NewWorker(r.st, r.conv, tr, r.tmp, time.Minute, opts...)
|
||||
}
|
||||
|
||||
func TestWorkerProgress_ASingleEngineJobShowsEveryPhaseDone(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Equal(t, []string{"received:done", "audio:done", "transcribe:done", "save:done"}, keysAndStates(r.st.lastSteps()))
|
||||
}
|
||||
|
||||
func TestWorkerProgress_ATranscriberThatPlansItsOwnPhasesGetsThemBetweenAudioAndSaving(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := newProgressWorker(t, r, plannedTr{r.tr})
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Equal(t, []string{"received:done", "audio:done", "engine_a:done", "engine_b:done", "save:done"}, keysAndStates(r.st.lastSteps()))
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AFailureMarksWhereItStoppedAndLeavesTheRestPending(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*wrig)
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
"conversion fails", func(r *wrig) { r.conv.err = errors.New("moov atom not found") },
|
||||
[]string{"received:done", "audio:failed", "transcribe:pending", "save:pending"},
|
||||
},
|
||||
{
|
||||
"every engine fails", func(r *wrig) { r.tr.err = errors.New("iguana down") },
|
||||
[]string{"received:done", "audio:done", "transcribe:failed", "save:pending"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
tt.setup(r)
|
||||
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Equal(t, tt.want, keysAndStates(r.st.lastSteps()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AShutdownIsNotRecordedAsAFailedStep(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); r.w.Process(ctx, r.spool(t, "a")) }()
|
||||
require.Eventually(t, func() bool {
|
||||
return strings.Contains(strings.Join(keysAndStates(r.st.lastSteps()), ","), "transcribe:running")
|
||||
}, 2*time.Second, time.Millisecond)
|
||||
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
assert.Contains(t, keysAndStates(r.st.lastSteps()), "transcribe:running",
|
||||
"a shutdown leaves the row for the start-up sweep; it must not pose as a failure of the step")
|
||||
}
|
||||
|
||||
func TestWorkerProgress_TheHeartbeatBeatsWhileAnEngineCallIsStuckAndStopsWhenTheJobEnds(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
r.st.beatDelay = 20 * time.Millisecond
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithHeartbeat(5*time.Millisecond, time.Hour))
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); w.Process(context.Background(), r.spool(t, "a")) }()
|
||||
|
||||
require.Eventually(t, func() bool { return r.st.beatCount() >= 3 }, 2*time.Second, 5*time.Millisecond,
|
||||
"the page tells a slow engine from a dead service by this heartbeat, so it must not wait on the engine")
|
||||
close(r.tr.block)
|
||||
<-done
|
||||
after := r.st.beatCount()
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
|
||||
assert.Equal(t, after, r.st.beatCount(), "no heartbeat may outlive its job")
|
||||
}
|
||||
|
||||
func TestWorkerProgress_OnASuccessfulJobTheLogNamesEveryPhaseInOrderAndNeverTheTranscriptOrThePath(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.result = domain.Transcription{Text: "zebrakaka secret meeting words", Endpoint: "iguana/kb-whisper", AudioSeconds: 3}
|
||||
buf := &syncLog{}
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithLogger(slog.New(slog.NewTextHandler(buf, nil))))
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
out := buf.String()
|
||||
audio := strings.Index(out, "step=audio")
|
||||
transcribe := strings.Index(out, "step=transcribe")
|
||||
save := strings.Index(out, "step=save")
|
||||
require.True(t, audio >= 0 && transcribe > audio && save > transcribe, "phases must be logged in the order they ran:\n%s", out)
|
||||
assert.Contains(t, out, "upload=a", "every line says which upload it is about")
|
||||
assert.NotContains(t, out, "zebrakaka", "no transcript text in the log")
|
||||
assert.NotContains(t, out, r.tmp, "no spool path, which carries the upload's identity, in the log")
|
||||
assert.Contains(t, out, "upload transcribed", "the job's own summary line")
|
||||
assert.Contains(t, out, "endpoint=iguana/kb-whisper", "which endpoint served it is a quality fact")
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AFailureLogsAKindNeverTheErrorTextOrThePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*wrig)
|
||||
line string
|
||||
}{
|
||||
{"conversion fails and names the spool file", func(r *wrig) {
|
||||
r.conv.err = errors.New("Error opening input file " + r.tmp + "/upload-a: Invalid data zebrakaka")
|
||||
}, "upload: convert"},
|
||||
{"every engine fails and the gateway echoed a word", func(r *wrig) {
|
||||
r.tr.err = errors.New("gateway echoed: zebrakaka secret words")
|
||||
}, "upload: transcribe"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
tt.setup(r)
|
||||
buf := &syncLog{}
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithLogger(slog.New(slog.NewTextHandler(buf, nil))))
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
out := buf.String()
|
||||
assert.NotContains(t, out, "zebrakaka", "provider text or file contents in the log")
|
||||
assert.NotContains(t, out, r.tmp, "the spool path in the log")
|
||||
assert.Contains(t, out, tt.line, "the failure must be logged at all, not only scrubbed")
|
||||
assert.Contains(t, out, "kind=error")
|
||||
assert.Contains(t, out, "upload=a")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerProgress_AStoreFailureLogsItsSQLStateNeverTheValueItQuotes(t *testing.T) {
|
||||
// A Postgres error can quote the offending value; for a transcript row that is the
|
||||
// transcript (fourth adversarial review). The log gets the SQLSTATE.
|
||||
r := newWRig(t, time.Minute)
|
||||
r.st.completeErr = dbErr{}
|
||||
buf := &syncLog{}
|
||||
w := newProgressWorker(t, r, r.tr, uploads.WithLogger(slog.New(slog.NewTextHandler(buf, nil))))
|
||||
|
||||
w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
out := buf.String()
|
||||
assert.NotContains(t, out, "zebrakaka")
|
||||
assert.Contains(t, out, "upload: store transcript")
|
||||
assert.Contains(t, out, "kind=sql_22P02")
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// The queue was append-only: 100,000 submits with the worker idle all succeeded, and each
|
||||
// queued job holds up to the upload cap on disk until its turn (adversarial review F6).
|
||||
|
||||
func queueWorker(t *testing.T, total, perUser int) *uploads.Worker {
|
||||
t.Helper()
|
||||
r := newWRig(t, time.Minute)
|
||||
return uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute, uploads.WithQueueLimits(total, perUser))
|
||||
}
|
||||
|
||||
func submit(w *uploads.Worker, user string, n int) error {
|
||||
return w.Submit(uploads.Job{UserID: user, UploadID: user + "-" + strconv.Itoa(n), Path: "/nonexistent"})
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_RefusesBeyondTheTotalQueueCap(t *testing.T) {
|
||||
w := queueWorker(t, 3, 10)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
require.NoError(t, submit(w, "u"+strconv.Itoa(i), i))
|
||||
}
|
||||
|
||||
err := submit(w, "u9", 9)
|
||||
assert.ErrorIs(t, err, uploads.ErrQueueFull)
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_OneUserCannotFillTheQueueForEveryoneElse(t *testing.T) {
|
||||
w := queueWorker(t, 100, 2)
|
||||
|
||||
require.NoError(t, submit(w, "alice", 1))
|
||||
require.NoError(t, submit(w, "alice", 2))
|
||||
errAlice := submit(w, "alice", 3)
|
||||
errBob := submit(w, "bob", 1)
|
||||
|
||||
assert.True(t, errors.Is(errAlice, uploads.ErrQueueFull), "a third waiting upload from one user is refused")
|
||||
assert.NoError(t, errBob, "another user is not affected by alice's queue")
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_ThereIsADefaultCapSoANeglectedWorkerCannotBeFilledForever(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute)
|
||||
|
||||
refused := 0
|
||||
for i := 0; i < 1000; i++ {
|
||||
if err := submit(w, "u"+strconv.Itoa(i), i); errors.Is(err, uploads.ErrQueueFull) {
|
||||
refused++
|
||||
}
|
||||
}
|
||||
|
||||
assert.Positive(t, refused, "with the defaults, 1,000 submits to an idle worker must not all be accepted")
|
||||
}
|
||||
|
||||
func TestWorkerSubmit_AJobThatHasLeftTheQueueFreesItsPlace(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute, uploads.WithQueueLimits(1, 1))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); w.Run(ctx) }()
|
||||
defer func() { cancel(); <-done }()
|
||||
|
||||
job := r.spool(t, "first")
|
||||
require.NoError(t, w.Submit(job))
|
||||
require.Eventually(t, func() bool { c, _ := r.st.snapshot(); _, ok := c["first"]; return ok },
|
||||
5*time.Second, 5*time.Millisecond, "the first job never completed")
|
||||
|
||||
assert.NoError(t, w.Submit(r.spool(t, "second")), "the cap counts jobs waiting, not jobs ever submitted")
|
||||
}
|
||||
|
||||
func TestWorkerHasRoom_SaysWhetherASubmitWouldBeAccepted(t *testing.T) {
|
||||
w := queueWorker(t, 3, 2)
|
||||
assert.True(t, w.HasRoom("alice"))
|
||||
require.NoError(t, submit(w, "alice", 1))
|
||||
require.NoError(t, submit(w, "alice", 2))
|
||||
|
||||
assert.False(t, w.HasRoom("alice"), "alice is at her share")
|
||||
assert.True(t, w.HasRoom("bob"), "bob is not")
|
||||
require.NoError(t, submit(w, "bob", 1))
|
||||
assert.False(t, w.HasRoom("carol"), "the queue as a whole is at its cap")
|
||||
}
|
||||
|
||||
func TestWorkerHasRoom_AgreesWithSubmitAtEveryStepAndIsFalseOnceStopped(t *testing.T) {
|
||||
w := queueWorker(t, 2, 1)
|
||||
for i, user := range []string{"a", "a", "b", "c"} {
|
||||
room := w.HasRoom(user)
|
||||
err := submit(w, user, i)
|
||||
assert.Equal(t, room, err == nil, "step %d (%s): HasRoom=%v but Submit returned %v", i, user, room, err)
|
||||
}
|
||||
|
||||
r := newWRig(t, time.Minute)
|
||||
stopped := uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, time.Minute)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
stopped.Run(ctx)
|
||||
assert.False(t, stopped.HasRoom("a"), "a worker that has stopped has no room")
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
type wStore struct {
|
||||
mu sync.Mutex
|
||||
started []string
|
||||
completed map[string]domain.Transcription
|
||||
failed map[string]domain.UploadErrorKind
|
||||
startErr error
|
||||
// completeErr makes CompleteUpload fail, as a database outage would.
|
||||
completeErr error
|
||||
saves [][]domain.ProgressStep
|
||||
beats int
|
||||
// beatDelay makes each heartbeat write slow, so one is still in flight when a job
|
||||
// ends -- which is what proves the worker waits for its heartbeat to finish.
|
||||
beatDelay time.Duration
|
||||
}
|
||||
|
||||
func newWStore() *wStore {
|
||||
return &wStore{completed: map[string]domain.Transcription{}, failed: map[string]domain.UploadErrorKind{}}
|
||||
}
|
||||
|
||||
func (s *wStore) StartTranscribing(_ context.Context, _, id string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.started = append(s.started, id)
|
||||
return s.startErr
|
||||
}
|
||||
|
||||
func (s *wStore) CompleteUpload(_ context.Context, _, id string, t domain.Transcription) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.completeErr != nil {
|
||||
return s.completeErr
|
||||
}
|
||||
s.completed[id] = t
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) FailUpload(_ context.Context, _, id string, k domain.UploadErrorKind) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.failed[id] = k
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) SaveProgress(_ context.Context, _, _ string, steps []domain.ProgressStep, _ time.Time) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.saves = append(s.saves, append([]domain.ProgressStep(nil), steps...))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) Heartbeat(context.Context, string, string) error {
|
||||
time.Sleep(s.beatDelay)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.beats++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *wStore) lastSteps() []domain.ProgressStep {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(s.saves) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.saves[len(s.saves)-1]
|
||||
}
|
||||
|
||||
func (s *wStore) beatCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.beats
|
||||
}
|
||||
|
||||
func (s *wStore) snapshot() (map[string]domain.Transcription, map[string]domain.UploadErrorKind) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
c := map[string]domain.Transcription{}
|
||||
f := map[string]domain.UploadErrorKind{}
|
||||
for k, v := range s.completed {
|
||||
c[k] = v
|
||||
}
|
||||
for k, v := range s.failed {
|
||||
f[k] = v
|
||||
}
|
||||
return c, f
|
||||
}
|
||||
|
||||
type fakeConverter struct {
|
||||
err error
|
||||
srcs []string
|
||||
dsts []string
|
||||
}
|
||||
|
||||
func (c *fakeConverter) ToWAV(_ context.Context, src, dst string) error {
|
||||
c.srcs = append(c.srcs, src)
|
||||
c.dsts = append(c.dsts, dst)
|
||||
if c.err != nil {
|
||||
return c.err
|
||||
}
|
||||
return os.WriteFile(dst, []byte("wav"), 0o600)
|
||||
}
|
||||
|
||||
type fakeTranscriber struct {
|
||||
result domain.Transcription
|
||||
err error
|
||||
paths []string
|
||||
// block, when non-nil, holds every call until it is closed or ctx ends.
|
||||
block chan struct{}
|
||||
running atomic.Int32
|
||||
maxSeen atomic.Int32
|
||||
}
|
||||
|
||||
func (f *fakeTranscriber) Transcribe(ctx context.Context, wav string) (domain.Transcription, error) {
|
||||
f.paths = append(f.paths, wav)
|
||||
n := f.running.Add(1)
|
||||
defer f.running.Add(-1)
|
||||
for {
|
||||
m := f.maxSeen.Load()
|
||||
if n <= m || f.maxSeen.CompareAndSwap(m, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if f.block != nil {
|
||||
select {
|
||||
case <-f.block:
|
||||
case <-ctx.Done():
|
||||
return domain.Transcription{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return f.result, f.err
|
||||
}
|
||||
|
||||
type wrig struct {
|
||||
w *uploads.Worker
|
||||
st *wStore
|
||||
conv *fakeConverter
|
||||
tr *fakeTranscriber
|
||||
tmp string
|
||||
}
|
||||
|
||||
func newWRig(t *testing.T, timeout time.Duration) *wrig {
|
||||
t.Helper()
|
||||
r := &wrig{
|
||||
st: newWStore(),
|
||||
conv: &fakeConverter{},
|
||||
tr: &fakeTranscriber{result: domain.Transcription{Text: "synthetic text", Endpoint: "iguana/kb-whisper", AudioSeconds: 3}},
|
||||
tmp: t.TempDir(),
|
||||
}
|
||||
r.w = uploads.NewWorker(r.st, r.conv, r.tr, r.tmp, timeout)
|
||||
return r
|
||||
}
|
||||
|
||||
// spool writes a fake uploaded file (what Intake would hand over) and returns a Job.
|
||||
func (r *wrig) spool(t *testing.T, id string) uploads.Job {
|
||||
t.Helper()
|
||||
p := filepath.Join(r.tmp, "upload-"+id)
|
||||
require.NoError(t, os.WriteFile(p, []byte("audio"), 0o600))
|
||||
return uploads.Job{UserID: "u1", UploadID: id, Path: p}
|
||||
}
|
||||
|
||||
func (r *wrig) leftovers(t *testing.T) []string {
|
||||
t.Helper()
|
||||
ents, err := os.ReadDir(r.tmp)
|
||||
require.NoError(t, err)
|
||||
var n []string
|
||||
for _, e := range ents {
|
||||
n = append(n, e.Name())
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestWorkerProcess_SuccessConvertsTranscribesStoresAndDeletesTheAudio(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
job := r.spool(t, "a")
|
||||
|
||||
r.w.Process(context.Background(), job)
|
||||
|
||||
c, f := r.st.snapshot()
|
||||
assert.Equal(t, []string{"a"}, r.st.started)
|
||||
assert.Equal(t, "synthetic text", c["a"].Text)
|
||||
assert.Equal(t, "iguana/kb-whisper", c["a"].Endpoint)
|
||||
assert.Empty(t, f)
|
||||
require.Len(t, r.conv.srcs, 1)
|
||||
assert.Equal(t, job.Path, r.conv.srcs[0], "the converter reads the uploaded file")
|
||||
assert.Equal(t, r.conv.dsts, r.tr.paths, "whisper is sent the converted WAV, not the original")
|
||||
assert.Equal(t, r.tmp, filepath.Dir(r.conv.dsts[0]))
|
||||
assert.Empty(t, r.leftovers(t), "the audio is never retained: both the upload and the WAV are deleted")
|
||||
}
|
||||
|
||||
func TestWorkerProcess_FailuresAreTypedAndStillDeleteTheAudio(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*wrig)
|
||||
}{
|
||||
{"conversion fails", func(r *wrig) { r.conv.err = errors.New("moov atom not found") }},
|
||||
{"every transcription endpoint fails", func(r *wrig) { r.tr.err = errors.New("iguana down; berget down") }},
|
||||
{"storing the finished transcript fails", func(r *wrig) { r.st.completeErr = errors.New("db down") }},
|
||||
{"the temporary WAV cannot be created", func(r *wrig) {
|
||||
r.w = uploads.NewWorker(r.st, r.conv, r.tr, filepath.Join(r.tmp, "missing"), time.Minute)
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
tt.setup(r)
|
||||
job := r.spool(t, "a")
|
||||
|
||||
r.w.Process(context.Background(), job)
|
||||
|
||||
c, f := r.st.snapshot()
|
||||
assert.Empty(t, c)
|
||||
assert.Equal(t, domain.UploadErrTranscriptionFailed, f["a"])
|
||||
assert.Empty(t, r.leftovers(t))
|
||||
})
|
||||
}
|
||||
t.Run("conversion failure never reaches whisper", func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.conv.err = errors.New("bad")
|
||||
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Empty(t, r.tr.paths)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkerProcess_ShutdownMidJobLeavesTheRowForTheStartupSweep(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
job := r.spool(t, "a")
|
||||
|
||||
go func() { r.w.Process(ctx, job); close(done) }()
|
||||
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
_, f := r.st.snapshot()
|
||||
assert.Empty(t, f, "a shutdown is not a transcription failure: the row stays for FailInterruptedUploads to mark interrupted")
|
||||
assert.Empty(t, r.leftovers(t), "but the audio is still deleted")
|
||||
}
|
||||
|
||||
func TestWorkerProcess_AJobThatOutlivesItsTimeoutIsFailedNotStuck(t *testing.T) {
|
||||
r := newWRig(t, 50*time.Millisecond)
|
||||
r.tr.block = make(chan struct{}) // never released
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() { r.w.Process(context.Background(), r.spool(t, "a")); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Process did not return: a hung endpoint was not bounded by the job timeout")
|
||||
}
|
||||
|
||||
_, f := r.st.snapshot()
|
||||
assert.Equal(t, domain.UploadErrTranscriptionFailed, f["a"], "a hung endpoint must end in failed, not an eternal spinner (ADR-025)")
|
||||
assert.Empty(t, r.leftovers(t))
|
||||
}
|
||||
|
||||
func TestWorkerProcess_StartFailureDropsTheJobAndDeletesTheAudio(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.st.startErr = errors.New("db down")
|
||||
|
||||
r.w.Process(context.Background(), r.spool(t, "a"))
|
||||
|
||||
assert.Empty(t, r.tr.paths, "never transcribe a job whose row could not be moved to transcribing")
|
||||
_, f := r.st.snapshot()
|
||||
assert.Contains(t, f, "a")
|
||||
assert.Empty(t, r.leftovers(t))
|
||||
}
|
||||
|
||||
func TestWorkerRun_ProcessesOneJobAtATime(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go r.w.Run(ctx)
|
||||
|
||||
for _, id := range []string{"a", "b", "c"} {
|
||||
require.NoError(t, r.w.Submit(r.spool(t, id)))
|
||||
}
|
||||
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||
time.Sleep(50 * time.Millisecond) // give a (wrong) second job time to start
|
||||
assert.Equal(t, int32(1), r.tr.running.Load(), "two simultaneous long jobs on iguana were never measured: concurrency is 1")
|
||||
|
||||
close(r.tr.block)
|
||||
require.Eventually(t, func() bool { c, _ := r.st.snapshot(); return len(c) == 3 }, 3*time.Second, 5*time.Millisecond)
|
||||
assert.Equal(t, int32(1), r.tr.maxSeen.Load())
|
||||
}
|
||||
|
||||
func TestWorkerRun_StoppingDeletesQueuedAudioAndRefusesNewJobs(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
r.tr.block = make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
stopped := make(chan struct{})
|
||||
go func() { r.w.Run(ctx); close(stopped) }()
|
||||
require.NoError(t, r.w.Submit(r.spool(t, "a")))
|
||||
require.Eventually(t, func() bool { return r.tr.running.Load() == 1 }, 2*time.Second, 5*time.Millisecond)
|
||||
require.NoError(t, r.w.Submit(r.spool(t, "b"))) // queued behind the running job
|
||||
|
||||
cancel()
|
||||
<-stopped
|
||||
|
||||
assert.Len(t, r.conv.srcs, 1, "a job still queued at shutdown must never be started: only the running job was converted")
|
||||
assert.Empty(t, r.leftovers(t), "audio of queued jobs is deleted on stop: the rows are swept to interrupted at next start")
|
||||
assert.ErrorIs(t, r.w.Submit(r.spool(t, "c")), uploads.ErrStopped)
|
||||
}
|
||||
|
||||
// dbErr is an error shaped like pgx's: a message that quotes a value, and an SQLSTATE.
|
||||
type dbErr struct{}
|
||||
|
||||
func (dbErr) Error() string {
|
||||
return `ERROR: invalid input syntax for type json: "zebrakaka secret words"`
|
||||
}
|
||||
func (dbErr) SQLState() string { return "22P02" }
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/a-h/templ"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/chat"
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||
)
|
||||
@@ -43,6 +45,28 @@ type chatView struct {
|
||||
History []chat.Turn
|
||||
Truncated bool
|
||||
Error string
|
||||
// PostURL and Subject let the same chat panel serve an UPLOADED recording
|
||||
// (ADR-033): where the form posts, and what the transcript is called. Empty
|
||||
// means a YouTube video (/v/{VideoID}/chat, "video"), so existing callers are
|
||||
// unchanged.
|
||||
PostURL templ.SafeURL
|
||||
Subject string
|
||||
}
|
||||
|
||||
// postURL is where the chat form posts.
|
||||
func (v chatView) postURL() templ.SafeURL {
|
||||
if v.PostURL != "" {
|
||||
return v.PostURL
|
||||
}
|
||||
return chatURL(v.VideoID)
|
||||
}
|
||||
|
||||
// subject is what the transcript is called in the panel's wording.
|
||||
func (v chatView) subject() string {
|
||||
if v.Subject != "" {
|
||||
return v.Subject
|
||||
}
|
||||
return "video"
|
||||
}
|
||||
|
||||
// handleChat renders the chat page for a summarized video (GET). Entry is scoped
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// Store is the read/write surface the web handlers depend on — a narrow port over
|
||||
@@ -53,6 +54,12 @@ type Store interface {
|
||||
// the most recent discovery pass, shown on the account page as a warning.
|
||||
ListChannelErrors(ctx context.Context, userID string) ([]store.ChannelError, error)
|
||||
|
||||
// Uploaded-audio transcripts (ADR-033): the caller's own PRIVATE uploads,
|
||||
// stored apart from the shared public captions. Both are user-scoped; a
|
||||
// non-owner's id is ErrUploadNotFound.
|
||||
GetUpload(ctx context.Context, userID, id string) (*domain.UploadedTranscript, error)
|
||||
ListUploads(ctx context.Context, userID string) ([]domain.UploadedTranscript, error)
|
||||
|
||||
// SetTranscriptStatus clears or updates a video's transcript backoff state.
|
||||
// Used by handleRetryNow to clear rate_limited_at before immediate processing.
|
||||
SetTranscriptStatus(ctx context.Context, userID, videoID, status string) error
|
||||
@@ -100,6 +107,26 @@ type App struct {
|
||||
// summary view shows no "dig deeper" affordance. It holds no caption-fetch
|
||||
// dependency, so an enabled chat cannot reach YouTube or the rate gate.
|
||||
Chat Chatter
|
||||
// Uploads, when non-nil, accepts audio recordings for private transcription
|
||||
// (ADR-033). Nil = the /uploads routes are not mounted. UploadLimits are the
|
||||
// same limits the Uploader enforces, shown on the form so they are stated up
|
||||
// front rather than discovered by failing.
|
||||
Uploads Uploader
|
||||
UploadLimits uploads.Limits
|
||||
// UploadChatMaxChars is the transcript budget for chat over an uploaded
|
||||
// recording. The caption default (18,000 chars, ~22 minutes of speech) would
|
||||
// keep only the first third of an hour-long meeting.
|
||||
UploadChatMaxChars int
|
||||
// UploadChatModel is the model chat over an uploaded recording opens with. Empty
|
||||
// means the Chatter's own default. It exists because that default is the
|
||||
// summarizer's (koala/phi4-mini, 8,192-token context) and cannot hold an hour of
|
||||
// speech.
|
||||
UploadChatModel string
|
||||
// UploadChatModels is the set of models chat over an uploaded recording offers and
|
||||
// accepts. The whole private transcript goes to the one chosen, so it must hold
|
||||
// LOCAL models only, and the caller decides that. Nil or empty fails closed: no
|
||||
// model is offered and none is called (ADR-035 review).
|
||||
UploadChatModels []string
|
||||
// Processing tracks in-flight immediate summarizations so the status endpoint
|
||||
// shows the animation until the summary lands. The zero value is ready to use.
|
||||
Processing ProcessingSet
|
||||
@@ -166,6 +193,20 @@ func (a *App) Router() http.Handler {
|
||||
app.HandleFunc("GET /v/{videoId}/chat", a.handleChat)
|
||||
app.HandleFunc("POST /v/{videoId}/chat", a.handleChatMessage)
|
||||
}
|
||||
// Audio upload → private transcript (ADR-033). Mounted only when an Uploader is
|
||||
// wired; every handler is user-scoped, so another user's upload is a 404.
|
||||
if a.Uploads != nil {
|
||||
app.HandleFunc("GET /uploads", a.handleUploadsPage)
|
||||
app.HandleFunc("POST /uploads", a.handleUploadPost)
|
||||
app.HandleFunc("GET /uploads/{id}", a.handleUploadDetail)
|
||||
app.HandleFunc("GET /uploads/{id}/status", a.handleUploadStatus)
|
||||
// Chat over the uploaded transcript reuses the ADR-027 Chatter; mounted only
|
||||
// when one is wired.
|
||||
if a.Chat != nil {
|
||||
app.HandleFunc("GET /uploads/{id}/chat", a.handleUploadChat)
|
||||
app.HandleFunc("POST /uploads/{id}/chat", a.handleUploadChatMessage)
|
||||
}
|
||||
}
|
||||
app.HandleFunc("GET /register", a.handleRegisterForm)
|
||||
app.HandleFunc("POST /register", a.handleRegister)
|
||||
|
||||
|
||||
@@ -0,0 +1,532 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/a-h/templ"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/chat"
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// Uploader accepts an uploaded recording for transcription (ADR-033).
|
||||
// *uploads.Intake satisfies it. It is the whole of what the handlers know about
|
||||
// validation, spooling and queueing.
|
||||
type Uploader interface {
|
||||
Accept(ctx context.Context, userID, title string, src io.Reader) (id string, err error)
|
||||
}
|
||||
|
||||
const (
|
||||
// uploadOverhead is headroom above the file-size cap for the multipart framing
|
||||
// and the title field; the cap itself is enforced on the file part by Intake.
|
||||
uploadOverhead = 1 << 20
|
||||
maxTitleRunes = 200
|
||||
// uploadPollEvery is how often an unfinished upload re-asks for its status.
|
||||
uploadPollEvery = "2s"
|
||||
// serviceSilentAfter is how long a job may go without any signal (heartbeat or
|
||||
// progress) before the page calls the service lost. The heartbeat is every 10 s, so
|
||||
// 45 s is four missed beats (ADR-035).
|
||||
serviceSilentAfter = 45 * time.Second
|
||||
// countedQuietAfter is how long a COUNTED phase (the deciding one) may go without
|
||||
// advancing before the page says it is quiet. Each decision has a two-minute timeout,
|
||||
// so five minutes is more than two in a row. A phase with no count is never quiet.
|
||||
countedQuietAfter = 5 * time.Minute
|
||||
)
|
||||
|
||||
// uploadsView is what the /uploads page renders.
|
||||
type uploadsView struct {
|
||||
Limits uploads.Limits
|
||||
Rows []domain.UploadedTranscript
|
||||
Error string // a synchronous rejection message, empty when none
|
||||
}
|
||||
|
||||
// handleUploadsPage renders the upload form and the caller's own uploads.
|
||||
func (a *App) handleUploadsPage(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := a.currentUserID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
a.renderUploadsPage(w, r, userID, http.StatusOK, "")
|
||||
}
|
||||
|
||||
func (a *App) renderUploadsPage(w http.ResponseWriter, r *http.Request, userID string, status int, msg string) {
|
||||
rows, err := a.Store.ListUploads(r.Context(), userID)
|
||||
if err != nil {
|
||||
a.serverError(w, r, "list uploads", err)
|
||||
return
|
||||
}
|
||||
a.renderStatus(w, r, status, UploadsPage(uploadsView{Limits: a.UploadLimits, Rows: rows, Error: msg}))
|
||||
}
|
||||
|
||||
// handleUploadPost streams the multipart body: the optional title field, then the
|
||||
// file part, which goes straight to Intake without being buffered in memory. A
|
||||
// rejection is synchronous — a status code and a message on the upload page, no
|
||||
// row, no timeout (ADR-033).
|
||||
func (a *App) handleUploadPost(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := a.currentUserID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, a.UploadLimits.MaxBytes+uploadOverhead)
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
a.renderUploadsPage(w, r, userID, http.StatusBadRequest, "Choose an audio file to upload.")
|
||||
return
|
||||
}
|
||||
title := ""
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
var tooBig *http.MaxBytesError
|
||||
if a.abandoned(w, r) {
|
||||
return
|
||||
}
|
||||
if !errors.As(err, &tooBig) {
|
||||
// A malformed multipart body (a header with a control byte, a truncated
|
||||
// part) is the sender's to fix: a 400, not a server error, and the error
|
||||
// text is never logged: it quotes the headers, which carry the file name.
|
||||
a.logger().Info("upload rejected", "reason", "malformed")
|
||||
a.renderUploadsPage(w, r, userID, http.StatusBadRequest, "That upload could not be read. Please try again.")
|
||||
return
|
||||
}
|
||||
a.uploadFailure(w, r, userID, err)
|
||||
return
|
||||
}
|
||||
switch part.FormName() {
|
||||
case "title":
|
||||
b, err := io.ReadAll(io.LimitReader(part, maxTitleRunes*utf8.UTFMax))
|
||||
if err != nil {
|
||||
if a.abandoned(w, r) {
|
||||
return
|
||||
}
|
||||
a.renderUploadsPage(w, r, userID, http.StatusBadRequest, "That upload could not be read. Please try again.")
|
||||
return
|
||||
}
|
||||
title = cleanTitle(string(b))
|
||||
case "file":
|
||||
if title == "" {
|
||||
title = titleFromFilename(part.FileName())
|
||||
}
|
||||
id, err := a.Uploads.Accept(r.Context(), userID, title, part)
|
||||
if err != nil {
|
||||
a.uploadFailure(w, r, userID, err)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/uploads/"+id, http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
}
|
||||
a.renderUploadsPage(w, r, userID, http.StatusBadRequest, "Choose an audio file to upload.")
|
||||
}
|
||||
|
||||
// abandoned reports, logs at INFO and answers 499, because the client has already gone: its request
|
||||
// context is cancelled, so there is nobody to answer and rendering a page would run its
|
||||
// database read on a dead context and log a false ERROR. A hang-up mid-upload is routine.
|
||||
func (a *App) abandoned(w http.ResponseWriter, r *http.Request) bool {
|
||||
if r.Context().Err() == nil {
|
||||
return false
|
||||
}
|
||||
a.logger().Info("upload abandoned", "reason", "client_gone")
|
||||
// 499 (the "client closed request" convention): a client that half-closed after sending
|
||||
// can still read, and an unwritten status would reach it as a false, empty 200.
|
||||
w.WriteHeader(499)
|
||||
return true
|
||||
}
|
||||
|
||||
// uploadFailure maps an Accept (or multipart) error to a response: a validation
|
||||
// rejection is the user's to fix (422, or 413 for size) and carries its own
|
||||
// message; anything else is ours.
|
||||
func (a *App) uploadFailure(w http.ResponseWriter, r *http.Request, userID string, err error) {
|
||||
if a.abandoned(w, r) {
|
||||
return
|
||||
}
|
||||
var rej *uploads.RejectError
|
||||
var tooBig *http.MaxBytesError
|
||||
switch {
|
||||
case errors.As(err, &rej):
|
||||
// Logged by reason only: the filename and title are private content.
|
||||
a.logger().Info("upload rejected", "reason", string(rej.Reason))
|
||||
status := http.StatusUnprocessableEntity
|
||||
switch rej.Reason {
|
||||
case uploads.ReasonTooLarge:
|
||||
status = http.StatusRequestEntityTooLarge
|
||||
case uploads.ReasonBusy:
|
||||
status = http.StatusTooManyRequests
|
||||
}
|
||||
a.renderUploadsPage(w, r, userID, status, rej.Message)
|
||||
case errors.As(err, &tooBig):
|
||||
a.logger().Info("upload rejected", "reason", string(uploads.ReasonTooLarge))
|
||||
a.renderUploadsPage(w, r, userID, http.StatusRequestEntityTooLarge,
|
||||
"This file is larger than the upload limit.")
|
||||
case errors.Is(err, uploads.ErrClientRead):
|
||||
// An error reading the CLIENT's body: theirs to retry, not a server fault. Only that
|
||||
// marker counts: io.ErrUnexpectedEOF alone is also what a database drop looks like.
|
||||
a.logger().Info("upload rejected", "reason", "truncated")
|
||||
a.renderUploadsPage(w, r, userID, http.StatusBadRequest, "The upload was cut off before it finished. Please try again.")
|
||||
default:
|
||||
// Not err: it can carry the file's name, a spool path or a database value.
|
||||
a.logger().Error("upload failed", "kind", errkind.Of(err))
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
// handleUploadDetail renders one upload's page (GET /uploads/{id}). The row is
|
||||
// read through the user-scoped store, so another user's id is a 404 — existence is
|
||||
// not disclosed, and the title never appears in the response.
|
||||
func (a *App) handleUploadDetail(w http.ResponseWriter, r *http.Request) {
|
||||
u, ok := a.loadOwnedUpload(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
a.render(w, r, UploadPage(*u, a.Chat != nil, a.now()))
|
||||
}
|
||||
|
||||
// handleUploadStatus renders the status fragment HTMX polls. Unfinished states
|
||||
// keep polling; terminal states render without hx-trigger so the page stops
|
||||
// asking (ADR-025: never a stuck spinner).
|
||||
func (a *App) handleUploadStatus(w http.ResponseWriter, r *http.Request) {
|
||||
u, ok := a.loadOwnedUpload(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
a.render(w, r, uploadStatus(*u, a.Chat != nil, a.now()))
|
||||
}
|
||||
|
||||
func (a *App) loadOwnedUpload(w http.ResponseWriter, r *http.Request) (*domain.UploadedTranscript, bool) {
|
||||
userID, ok := a.currentUserID(w, r)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
id := r.PathValue("id")
|
||||
if !uuidShape.MatchString(id) {
|
||||
// Not an id at all: the same 404 as an unknown or foreign one, never a database
|
||||
// error that surfaces as a 500 (and no oracle about which ids exist).
|
||||
http.NotFound(w, r)
|
||||
return nil, false
|
||||
}
|
||||
u, err := a.Store.GetUpload(r.Context(), userID, id)
|
||||
if errors.Is(err, store.ErrUploadNotFound) {
|
||||
http.NotFound(w, r)
|
||||
return nil, false
|
||||
}
|
||||
if err != nil {
|
||||
a.serverError(w, r, "get upload", err)
|
||||
return nil, false
|
||||
}
|
||||
return u, true
|
||||
}
|
||||
|
||||
// titleFromFilename is the default title: the file's name without its extension.
|
||||
func titleFromFilename(name string) string {
|
||||
base := filepath.Base(name)
|
||||
t := cleanTitle(strings.TrimSuffix(base, filepath.Ext(base)))
|
||||
if t == "" || t == "." {
|
||||
return "Untitled recording"
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// hasNote reports whether any phase carries a note. A note is the only warning for a
|
||||
// recording that is shorter than its file claimed, so a record that has one is shown open.
|
||||
func hasNote(steps []domain.ProgressStep) bool {
|
||||
for _, s := range steps {
|
||||
if s.Note != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// uuidShape is the form of an upload id. Anything else cannot be one, so it is a 404.
|
||||
var uuidShape = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||
|
||||
// isBidiControl reports the invisible characters that change text direction. Emoji joiners
|
||||
// (ZWJ, ZWNJ) and variation selectors are not among them and are kept.
|
||||
func isBidiControl(r rune) bool {
|
||||
return r >= 0x202A && r <= 0x202E || r >= 0x2066 && r <= 0x2069 || r == 0x200E || r == 0x200F || r == 0x061C
|
||||
}
|
||||
|
||||
// cleanTitle makes a title safe to store and show: invalid UTF-8 dropped, control
|
||||
// characters (NUL, newlines, escapes) turned into spaces, runs of space collapsed, trimmed
|
||||
// and clipped. A title is cosmetic; bad bytes in it must never fail an upload, and Postgres
|
||||
// refuses NUL in text.
|
||||
func cleanTitle(s string) string {
|
||||
s = strings.ToValidUTF8(s, "")
|
||||
s = strings.Map(func(r rune) rune {
|
||||
switch {
|
||||
case isBidiControl(r):
|
||||
return -1 // dropped: U+202E would reverse the rest of the title on its owner's page
|
||||
case unicode.IsControl(r):
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
return clipRunes(strings.Join(strings.Fields(s), " "), maxTitleRunes)
|
||||
}
|
||||
|
||||
func clipRunes(s string, n int) string {
|
||||
if utf8.RuneCountInString(s) <= n {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:n])
|
||||
}
|
||||
|
||||
// uploadFailureMessage turns a typed failure kind into words for the user. It is
|
||||
// keyed on OUR kinds, never on a provider's error string (ADR-033).
|
||||
func uploadFailureMessage(k domain.UploadErrorKind) string {
|
||||
switch k {
|
||||
case domain.UploadErrInterrupted:
|
||||
return "The server restarted before this recording was transcribed. Please upload it again."
|
||||
case domain.UploadErrBusy:
|
||||
return "Too many recordings were waiting to be transcribed when this one was uploaded. Please upload it again in a few minutes."
|
||||
case domain.UploadErrTooLong:
|
||||
return "This recording is longer than the limit once its audio is read (the file's header said otherwise). Please upload a shorter recording."
|
||||
case domain.UploadErrTranscriptionFailed:
|
||||
return "Transcription failed on every available service. Please upload it again later."
|
||||
default:
|
||||
return "This upload failed. Please upload it again."
|
||||
}
|
||||
}
|
||||
|
||||
// handleUploadChat renders the chat page for a FINISHED upload (GET). The row is
|
||||
// read through the user-scoped store, so another user's id is a 404 before any
|
||||
// model is involved. An unfinished upload has no transcript to ask about yet, so
|
||||
// it goes back to its status page.
|
||||
func (a *App) handleUploadChat(w http.ResponseWriter, r *http.Request) {
|
||||
u, ok := a.loadOwnedUpload(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if u.Status != domain.UploadDone {
|
||||
http.Redirect(w, r, "/uploads/"+u.ID, http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
model, _ := a.resolveUploadModel("")
|
||||
a.render(w, r, UploadChatPage(*u, a.uploadChatView(*u, model, nil, false, "")))
|
||||
}
|
||||
|
||||
// handleUploadChatMessage answers one question against the owner's uploaded
|
||||
// transcript (POST), reusing the ADR-027 Chatter. Nothing is persisted: the prior
|
||||
// turns ride in the form. The transcript goes to the model with the UPLOAD budget,
|
||||
// not the caption default. A model failure is inline, not a 500.
|
||||
func (a *App) handleUploadChatMessage(w http.ResponseWriter, r *http.Request) {
|
||||
u, ok := a.loadOwnedUpload(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if u.Status != domain.UploadDone {
|
||||
http.Redirect(w, r, "/uploads/"+u.ID, http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "bad form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
model, ok := a.resolveUploadModel(r.FormValue("model"))
|
||||
if !ok {
|
||||
a.logger().Error("upload chat: no local model is configured; nothing was sent", "upload", u.ID)
|
||||
view := a.uploadChatView(*u, "", nil, false, "No local model is available for chat about a recording.")
|
||||
a.render(w, r, UploadChatPage(*u, view))
|
||||
return
|
||||
}
|
||||
history := parseHistory(r.Form["hq"], r.Form["ha"])
|
||||
question := strings.TrimSpace(r.FormValue("question"))
|
||||
|
||||
view := a.uploadChatView(*u, model, history, false, "")
|
||||
if question != "" {
|
||||
reply, err := a.Chat.Answer(r.Context(), chat.Request{
|
||||
Model: model,
|
||||
Transcript: u.Content,
|
||||
History: history,
|
||||
Question: question,
|
||||
MaxChars: a.UploadChatMaxChars,
|
||||
})
|
||||
if err != nil {
|
||||
a.logger().Error("upload chat answer", "upload", u.ID, "model", model, "kind", errkind.Of(err))
|
||||
view.Error = "That model couldn't answer just now. Try again, or switch models."
|
||||
} else {
|
||||
view.History = appendTurn(history, chat.Turn{Question: question, Answer: reply.Answer})
|
||||
view.Truncated = reply.Truncated
|
||||
}
|
||||
}
|
||||
if isHTMX(r) {
|
||||
a.render(w, r, chatPanel(view))
|
||||
return
|
||||
}
|
||||
a.render(w, r, UploadChatPage(*u, view))
|
||||
}
|
||||
|
||||
func (a *App) uploadChatView(u domain.UploadedTranscript, model string, history []chat.Turn, truncated bool, errMsg string) chatView {
|
||||
return chatView{
|
||||
VideoID: u.ID,
|
||||
Title: u.Title,
|
||||
Available: true,
|
||||
Models: a.UploadChatModels,
|
||||
Selected: model,
|
||||
History: history,
|
||||
Truncated: truncated,
|
||||
Error: errMsg,
|
||||
PostURL: templ.SafeURL("/uploads/" + u.ID + "/chat"),
|
||||
Subject: "recording",
|
||||
}
|
||||
}
|
||||
|
||||
// maxSpotsShown is how many spots to check the page lists. The stored list holds every
|
||||
// judged spot and the counts are exact; a page of two hundred "check this" items is not a
|
||||
// page anyone reads, so the rest are counted, not listed.
|
||||
const maxSpotsShown = 50
|
||||
|
||||
// minuteMark renders where in the recording a spot is, as the nearest minute.
|
||||
func minuteMark(seconds float64) string {
|
||||
return fmt.Sprintf("~%d min", int(math.Round(seconds/60)))
|
||||
}
|
||||
|
||||
// shownSpots is the part of the stored list the page renders: the spots a reader should
|
||||
// check, changed or uncertain, in order, capped. A spot the model confidently kept is
|
||||
// stored (ADR-035) but is not one to check, so it is not listed.
|
||||
func shownSpots(r *domain.Reconciliation) []domain.ContestedSpot {
|
||||
var out []domain.ContestedSpot
|
||||
for _, c := range r.Contested {
|
||||
if c.Used == "alt" || c.Unsure {
|
||||
out = append(out, c)
|
||||
if len(out) == maxSpotsShown {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moreSpots is how many spots to check are not listed, from the exact counts.
|
||||
func moreSpots(r *domain.Reconciliation) int {
|
||||
return max(0, r.Changed+r.Unsure-len(shownSpots(r)))
|
||||
}
|
||||
|
||||
// jobHealth is what the page can honestly say about a job it is still polling.
|
||||
type jobHealth string
|
||||
|
||||
const (
|
||||
// healthRunning: the service is reporting and work is moving, or there is nothing yet
|
||||
// to say otherwise.
|
||||
healthRunning jobHealth = "running"
|
||||
// healthQuiet: the service is alive but a counted phase has not advanced for a while.
|
||||
healthQuiet jobHealth = "quiet"
|
||||
// healthLost: nothing has been heard from the service for too long. It may have
|
||||
// restarted or crashed; the start-up sweep fails the row once it is back.
|
||||
healthLost jobHealth = "lost"
|
||||
)
|
||||
|
||||
// uploadHealth tells running from quiet from lost for an upload being transcribed (the
|
||||
// page calls it for no other state: a queued or finished job has no heartbeat to judge). The
|
||||
// two timestamps mean different things (ADR-035): alive_at is the heartbeat, which says
|
||||
// the service is still reporting; progress_at moves only when work really moved. A
|
||||
// single blocking engine call has no sub-progress, so only a counted phase can be quiet.
|
||||
// silent is how long since the signal that decided it.
|
||||
func uploadHealth(u domain.UploadedTranscript, now time.Time) (health jobHealth, silent time.Duration) {
|
||||
last := u.AliveAt
|
||||
if u.ProgressAt.After(last) {
|
||||
last = u.ProgressAt
|
||||
}
|
||||
if last.IsZero() {
|
||||
return healthRunning, 0
|
||||
}
|
||||
if d := now.Sub(last); d > serviceSilentAfter {
|
||||
return healthLost, d
|
||||
}
|
||||
for _, s := range u.Progress {
|
||||
if s.State == domain.StepRunning && s.Total > 0 && !u.ProgressAt.IsZero() {
|
||||
if d := now.Sub(u.ProgressAt); d > countedQuietAfter {
|
||||
return healthQuiet, d
|
||||
}
|
||||
}
|
||||
}
|
||||
return healthRunning, 0
|
||||
}
|
||||
|
||||
// stepDuration renders how long a phase took or has run: "<1s", "42s", "3m 12s", "1h 02m".
|
||||
func stepDuration(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return "<1s"
|
||||
case d < time.Minute:
|
||||
return fmt.Sprintf("%ds", int(d.Seconds()))
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%dm %02ds", int(d.Minutes()), int(d.Seconds())%60)
|
||||
default:
|
||||
return fmt.Sprintf("%dh %02dm", int(d.Hours()), int(d.Minutes())%60)
|
||||
}
|
||||
}
|
||||
|
||||
// stepTime is the time shown beside a phase: how long it has run, or how long it took.
|
||||
func stepTime(s domain.ProgressStep, now time.Time) string {
|
||||
switch s.State {
|
||||
case domain.StepRunning:
|
||||
return stepDuration(now.Sub(s.StartedAt))
|
||||
case domain.StepDone, domain.StepFailed:
|
||||
return stepDuration(s.FinishedAt.Sub(s.StartedAt))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stepMark is the glyph at the head of a phase line.
|
||||
func stepMark(s domain.ProgressStep) string {
|
||||
switch s.State {
|
||||
case domain.StepDone:
|
||||
return "✓"
|
||||
case domain.StepFailed:
|
||||
return "✗"
|
||||
case domain.StepSkipped:
|
||||
return "–"
|
||||
}
|
||||
return "·"
|
||||
}
|
||||
|
||||
// minutesAgo is a whole number of minutes for the health notes: "6 min", or "under a
|
||||
// minute" when it is less.
|
||||
func minutesAgo(d time.Duration) string {
|
||||
if m := int(d.Minutes()); m >= 1 {
|
||||
return fmt.Sprintf("%d min", m)
|
||||
}
|
||||
return "under a minute"
|
||||
}
|
||||
|
||||
// resolveUploadModel picks the model for chat over an uploaded recording from the
|
||||
// offered LOCAL set only: the posted one when it is in the set, else the configured
|
||||
// default when it is, else the first. ok is false when no model is offered, in which
|
||||
// case nothing may be sent anywhere (the whole private transcript is the payload).
|
||||
func (a *App) resolveUploadModel(posted string) (model string, ok bool) {
|
||||
offered := func(m string) bool {
|
||||
for _, o := range a.UploadChatModels {
|
||||
if o == m {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case posted != "" && offered(posted):
|
||||
return posted, true
|
||||
case a.UploadChatModel != "" && offered(a.UploadChatModel):
|
||||
return a.UploadChatModel, true
|
||||
case len(a.UploadChatModels) > 0:
|
||||
return a.UploadChatModels[0], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// UploadsPage is the upload form plus the caller's own uploads (ADR-033).
|
||||
templ UploadsPage(v uploadsView) {
|
||||
@Layout("Tapir — Upload a recording") {
|
||||
<article class="account">
|
||||
<h1>Upload a recording</h1>
|
||||
<p class="muted">
|
||||
Audio only. Up to { fmt.Sprintf("%d MB", v.Limits.MaxBytes>>20) } and
|
||||
{ uploads.FormatDuration(v.Limits.MaxDuration) }. The transcript is private to you;
|
||||
the audio is deleted once it has been transcribed.
|
||||
</p>
|
||||
if v.Error != "" {
|
||||
<div class="flash flash-error" role="alert">{ v.Error }</div>
|
||||
}
|
||||
<form method="post" action="/uploads" enctype="multipart/form-data">
|
||||
<p><label>Title (optional) <input type="text" name="title" maxlength="200"/></label></p>
|
||||
<p><input type="file" name="file" accept="audio/*,.m4a,.mp3,.wav,.opus,.ogg,.mp4" required/></p>
|
||||
<p><button type="submit">Upload and transcribe</button></p>
|
||||
</form>
|
||||
if len(v.Rows) > 0 {
|
||||
<section>
|
||||
<h2>Your recordings</h2>
|
||||
<ul>
|
||||
for _, u := range v.Rows {
|
||||
<li>
|
||||
<a href={ templ.SafeURL("/uploads/" + u.ID) }>{ u.Title }</a>
|
||||
<span class="chip">{ string(u.Status) }</span>
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
</section>
|
||||
}
|
||||
</article>
|
||||
}
|
||||
}
|
||||
|
||||
// UploadPage is one upload's page: its title and the polling status region.
|
||||
templ UploadPage(u domain.UploadedTranscript, chatEnabled bool, now time.Time) {
|
||||
@Layout("Tapir — " + u.Title) {
|
||||
<article class="account">
|
||||
<h1>{ u.Title }</h1>
|
||||
@uploadStatus(u, chatEnabled, now)
|
||||
<p><a href="/uploads">All recordings</a></p>
|
||||
</article>
|
||||
}
|
||||
}
|
||||
|
||||
// uploadStatus is the region HTMX polls. Unfinished states carry hx-trigger so the
|
||||
// page re-asks; done and failed render without it, so nothing polls forever.
|
||||
templ uploadStatus(u domain.UploadedTranscript, chatEnabled bool, now time.Time) {
|
||||
switch u.Status {
|
||||
case domain.UploadDone:
|
||||
<div class="upload-status upload-done" role="status">
|
||||
@videoNotice(u.HadVideo)
|
||||
<p>
|
||||
Transcribed { uploads.FormatDuration(time.Duration(u.AudioSeconds * float64(time.Second))) }
|
||||
of audio by <span class="chip">{ u.Endpoint }</span>.
|
||||
</p>
|
||||
@reconciliationNote(u.Reconciliation)
|
||||
if len(u.Progress) > 0 {
|
||||
<details class="what-was-done" if hasNote(u.Progress) { open }>
|
||||
<summary>What was done</summary>
|
||||
@progressSteps(u.Progress, now, false)
|
||||
</details>
|
||||
}
|
||||
if chatEnabled {
|
||||
<p><a class="btn" href={ templ.SafeURL("/uploads/" + u.ID + "/chat") }>Ask about this recording</a></p>
|
||||
}
|
||||
<details open>
|
||||
<summary>Transcript</summary>
|
||||
<pre class="transcript">{ u.Content }</pre>
|
||||
</details>
|
||||
</div>
|
||||
case domain.UploadFailed:
|
||||
<div class="upload-status upload-failed flash flash-error" role="alert">
|
||||
@videoNotice(u.HadVideo)
|
||||
<p>{ uploadFailureMessage(u.ErrorKind) }</p>
|
||||
if len(u.Progress) > 0 {
|
||||
<details class="what-was-done" open>
|
||||
<summary>Where it stopped</summary>
|
||||
@progressSteps(u.Progress, now, false)
|
||||
</details>
|
||||
}
|
||||
</div>
|
||||
default:
|
||||
<div
|
||||
class="upload-status upload-working"
|
||||
role="status"
|
||||
hx-get={ "/uploads/" + u.ID + "/status" }
|
||||
hx-trigger={ "every " + uploadPollEvery }
|
||||
hx-swap="outerHTML"
|
||||
>
|
||||
@videoNotice(u.HadVideo)
|
||||
if u.Status == domain.UploadTranscribing {
|
||||
{{ health, silent := uploadHealth(u, now) }}
|
||||
if len(u.Progress) > 0 {
|
||||
@progressSteps(u.Progress, now, health != healthLost)
|
||||
@healthNote(health, silent)
|
||||
} else {
|
||||
<p>Transcribing… a long recording takes a few minutes.</p>
|
||||
}
|
||||
} else {
|
||||
<p>Queued — waiting for its turn to be transcribed.</p>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
}
|
||||
|
||||
// UploadChatPage is the no-JS full page for chatting with an uploaded transcript:
|
||||
// the same chat panel as a video's (ADR-027), scoped to this recording.
|
||||
templ UploadChatPage(u domain.UploadedTranscript, v chatView) {
|
||||
@Layout("Tapir — Ask about " + u.Title) {
|
||||
<article class="account">
|
||||
<h1>{ u.Title }</h1>
|
||||
<section id="chat-section" class="chat-dock chat-dock-open">
|
||||
<h2 class="chat-heading">Ask about this recording</h2>
|
||||
<p class="chat-scope muted">Answers come only from this recording's transcript, which is private to you.</p>
|
||||
@chatPanel(v)
|
||||
</section>
|
||||
<p><a href={ templ.SafeURL("/uploads/" + u.ID) }>Back to the transcript</a></p>
|
||||
</article>
|
||||
}
|
||||
}
|
||||
|
||||
// videoNotice tells the user a video track was present and what became of it. A
|
||||
// Teams meeting with the camera off is still a video file; it is accepted, only the
|
||||
// audio is transcribed, and the picture is thrown away — so say so, in every state,
|
||||
// instead of leaving the user to wonder (ADR-033).
|
||||
templ videoNotice(hadVideo bool) {
|
||||
if hadVideo {
|
||||
<p class="muted video-notice">
|
||||
This file contained a video track. Only its audio was used; the video was discarded and nothing from it was kept.
|
||||
</p>
|
||||
}
|
||||
}
|
||||
|
||||
// reconciliationNote says what was done when two engines transcribed the audio
|
||||
// (ADR-034), and never claims a second opinion that was not there: nil means a single
|
||||
// engine and renders nothing. Spots the model changed or was unsure about are listed
|
||||
// with a minute mark and both readings so the user can check them against the audio.
|
||||
templ reconciliationNote(r *domain.Reconciliation) {
|
||||
if r != nil {
|
||||
switch r.Status {
|
||||
case domain.ReconcileDone:
|
||||
<div class="reconciliation">
|
||||
<p>
|
||||
Reconciled from two transcripts ({ r.Base } and { r.Alt }):
|
||||
{ strconv.Itoa(r.Disputes) } differences,
|
||||
{ strconv.Itoa(r.AutoResolved) } settled automatically,
|
||||
{ strconv.Itoa(r.Adjudicated) } decided by the model ({ r.Reconciler }),
|
||||
{ strconv.Itoa(r.Unsure) } uncertain.
|
||||
</p>
|
||||
if len(shownSpots(r)) > 0 {
|
||||
<details>
|
||||
<summary>Spots to check against the audio</summary>
|
||||
<ul>
|
||||
for _, c := range shownSpots(r) {
|
||||
<li>
|
||||
{ minuteMark(c.Seconds) }:
|
||||
if c.Before != "" {
|
||||
<span class="muted">…{ c.Before }</span>
|
||||
}
|
||||
{ c.Base } / { c.Alt } —
|
||||
if c.Unsure {
|
||||
<span class="chip chip-warn">uncertain</span> kept { c.Used }
|
||||
} else {
|
||||
<span class="chip">changed</span> used { c.Used }
|
||||
}
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
if moreSpots(r) > 0 {
|
||||
<p class="muted">and { strconv.Itoa(moreSpots(r)) } more not listed</p>
|
||||
}
|
||||
</details>
|
||||
}
|
||||
</div>
|
||||
case domain.ReconcileSingle:
|
||||
<p class="muted reconciliation">Only one engine produced this transcript ({ r.Reason }), so there was no second opinion to reconcile it with.</p>
|
||||
case domain.ReconcileSkipped:
|
||||
<p class="muted reconciliation">Two transcripts were made but could not be reconciled ({ r.Reason }); the text shown is { r.Base }'s, unreconciled.</p>
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// progressSteps is the job's phases as a log that fills in as it runs: finished ones
|
||||
// collapse to a line with how long they took, the running one has a spinner and its
|
||||
// elapsed time, and what has not started is dimmed. live is false when the spinner would
|
||||
// be a lie (the service is lost, or the job is over), so the running phase shows a
|
||||
// warning mark instead. Only a counted phase gets a bar (ADR-035).
|
||||
templ progressSteps(steps []domain.ProgressStep, now time.Time, live bool) {
|
||||
<ol class="steps" aria-label="Progress">
|
||||
for _, s := range steps {
|
||||
<li
|
||||
class={ "step step-" + string(s.State) }
|
||||
if s.State == domain.StepRunning {
|
||||
aria-current="step"
|
||||
}
|
||||
>
|
||||
if s.State == domain.StepRunning && live {
|
||||
<span class="step-mark" aria-hidden="true"><span class="spinner"></span></span>
|
||||
} else if s.State == domain.StepRunning {
|
||||
<span class="step-mark step-mark-warn" aria-hidden="true">!</span>
|
||||
} else {
|
||||
<span class="step-mark" aria-hidden="true">{ stepMark(s) }</span>
|
||||
}
|
||||
<span class="step-label">{ s.Label }</span>
|
||||
if t := stepTime(s, now); t != "" {
|
||||
<span class="step-time">{ t }</span>
|
||||
}
|
||||
if s.Note != "" {
|
||||
<span class="step-note">{ s.Note }</span>
|
||||
}
|
||||
if s.State == domain.StepRunning && s.Total > 0 {
|
||||
<progress class="step-bar" value={ strconv.Itoa(s.Done) } max={ strconv.Itoa(s.Total) }></progress>
|
||||
<span class="step-count">{ strconv.Itoa(s.Done) } of { strconv.Itoa(s.Total) }</span>
|
||||
}
|
||||
</li>
|
||||
}
|
||||
</ol>
|
||||
}
|
||||
|
||||
// healthNote says so when the job is not simply running: the service stopped reporting,
|
||||
// or a counted phase has gone quiet. Running says nothing: the spinner does.
|
||||
templ healthNote(h jobHealth, silent time.Duration) {
|
||||
switch h {
|
||||
case healthLost:
|
||||
<p class="step-health step-health-lost" role="alert">
|
||||
The service stopped reporting { minutesAgo(silent) } ago. It may have restarted or crashed.
|
||||
This page keeps checking; if nothing changes, upload the recording again.
|
||||
</p>
|
||||
case healthQuiet:
|
||||
<p class="step-health step-health-quiet">
|
||||
Nothing new for { minutesAgo(silent) }: the model may be slow. The service is still running,
|
||||
and the job gives up on its own if it takes too long.
|
||||
</p>
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Input that cannot be right is the caller's to fix: a 4xx, never a 500 (adversarial
|
||||
// review F7). No cross-tenant oracle either way: unknown and foreign ids look the same.
|
||||
|
||||
func TestUploadRoutes_ANonUUIDIdIs404NotA500(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
for _, path := range []string{"/uploads/not-a-uuid", "/uploads/not-a-uuid/status", "/uploads/not-a-uuid/chat", "/uploads/%00/status", "/uploads/1/status"} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
rec := do(t, app, get(path))
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, rec.Code)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPost_ATitleWithControlCharactersOrInvalidUTF8IsCleanedNotA500(t *testing.T) {
|
||||
tests := []struct{ name, title, filename string }{
|
||||
{"a NUL in the title", "Sprint\x00review", "a.m4a"},
|
||||
{"invalid UTF-8 in the title", "Sprint \xff\xfe review", "a.m4a"},
|
||||
{"a newline in the title", "Sprint\nreview", "a.m4a"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
app, sub := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
rec := do(t, app, uploadReq(t, tt.title, tt.filename, []byte("synthetic audio bytes")))
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, rec.Code, "bad bytes in a cosmetic title must not fail the upload: %s", body(t, rec))
|
||||
require.Len(t, sub.jobs, 1)
|
||||
var title string
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(), `SELECT title FROM uploaded_transcripts WHERE id = $1`, sub.jobs[0].UploadID).Scan(&title))
|
||||
assert.NotContains(t, title, "\x00")
|
||||
assert.NotContains(t, title, "\n")
|
||||
assert.True(t, len(title) > 0, "a title is still shown")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPost_AMalformedMultipartHeaderIsA400AndItsTextIsNotLogged(t *testing.T) {
|
||||
// A control byte in the part's file name makes the multipart body malformed. It is the
|
||||
// sender's to fix, and the parser's error quotes the header, which carries the name.
|
||||
app, sub := newUploadApp(t, audioOK, 200<<20)
|
||||
buf := &lockedBuf{}
|
||||
app.Log = slog.New(slog.NewTextHandler(buf, nil))
|
||||
|
||||
rec := do(t, app, uploadReq(t, "", "zebrakaka\x00weekly.m4a", []byte("x")))
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code)
|
||||
assert.Empty(t, sub.jobs)
|
||||
assert.NotContains(t, buf.String(), "zebrakaka", "a file name is private content")
|
||||
}
|
||||
|
||||
func TestUploadPost_EveryBidirectionalControlCharacterIsStrippedFromATitle(t *testing.T) {
|
||||
// U+202E reverses the text that follows it, so a title could render reversed on its
|
||||
// owner's own page (second adversarial review). Every one of the twelve is pinned.
|
||||
controls := []rune{0x202A, 0x202B, 0x202C, 0x202D, 0x202E, 0x2066, 0x2067, 0x2068, 0x2069, 0x200E, 0x200F, 0x061C}
|
||||
for _, c := range controls {
|
||||
t.Run(fmt.Sprintf("U+%04X", c), func(t *testing.T) {
|
||||
app, sub := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
rec := do(t, app, uploadReq(t, "Sprint"+string(c)+"review", "a.m4a", []byte("x")))
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, rec.Code)
|
||||
var title string
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(), `SELECT title FROM uploaded_transcripts WHERE id = $1`, sub.jobs[0].UploadID).Scan(&title))
|
||||
assert.Equal(t, "Sprintreview", title, "dropped, not replaced by a space: a space would leave a visible gap")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPost_SwedishLettersAndEmojiSurviveTheTitleCleaning(t *testing.T) {
|
||||
app, sub := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
// Includes a ZWJ sequence (woman + ZWJ + laptop) and a ZWNJ: joiners are not direction controls.
|
||||
rec := do(t, app, uploadReq(t, "Möte åäö 🎙️ 👩\u200d💻 a\u200cb", "a.m4a", []byte("x")))
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, rec.Code)
|
||||
var title string
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(), `SELECT title FROM uploaded_transcripts WHERE id = $1`, sub.jobs[0].UploadID).Scan(&title))
|
||||
assert.Equal(t, "Möte åäö 🎙️ 👩\u200d💻 a\u200cb", title, "joiners and variation selectors belong to the text")
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
func TestUploadPost_ABusyRejectionIs429WithItsMessage(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
app.Uploads = failingUploader{err: &uploads.RejectError{Reason: uploads.ReasonBusy, Message: "Too many recordings are waiting. Try again in a few minutes."}}
|
||||
|
||||
rec := do(t, app, uploadReq(t, "t", "a.m4a", []byte("x")))
|
||||
|
||||
require.Equal(t, http.StatusTooManyRequests, rec.Code, "the user's to retry: not a 500, not a 422")
|
||||
assert.Contains(t, body(t, rec), "Too many recordings are waiting")
|
||||
}
|
||||
|
||||
func TestUploadStatus_ABusyFailureIsExplainedInWords(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "failed", "queue_full")
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "waiting")
|
||||
assert.NotContains(t, b, "This upload failed. Please upload it again.")
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/chat"
|
||||
"git.d-ma.be/mathias/tapir/internal/web"
|
||||
)
|
||||
|
||||
// Chat over an uploaded transcript REUSES the ADR-027 chat path (the same
|
||||
// Chatter, the same panel) — only the lookup differs: the row comes from the
|
||||
// user-scoped private store, so another user's upload is a 404 and, crucially, no
|
||||
// model is ever called with it. Transcript text is synthetic; this repo is public.
|
||||
|
||||
const meetingText = "Anders ansvarar för virket. Maria undersöker leverantörer."
|
||||
|
||||
func newUploadChatApp(t *testing.T) (*web.App, *fakeChatter) {
|
||||
t.Helper()
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fc := &fakeChatter{
|
||||
models: []string{"koala/phi4-mini", "iguana/gemma4-26b"},
|
||||
reply: chat.Reply{Answer: "Anders owns the timber order."},
|
||||
}
|
||||
app.Chat = fc
|
||||
app.UploadChatMaxChars = 120000
|
||||
app.UploadChatModels = fc.models
|
||||
return app, fc
|
||||
}
|
||||
|
||||
func chatPost(path string, kv url.Values) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(kv.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return req
|
||||
}
|
||||
|
||||
func TestUploadChatPage_OwnerOfAFinishedUploadSeesThePanelPostingToTheUploadRoute(t *testing.T) {
|
||||
app, _ := newUploadChatApp(t)
|
||||
id := seedUpload(t, userID, "Roadmap sync", "done", meetingText)
|
||||
|
||||
rec := do(t, app, get("/uploads/"+id+"/chat"))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
b := body(t, rec)
|
||||
assert.Contains(t, b, `action="/uploads/`+id+`/chat"`, "the form posts to the UPLOAD route, not /v/{id}/chat")
|
||||
assert.Contains(t, b, "Roadmap sync")
|
||||
assert.Contains(t, b, "koala/phi4-mini")
|
||||
assert.Contains(t, b, "recording", "wording says recording, not video")
|
||||
}
|
||||
|
||||
func TestUploadChatPage_AnUnfinishedUploadRedirectsToItsStatusPage(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
|
||||
rec := do(t, app, get("/uploads/"+id+"/chat"))
|
||||
|
||||
assert.Equal(t, http.StatusSeeOther, rec.Code)
|
||||
assert.Equal(t, "/uploads/"+id, rec.Header().Get("Location"))
|
||||
assert.Zero(t, fc.callCount)
|
||||
}
|
||||
|
||||
func TestUploadChat_AnotherUsersUploadIs404AndNoModelIsCalled(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
theirs := seedUpload(t, otherUserID, "Their private meeting", "done", meetingText)
|
||||
|
||||
getRec := do(t, app, get("/uploads/"+theirs+"/chat"))
|
||||
postRec := do(t, app, chatPost("/uploads/"+theirs+"/chat", url.Values{"question": {"what did they say?"}}))
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, getRec.Code)
|
||||
assert.Equal(t, http.StatusNotFound, postRec.Code)
|
||||
assert.NotContains(t, body(t, getRec), "Their private meeting")
|
||||
assert.Zero(t, fc.callCount, "someone else's transcript must never reach a model")
|
||||
}
|
||||
|
||||
func TestUploadChatPost_SendsTheOwnersTranscriptWithTheUploadBudget(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
app.UploadChatMaxChars = 77777
|
||||
id := seedUpload(t, userID, "Roadmap sync", "done", meetingText)
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{
|
||||
"question": {"Who owns the timber order?"},
|
||||
"model": {"iguana/gemma4-26b"},
|
||||
}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Equal(t, 1, fc.callCount)
|
||||
assert.Equal(t, meetingText, fc.gotReq.Transcript)
|
||||
assert.Equal(t, "Who owns the timber order?", fc.gotReq.Question)
|
||||
assert.Equal(t, "iguana/gemma4-26b", fc.gotReq.Model)
|
||||
assert.Equal(t, 77777, fc.gotReq.MaxChars,
|
||||
"an uploaded transcript takes its own budget: the 18,000-char caption default would cut an hour to its first third")
|
||||
assert.Contains(t, body(t, rec), "Anders owns the timber order.")
|
||||
}
|
||||
|
||||
func TestUploadChatPost_AnUnofferedModelFallsBackToTheDefault(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
id := seedUpload(t, userID, "m", "done", meetingText)
|
||||
|
||||
do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"q"}, "model": {"evil/model"}}))
|
||||
|
||||
assert.Equal(t, "koala/phi4-mini", fc.gotReq.Model)
|
||||
}
|
||||
|
||||
func TestUploadChatPost_UnfinishedUploadCallsNoModel(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
id := seedUpload(t, userID, "m", "queued", "")
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"q"}}))
|
||||
|
||||
assert.Equal(t, http.StatusSeeOther, rec.Code)
|
||||
assert.Zero(t, fc.callCount)
|
||||
}
|
||||
|
||||
func TestUploadChatPost_PriorTurnsRideAlongAndATruncatedReplyIsSaidSo(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
fc.reply = chat.Reply{Answer: "second answer", Truncated: true}
|
||||
id := seedUpload(t, userID, "m", "done", meetingText)
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{
|
||||
"question": {"and who else?"},
|
||||
"hq": {"first question"},
|
||||
"ha": {"first answer"},
|
||||
}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Equal(t, []chat.Turn{{Question: "first question", Answer: "first answer"}}, fc.gotReq.History)
|
||||
b := body(t, rec)
|
||||
assert.Contains(t, b, "first answer")
|
||||
assert.Contains(t, b, "second answer")
|
||||
assert.Contains(t, b, "end of the recording", "a long recording cut to a budget must say so, in its own words")
|
||||
}
|
||||
|
||||
func TestUploadChatPost_AModelFailureIsShownInlineNotAs500(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
fc.err = errors.New("gateway down")
|
||||
id := seedUpload(t, userID, "m", "done", meetingText)
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"q"}, "hq": {"earlier"}, "ha": {"kept"}}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
b := body(t, rec)
|
||||
assert.Contains(t, b, "answer just now", "the failure is inline (the apostrophe is HTML-escaped, so match the tail)")
|
||||
assert.Contains(t, b, "kept", "the conversation survives a failed turn")
|
||||
}
|
||||
|
||||
func TestUploadChatRoutes_AreNotMountedWithoutAChatBackend(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", meetingText)
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, do(t, app, get("/uploads/"+id+"/chat")).Code)
|
||||
}
|
||||
|
||||
func TestUploadStatus_DoneOffersTheChatLinkOnlyWhenChatIsWired(t *testing.T) {
|
||||
appWith, _ := newUploadChatApp(t)
|
||||
id := seedUpload(t, userID, "m", "done", meetingText)
|
||||
assert.Contains(t, body(t, do(t, appWith, get("/uploads/"+id+"/status"))), `href="/uploads/`+id+`/chat"`)
|
||||
|
||||
appWithout, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id2 := seedUpload(t, userID, "m2", "done", meetingText)
|
||||
assert.NotContains(t, body(t, do(t, appWithout, get("/uploads/"+id2+"/status"))), "/chat")
|
||||
}
|
||||
|
||||
func TestUploadChat_OpensWithTheUploadChatModelNotTheSummarizerDefault(t *testing.T) {
|
||||
// koala/phi4-mini is first in the offered list (it is the summarizer default) and
|
||||
// has an 8,192-token context: an hour of speech returns HTTP 400 there. Chat over
|
||||
// an upload must open with the model configured for it.
|
||||
app, fc := newUploadChatApp(t)
|
||||
app.UploadChatModel = "iguana/gemma4-26b"
|
||||
id := seedUpload(t, userID, "m", "done", meetingText)
|
||||
|
||||
page := body(t, do(t, app, get("/uploads/"+id+"/chat")))
|
||||
do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"q"}}))
|
||||
|
||||
assert.Contains(t, page, `<option value="iguana/gemma4-26b" selected`, "the page opens on the upload chat model")
|
||||
assert.Equal(t, "iguana/gemma4-26b", fc.gotReq.Model, "a post that names no model uses it too")
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/adapters/chat"
|
||||
"git.d-ma.be/mathias/tapir/internal/web"
|
||||
)
|
||||
|
||||
// The whole private transcript goes to the model chosen in the form, so the upload chat
|
||||
// offers and accepts local models only, even when the shared chat service knows a cloud
|
||||
// one (ADR-035 review). The cloud model here is only a name; no network is involved.
|
||||
|
||||
func newUploadChatAppWithCloud(t *testing.T) (*web.App, *fakeChatter) {
|
||||
t.Helper()
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fc := &fakeChatter{
|
||||
models: []string{"koala/phi4-mini", "iguana/gemma4-26b", "berget/mistral-small"},
|
||||
reply: chat.Reply{Answer: "Anders owns the timber order."},
|
||||
}
|
||||
app.Chat = fc
|
||||
app.UploadChatMaxChars = 120000
|
||||
app.UploadChatModels = []string{"koala/phi4-mini", "iguana/gemma4-26b"}
|
||||
return app, fc
|
||||
}
|
||||
|
||||
func TestUploadChatPage_DoesNotOfferACloudModelEvenThoughTheServiceKnowsOne(t *testing.T) {
|
||||
app, _ := newUploadChatAppWithCloud(t)
|
||||
id := seedUpload(t, userID, "Roadmap sync", "done", meetingText)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/chat")))
|
||||
|
||||
assert.Contains(t, b, "iguana/gemma4-26b")
|
||||
assert.NotContains(t, b, "berget/mistral-small", "the switcher must not list a model the transcript must not be sent to")
|
||||
}
|
||||
|
||||
func TestUploadChatPost_AForgedCloudModelIsReplacedByALocalOneBeforeTheTranscriptIsSent(t *testing.T) {
|
||||
app, fc := newUploadChatAppWithCloud(t)
|
||||
id := seedUpload(t, userID, "Roadmap sync", "done", meetingText)
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"Who owns timber?"}, "model": {"berget/mistral-small"}}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
require.Equal(t, 1, fc.callCount)
|
||||
assert.NotEqual(t, "berget/mistral-small", fc.gotReq.Model, "a cloud model reached the chat service with the private transcript")
|
||||
assert.Contains(t, []string{"koala/phi4-mini", "iguana/gemma4-26b"}, fc.gotReq.Model)
|
||||
}
|
||||
|
||||
func TestUploadChatPost_ALocalModelThatIsOfferedIsHonoured(t *testing.T) {
|
||||
app, fc := newUploadChatAppWithCloud(t)
|
||||
id := seedUpload(t, userID, "Roadmap sync", "done", meetingText)
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"q"}, "model": {"koala/phi4-mini"}}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Equal(t, "koala/phi4-mini", fc.gotReq.Model)
|
||||
}
|
||||
|
||||
func TestUploadChat_WithNoLocalModelConfiguredNothingIsSentAnywhere(t *testing.T) {
|
||||
app, fc := newUploadChatAppWithCloud(t)
|
||||
app.UploadChatModels = nil
|
||||
id := seedUpload(t, userID, "Roadmap sync", "done", meetingText)
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"q"}, "model": {"berget/mistral-small"}}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Zero(t, fc.callCount, "failing closed: no local model offered means no model is called")
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A client that hangs up in the middle of an upload is routine, not a server fault. The
|
||||
// in-process recorder cannot see it (its context never cancels), so this talks to a real
|
||||
// listener (third adversarial review: over a socket every such case logged an ERROR and
|
||||
// half-close returned a 500, because the error page was rendered on a cancelled context).
|
||||
|
||||
// body positions at which the client can hang up: inside the title field, inside the file
|
||||
// part's own headers, and mid-file. Every one must be handled alike (fourth adversarial
|
||||
// review: the title branch had no guard and still logged an ERROR).
|
||||
const (
|
||||
inTitle = "in the title"
|
||||
inHeaders = "in the file part's headers"
|
||||
midFile = "mid-file"
|
||||
)
|
||||
|
||||
func partialUpload(t *testing.T, addr, where string, hangUp func(*net.TCPConn)) {
|
||||
t.Helper()
|
||||
c, err := net.Dial("tcp", addr)
|
||||
require.NoError(t, err)
|
||||
tc := c.(*net.TCPConn)
|
||||
const boundary = "xBOUNDARYx"
|
||||
var body string
|
||||
switch where {
|
||||
case inTitle:
|
||||
body = "--" + boundary + "\r\nContent-Disposition: form-data; name=\"title\"\r\n\r\nhalf a tit"
|
||||
case inHeaders:
|
||||
body = "--" + boundary + "\r\nContent-Disposition: form-data; name=\"file\"; filena"
|
||||
default:
|
||||
body = "--" + boundary + "\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a.m4a\"\r\nContent-Type: audio/mp4\r\n\r\n" + strings.Repeat("a", 4000)
|
||||
}
|
||||
_, err = fmt.Fprintf(tc, "POST /uploads HTTP/1.1\r\nHost: x\r\nContent-Type: multipart/form-data; boundary=%s\r\nContent-Length: 200000\r\n\r\n%s", boundary, body)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond) // let the server start reading
|
||||
hangUp(tc)
|
||||
}
|
||||
|
||||
func TestUploadPost_AClientThatHangsUpAnywhereInTheBodyIsLoggedAtInfoAndNeverAsAnError(t *testing.T) {
|
||||
hangUps := map[string]func(*net.TCPConn){
|
||||
"a half close": func(c *net.TCPConn) { _ = c.CloseWrite(); time.Sleep(100 * time.Millisecond); _ = c.Close() },
|
||||
"a full close": func(c *net.TCPConn) { _ = c.Close() },
|
||||
"a reset (RST)": func(c *net.TCPConn) { _ = c.SetLinger(0); _ = c.Close() },
|
||||
}
|
||||
for _, where := range []string{inTitle, inHeaders, midFile} {
|
||||
for name, hangUp := range hangUps {
|
||||
t.Run(where+", "+name, func(t *testing.T) {
|
||||
app, sub := newUploadApp(t, audioOK, 200<<20)
|
||||
buf := &lockedBuf{}
|
||||
app.Log = slog.New(slog.NewTextHandler(buf, nil))
|
||||
srv := httptest.NewServer(app.Router())
|
||||
defer srv.Close()
|
||||
|
||||
partialUpload(t, strings.TrimPrefix(srv.URL, "http://"), where, hangUp)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
out := buf.String()
|
||||
return strings.Contains(out, "upload abandoned") || strings.Contains(out, "upload rejected") || strings.Contains(out, "level=ERROR")
|
||||
}, 3*time.Second, 20*time.Millisecond, "the server never finished handling the aborted upload")
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
out := buf.String()
|
||||
assert.NotContains(t, out, "level=ERROR", "a client hanging up is not a server fault:\n%s", out)
|
||||
assert.Empty(t, sub.jobs)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// goneMidAccept is an Uploader during whose Accept the client disappears: the request's
|
||||
// context is cancelled, then Accept fails the way a read from a dead connection does.
|
||||
type goneMidAccept struct{ cancel context.CancelFunc }
|
||||
|
||||
func (g goneMidAccept) Accept(context.Context, string, string, io.Reader) (string, error) {
|
||||
g.cancel()
|
||||
return "", fmt.Errorf("receive: %w", io.ErrClosedPipe)
|
||||
}
|
||||
|
||||
func TestUploadPost_AnAbandonedRequestGetsA499NotAnImplicitEmpty200(t *testing.T) {
|
||||
// If the context is cancelled but the connection can still be written to (a client that
|
||||
// half-closed after sending), a missing status becomes 200 with an empty body: a false
|
||||
// success (fourth adversarial review).
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
buf := &lockedBuf{}
|
||||
app.Log = slog.New(slog.NewTextHandler(buf, nil))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
app.Uploads = goneMidAccept{cancel: cancel}
|
||||
|
||||
rec := do(t, app, uploadReq(t, "t", "a.m4a", []byte("x")).WithContext(ctx))
|
||||
|
||||
assert.Equal(t, 499, rec.Code, "never an implicit 200 for an upload that was not accepted")
|
||||
assert.Contains(t, buf.String(), "upload abandoned")
|
||||
assert.NotContains(t, buf.String(), "level=ERROR")
|
||||
}
|
||||
|
||||
func TestUploadPost_ADatabaseDropThatLooksLikeAnEOFIsStillAServerError(t *testing.T) {
|
||||
// pgx turns a connection that drops mid-message into io.ErrUnexpectedEOF. Mapping that
|
||||
// sentinel to "your upload was cut off" (a 400) would hide a real fault: only an error
|
||||
// from reading the CLIENT's body is theirs (third adversarial review).
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
buf := &lockedBuf{}
|
||||
app.Log = slog.New(slog.NewTextHandler(buf, nil))
|
||||
app.Uploads = failingUploader{err: fmt.Errorf("failed to receive message: %w", io.ErrUnexpectedEOF)}
|
||||
|
||||
rec := do(t, app, uploadReq(t, "t", "a.m4a", []byte("x")))
|
||||
|
||||
assert.Equal(t, http.StatusInternalServerError, rec.Code)
|
||||
assert.Contains(t, buf.String(), "level=ERROR")
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
"git.d-ma.be/mathias/tapir/internal/web"
|
||||
)
|
||||
|
||||
// Upload route tests drive the real Router, the real Intake and the real Postgres
|
||||
// store; only the process boundary (ffprobe) and the transcription worker are
|
||||
// faked. All audio bytes and transcript text are synthetic — this repo is public
|
||||
// and no real recording or transcript may appear in it (#33).
|
||||
|
||||
const otherUserID = "22222222-2222-2222-2222-222222222222"
|
||||
|
||||
type uploadProber struct{ probe uploads.Probe }
|
||||
|
||||
func (p uploadProber) Probe(context.Context, string) (uploads.Probe, error) { return p.probe, nil }
|
||||
|
||||
type uploadSubmitter struct{ jobs []uploads.Job }
|
||||
|
||||
func (s *uploadSubmitter) Submit(j uploads.Job) error { s.jobs = append(s.jobs, j); return nil }
|
||||
|
||||
func newUploadApp(t *testing.T, probe uploads.Probe, maxBytes int64) (*web.App, *uploadSubmitter) {
|
||||
t.Helper()
|
||||
app := newApp(t)
|
||||
resetDB(t, rawPool(t))
|
||||
sub := &uploadSubmitter{}
|
||||
limits := uploads.Limits{MaxBytes: maxBytes, MaxDuration: 90 * time.Minute}
|
||||
s := newStore(t)
|
||||
app.Uploads = uploads.NewIntake(s, uploadProber{probe: probe}, sub, limits, t.TempDir())
|
||||
app.UploadLimits = limits
|
||||
return app, sub
|
||||
}
|
||||
|
||||
var audioOK = uploads.Probe{Duration: 45 * time.Minute, HasAudio: true}
|
||||
|
||||
func uploadReq(t *testing.T, title, filename string, content []byte) *http.Request {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
if title != "" {
|
||||
require.NoError(t, mw.WriteField("title", title))
|
||||
}
|
||||
if filename != "" {
|
||||
fw, err := mw.CreateFormFile("file", filename)
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write(content)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, mw.Close())
|
||||
req := httptest.NewRequest(http.MethodPost, "/uploads", &buf)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
return req
|
||||
}
|
||||
|
||||
func seedUpload(t *testing.T, owner, title, status, content string) string {
|
||||
t.Helper()
|
||||
p := rawPool(t)
|
||||
_, err := p.Exec(context.Background(), `INSERT INTO users (id) VALUES ($1) ON CONFLICT DO NOTHING`, owner)
|
||||
require.NoError(t, err)
|
||||
var id string
|
||||
switch status {
|
||||
case "done":
|
||||
require.NoError(t, p.QueryRow(context.Background(),
|
||||
`INSERT INTO uploaded_transcripts (user_id, title, status, content, endpoint, audio_seconds, finished_at)
|
||||
VALUES ($1, $2, 'done', $3, 'berget/kb-whisper', 2700, NOW()) RETURNING id`,
|
||||
owner, title, content).Scan(&id))
|
||||
case "failed":
|
||||
require.NoError(t, p.QueryRow(context.Background(),
|
||||
`INSERT INTO uploaded_transcripts (user_id, title, status, error_kind, finished_at)
|
||||
VALUES ($1, $2, 'failed', $3, NOW()) RETURNING id`,
|
||||
owner, title, content).Scan(&id)) // content carries the error_kind here
|
||||
default:
|
||||
require.NoError(t, p.QueryRow(context.Background(),
|
||||
`INSERT INTO uploaded_transcripts (user_id, title, status) VALUES ($1, $2, $3) RETURNING id`,
|
||||
owner, title, status).Scan(&id))
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func get(path string) *http.Request { return httptest.NewRequest(http.MethodGet, path, nil) }
|
||||
|
||||
func TestUploadsPage_ShowsAMultipartFormWithAFileField(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
rec := do(t, app, get("/uploads"))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
b := body(t, rec)
|
||||
assert.Contains(t, b, `enctype="multipart/form-data"`)
|
||||
assert.Contains(t, b, `name="file"`, "the multipart field is `file`")
|
||||
assert.Contains(t, b, "200 MB", "the size limit is stated up front, not discovered by failing")
|
||||
assert.Contains(t, b, "1 h 30 min", "the length limit is stated up front")
|
||||
}
|
||||
|
||||
func TestUploadsPage_ListsOnlyTheCallersUploads(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
seedUpload(t, userID, "My weekly sync", "queued", "")
|
||||
seedUpload(t, otherUserID, "Someone else's board meeting", "queued", "")
|
||||
|
||||
b := body(t, do(t, app, get("/uploads")))
|
||||
|
||||
assert.Contains(t, b, "My weekly sync")
|
||||
assert.NotContains(t, b, "Someone else's board meeting")
|
||||
}
|
||||
|
||||
func TestUploadPost_AcceptedAudioQueuesARowAndRedirectsToIt(t *testing.T) {
|
||||
app, sub := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
rec := do(t, app, uploadReq(t, "Sprint review", "recording.m4a", []byte("synthetic audio bytes")))
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, rec.Code)
|
||||
var id, status, title string
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT id, status, title FROM uploaded_transcripts WHERE user_id = $1`, userID).Scan(&id, &status, &title))
|
||||
assert.Equal(t, "queued", status)
|
||||
assert.Equal(t, "Sprint review", title)
|
||||
assert.Equal(t, "/uploads/"+id, rec.Header().Get("Location"))
|
||||
require.Len(t, sub.jobs, 1)
|
||||
assert.Equal(t, uploads.Job{UserID: userID, UploadID: id, Path: sub.jobs[0].Path, Duration: audioOK.Duration}, sub.jobs[0])
|
||||
spooled, err := os.ReadFile(sub.jobs[0].Path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "synthetic audio bytes", string(spooled))
|
||||
}
|
||||
|
||||
func TestUploadPost_WithoutATitleUsesTheFilenameMinusItsExtension(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
rec := do(t, app, uploadReq(t, "", "Teams meeting 2026-09-30.mp4", []byte("x")))
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, rec.Code)
|
||||
var title string
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT title FROM uploaded_transcripts WHERE user_id = $1`, userID).Scan(&title))
|
||||
assert.Equal(t, "Teams meeting 2026-09-30", title)
|
||||
}
|
||||
|
||||
func TestUploadPost_RejectionsAreSynchronousWithAMessageAndNoRow(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
probe uploads.Probe
|
||||
maxBytes int64
|
||||
content string
|
||||
wantStatus int
|
||||
wantInBody []string
|
||||
}{
|
||||
{
|
||||
name: "video with no audio is rejected for having no audio",
|
||||
probe: uploads.Probe{Duration: time.Minute, HasVideo: true},
|
||||
maxBytes: 1 << 20,
|
||||
content: "x",
|
||||
wantStatus: http.StatusUnprocessableEntity,
|
||||
wantInBody: []string{"no audio"},
|
||||
},
|
||||
{
|
||||
name: "too long states both lengths",
|
||||
probe: uploads.Probe{Duration: 2 * time.Hour, HasAudio: true},
|
||||
maxBytes: 1 << 20,
|
||||
content: "x",
|
||||
wantStatus: http.StatusUnprocessableEntity,
|
||||
wantInBody: []string{"2 h", "1 h 30 min"},
|
||||
},
|
||||
{
|
||||
name: "too large is 413",
|
||||
probe: audioOK,
|
||||
maxBytes: 4,
|
||||
content: "more than four bytes",
|
||||
wantStatus: http.StatusRequestEntityTooLarge,
|
||||
wantInBody: []string{"limit"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
app, sub := newUploadApp(t, tt.probe, tt.maxBytes)
|
||||
|
||||
rec := do(t, app, uploadReq(t, "t", "r.m4a", []byte(tt.content)))
|
||||
|
||||
assert.Equal(t, tt.wantStatus, rec.Code)
|
||||
b := body(t, rec)
|
||||
for _, s := range tt.wantInBody {
|
||||
assert.Contains(t, b, s)
|
||||
}
|
||||
var n int
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT count(*) FROM uploaded_transcripts`).Scan(&n))
|
||||
assert.Zero(t, n, "a rejected upload leaves no row")
|
||||
assert.Empty(t, sub.jobs)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPost_WithNoFilePartIs400(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
rec := do(t, app, uploadReq(t, "only a title", "", nil))
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code)
|
||||
assert.Contains(t, body(t, rec), "Choose an audio file")
|
||||
}
|
||||
|
||||
func TestUploadRoutes_AreNotMountedWithoutAnUploader(t *testing.T) {
|
||||
app := newApp(t)
|
||||
resetDB(t, rawPool(t))
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, do(t, app, get("/uploads")).Code)
|
||||
}
|
||||
|
||||
func TestUploadPage_OwnerSeesTitleStatusAndNonOwnerGetsNotFound(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
mine := seedUpload(t, userID, "Roadmap sync", "queued", "")
|
||||
theirs := seedUpload(t, otherUserID, "Their private meeting", "queued", "")
|
||||
|
||||
own := do(t, app, get("/uploads/"+mine))
|
||||
require.Equal(t, http.StatusOK, own.Code)
|
||||
assert.Contains(t, body(t, own), "Roadmap sync")
|
||||
|
||||
for _, path := range []string{"/uploads/" + theirs, "/uploads/" + theirs + "/status"} {
|
||||
rec := do(t, app, get(path))
|
||||
assert.Equal(t, http.StatusNotFound, rec.Code, path)
|
||||
assert.NotContains(t, body(t, rec), "Their private meeting", "a 404 must not leak the title")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadStatus_UnfinishedStatesKeepPollingTerminalStatesStop(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
tests := []struct {
|
||||
name, status, content string
|
||||
polls bool
|
||||
wantInBody string
|
||||
}{
|
||||
{"queued", "queued", "", true, "Queued"},
|
||||
{"transcribing", "transcribing", "", true, "ranscrib"},
|
||||
{"done", "done", "Synthetic transcript text.", false, "Synthetic transcript text."},
|
||||
{"failed interrupted", "failed", "interrupted", false, "upload it again"},
|
||||
{"failed transcription", "failed", "transcription_failed", false, "Transcription failed"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
id := seedUpload(t, userID, "m-"+tt.name, tt.status, tt.content)
|
||||
|
||||
rec := do(t, app, get("/uploads/"+id+"/status"))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
b := body(t, rec)
|
||||
assert.Contains(t, b, tt.wantInBody)
|
||||
if tt.polls {
|
||||
assert.Contains(t, b, `hx-trigger="every`, "an unfinished upload must keep polling")
|
||||
} else {
|
||||
assert.NotContains(t, b, "hx-trigger", "a finished upload must not poll forever: no stuck spinner")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadStatus_DoneNamesTheEndpointThatServedIt(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript text.")
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "berget/kb-whisper",
|
||||
"which backend served a transcript is a quality fact, and berget means the audio left the homelab")
|
||||
assert.True(t, strings.Contains(b, "45 min"), "the audio length is shown")
|
||||
}
|
||||
|
||||
func markHadVideo(t *testing.T, id string) {
|
||||
t.Helper()
|
||||
_, err := rawPool(t).Exec(context.Background(), `UPDATE uploaded_transcripts SET had_video = TRUE WHERE id = $1`, id)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestUploadPost_AFileWithAVideoTrackIsAcceptedAndTheRowRemembersIt(t *testing.T) {
|
||||
// A Teams meeting with the camera off is still an .mp4 with a video track.
|
||||
app, sub := newUploadApp(t, uploads.Probe{Duration: 62 * time.Minute, HasAudio: true, HasVideo: true}, 200<<20)
|
||||
|
||||
rec := do(t, app, uploadReq(t, "Project notes", "Teams meeting.mp4", []byte("synthetic bytes")))
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, rec.Code, "it must be accepted, not rejected as video")
|
||||
var hadVideo bool
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT had_video FROM uploaded_transcripts WHERE user_id = $1`, userID).Scan(&hadVideo))
|
||||
assert.True(t, hadVideo)
|
||||
assert.Len(t, sub.jobs, 1)
|
||||
}
|
||||
|
||||
func TestUploadPost_AudioOnlyIsNotMarkedAsHavingHadVideo(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
|
||||
do(t, app, uploadReq(t, "Voice memo", "memo.m4a", []byte("x")))
|
||||
|
||||
var hadVideo bool
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT had_video FROM uploaded_transcripts WHERE user_id = $1`, userID).Scan(&hadVideo))
|
||||
assert.False(t, hadVideo)
|
||||
}
|
||||
|
||||
func TestUploadStatus_TellsTheUserTheVideoWasIgnoredInEveryStateAndOnlyWhenItWas(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
states := []struct{ status, content string }{
|
||||
{"queued", ""}, {"transcribing", ""}, {"done", "Synthetic transcript."}, {"failed", "interrupted"},
|
||||
}
|
||||
for _, st := range states {
|
||||
t.Run(st.status, func(t *testing.T) {
|
||||
withVideo := seedUpload(t, userID, "camera off "+st.status, st.status, st.content)
|
||||
markHadVideo(t, withVideo)
|
||||
audioOnly := seedUpload(t, userID, "voice only "+st.status, st.status, st.content)
|
||||
|
||||
notice := body(t, do(t, app, get("/uploads/"+withVideo+"/status")))
|
||||
plain := body(t, do(t, app, get("/uploads/"+audioOnly+"/status")))
|
||||
|
||||
assert.Contains(t, notice, "video track")
|
||||
assert.Contains(t, notice, "Only its audio was used", "say what happened, not just that something did")
|
||||
assert.Contains(t, notice, "discarded")
|
||||
assert.NotContains(t, plain, "video track", "no notice when there was no video")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPage_AlsoCarriesTheVideoNotice(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "camera off", "done", "Synthetic transcript.")
|
||||
markHadVideo(t, id)
|
||||
|
||||
assert.Contains(t, body(t, do(t, app, get("/uploads/"+id))), "Only its audio was used")
|
||||
}
|
||||
|
||||
func TestUploadPost_ARejectionIsLoggedWithItsReasonAndNeverTheFilenameOrTitle(t *testing.T) {
|
||||
// Rejections are synchronous and leave no row, so without a log line an operator
|
||||
// cannot tell "the user never retried" from "the page refused it". The reason is
|
||||
// operational; the filename and title are the user's private content.
|
||||
var buf bytes.Buffer
|
||||
app, _ := newUploadApp(t, uploads.Probe{Duration: time.Minute, HasVideo: true}, 1<<20)
|
||||
app.Log = slog.New(slog.NewTextHandler(&buf, nil))
|
||||
|
||||
rec := do(t, app, uploadReq(t, "Secret project notes", "client-acme-roadmap.mp4", []byte("x")))
|
||||
|
||||
require.Equal(t, http.StatusUnprocessableEntity, rec.Code)
|
||||
logged := buf.String()
|
||||
assert.Contains(t, logged, "upload rejected")
|
||||
assert.Contains(t, logged, "reason=no_audio")
|
||||
assert.NotContains(t, logged, "Secret project notes", "the title is private content")
|
||||
assert.NotContains(t, logged, "client-acme-roadmap", "the filename is private content")
|
||||
}
|
||||
|
||||
func setReconciliation(t *testing.T, id, json string) {
|
||||
t.Helper()
|
||||
_, err := rawPool(t).Exec(context.Background(), `UPDATE uploaded_transcripts SET reconciliation = $2::jsonb WHERE id = $1`, id, json)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
const reconciledJSON = `{"status":"reconciled","base":"iguana/pianissimo","alt":"iguana/kb-whisper","reconciler":"iguana/gemma4-26b",
|
||||
"disputes":380,"auto_resolved":90,"adjudicated":285,"changed":120,"unsure":5,"errors":0,
|
||||
"contested":[
|
||||
{"seconds":2210.5,"base":"Axel","alt":"Alex","used":"alt","unsure":false},
|
||||
{"seconds":3400,"base":"A+2","alt":"en A","used":"base","unsure":true}]}`
|
||||
|
||||
func TestUploadStatus_ReconciledSaysWhatWasDoneInPlainWords(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, reconciledJSON)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "Reconciled from two transcripts")
|
||||
assert.Contains(t, b, "iguana/pianissimo")
|
||||
assert.Contains(t, b, "iguana/kb-whisper")
|
||||
assert.Contains(t, b, "iguana/gemma4-26b", "who decided is part of the account")
|
||||
assert.Contains(t, b, "380 differences")
|
||||
assert.Contains(t, b, "90 settled automatically")
|
||||
assert.Contains(t, b, "285 decided by the model")
|
||||
assert.Contains(t, b, "5 uncertain")
|
||||
}
|
||||
|
||||
func TestUploadStatus_ListsTheContestedSpotsWithAMinuteMarkBothReadingsAndWhichWasUsed(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, reconciledJSON)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "~37 min", "2210.5 s")
|
||||
assert.Contains(t, b, "Axel")
|
||||
assert.Contains(t, b, "Alex")
|
||||
assert.Contains(t, b, "~57 min", "3400 s")
|
||||
assert.Contains(t, b, "A+2")
|
||||
assert.Contains(t, b, "en A")
|
||||
assert.Contains(t, b, "uncertain", "an unsure spot is marked so the reader knows to check it")
|
||||
}
|
||||
|
||||
func TestUploadStatus_ASingleEngineTranscriptDoesNotClaimASecondOpinion(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
|
||||
assert.NotContains(t, body(t, do(t, app, get("/uploads/"+id+"/status"))), "Reconciled", "no reconciliation, no claim")
|
||||
}
|
||||
|
||||
func TestUploadStatus_WhenOnlyOneEngineWorkedItSaysWhyTheSecondOpinionIsMissing(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, `{"status":"single_engine","reason":"iguana/pianissimo failed: wedged","base":"iguana/pianissimo","alt":"iguana/kb-whisper","reconciler":"iguana/gemma4-26b"}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "Only one engine")
|
||||
assert.Contains(t, b, "iguana/pianissimo failed")
|
||||
assert.NotContains(t, b, "Reconciled from two")
|
||||
}
|
||||
|
||||
func TestUploadStatus_WhenTheModelWasUnavailableItSaysTheTextIsUnreconciled(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, `{"status":"skipped","reason":"the model failed 3 times in a row: gateway down","base":"iguana/pianissimo","alt":"iguana/kb-whisper","reconciler":"iguana/gemma4-26b","disputes":380,"errors":3}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "could not be reconciled")
|
||||
assert.Contains(t, b, "iguana/pianissimo", "which engine's text is being shown")
|
||||
assert.Contains(t, b, "gateway down")
|
||||
}
|
||||
|
||||
func TestUploadStatus_ContestedReadingsAreEscaped(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, `{"status":"reconciled","base":"a","alt":"b","reconciler":"c","disputes":1,"unsure":1,
|
||||
"contested":[{"seconds":60,"base":"<script>alert(1)</script>","alt":"ok","used":"base","unsure":true}]}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.NotContains(t, b, "<script>alert(1)</script>", "a reading is words from a recording, never markup")
|
||||
assert.Contains(t, b, "<script>")
|
||||
}
|
||||
|
||||
func TestUploadStatus_AVeryLongListOfContestedSpotsIsShortenedWithItsTrueCount(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
var spots []string
|
||||
for i := 0; i < 120; i++ {
|
||||
spots = append(spots, fmt.Sprintf(`{"seconds":%d,"base":"b%d","alt":"a%d","used":"base","unsure":true}`, i*10, i, i))
|
||||
}
|
||||
setReconciliation(t, id, `{"status":"reconciled","base":"a","alt":"b","reconciler":"c","disputes":400,"unsure":120,"contested":[`+strings.Join(spots, ",")+`]}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "b0", "the first spots are shown")
|
||||
assert.NotContains(t, b, ">b119<", "the page does not render every one")
|
||||
assert.Contains(t, b, "70 more", "120 stored, 50 shown")
|
||||
}
|
||||
|
||||
func TestUploadStatus_ALengthFailureIsExplainedInWordsAndNotAsAGenericError(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
tests := []struct{ kind, want string }{
|
||||
{"audio_too_long", "longer than the limit"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.kind, func(t *testing.T) {
|
||||
id := seedUpload(t, userID, "m-"+tt.kind, "failed", tt.kind)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, tt.want)
|
||||
assert.NotContains(t, b, "This upload failed. Please upload it again.")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Error paths of the upload routes log a kind, never the error's text, which can carry
|
||||
// a file name, a path, a database value or a provider's output (adversarial review F5).
|
||||
|
||||
type lockedBuf struct {
|
||||
mu sync.Mutex
|
||||
b bytes.Buffer
|
||||
}
|
||||
|
||||
func (l *lockedBuf) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.b.Write(p)
|
||||
}
|
||||
|
||||
func (l *lockedBuf) String() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.b.String()
|
||||
}
|
||||
|
||||
type failingUploader struct{ err error }
|
||||
|
||||
func (f failingUploader) Accept(context.Context, string, string, io.Reader) (string, error) {
|
||||
return "", f.err
|
||||
}
|
||||
|
||||
func TestUploadPost_AnUnexpectedFailureLogsAKindAndNotItsText(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
buf := &lockedBuf{}
|
||||
app.Log = slog.New(slog.NewTextHandler(buf, nil))
|
||||
app.Uploads = failingUploader{err: errors.New(`ffprobe: Error opening input file /spool/upload-555 for "zebrakaka-meeting.m4a"`)}
|
||||
|
||||
rec := do(t, app, uploadReq(t, "t", "zebrakaka-meeting.m4a", []byte("x")))
|
||||
|
||||
assert.Equal(t, http.StatusInternalServerError, rec.Code)
|
||||
out := buf.String()
|
||||
assert.NotContains(t, out, "zebrakaka", "a file name is private content")
|
||||
assert.NotContains(t, out, "/spool/", "a spool path carries the upload's identity")
|
||||
assert.Contains(t, out, "kind=error")
|
||||
}
|
||||
|
||||
func TestUploadChatPost_AModelFailureLogsAKindAndNotItsText(t *testing.T) {
|
||||
app, fc := newUploadChatApp(t)
|
||||
buf := &lockedBuf{}
|
||||
app.Log = slog.New(slog.NewTextHandler(buf, nil))
|
||||
fc.err = errors.New("LLM returned 400: the prompt was zebrakaka secret words")
|
||||
id := seedUpload(t, userID, "m", "done", meetingText)
|
||||
|
||||
rec := do(t, app, chatPost("/uploads/"+id+"/chat", url.Values{"question": {"q"}}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
out := buf.String()
|
||||
assert.NotContains(t, out, "zebrakaka")
|
||||
assert.Contains(t, out, "kind=error")
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/web"
|
||||
)
|
||||
|
||||
// The page's progress display (ADR-035, #42): real phases from the row, and the three
|
||||
// honest states the page can tell apart -- running, quiet (counted phase not advancing)
|
||||
// and lost (the service stopped reporting). Real router, real Postgres, a fixed clock.
|
||||
|
||||
var progressNow = time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
func fixedClock(app *web.App) { app.Now = func() time.Time { return progressNow } }
|
||||
|
||||
// seedProgress writes the job's phases and the two timestamps onto an existing row.
|
||||
func seedProgress(t *testing.T, id string, steps []domain.ProgressStep, progressAt, aliveAt time.Time) {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(steps)
|
||||
require.NoError(t, err)
|
||||
var alive any
|
||||
if !aliveAt.IsZero() {
|
||||
alive = aliveAt
|
||||
}
|
||||
_, err = rawPool(t).Exec(context.Background(),
|
||||
`UPDATE uploaded_transcripts SET progress = $2, progress_at = $3, alive_at = $4 WHERE id = $1`,
|
||||
id, b, progressAt, alive)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func step(key, label string, state domain.StepState, startedAgo, ranFor time.Duration, done, total int) domain.ProgressStep {
|
||||
s := domain.ProgressStep{Key: key, Label: label, State: state, Done: done, Total: total}
|
||||
if state != domain.StepPending {
|
||||
s.StartedAt = progressNow.Add(-startedAgo)
|
||||
}
|
||||
if state == domain.StepDone || state == domain.StepFailed {
|
||||
s.FinishedAt = s.StartedAt.Add(ranFor)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// twoEngineRun is a job in its second engine: three steps done, one running, three to come.
|
||||
func twoEngineRun() []domain.ProgressStep {
|
||||
return []domain.ProgressStep{
|
||||
step("received", "Received", domain.StepDone, 5*time.Minute, 0, 0, 0),
|
||||
step("audio", "Preparing the audio", domain.StepDone, 5*time.Minute, 40*time.Second, 0, 0),
|
||||
step("engine_a", "Transcribing with iguana/pianissimo", domain.StepDone, 4*time.Minute, 38*time.Second, 0, 0),
|
||||
step("engine_b", "Transcribing with iguana/kb-whisper", domain.StepRunning, 3*time.Minute+12*time.Second, 0, 0, 0),
|
||||
step("compare", "Comparing the two transcripts", domain.StepPending, 0, 0, 0, 0),
|
||||
step("decide", "Deciding the differences", domain.StepPending, 0, 0, 0, 0),
|
||||
step("save", "Saving the transcript", domain.StepPending, 0, 0, 0, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func statusBody(t *testing.T, app *web.App, id string) string {
|
||||
t.Helper()
|
||||
rec := do(t, app, get("/uploads/"+id+"/status"))
|
||||
require.Equal(t, 200, rec.Code)
|
||||
return body(t, rec)
|
||||
}
|
||||
|
||||
func TestUploadProgress_ARunningJobShowsEveryPhaseInOrderWithTheRunningOneMarked(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
seedProgress(t, id, twoEngineRun(), progressNow.Add(-10*time.Second), progressNow.Add(-4*time.Second))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
last := -1
|
||||
for _, label := range []string{
|
||||
"Received", "Preparing the audio", "Transcribing with iguana/pianissimo", "Transcribing with iguana/kb-whisper",
|
||||
"Comparing the two transcripts", "Deciding the differences", "Saving the transcript",
|
||||
} {
|
||||
i := strings.Index(b, label)
|
||||
require.Greater(t, i, last, "phase %q missing or out of order:\n%s", label, b)
|
||||
last = i
|
||||
}
|
||||
assert.Equal(t, 3, strings.Count(b, "step-done"), "three finished phases collapse to a done line")
|
||||
assert.Equal(t, 1, strings.Count(b, `aria-current="step"`), "exactly one phase is the current one")
|
||||
assert.Contains(t, b, "3m 12s", "the running phase shows how long it has run")
|
||||
assert.Contains(t, b, "38s", "a finished phase shows how long it took")
|
||||
assert.Contains(t, b, `hx-trigger="every 2s"`, "an unfinished job re-asks every two seconds")
|
||||
}
|
||||
|
||||
func TestUploadProgress_OnlyACountedPhaseGetsABarAndItShowsRealCounts(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
steps := twoEngineRun()
|
||||
steps[3] = step("engine_b", "Transcribing with iguana/kb-whisper", domain.StepDone, 3*time.Minute, 2*time.Minute, 0, 0)
|
||||
steps[4] = step("compare", "Comparing the two transcripts", domain.StepDone, 1*time.Minute, time.Second, 0, 0)
|
||||
steps[5] = step("decide", "Deciding the differences", domain.StepRunning, 50*time.Second, 0, 142, 282)
|
||||
seedProgress(t, id, steps, progressNow.Add(-5*time.Second), progressNow.Add(-3*time.Second))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
assert.Contains(t, b, `<progress`)
|
||||
assert.Contains(t, b, `value="142"`)
|
||||
assert.Contains(t, b, `max="282"`)
|
||||
assert.Contains(t, b, "142 of 282")
|
||||
|
||||
// ...and a phase with no honest count never gets one: no percentage out of thin air.
|
||||
app2, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app2)
|
||||
id2 := seedUpload(t, userID, "m2", "transcribing", "")
|
||||
seedProgress(t, id2, twoEngineRun(), progressNow.Add(-10*time.Second), progressNow.Add(-4*time.Second))
|
||||
assert.NotContains(t, statusBody(t, app2, id2), "<progress", "an engine call is one request: there is nothing to count")
|
||||
}
|
||||
|
||||
func TestUploadProgress_ALongEngineCallWithAFreshHeartbeatIsStillJustRunning(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
steps := twoEngineRun()
|
||||
steps[3] = step("engine_b", "Transcribing with iguana/kb-whisper", domain.StepRunning, 25*time.Minute, 0, 0, 0)
|
||||
seedProgress(t, id, steps, progressNow.Add(-25*time.Minute), progressNow.Add(-3*time.Second))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
assert.Contains(t, b, "25m 00s")
|
||||
assert.NotContains(t, b, "step-health-lost")
|
||||
assert.NotContains(t, b, "step-health-quiet",
|
||||
"a single blocking request has no sub-progress to go quiet about; the heartbeat says the service is alive")
|
||||
}
|
||||
|
||||
func TestUploadProgress_AHeartbeatThatStoppedMeansTheServiceIsLostAndTheJobIsNotCalledRunning(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
seedProgress(t, id, twoEngineRun(), progressNow.Add(-3*time.Minute), progressNow.Add(-3*time.Minute))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
assert.Contains(t, b, "step-health-lost")
|
||||
assert.Contains(t, b, "stopped reporting")
|
||||
assert.Contains(t, b, "3 min", "how long ago it was last heard from")
|
||||
assert.NotContains(t, b, "spinner", "a lost job must not wear the running spinner")
|
||||
assert.Contains(t, b, "step-mark-warn", "the running phase of a lost job carries a warning mark, in the error colour, not the accent")
|
||||
assert.Contains(t, b, `hx-trigger="every`, "it keeps asking: a restarted service comes back, or the start-up sweep fails the row")
|
||||
}
|
||||
|
||||
func TestUploadProgress_ACountedPhaseThatStopsAdvancingGoesQuietNotLost(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
steps := twoEngineRun()
|
||||
steps[5] = step("decide", "Deciding the differences", domain.StepRunning, 20*time.Minute, 0, 142, 282)
|
||||
seedProgress(t, id, steps, progressNow.Add(-6*time.Minute), progressNow.Add(-3*time.Second))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
assert.Contains(t, b, "step-health-quiet")
|
||||
assert.Contains(t, b, "Nothing new for 6 min")
|
||||
assert.NotContains(t, b, "stopped reporting", "the service is alive; it is the model that is slow")
|
||||
}
|
||||
|
||||
func TestUploadProgress_BeforeTheFirstHeartbeatAJobIsRunningUntilItHasBeenSilentTooLong(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
fresh := seedUpload(t, userID, "fresh", "transcribing", "")
|
||||
seedProgress(t, fresh, twoEngineRun(), progressNow.Add(-5*time.Second), time.Time{})
|
||||
silent := seedUpload(t, userID, "silent", "transcribing", "")
|
||||
seedProgress(t, silent, twoEngineRun(), progressNow.Add(-2*time.Minute), time.Time{})
|
||||
|
||||
assert.NotContains(t, statusBody(t, app, fresh), "step-health-lost", "the first beat has not been due yet")
|
||||
assert.Contains(t, statusBody(t, app, silent), "step-health-lost", "no beat and no progress for two minutes is a lost service")
|
||||
}
|
||||
|
||||
func TestUploadProgress_AQueuedJobShowsNoPhasesAndNoLostWarning(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "queued", "")
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
assert.Contains(t, b, "Queued")
|
||||
assert.NotContains(t, b, `class="steps"`)
|
||||
assert.NotContains(t, b, "step-health")
|
||||
assert.Contains(t, b, `hx-trigger="every 2s"`)
|
||||
}
|
||||
|
||||
func TestUploadProgress_AFailedJobShowsWhichPhaseFailedAndWhatNeverStarted(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "failed", "transcription_failed")
|
||||
steps := twoEngineRun()
|
||||
steps[3] = step("engine_b", "Transcribing with iguana/kb-whisper", domain.StepFailed, 3*time.Minute, time.Minute, 0, 0)
|
||||
seedProgress(t, id, steps, progressNow.Add(-2*time.Minute), progressNow.Add(-2*time.Minute))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
assert.Contains(t, b, "Transcription failed")
|
||||
assert.Contains(t, b, "step-failed")
|
||||
assert.Contains(t, b, "step-pending", "what never started says where the job stopped")
|
||||
assert.NotContains(t, b, "hx-trigger", "a failed job stops polling")
|
||||
assert.NotContains(t, b, "step-health", "a finished job is not 'lost'")
|
||||
}
|
||||
|
||||
func TestUploadProgress_ADoneJobKeepsItsRecordOfWhatWasDoneAndHowLongEachTook(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript text.")
|
||||
steps := []domain.ProgressStep{
|
||||
step("received", "Received", domain.StepDone, 20*time.Minute, 0, 0, 0),
|
||||
step("audio", "Preparing the audio", domain.StepDone, 20*time.Minute, 40*time.Second, 0, 0),
|
||||
step("decide", "Deciding the differences", domain.StepDone, 15*time.Minute, 9*time.Minute+30*time.Second, 282, 282),
|
||||
step("save", "Saving the transcript", domain.StepSkipped, 0, 0, 0, 0),
|
||||
}
|
||||
seedProgress(t, id, steps, progressNow.Add(-time.Minute), progressNow.Add(-time.Minute))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
assert.Contains(t, b, "Synthetic transcript text.")
|
||||
assert.Contains(t, b, "What was done")
|
||||
assert.Contains(t, b, "9m 30s")
|
||||
assert.Contains(t, b, "step-skipped")
|
||||
assert.NotContains(t, b, "hx-trigger", "a finished upload must not poll forever")
|
||||
assert.NotContains(t, b, "step-health", "a finished job is never 'lost' however old its heartbeat")
|
||||
assert.NotContains(t, b, "spinner")
|
||||
}
|
||||
|
||||
// stepsBlock is the progress list of a page: from its opening tag to its closing one.
|
||||
func stepsBlock(t *testing.T, html string) string {
|
||||
t.Helper()
|
||||
start := strings.Index(html, `<ol class="steps"`)
|
||||
require.GreaterOrEqual(t, start, 0, "no progress list in:\n%s", html)
|
||||
end := strings.Index(html[start:], "</ol>")
|
||||
require.GreaterOrEqual(t, end, 0)
|
||||
return html[start : start+end+len("</ol>")]
|
||||
}
|
||||
|
||||
func TestUploadProgress_TheFullPageCarriesTheVerySameProgressListAsTheFragment(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
seedProgress(t, id, twoEngineRun(), progressNow.Add(-10*time.Second), progressNow.Add(-4*time.Second))
|
||||
|
||||
full := body(t, do(t, app, get("/uploads/"+id)))
|
||||
fragment := statusBody(t, app, id)
|
||||
|
||||
assert.Equal(t, stepsBlock(t, fragment), stepsBlock(t, full),
|
||||
"the page a user opens and the fragment HTMX swaps in must show the same list, or the display jumps on the first poll")
|
||||
}
|
||||
|
||||
func TestUploadProgress_AStepNoteIsShownDirectlyUnderItsStepAndEscaped(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
id := seedUpload(t, userID, "m", "transcribing", "")
|
||||
steps := twoEngineRun()
|
||||
steps[1].Note = "The audio is 20 min long; the file claimed 1 h 19 min. <b>cut off</b>?"
|
||||
seedProgress(t, id, steps, progressNow.Add(-10*time.Second), progressNow.Add(-4*time.Second))
|
||||
|
||||
b := statusBody(t, app, id)
|
||||
|
||||
audio := strings.Index(b, "Preparing the audio")
|
||||
note := strings.Index(b, "The audio is 20 min long")
|
||||
next := strings.Index(b, "Transcribing with iguana/pianissimo")
|
||||
require.True(t, audio >= 0 && note > audio && next > note, "the note must sit between its own step and the next one:\n%s", b)
|
||||
assert.Contains(t, b, "step-note")
|
||||
assert.NotContains(t, b, "<b>cut off</b>", "a note is words, never markup")
|
||||
}
|
||||
|
||||
func TestUploadProgress_ADonePageShowsItsRecordOpenWhenAStepHasANote(t *testing.T) {
|
||||
// A truncated recording must not look like a clean one: the note that replaced the
|
||||
// refusal is the only warning, so it cannot sit inside a collapsed list (third review).
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
noted := seedUpload(t, userID, "noted", "done", "Synthetic transcript text.")
|
||||
steps := []domain.ProgressStep{step("audio", "Preparing the audio", domain.StepDone, 20*time.Minute, 40*time.Second, 0, 0)}
|
||||
steps[0].Note = "The audio is 20 min long; the file claimed 1 h 19 min."
|
||||
seedProgress(t, noted, steps, progressNow.Add(-time.Minute), progressNow.Add(-time.Minute))
|
||||
clean := seedUpload(t, userID, "clean", "done", "Synthetic transcript text.")
|
||||
seedProgress(t, clean, []domain.ProgressStep{step("audio", "Preparing the audio", domain.StepDone, 20*time.Minute, 40*time.Second, 0, 0)}, progressNow.Add(-time.Minute), progressNow.Add(-time.Minute))
|
||||
|
||||
assert.Contains(t, statusBody(t, app, noted), `<details class="what-was-done" open>`)
|
||||
assert.NotContains(t, statusBody(t, app, clean), `<details class="what-was-done" open>`, "a clean record stays collapsed")
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// TestRenderUploadProgressStatesForReview writes the upload page in each progress state
|
||||
// to $TAPIR_SHOT_DIR as standalone HTML (light and dark), for a screenshot review of the
|
||||
// visual claims tests cannot assert: that it looks like tapir and reads like a log of
|
||||
// phases (ADR-035, #42). Opt-in: it does nothing unless the variable is set. The text is
|
||||
// synthetic. Screenshot with any browser, for example:
|
||||
//
|
||||
// TAPIR_SHOT_DIR=/tmp/shots go test -run TestRenderUploadProgressStatesForReview ./internal/web
|
||||
// chromium --headless --screenshot=/tmp/shots/running.png --window-size=900,900 /tmp/shots/running-light.html
|
||||
func TestRenderUploadProgressStatesForReview(t *testing.T) {
|
||||
dir := os.Getenv("TAPIR_SHOT_DIR")
|
||||
if dir == "" {
|
||||
t.Skip("set TAPIR_SHOT_DIR to render the progress states for a screenshot review")
|
||||
}
|
||||
require.NoError(t, os.MkdirAll(dir, 0o750))
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
fixedClock(app)
|
||||
|
||||
decide := twoEngineRun()
|
||||
decide[3] = step("engine_b", "Transcribing with iguana/kb-whisper", domain.StepDone, 9*time.Minute, 4*time.Minute, 0, 0)
|
||||
decide[4] = step("compare", "Comparing the two transcripts", domain.StepDone, 5*time.Minute, time.Second, 0, 0)
|
||||
decide[5] = step("decide", "Deciding the differences", domain.StepRunning, 4*time.Minute+30*time.Second, 0, 142, 282)
|
||||
|
||||
failed := twoEngineRun()
|
||||
failed[3] = step("engine_b", "Transcribing with iguana/kb-whisper", domain.StepFailed, 3*time.Minute, time.Minute, 0, 0)
|
||||
|
||||
done := []domain.ProgressStep{
|
||||
step("received", "Received", domain.StepDone, 22*time.Minute, 0, 0, 0),
|
||||
step("audio", "Preparing the audio", domain.StepDone, 22*time.Minute, 41*time.Second, 0, 0),
|
||||
step("engine_a", "Transcribing with iguana/pianissimo", domain.StepDone, 21*time.Minute, 2*time.Minute+10*time.Second, 0, 0),
|
||||
step("engine_b", "Transcribing with iguana/kb-whisper", domain.StepDone, 19*time.Minute, 4*time.Minute+5*time.Second, 0, 0),
|
||||
step("compare", "Comparing the two transcripts", domain.StepDone, 15*time.Minute, 2*time.Second, 0, 0),
|
||||
step("decide", "Deciding the differences", domain.StepDone, 15*time.Minute, 14*time.Minute, 282, 282),
|
||||
step("save", "Saving the transcript", domain.StepDone, time.Minute, time.Second, 0, 0),
|
||||
}
|
||||
|
||||
noted := twoEngineRun()
|
||||
noted[1].Note = "The audio is 20 min long; the file claimed 1 h 19 min. If the recording was cut off, upload the complete one."
|
||||
|
||||
cases := []struct {
|
||||
name, status, content string
|
||||
steps []domain.ProgressStep
|
||||
progressAgo, aliveAgo time.Duration
|
||||
}{
|
||||
{"running", "transcribing", "", twoEngineRun(), 10 * time.Second, 4 * time.Second},
|
||||
{"deciding", "transcribing", "", decide, 3 * time.Second, 3 * time.Second},
|
||||
{"quiet", "transcribing", "", decide, 6 * time.Minute, 4 * time.Second},
|
||||
{"lost", "transcribing", "", twoEngineRun(), 3 * time.Minute, 3 * time.Minute},
|
||||
{"failed", "failed", "transcription_failed", failed, 2 * time.Minute, 2 * time.Minute},
|
||||
{"done", "done", "Synthetic transcript text for the review.", done, time.Minute, time.Minute},
|
||||
{"noted", "transcribing", "", noted, 10 * time.Second, 4 * time.Second},
|
||||
{"queued", "queued", "", nil, 0, 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
id := seedUpload(t, userID, "Project notes (synthetic)", c.status, c.content)
|
||||
if c.steps != nil {
|
||||
seedProgress(t, id, c.steps, progressNow.Add(-c.progressAgo), progressNow.Add(-c.aliveAgo))
|
||||
}
|
||||
page := body(t, do(t, app, get("/uploads/"+id)))
|
||||
for _, theme := range []string{"light", "dark"} {
|
||||
html := page
|
||||
if theme == "dark" {
|
||||
html = strings.Replace(html, "<html", `<html data-theme="dark"`, 1)
|
||||
}
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, c.name+"-"+theme+".html"), []byte(html), 0o600))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The stored judged spots include the ones the model kept as the base (ADR-035), for a
|
||||
// later evaluation. The page lists only the ones a reader should check: changed and
|
||||
// uncertain.
|
||||
|
||||
func TestUploadStatus_OnlyChangedAndUncertainSpotsAreListedNotTheKeptOnes(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, `{"status":"reconciled","base":"a","alt":"b","reconciler":"c","disputes":3,"adjudicated":3,"changed":1,"unsure":1,
|
||||
"contested":[
|
||||
{"seconds":60,"base":"CHANGEDBASE","alt":"CHANGEDALT","used":"alt","unsure":false},
|
||||
{"seconds":120,"base":"KEPTBASE","alt":"KEPTALT","used":"base","unsure":false},
|
||||
{"seconds":180,"base":"UNSUREBASE","alt":"UNSUREALT","used":"base","unsure":true}]}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "CHANGEDBASE")
|
||||
assert.Contains(t, b, "UNSUREALT")
|
||||
assert.NotContains(t, b, "KEPTBASE", "a spot the model confidently kept is not one for the reader to check")
|
||||
assert.NotContains(t, b, "KEPTALT")
|
||||
}
|
||||
|
||||
func TestUploadStatus_TheMoreCountIgnoresKeptSpotsAndMatchesTheListedKind(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
var spots []string
|
||||
for i := 0; i < 60; i++ {
|
||||
spots = append(spots, fmt.Sprintf(`{"seconds":%d,"base":"b%d","alt":"a%d","used":"alt","unsure":false}`, i*10, i, i))
|
||||
spots = append(spots, fmt.Sprintf(`{"seconds":%d,"base":"k%d","alt":"j%d","used":"base","unsure":false}`, i*10+5, i, i))
|
||||
}
|
||||
setReconciliation(t, id, `{"status":"reconciled","base":"a","alt":"b","reconciler":"c","disputes":400,"changed":60,"contested":[`+strings.Join(spots, ",")+`]}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "10 more", "60 changed, 50 shown: the 60 kept ones do not count")
|
||||
assert.NotContains(t, b, "k0 / j0", "a kept spot is not rendered even when it is among the first stored")
|
||||
}
|
||||
|
||||
func TestUploadStatus_WhenNothingNeedsCheckingThereIsNoSpotsSection(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, `{"status":"reconciled","base":"a","alt":"b","reconciler":"c","disputes":1,"adjudicated":1,
|
||||
"contested":[{"seconds":60,"base":"KEPTBASE","alt":"KEPTALT","used":"base","unsure":false}]}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
require.Contains(t, b, "Reconciled from two transcripts")
|
||||
assert.NotContains(t, b, "Spots to check", "an empty list would invite checking nothing")
|
||||
}
|
||||
|
||||
func TestUploadStatus_ASpotShowsTheWordsJustBeforeIt(t *testing.T) {
|
||||
app, _ := newUploadApp(t, audioOK, 200<<20)
|
||||
id := seedUpload(t, userID, "m", "done", "Synthetic transcript.")
|
||||
setReconciliation(t, id, `{"status":"reconciled","base":"a","alt":"b","reconciler":"c","disputes":1,"changed":1,
|
||||
"contested":[{"seconds":2640,"base":"vanligt","alt":"KORT","used":"alt","unsure":false,"before":"grannen målar ett"}]}`)
|
||||
|
||||
b := body(t, do(t, app, get("/uploads/"+id+"/status")))
|
||||
|
||||
assert.Contains(t, b, "grannen målar ett", "the words before the spot are how a reader finds it in the audio")
|
||||
assert.Contains(t, b, "vanligt")
|
||||
assert.Contains(t, b, "KORT")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -765,6 +765,32 @@ a.btn, a.btn:visited { color: var(--accent-fg); }
|
||||
.flash-success { background: var(--accent-weak); color: var(--accent); border-color: var(--accent); }
|
||||
.flash-error { background: var(--err-bg); color: var(--err-fg); border-color: var(--err-line); }
|
||||
|
||||
/* upload progress (ADR-035): a log of phases that fills in as the job runs */
|
||||
.steps { list-style: none; margin: var(--s3) 0; padding: 0; font-family: var(--mono); font-size: .86rem; }
|
||||
.step { display: flex; align-items: baseline; gap: var(--s2); padding: .2rem 0; color: var(--muted); flex-wrap: wrap; }
|
||||
.step-mark { flex: none; width: 1.1rem; text-align: center; }
|
||||
.step-label { flex: 1 1 auto; }
|
||||
.step-time { margin-left: auto; color: var(--muted); font-weight: 400; font-variant-numeric: tabular-nums; }
|
||||
.step-done { color: var(--fg); }
|
||||
.step-done .step-mark { color: var(--mint); }
|
||||
.step-running { color: var(--fg); font-weight: 600; }
|
||||
.step-running .step-mark { color: var(--accent); }
|
||||
.step-failed { color: var(--err-fg); }
|
||||
.step-skipped, .step-pending { color: var(--muted); }
|
||||
.step-skipped { opacity: .7; }
|
||||
.step-pending { opacity: .75; }
|
||||
.step-bar { flex: none; width: calc(100% - 1.6rem); margin: 0 0 0 1.6rem; height: .5rem; accent-color: var(--accent); }
|
||||
.step-note { flex: none; width: calc(100% - 1.6rem); margin-left: 1.6rem; font-weight: 400; color: var(--muted); font-family: system-ui, -apple-system, sans-serif; font-size: .82rem; }
|
||||
.step-count { flex: none; width: calc(100% - 1.6rem); margin-left: 1.6rem; font-weight: 400; color: var(--muted); }
|
||||
.step-running .step-mark-warn { color: var(--err-fg); }
|
||||
.spinner { display: inline-block; vertical-align: -.1em; width: .8rem; height: .8rem; box-sizing: border-box; border: 2px solid var(--line); border-top-color: var(--accent); border-radius: 50%; animation: tapir-spin 1s linear infinite; }
|
||||
@keyframes tapir-spin { to { transform: rotate(360deg); } }
|
||||
@media (prefers-reduced-motion: reduce) { .spinner { animation: none; border-color: var(--accent); } }
|
||||
.step-health { margin: var(--s2) 0; padding: var(--s2) var(--s3); border-radius: var(--radius); font-size: .9rem; border: 1px solid var(--line); }
|
||||
.step-health-lost { background: var(--err-bg); color: var(--err-fg); border-color: var(--err-line); }
|
||||
.step-health-quiet { background: var(--accent-weak); color: var(--fg); border-color: var(--accent); }
|
||||
.what-was-done summary { cursor: pointer; color: var(--muted); }
|
||||
|
||||
/* htmx loading feedback */
|
||||
.htmx-indicator { opacity: 0; transition: opacity .2s; color: var(--muted); font-size: .8rem; }
|
||||
.htmx-request .htmx-indicator, .htmx-request.htmx-indicator { opacity: 1; }
|
||||
|
||||
@@ -583,7 +583,7 @@ templ chatPanel(v chatView) {
|
||||
</div>
|
||||
}
|
||||
if v.Truncated {
|
||||
<p class="chat-note muted">Working from a bounded portion of a long transcript — answers about the end of the video may be incomplete.</p>
|
||||
<p class="chat-note muted">Working from a bounded portion of a long transcript — answers about the end of the { v.subject() } may be incomplete.</p>
|
||||
}
|
||||
if v.Error != "" {
|
||||
<p class="chat-error" role="alert">{ v.Error }</p>
|
||||
@@ -591,8 +591,8 @@ templ chatPanel(v chatView) {
|
||||
<form
|
||||
class="chat-form"
|
||||
method="post"
|
||||
action={ chatURL(v.VideoID) }
|
||||
hx-post={ string(chatURL(v.VideoID)) }
|
||||
action={ v.postURL() }
|
||||
hx-post={ string(v.postURL()) }
|
||||
hx-target="#chat-panel"
|
||||
hx-swap="outerHTML"
|
||||
>
|
||||
@@ -609,7 +609,7 @@ templ chatPanel(v chatView) {
|
||||
</select>
|
||||
<span class="chat-model-hint muted">Switch models to compare answers on the same transcript.</span>
|
||||
</label>
|
||||
<textarea name="question" rows="3" placeholder="Ask a question about this video…" required aria-label="Your question"></textarea>
|
||||
<textarea name="question" rows="3" placeholder={ "Ask a question about this " + v.subject() + "…" } required aria-label="Your question"></textarea>
|
||||
<button type="submit" class="btn">Ask</button>
|
||||
<span class="htmx-indicator chat-thinking">thinking…</span>
|
||||
</form>
|
||||
|
||||
+325
-297
@@ -5,8 +5,10 @@ package web
|
||||
|
||||
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||
|
||||
import "github.com/a-h/templ"
|
||||
import templruntime "github.com/a-h/templ/runtime"
|
||||
import (
|
||||
"github.com/a-h/templ"
|
||||
templruntime "github.com/a-h/templ/runtime"
|
||||
)
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -292,7 +294,7 @@ func flashBanner(code string) templ.Component {
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
if f, ok := flashFor(code); ok {
|
||||
var templ_7745c5c3_Var9 = []any{"flash", "flash-" + f.Kind}
|
||||
templ_7745c5c3_Var9 := []any{"flash", "flash-" + f.Kind}
|
||||
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var9...)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
@@ -824,7 +826,7 @@ func VideoCard(r store.SummaryRow) templ.Component {
|
||||
templ_7745c5c3_Var25 = templ.NopComponent
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
var templ_7745c5c3_Var26 = []any{"card", templ.KV("card-pending", !r.Summarized)}
|
||||
templ_7745c5c3_Var26 := []any{"card", templ.KV("card-pending", !r.Summarized)}
|
||||
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var26...)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
@@ -2152,148 +2154,174 @@ func chatPanel(v chatView) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if v.Truncated {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 173, "<p class=\"chat-note muted\">Working from a bounded portion of a long transcript — answers about the end of the video may be incomplete.</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 174, " ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if v.Error != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 175, "<p class=\"chat-error\" role=\"alert\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 173, "<p class=\"chat-note muted\">Working from a bounded portion of a long transcript — answers about the end of the ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var88 string
|
||||
templ_7745c5c3_Var88, templ_7745c5c3_Err = templ.JoinStringErrs(v.Error)
|
||||
templ_7745c5c3_Var88, templ_7745c5c3_Err = templ.JoinStringErrs(v.subject())
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 589, Col: 48}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 586, Col: 129}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var88))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 176, "</p>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 174, " may be incomplete.</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 177, " <form class=\"chat-form\" method=\"post\" action=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 175, " ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var89 templ.SafeURL
|
||||
templ_7745c5c3_Var89, templ_7745c5c3_Err = templ.JoinURLErrs(chatURL(v.VideoID))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 594, Col: 31}
|
||||
if v.Error != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 176, "<p class=\"chat-error\" role=\"alert\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var89 string
|
||||
templ_7745c5c3_Var89, templ_7745c5c3_Err = templ.JoinStringErrs(v.Error)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 589, Col: 48}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var89))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 177, "</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var89))
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 178, " <form class=\"chat-form\" method=\"post\" action=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 178, "\" hx-post=\"")
|
||||
var templ_7745c5c3_Var90 templ.SafeURL
|
||||
templ_7745c5c3_Var90, templ_7745c5c3_Err = templ.JoinURLErrs(v.postURL())
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 594, Col: 24}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var90))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var90 string
|
||||
templ_7745c5c3_Var90, templ_7745c5c3_Err = templ.ResolveAttributeValue(string(chatURL(v.VideoID)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 595, Col: 40}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var90)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 179, "\" hx-post=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 179, "\" hx-target=\"#chat-panel\" hx-swap=\"outerHTML\">")
|
||||
var templ_7745c5c3_Var91 string
|
||||
templ_7745c5c3_Var91, templ_7745c5c3_Err = templ.ResolveAttributeValue(string(v.postURL()))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 595, Col: 33}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var91)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 180, "\" hx-target=\"#chat-panel\" hx-swap=\"outerHTML\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, t := range v.History {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 180, "<input type=\"hidden\" name=\"hq\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var91 string
|
||||
templ_7745c5c3_Var91, templ_7745c5c3_Err = templ.ResolveAttributeValue(t.Question)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 600, Col: 54}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var91)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 181, "\"> <input type=\"hidden\" name=\"ha\" value=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 181, "<input type=\"hidden\" name=\"hq\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var92 string
|
||||
templ_7745c5c3_Var92, templ_7745c5c3_Err = templ.ResolveAttributeValue(t.Answer)
|
||||
templ_7745c5c3_Var92, templ_7745c5c3_Err = templ.ResolveAttributeValue(t.Question)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 601, Col: 52}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 600, Col: 54}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var92)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 182, "\"> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 183, "<label class=\"chat-model\">Model <select name=\"model\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, m := range v.Models {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 184, "<option value=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 182, "\"> <input type=\"hidden\" name=\"ha\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var93 string
|
||||
templ_7745c5c3_Var93, templ_7745c5c3_Err = templ.ResolveAttributeValue(m)
|
||||
templ_7745c5c3_Var93, templ_7745c5c3_Err = templ.ResolveAttributeValue(t.Answer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 607, Col: 24}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 601, Col: 52}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var93)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 185, "\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 183, "\"> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if m == v.Selected {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 186, " selected")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 187, ">")
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 184, "<label class=\"chat-model\">Model <select name=\"model\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, m := range v.Models {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 185, "<option value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var94 string
|
||||
templ_7745c5c3_Var94, templ_7745c5c3_Err = templ.JoinStringErrs(m)
|
||||
templ_7745c5c3_Var94, templ_7745c5c3_Err = templ.ResolveAttributeValue(m)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 607, Col: 60}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 607, Col: 24}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var94))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var94)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 188, "</option>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 186, "\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if m == v.Selected {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 187, " selected")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 188, ">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var95 string
|
||||
templ_7745c5c3_Var95, templ_7745c5c3_Err = templ.JoinStringErrs(m)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 607, Col: 60}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var95))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 189, "</option>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 189, "</select> <span class=\"chat-model-hint muted\">Switch models to compare answers on the same transcript.</span></label> <textarea name=\"question\" rows=\"3\" placeholder=\"Ask a question about this video…\" required aria-label=\"Your question\"></textarea> <button type=\"submit\" class=\"btn\">Ask</button> <span class=\"htmx-indicator chat-thinking\">thinking…</span></form>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 190, "</select> <span class=\"chat-model-hint muted\">Switch models to compare answers on the same transcript.</span></label> <textarea name=\"question\" rows=\"3\" placeholder=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var96 string
|
||||
templ_7745c5c3_Var96, templ_7745c5c3_Err = templ.ResolveAttributeValue("Ask a question about this " + v.subject() + "…")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 612, Col: 103}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var96)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 191, "\" required aria-label=\"Your question\"></textarea> <button type=\"submit\" class=\"btn\">Ask</button> <span class=\"htmx-indicator chat-thinking\">thinking…</span></form>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 190, "</div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 192, "</div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2320,12 +2348,12 @@ func RegisterPage(email, errMsg string) templ.Component {
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
templ_7745c5c3_Var95 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var95 == nil {
|
||||
templ_7745c5c3_Var95 = templ.NopComponent
|
||||
templ_7745c5c3_Var97 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var97 == nil {
|
||||
templ_7745c5c3_Var97 = templ.NopComponent
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Var96 := templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_Var98 := templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
@@ -2337,59 +2365,59 @@ func RegisterPage(email, errMsg string) templ.Component {
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 191, "<article class=\"register\"><h1>Complete your registration</h1>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 193, "<article class=\"register\"><h1>Complete your registration</h1>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if email != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 192, "<p class=\"meta\">Signed in as ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 194, "<p class=\"meta\">Signed in as ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var97 string
|
||||
templ_7745c5c3_Var97, templ_7745c5c3_Err = templ.JoinStringErrs(email)
|
||||
var templ_7745c5c3_Var99 string
|
||||
templ_7745c5c3_Var99, templ_7745c5c3_Err = templ.JoinStringErrs(email)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 628, Col: 40}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var97))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var99))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 193, ".</p>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 195, ".</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 194, "<p>Choose a display name to finish setting up your Tapir account.</p>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 196, "<p>Choose a display name to finish setting up your Tapir account.</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if errMsg != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 195, "<p class=\"error\" role=\"alert\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 197, "<p class=\"error\" role=\"alert\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var98 string
|
||||
templ_7745c5c3_Var98, templ_7745c5c3_Err = templ.JoinStringErrs(errMsg)
|
||||
var templ_7745c5c3_Var100 string
|
||||
templ_7745c5c3_Var100, templ_7745c5c3_Err = templ.JoinStringErrs(errMsg)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 632, Col: 42}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var98))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var100))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 196, "</p>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 198, "</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 197, "<form method=\"post\" action=\"/register\" class=\"register-form\"><label>Display name <input type=\"text\" name=\"display_name\" required autofocus></label> <button type=\"submit\" class=\"btn\">Register</button></form></article>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 199, "<form method=\"post\" action=\"/register\" class=\"register-form\"><label>Display name <input type=\"text\" name=\"display_name\" required autofocus></label> <button type=\"submit\" class=\"btn\">Register</button></form></article>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
templ_7745c5c3_Err = Layout("Tapir — Register").Render(templ.WithChildren(ctx, templ_7745c5c3_Var96), templ_7745c5c3_Buffer)
|
||||
templ_7745c5c3_Err = Layout("Tapir — Register").Render(templ.WithChildren(ctx, templ_7745c5c3_Var98), templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2417,12 +2445,12 @@ func AccountPage(displayName, email string, conns []store.Connection, autoSummar
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
templ_7745c5c3_Var99 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var99 == nil {
|
||||
templ_7745c5c3_Var99 = templ.NopComponent
|
||||
templ_7745c5c3_Var101 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var101 == nil {
|
||||
templ_7745c5c3_Var101 = templ.NopComponent
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Var100 := templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_Var102 := templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
@@ -2438,43 +2466,43 @@ func AccountPage(displayName, email string, conns []store.Connection, autoSummar
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 198, " <article class=\"account\"><h1>Account</h1><dl class=\"account-meta\"><dt>Display name</dt><dd>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 200, " <article class=\"account\"><h1>Account</h1><dl class=\"account-meta\"><dt>Display name</dt><dd>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var101 string
|
||||
templ_7745c5c3_Var101, templ_7745c5c3_Err = templ.JoinStringErrs(displayNameOr(displayName))
|
||||
var templ_7745c5c3_Var103 string
|
||||
templ_7745c5c3_Var103, templ_7745c5c3_Err = templ.JoinStringErrs(displayNameOr(displayName))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 656, Col: 36}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var101))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var103))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 199, "</dd>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 201, "</dd>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if email != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 200, "<dt>Signed in as</dt><dd>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 202, "<dt>Signed in as</dt><dd>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var102 string
|
||||
templ_7745c5c3_Var102, templ_7745c5c3_Err = templ.JoinStringErrs(email)
|
||||
var templ_7745c5c3_Var104 string
|
||||
templ_7745c5c3_Var104, templ_7745c5c3_Err = templ.JoinStringErrs(email)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 659, Col: 16}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var102))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var104))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 201, "</dd>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 203, "</dd>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 202, "</dl><section><h2>Summarization</h2><p class=\"muted\">Automatic summarizes new videos from about the last week as they are discovered. Older videos stay browsable — summarize them on demand. Manual lets you pick which videos to summarize — every new video appears in your list with a Summarize button.</p>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 204, "</dl><section><h2>Summarization</h2><p class=\"muted\">Automatic summarizes new videos from about the last week as they are discovered. Older videos stay browsable — summarize them on demand. Manual lets you pick which videos to summarize — every new video appears in your list with a Summarize button.</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2482,178 +2510,178 @@ func AccountPage(displayName, email string, conns []store.Connection, autoSummar
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 203, "</section>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 205, "</section>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if len(channelErrors) > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 204, "<section class=\"channel-errors\"><h2>Unavailable channels</h2><p class=\"muted\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 206, "<section class=\"channel-errors\"><h2>Unavailable channels</h2><p class=\"muted\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var103 string
|
||||
templ_7745c5c3_Var103, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d channel(s) returned errors on the last discovery pass.", len(channelErrors)))
|
||||
var templ_7745c5c3_Var105 string
|
||||
templ_7745c5c3_Var105, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d channel(s) returned errors on the last discovery pass.", len(channelErrors)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 676, Col: 100}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var103))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var105))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 205, " These may have been deleted or made private on YouTube.</p><ul class=\"channel-error-list\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 207, " These may have been deleted or made private on YouTube.</p><ul class=\"channel-error-list\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, ce := range channelErrors {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 206, "<li><span class=\"channel-error-name\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var104 string
|
||||
templ_7745c5c3_Var104, templ_7745c5c3_Err = templ.JoinStringErrs(ce.ChannelName)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 682, Col: 57}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var104))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 207, "</span> <span class=\"chip chip-warn\">unavailable</span> <span class=\"muted channel-error-since\">since ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var105 string
|
||||
templ_7745c5c3_Var105, templ_7745c5c3_Err = templ.JoinStringErrs(ce.FirstSeen.Format("2006-01-02"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 684, Col: 89}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var105))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 208, "</span></li>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 209, "</ul></section>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 210, "<section><h2>Connected accounts</h2>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if len(conns) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 211, "<p class=\"muted\">No connected video accounts yet.</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 212, "<ul class=\"conn-list\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, c := range conns {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 213, "<li class=\"conn\"><div class=\"conn-main\"><span class=\"conn-provider\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 208, "<li><span class=\"channel-error-name\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var106 string
|
||||
templ_7745c5c3_Var106, templ_7745c5c3_Err = templ.JoinStringErrs(providerLabel(c.Provider))
|
||||
templ_7745c5c3_Var106, templ_7745c5c3_Err = templ.JoinStringErrs(ce.ChannelName)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 699, Col: 64}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 682, Col: 57}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var106))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 214, "</span> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 209, "</span> <span class=\"chip chip-warn\">unavailable</span> <span class=\"muted channel-error-since\">since ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if c.ProviderAccount != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 215, "<span class=\"muted\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var107 string
|
||||
templ_7745c5c3_Var107, templ_7745c5c3_Err = templ.JoinStringErrs(c.ProviderAccount)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 701, Col: 49}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var107))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 216, "</span> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var107 string
|
||||
templ_7745c5c3_Var107, templ_7745c5c3_Err = templ.JoinStringErrs(ce.FirstSeen.Format("2006-01-02"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 684, Col: 89}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 217, "<span class=\"chip\">")
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var107))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 210, "</span></li>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 211, "</ul></section>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 212, "<section><h2>Connected accounts</h2>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if len(conns) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 213, "<p class=\"muted\">No connected video accounts yet.</p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 214, "<ul class=\"conn-list\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, c := range conns {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 215, "<li class=\"conn\"><div class=\"conn-main\"><span class=\"conn-provider\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var108 string
|
||||
templ_7745c5c3_Var108, templ_7745c5c3_Err = templ.JoinStringErrs(c.Status)
|
||||
templ_7745c5c3_Var108, templ_7745c5c3_Err = templ.JoinStringErrs(providerLabel(c.Provider))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 703, Col: 38}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 699, Col: 64}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var108))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 218, "</span></div><div class=\"conn-meta muted\">connected ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 216, "</span> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var109 string
|
||||
templ_7745c5c3_Var109, templ_7745c5c3_Err = templ.JoinStringErrs(c.ConnectedAt.Format("2006-01-02"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 705, Col: 83}
|
||||
if c.ProviderAccount != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 217, "<span class=\"muted\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var109 string
|
||||
templ_7745c5c3_Var109, templ_7745c5c3_Err = templ.JoinStringErrs(c.ProviderAccount)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 701, Col: 49}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var109))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 218, "</span> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var109))
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 219, "<span class=\"chip\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 219, "</div><form method=\"post\" action=\"")
|
||||
var templ_7745c5c3_Var110 string
|
||||
templ_7745c5c3_Var110, templ_7745c5c3_Err = templ.JoinStringErrs(c.Status)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var110 templ.SafeURL
|
||||
templ_7745c5c3_Var110, templ_7745c5c3_Err = templ.JoinURLErrs(disconnectURL(c.Provider))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 706, Col: 62}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 703, Col: 38}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var110))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 220, "\"><button type=\"submit\" class=\"btn-secondary\">Disconnect</button></form></li>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 220, "</span></div><div class=\"conn-meta muted\">connected ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var111 string
|
||||
templ_7745c5c3_Var111, templ_7745c5c3_Err = templ.JoinStringErrs(c.ConnectedAt.Format("2006-01-02"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 705, Col: 83}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var111))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 221, "</div><form method=\"post\" action=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var112 templ.SafeURL
|
||||
templ_7745c5c3_Var112, templ_7745c5c3_Err = templ.JoinURLErrs(disconnectURL(c.Provider))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 706, Col: 62}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var112))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 222, "\"><button type=\"submit\" class=\"btn-secondary\">Disconnect</button></form></li>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 221, "</ul>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 223, "</ul>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if !hasYouTube(conns) {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 222, "<p><a class=\"btn\" href=\"/oauth/youtube/connect\">Connect YouTube</a></p>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 224, "<p><a class=\"btn\" href=\"/oauth/youtube/connect\">Connect YouTube</a></p>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 223, "</section><section class=\"danger-zone\"><h2>Delete account</h2><p class=\"muted\">Permanently remove your Tapir account and all of its data — summaries, watch/skip/save actions, and connected accounts. This cannot be undone.</p><details class=\"confirm-delete\"><summary class=\"btn-danger\">Delete account…</summary><div class=\"confirm-body\"><p>This permanently deletes your account and all data. Are you sure?</p><form method=\"post\" action=\"/account/delete\"><button type=\"submit\" class=\"btn-danger\">Yes, permanently delete my account</button></form></div></details></section></article>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 225, "</section><section class=\"danger-zone\"><h2>Delete account</h2><p class=\"muted\">Permanently remove your Tapir account and all of its data — summaries, watch/skip/save actions, and connected accounts. This cannot be undone.</p><details class=\"confirm-delete\"><summary class=\"btn-danger\">Delete account…</summary><div class=\"confirm-body\"><p>This permanently deletes your account and all data. Are you sure?</p><form method=\"post\" action=\"/account/delete\"><button type=\"submit\" class=\"btn-danger\">Yes, permanently delete my account</button></form></div></details></section></article>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
templ_7745c5c3_Err = Layout("Tapir — Account").Render(templ.WithChildren(ctx, templ_7745c5c3_Var100), templ_7745c5c3_Buffer)
|
||||
templ_7745c5c3_Err = Layout("Tapir — Account").Render(templ.WithChildren(ctx, templ_7745c5c3_Var102), templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2681,51 +2709,51 @@ func summarizeModeControl(auto bool) templ.Component {
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
templ_7745c5c3_Var111 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var111 == nil {
|
||||
templ_7745c5c3_Var111 = templ.NopComponent
|
||||
templ_7745c5c3_Var113 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var113 == nil {
|
||||
templ_7745c5c3_Var113 = templ.NopComponent
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 224, "<div id=\"summarize-mode\" class=\"summarize-mode\"><p>Current mode: <strong>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var112 string
|
||||
templ_7745c5c3_Var112, templ_7745c5c3_Err = templ.JoinStringErrs(summarizeModeLabel(auto))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 743, Col: 53}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var112))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 225, "</strong></p><form method=\"post\" action=\"/account/summarize-mode\" hx-post=\"/account/summarize-mode\" hx-target=\"#summarize-mode\" hx-swap=\"outerHTML\"><input type=\"hidden\" name=\"enabled\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var113 string
|
||||
templ_7745c5c3_Var113, templ_7745c5c3_Err = templ.ResolveAttributeValue(boolStr(!auto))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 751, Col: 61}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var113)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 226, "\"> <button type=\"submit\" class=\"btn-secondary\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 226, "<div id=\"summarize-mode\" class=\"summarize-mode\"><p>Current mode: <strong>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var114 string
|
||||
templ_7745c5c3_Var114, templ_7745c5c3_Err = templ.JoinStringErrs(summarizeModeToggleLabel(auto))
|
||||
templ_7745c5c3_Var114, templ_7745c5c3_Err = templ.JoinStringErrs(summarizeModeLabel(auto))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 752, Col: 79}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 743, Col: 53}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var114))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 227, "</button></form></div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 227, "</strong></p><form method=\"post\" action=\"/account/summarize-mode\" hx-post=\"/account/summarize-mode\" hx-target=\"#summarize-mode\" hx-swap=\"outerHTML\"><input type=\"hidden\" name=\"enabled\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var115 string
|
||||
templ_7745c5c3_Var115, templ_7745c5c3_Err = templ.ResolveAttributeValue(boolStr(!auto))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 751, Col: 61}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var115)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 228, "\"> <button type=\"submit\" class=\"btn-secondary\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var116 string
|
||||
templ_7745c5c3_Var116, templ_7745c5c3_Err = templ.JoinStringErrs(summarizeModeToggleLabel(auto))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 752, Col: 79}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var116))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 229, "</button></form></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2753,38 +2781,38 @@ func ActionButtons(videoID string, active map[string]bool) templ.Component {
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
templ_7745c5c3_Var115 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var115 == nil {
|
||||
templ_7745c5c3_Var115 = templ.NopComponent
|
||||
templ_7745c5c3_Var117 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var117 == nil {
|
||||
templ_7745c5c3_Var117 = templ.NopComponent
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 228, "<form id=\"action-buttons\" class=\"actions\" method=\"post\" action=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 230, "<form id=\"action-buttons\" class=\"actions\" method=\"post\" action=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var116 templ.SafeURL
|
||||
templ_7745c5c3_Var116, templ_7745c5c3_Err = templ.JoinURLErrs(actionURL(videoID))
|
||||
var templ_7745c5c3_Var118 templ.SafeURL
|
||||
templ_7745c5c3_Var118, templ_7745c5c3_Err = templ.JoinURLErrs(actionURL(videoID))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 766, Col: 29}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var116))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var118))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 229, "\" hx-post=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 231, "\" hx-post=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var117 string
|
||||
templ_7745c5c3_Var117, templ_7745c5c3_Err = templ.ResolveAttributeValue(string(actionURL(videoID)))
|
||||
var templ_7745c5c3_Var119 string
|
||||
templ_7745c5c3_Var119, templ_7745c5c3_Err = templ.ResolveAttributeValue(string(actionURL(videoID)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 767, Col: 38}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var117)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var119)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 230, "\" hx-target=\"#action-buttons\" hx-swap=\"outerHTML\"><span class=\"segmented\" role=\"group\" aria-label=\"Watched or skipped\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 232, "\" hx-target=\"#action-buttons\" hx-swap=\"outerHTML\"><span class=\"segmented\" role=\"group\" aria-label=\"Watched or skipped\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2796,7 +2824,7 @@ func ActionButtons(videoID string, active map[string]bool) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 231, "</span>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 233, "</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2804,7 +2832,7 @@ func ActionButtons(videoID string, active map[string]bool) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 232, "</form>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 234, "</form>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -2830,81 +2858,81 @@ func actionButton(verb string, isActive bool) templ.Component {
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
templ_7745c5c3_Var118 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var118 == nil {
|
||||
templ_7745c5c3_Var118 = templ.NopComponent
|
||||
templ_7745c5c3_Var120 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var120 == nil {
|
||||
templ_7745c5c3_Var120 = templ.NopComponent
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
var templ_7745c5c3_Var119 = []any{"action", templ.KV("active", isActive)}
|
||||
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var119...)
|
||||
templ_7745c5c3_Var121 := []any{"action", templ.KV("active", isActive)}
|
||||
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var121...)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 233, "<button type=\"submit\" name=\"action\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var120 string
|
||||
templ_7745c5c3_Var120, templ_7745c5c3_Err = templ.ResolveAttributeValue(verb)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 788, Col: 14}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var120)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 234, "\" class=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var121 string
|
||||
templ_7745c5c3_Var121, templ_7745c5c3_Err = templ.ResolveAttributeValue(templ.CSSClasses(templ_7745c5c3_Var119).String())
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 1, Col: 0}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var121)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 235, "\" aria-pressed=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 235, "<button type=\"submit\" name=\"action\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var122 string
|
||||
templ_7745c5c3_Var122, templ_7745c5c3_Err = templ.ResolveAttributeValue(ariaPressed(isActive))
|
||||
templ_7745c5c3_Var122, templ_7745c5c3_Err = templ.ResolveAttributeValue(verb)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 790, Col: 38}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 788, Col: 14}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var122)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 236, "\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 236, "\" class=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var123 string
|
||||
templ_7745c5c3_Var123, templ_7745c5c3_Err = templ.ResolveAttributeValue(templ.CSSClasses(templ_7745c5c3_Var121).String())
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 1, Col: 0}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var123)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 237, "\" aria-pressed=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var124 string
|
||||
templ_7745c5c3_Var124, templ_7745c5c3_Err = templ.ResolveAttributeValue(ariaPressed(isActive))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 790, Col: 38}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var124)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 238, "\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isActive {
|
||||
var templ_7745c5c3_Var123 string
|
||||
templ_7745c5c3_Var123, templ_7745c5c3_Err = templ.JoinStringErrs("✓ " + actionLabel(verb))
|
||||
var templ_7745c5c3_Var125 string
|
||||
templ_7745c5c3_Var125, templ_7745c5c3_Err = templ.JoinStringErrs("✓ " + actionLabel(verb))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 793, Col: 31}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var123))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var125))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
var templ_7745c5c3_Var124 string
|
||||
templ_7745c5c3_Var124, templ_7745c5c3_Err = templ.JoinStringErrs(actionLabel(verb))
|
||||
var templ_7745c5c3_Var126 string
|
||||
templ_7745c5c3_Var126, templ_7745c5c3_Err = templ.JoinStringErrs(actionLabel(verb))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/views.templ`, Line: 795, Col: 22}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var124))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var126))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 237, "</button>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 239, "</button>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user