Upload a recording (audio, or video with the camera off), get a private Swedish transcript, and chat about it. - ADR-033: RLS-scoped private transcript store; ffprobe/ffmpeg intake; gateway transcription (iguana kb-whisper, berget fallback); in-process single-replica worker; chat over the transcript (local models only). - ADR-034: optional two-engine mode (pianissimo + kb-whisper) reconciled by a LOCAL model; off unless TAPIR_RECONCILE_BASE_MODEL is set. - ADR-035: real progress (phases, running/quiet/lost), phase logging, every judged spot stored, merged segments. - Hardening from four adversarial reviews: errors logged and stored as kinds never text; per-engine deadlines; converted-length check; bounded queue; client hang-ups are not errors; strict local-alias check. - Dockerfile: static ffmpeg/ffprobe 7.0.2 pinned by sha256 (#34). Built and inspected by hand; CI does not build the image on a PR, so its first CI build is this merge. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set in the deployment. Closes #33, #34, #42, #43, #44. Co-authored-by: Mathias <mthbqv@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
package uploads_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||
)
|
||||
|
||||
// The duration limit is a claim in the file's header. The worker measures what was
|
||||
// really converted and refuses both a recording that is longer than the limit (the
|
||||
// header lied low) and one that ends far earlier than the file claimed (it was truncated
|
||||
// and would otherwise yield a partial transcript with no warning) -- adversarial review F2.
|
||||
|
||||
// sizedConverter writes a (sparse) 16 kHz mono PCM WAV of the given length, without
|
||||
// actually allocating it: 32,000 bytes per second plus a 44-byte header.
|
||||
type sizedConverter struct{ seconds float64 }
|
||||
|
||||
func (c sizedConverter) ToWAV(_ context.Context, _, dst string) error {
|
||||
f, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
return f.Truncate(44 + int64(c.seconds*32000))
|
||||
}
|
||||
|
||||
func durationWorker(t *testing.T, r *wrig, wavSeconds float64, max time.Duration) *uploads.Worker {
|
||||
t.Helper()
|
||||
opts := []uploads.WorkerOption{}
|
||||
if max > 0 {
|
||||
opts = append(opts, uploads.WithMaxDuration(max))
|
||||
}
|
||||
return uploads.NewWorker(r.st, sizedConverter{seconds: wavSeconds}, r.tr, r.tmp, time.Minute, opts...)
|
||||
}
|
||||
|
||||
func jobWith(r *wrig, t *testing.T, claimed time.Duration) uploads.Job {
|
||||
j := r.spool(t, "a")
|
||||
j.Duration = claimed
|
||||
return j
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AConvertedLengthOverTheLimitFailsTheJobBeforeAnyEngineSeesIt(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 100*60, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 10*time.Second))
|
||||
|
||||
_, failed := r.st.snapshot()
|
||||
assert.Equal(t, domain.UploadErrTooLong, failed["a"], "a header that claimed 10 s must not let 100 minutes through")
|
||||
assert.Empty(t, r.tr.paths, "no engine may be sent audio that is over the limit")
|
||||
assert.Empty(t, r.leftovers(t), "the audio is still deleted")
|
||||
}
|
||||
|
||||
func TestWorkerDuration_JustOverTheLimitWithinTheSlackStillTranscribes(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 90*60+20, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 90*time.Minute))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed)
|
||||
assert.Contains(t, completed, "a", "ffmpeg rounds: 20 s over a 90-minute limit is not a lie")
|
||||
}
|
||||
|
||||
func audioStepNote(r *wrig) string {
|
||||
for _, st := range r.st.lastSteps() {
|
||||
if st.Key == "audio" {
|
||||
return st.Note
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AFileThatEndsFarEarlierThanItClaimedIsStillTranscribedAndSaysSo(t *testing.T) {
|
||||
// A container header can over-estimate a complete file (VBR MP3 without a Xing header
|
||||
// claimed 79 minutes for 20 real ones). Refusing it was wrong; staying silent about a
|
||||
// truncated one is wrong too. So it is transcribed, and the audio step carries a note.
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 20*60, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 79*time.Minute))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed, "a complete recording with a bad header estimate must not be refused")
|
||||
assert.Contains(t, completed, "a")
|
||||
note := audioStepNote(r)
|
||||
assert.Contains(t, note, "20 min", "what was really transcribed")
|
||||
assert.Contains(t, note, "1 h 19 min", "what the file claimed")
|
||||
assert.Contains(t, note, "cut off", "and what to do if that is not a surprise")
|
||||
}
|
||||
|
||||
func TestWorkerDuration_TheShortfallNoteIsPinnedAtEightyPercentAndTwoMinutes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
claimed time.Duration
|
||||
wav float64
|
||||
noted bool
|
||||
}{
|
||||
{"79 percent of an hour", 60 * time.Minute, 0.79 * 3600, true},
|
||||
{"exactly 80 percent of an hour", 60 * time.Minute, 2880, false},
|
||||
{"one second under 80 percent of an hour", 60 * time.Minute, 2879, true},
|
||||
{"81 percent of an hour", 60 * time.Minute, 0.81 * 3600, false},
|
||||
{"exactly two minutes short and under 80 percent", 6 * time.Minute, 4 * 60, false},
|
||||
{"two minutes and a second short and under 80 percent", 6 * time.Minute, 4*60 - 1, true},
|
||||
{"unknown claimed length", 0, 20 * 60, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, tt.claimed))
|
||||
|
||||
assert.Equal(t, tt.noted, audioStepNote(r) != "")
|
||||
completed, _ := r.st.snapshot()
|
||||
assert.Contains(t, completed, "a", "a note never stops the job")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_TheTooLongToleranceIsExactlyThirtySeconds(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
wav float64
|
||||
fail bool
|
||||
}{
|
||||
{"at the limit plus 30 s", 90*60 + 30, false},
|
||||
{"at the limit plus 31 s", 90*60 + 31, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 90*time.Minute))
|
||||
|
||||
_, failed := r.st.snapshot()
|
||||
if tt.fail {
|
||||
assert.Equal(t, domain.UploadErrTooLong, failed["a"])
|
||||
} else {
|
||||
assert.Empty(t, failed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_AnEstimateThatIsMerelyALittleOffIsNotTruncation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
claimed time.Duration
|
||||
wav float64
|
||||
}{
|
||||
{"five percent short", 60 * time.Minute, 57 * 60},
|
||||
{"under two minutes short of a long file", 60 * time.Minute, 59 * 60},
|
||||
{"unknown claimed length", 0, 20 * 60},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, tt.wav, 90*time.Minute)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, tt.claimed))
|
||||
|
||||
completed, failed := r.st.snapshot()
|
||||
assert.Empty(t, failed)
|
||||
assert.Contains(t, completed, "a")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerDuration_WithNoMaximumConfiguredNothingIsMeasured(t *testing.T) {
|
||||
r := newWRig(t, time.Minute)
|
||||
w := durationWorker(t, r, 100*60, 0)
|
||||
|
||||
w.Process(context.Background(), jobWith(r, t, 0))
|
||||
|
||||
completed, _ := r.st.snapshot()
|
||||
assert.Contains(t, completed, "a")
|
||||
require.NotNil(t, w)
|
||||
}
|
||||
Reference in New Issue
Block a user