Upload a recording (audio, or video with the camera off), get a private Swedish transcript, and chat about it. - ADR-033: RLS-scoped private transcript store; ffprobe/ffmpeg intake; gateway transcription (iguana kb-whisper, berget fallback); in-process single-replica worker; chat over the transcript (local models only). - ADR-034: optional two-engine mode (pianissimo + kb-whisper) reconciled by a LOCAL model; off unless TAPIR_RECONCILE_BASE_MODEL is set. - ADR-035: real progress (phases, running/quiet/lost), phase logging, every judged spot stored, merged segments. - Hardening from four adversarial reviews: errors logged and stored as kinds never text; per-engine deadlines; converted-length check; bounded queue; client hang-ups are not errors; strict local-alias check. - Dockerfile: static ffmpeg/ffprobe 7.0.2 pinned by sha256 (#34). Built and inspected by hand; CI does not build the image on a PR, so its first CI build is this merge. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set in the deployment. Closes #33, #34, #42, #43, #44. Co-authored-by: Mathias <mthbqv@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
type fakeCompleter struct {
|
||||
replies []string
|
||||
err error
|
||||
systems []string
|
||||
users []string
|
||||
}
|
||||
|
||||
func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) {
|
||||
f.systems = append(f.systems, system)
|
||||
f.users = append(f.users, user)
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
i := len(f.users) - 1
|
||||
if i >= len(f.replies) {
|
||||
i = len(f.replies) - 1
|
||||
}
|
||||
return f.replies[i], nil
|
||||
}
|
||||
|
||||
var req = reconcile.Request{Before: "grannarna ska då", After: "De ska kunna måla", Base: "målade", Alt: "mölade"}
|
||||
|
||||
func TestLLMAdjudicator_SendsTheMeasuredPromptWithTheReadingsInTheRequestedOrder(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
a := reconcile.NewLLMAdjudicator(fc)
|
||||
|
||||
_, err := a.Adjudicate(context.Background(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fc.users, 1)
|
||||
assert.Contains(t, fc.systems[0], "two automatic transcriptions of the same Swedish speech")
|
||||
assert.Contains(t, fc.systems[0], "ANSWER: 1")
|
||||
u := fc.users[0]
|
||||
assert.Contains(t, u, "Text before the spot:\ngrannarna ska då")
|
||||
assert.Contains(t, u, "[[ THE SPOT ]]")
|
||||
assert.Contains(t, u, "Text after the spot:\nDe ska kunna måla")
|
||||
assert.Contains(t, u, "Reading 1: målade\nReading 2: mölade", "base first when AltFirst is false")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_AltFirstSwapsWhichReadingIsNumberOne(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
r := req
|
||||
r.AltFirst = true
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, fc.users[0], "Reading 1: mölade\nReading 2: målade")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
altFirst bool
|
||||
reply string
|
||||
want reconcile.Verdict
|
||||
}{
|
||||
{"1 with base first is base", false, "ANSWER: 1", reconcile.VerdictBase},
|
||||
{"2 with base first is alt", false, "ANSWER: 2", reconcile.VerdictAlt},
|
||||
{"1 with alt first is alt", true, "ANSWER: 1", reconcile.VerdictAlt},
|
||||
{"2 with alt first is base", true, "ANSWER: 2", reconcile.VerdictBase},
|
||||
{"unsure is unsure", false, "ANSWER: UNSURE", reconcile.VerdictUnsure},
|
||||
{"case and trailing reasoning are tolerated", false, "answer: 2\nThe second reading fits the context.", reconcile.VerdictAlt},
|
||||
{"reasoning before the answer line is tolerated", true, "Hmm.\nANSWER: 2", reconcile.VerdictBase},
|
||||
{"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure},
|
||||
{"an empty reply is unsure", false, "", reconcile.VerdictUnsure},
|
||||
{"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure},
|
||||
{"a number that merely starts with 1 is not 1", false, "ANSWER: 12", reconcile.VerdictUnsure},
|
||||
{"a word that merely starts with 2 is not 2", false, "ANSWER: 2nd option", reconcile.VerdictUnsure},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := req
|
||||
r.AltFirst = tt.altFirst
|
||||
|
||||
got, err := reconcile.NewLLMAdjudicator(&fakeCompleter{replies: []string{tt.reply}}).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_AnEmptyReadingIsDescribedNotLeftBlank(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
r := req
|
||||
r.Base = ""
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, fc.users[0], "Reading 1: (nothing: the words are simply absent)")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_ACompleterErrorIsReturnedNotTurnedIntoAVerdict(t *testing.T) {
|
||||
boom := errors.New("gateway down")
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(&fakeCompleter{err: boom}).Adjudicate(context.Background(), req)
|
||||
|
||||
assert.ErrorIs(t, err, boom, "the caller must be able to tell 'the model is down' from 'the model is unsure'")
|
||||
}
|
||||
|
||||
func TestOrderChecked_AsksTwiceWithTheOrderSwappedAndKeepsOnlyAgreement(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
replies []string // for [base-first call, alt-first call]
|
||||
want reconcile.Verdict
|
||||
}{
|
||||
{"both orders pick the base", []string{"ANSWER: 1", "ANSWER: 2"}, reconcile.VerdictBase},
|
||||
{"both orders pick the alternative", []string{"ANSWER: 2", "ANSWER: 1"}, reconcile.VerdictAlt},
|
||||
{"it flips with the order, so it is position bias, not a judgement", []string{"ANSWER: 1", "ANSWER: 1"}, reconcile.VerdictUnsure},
|
||||
{"one order unsure is unsure", []string{"ANSWER: 1", "ANSWER: UNSURE"}, reconcile.VerdictUnsure},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: tt.replies}
|
||||
|
||||
got, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(fc)).Adjudicate(context.Background(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
require.Len(t, fc.users, 2)
|
||||
assert.True(t, strings.Contains(fc.users[0], "Reading 1: målade"), "first call shows the base first")
|
||||
assert.True(t, strings.Contains(fc.users[1], "Reading 1: mölade"), "second call shows the alternative first")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrderChecked_AnErrorOnEitherCallIsAnError(t *testing.T) {
|
||||
_, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(&fakeCompleter{err: errors.New("down")})).Adjudicate(context.Background(), req)
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
Reference in New Issue
Block a user