package reconcile_test import ( "context" "errors" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "git.d-ma.be/mathias/tapir/internal/reconcile" ) type fakeCompleter struct { replies []string err error systems []string users []string } func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) { f.systems = append(f.systems, system) f.users = append(f.users, user) if f.err != nil { return "", f.err } i := len(f.users) - 1 if i >= len(f.replies) { i = len(f.replies) - 1 } return f.replies[i], nil } var req = reconcile.Request{Before: "grannarna ska då", After: "De ska kunna måla", Base: "målade", Alt: "mölade"} func TestLLMAdjudicator_SendsTheMeasuredPromptWithTheReadingsInTheRequestedOrder(t *testing.T) { fc := &fakeCompleter{replies: []string{"ANSWER: 1"}} a := reconcile.NewLLMAdjudicator(fc) _, err := a.Adjudicate(context.Background(), req) require.NoError(t, err) require.Len(t, fc.users, 1) assert.Contains(t, fc.systems[0], "two automatic transcriptions of the same Swedish speech") assert.Contains(t, fc.systems[0], "ANSWER: 1") u := fc.users[0] assert.Contains(t, u, "Text before the spot:\ngrannarna ska då") assert.Contains(t, u, "[[ THE SPOT ]]") assert.Contains(t, u, "Text after the spot:\nDe ska kunna måla") assert.Contains(t, u, "Reading 1: målade\nReading 2: mölade", "base first when AltFirst is false") } func TestLLMAdjudicator_AltFirstSwapsWhichReadingIsNumberOne(t *testing.T) { fc := &fakeCompleter{replies: []string{"ANSWER: 1"}} r := req r.AltFirst = true _, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r) require.NoError(t, err) assert.Contains(t, fc.users[0], "Reading 1: mölade\nReading 2: målade") } func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *testing.T) { tests := []struct { name string altFirst bool reply string want reconcile.Verdict }{ {"1 with base first is base", false, "ANSWER: 1", reconcile.VerdictBase}, {"2 with base first is alt", false, "ANSWER: 2", reconcile.VerdictAlt}, {"1 with alt first is alt", true, "ANSWER: 1", reconcile.VerdictAlt}, {"2 with alt first is base", true, "ANSWER: 2", reconcile.VerdictBase}, {"unsure is unsure", false, "ANSWER: UNSURE", reconcile.VerdictUnsure}, {"case and trailing reasoning are tolerated", false, "answer: 2\nThe second reading fits the context.", reconcile.VerdictAlt}, {"reasoning before the answer line is tolerated", true, "Hmm.\nANSWER: 2", reconcile.VerdictBase}, {"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure}, {"an empty reply is unsure", false, "", reconcile.VerdictUnsure}, {"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure}, {"a number that merely starts with 1 is not 1", false, "ANSWER: 12", reconcile.VerdictUnsure}, {"a word that merely starts with 2 is not 2", false, "ANSWER: 2nd option", reconcile.VerdictUnsure}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r := req r.AltFirst = tt.altFirst got, err := reconcile.NewLLMAdjudicator(&fakeCompleter{replies: []string{tt.reply}}).Adjudicate(context.Background(), r) require.NoError(t, err) assert.Equal(t, tt.want, got) }) } } func TestLLMAdjudicator_AnEmptyReadingIsDescribedNotLeftBlank(t *testing.T) { fc := &fakeCompleter{replies: []string{"ANSWER: 1"}} r := req r.Base = "" _, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r) require.NoError(t, err) assert.Contains(t, fc.users[0], "Reading 1: (nothing: the words are simply absent)") } func TestLLMAdjudicator_ACompleterErrorIsReturnedNotTurnedIntoAVerdict(t *testing.T) { boom := errors.New("gateway down") _, err := reconcile.NewLLMAdjudicator(&fakeCompleter{err: boom}).Adjudicate(context.Background(), req) assert.ErrorIs(t, err, boom, "the caller must be able to tell 'the model is down' from 'the model is unsure'") } func TestOrderChecked_AsksTwiceWithTheOrderSwappedAndKeepsOnlyAgreement(t *testing.T) { tests := []struct { name string replies []string // for [base-first call, alt-first call] want reconcile.Verdict }{ {"both orders pick the base", []string{"ANSWER: 1", "ANSWER: 2"}, reconcile.VerdictBase}, {"both orders pick the alternative", []string{"ANSWER: 2", "ANSWER: 1"}, reconcile.VerdictAlt}, {"it flips with the order, so it is position bias, not a judgement", []string{"ANSWER: 1", "ANSWER: 1"}, reconcile.VerdictUnsure}, {"one order unsure is unsure", []string{"ANSWER: 1", "ANSWER: UNSURE"}, reconcile.VerdictUnsure}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fc := &fakeCompleter{replies: tt.replies} got, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(fc)).Adjudicate(context.Background(), req) require.NoError(t, err) assert.Equal(t, tt.want, got) require.Len(t, fc.users, 2) assert.True(t, strings.Contains(fc.users[0], "Reading 1: målade"), "first call shows the base first") assert.True(t, strings.Contains(fc.users[1], "Reading 1: mölade"), "second call shows the alternative first") }) } } func TestOrderChecked_AnErrorOnEitherCallIsAnError(t *testing.T) { _, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(&fakeCompleter{err: errors.New("down")})).Adjudicate(context.Background(), req) assert.Error(t, err) }