internal/reconcile/adjudicate_test.go

@@ -0,0 +1,145 @@
+package reconcile_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"git.d-ma.be/mathias/tapir/internal/reconcile"
+)
+
+type fakeCompleter struct {
+	replies []string
+	err     error
+	systems []string
+	users   []string
+}
+
+func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) {
+	f.systems = append(f.systems, system)
+	f.users = append(f.users, user)
+	if f.err != nil {
+		return "", f.err
+	}
+	i := len(f.users) - 1
+	if i >= len(f.replies) {
+		i = len(f.replies) - 1
+	}
+	return f.replies[i], nil
+}
+
+var req = reconcile.Request{Before: "kunderna ska då", After: "De ska kunna betala", Base: "onboardade", Alt: "omvårdade"}
+
+func TestLLMAdjudicator_SendsTheMeasuredPromptWithTheReadingsInTheRequestedOrder(t *testing.T) {
+	fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
+	a := reconcile.NewLLMAdjudicator(fc)
+
+	_, err := a.Adjudicate(context.Background(), req)
+
+	require.NoError(t, err)
+	require.Len(t, fc.users, 1)
+	assert.Contains(t, fc.systems[0], "two automatic transcriptions of the same Swedish speech")
+	assert.Contains(t, fc.systems[0], "ANSWER: 1")
+	u := fc.users[0]
+	assert.Contains(t, u, "Text before the spot:\nkunderna ska då")
+	assert.Contains(t, u, "[[ THE SPOT ]]")
+	assert.Contains(t, u, "Text after the spot:\nDe ska kunna betala")
+	assert.Contains(t, u, "Reading 1: onboardade\nReading 2: omvårdade", "base first when AltFirst is false")
+}
+
+func TestLLMAdjudicator_AltFirstSwapsWhichReadingIsNumberOne(t *testing.T) {
+	fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
+	r := req
+	r.AltFirst = true
+
+	_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
+
+	require.NoError(t, err)
+	assert.Contains(t, fc.users[0], "Reading 1: omvårdade\nReading 2: onboardade")
+}
+
+func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *testing.T) {
+	tests := []struct {
+		name     string
+		altFirst bool
+		reply    string
+		want     reconcile.Verdict
+	}{
+		{"1 with base first is base", false, "ANSWER: 1", reconcile.VerdictBase},
+		{"2 with base first is alt", false, "ANSWER: 2", reconcile.VerdictAlt},
+		{"1 with alt first is alt", true, "ANSWER: 1", reconcile.VerdictAlt},
+		{"2 with alt first is base", true, "ANSWER: 2", reconcile.VerdictBase},
+		{"unsure is unsure", false, "ANSWER: UNSURE", reconcile.VerdictUnsure},
+		{"case and trailing reasoning are tolerated", false, "answer: 2\nThe second reading fits the context.", reconcile.VerdictAlt},
+		{"reasoning before the answer line is tolerated", true, "Hmm.\nANSWER: 2", reconcile.VerdictBase},
+		{"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure},
+		{"an empty reply is unsure", false, "", reconcile.VerdictUnsure},
+		{"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := req
+			r.AltFirst = tt.altFirst
+
+			got, err := reconcile.NewLLMAdjudicator(&fakeCompleter{replies: []string{tt.reply}}).Adjudicate(context.Background(), r)
+
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestLLMAdjudicator_AnEmptyReadingIsDescribedNotLeftBlank(t *testing.T) {
+	fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
+	r := req
+	r.Base = ""
+
+	_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
+
+	require.NoError(t, err)
+	assert.Contains(t, fc.users[0], "Reading 1: (nothing: the words are simply absent)")
+}
+
+func TestLLMAdjudicator_ACompleterErrorIsReturnedNotTurnedIntoAVerdict(t *testing.T) {
+	boom := errors.New("gateway down")
+
+	_, err := reconcile.NewLLMAdjudicator(&fakeCompleter{err: boom}).Adjudicate(context.Background(), req)
+
+	assert.ErrorIs(t, err, boom, "the caller must be able to tell 'the model is down' from 'the model is unsure'")
+}
+
+func TestOrderChecked_AsksTwiceWithTheOrderSwappedAndKeepsOnlyAgreement(t *testing.T) {
+	tests := []struct {
+		name    string
+		replies []string // for [base-first call, alt-first call]
+		want    reconcile.Verdict
+	}{
+		{"both orders pick the base", []string{"ANSWER: 1", "ANSWER: 2"}, reconcile.VerdictBase},
+		{"both orders pick the alternative", []string{"ANSWER: 2", "ANSWER: 1"}, reconcile.VerdictAlt},
+		{"it flips with the order, so it is position bias, not a judgement", []string{"ANSWER: 1", "ANSWER: 1"}, reconcile.VerdictUnsure},
+		{"one order unsure is unsure", []string{"ANSWER: 1", "ANSWER: UNSURE"}, reconcile.VerdictUnsure},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fc := &fakeCompleter{replies: tt.replies}
+
+			got, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(fc)).Adjudicate(context.Background(), req)
+
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+			require.Len(t, fc.users, 2)
+			assert.True(t, strings.Contains(fc.users[0], "Reading 1: onboardade"), "first call shows the base first")
+			assert.True(t, strings.Contains(fc.users[1], "Reading 1: omvårdade"), "second call shows the alternative first")
+		})
+	}
+}
+
+func TestOrderChecked_AnErrorOnEitherCallIsAnError(t *testing.T) {
+	_, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(&fakeCompleter{err: errors.New("down")})).Adjudicate(context.Background(), req)
+
+	assert.Error(t, err)
+}