tapir/internal/reconcile/adjudicate_test.go

package reconcile_test

import (
	"context"
	"errors"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"git.d-ma.be/mathias/tapir/internal/reconcile"
)

type fakeCompleter struct {
	replies []string
	err     error
	systems []string
	users   []string
}

func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) {
	f.systems = append(f.systems, system)
	f.users = append(f.users, user)
	if f.err != nil {
		return "", f.err
	}
	i := len(f.users) - 1
	if i >= len(f.replies) {
		i = len(f.replies) - 1
	}
	return f.replies[i], nil
}

var req = reconcile.Request{Before: "grannarna ska då", After: "De ska kunna måla", Base: "målade", Alt: "mölade"}

func TestLLMAdjudicator_SendsTheMeasuredPromptWithTheReadingsInTheRequestedOrder(t *testing.T) {
	fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
	a := reconcile.NewLLMAdjudicator(fc)

	_, err := a.Adjudicate(context.Background(), req)

	require.NoError(t, err)
	require.Len(t, fc.users, 1)
	assert.Contains(t, fc.systems[0], "two automatic transcriptions of the same Swedish speech")
	assert.Contains(t, fc.systems[0], "ANSWER: 1")
	u := fc.users[0]
	assert.Contains(t, u, "Text before the spot:\ngrannarna ska då")
	assert.Contains(t, u, "[[ THE SPOT ]]")
	assert.Contains(t, u, "Text after the spot:\nDe ska kunna måla")
	assert.Contains(t, u, "Reading 1: målade\nReading 2: mölade", "base first when AltFirst is false")
}

func TestLLMAdjudicator_AltFirstSwapsWhichReadingIsNumberOne(t *testing.T) {
	fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
	r := req
	r.AltFirst = true

	_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)

	require.NoError(t, err)
	assert.Contains(t, fc.users[0], "Reading 1: mölade\nReading 2: målade")
}

func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *testing.T) {
	tests := []struct {
		name     string
		altFirst bool
		reply    string
		want     reconcile.Verdict
	}{
		{"1 with base first is base", false, "ANSWER: 1", reconcile.VerdictBase},
		{"2 with base first is alt", false, "ANSWER: 2", reconcile.VerdictAlt},
		{"1 with alt first is alt", true, "ANSWER: 1", reconcile.VerdictAlt},
		{"2 with alt first is base", true, "ANSWER: 2", reconcile.VerdictBase},
		{"unsure is unsure", false, "ANSWER: UNSURE", reconcile.VerdictUnsure},
		{"case and trailing reasoning are tolerated", false, "answer: 2\nThe second reading fits the context.", reconcile.VerdictAlt},
		{"reasoning before the answer line is tolerated", true, "Hmm.\nANSWER: 2", reconcile.VerdictBase},
		{"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure},
		{"an empty reply is unsure", false, "", reconcile.VerdictUnsure},
		{"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure},
		{"a number that merely starts with 1 is not 1", false, "ANSWER: 12", reconcile.VerdictUnsure},
		{"a word that merely starts with 2 is not 2", false, "ANSWER: 2nd option", reconcile.VerdictUnsure},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			r := req
			r.AltFirst = tt.altFirst

			got, err := reconcile.NewLLMAdjudicator(&fakeCompleter{replies: []string{tt.reply}}).Adjudicate(context.Background(), r)

			require.NoError(t, err)
			assert.Equal(t, tt.want, got)
		})
	}
}

func TestLLMAdjudicator_AnEmptyReadingIsDescribedNotLeftBlank(t *testing.T) {
	fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
	r := req
	r.Base = ""

	_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)

	require.NoError(t, err)
	assert.Contains(t, fc.users[0], "Reading 1: (nothing: the words are simply absent)")
}

func TestLLMAdjudicator_ACompleterErrorIsReturnedNotTurnedIntoAVerdict(t *testing.T) {
	boom := errors.New("gateway down")

	_, err := reconcile.NewLLMAdjudicator(&fakeCompleter{err: boom}).Adjudicate(context.Background(), req)

	assert.ErrorIs(t, err, boom, "the caller must be able to tell 'the model is down' from 'the model is unsure'")
}

func TestOrderChecked_AsksTwiceWithTheOrderSwappedAndKeepsOnlyAgreement(t *testing.T) {
	tests := []struct {
		name    string
		replies []string // for [base-first call, alt-first call]
		want    reconcile.Verdict
	}{
		{"both orders pick the base", []string{"ANSWER: 1", "ANSWER: 2"}, reconcile.VerdictBase},
		{"both orders pick the alternative", []string{"ANSWER: 2", "ANSWER: 1"}, reconcile.VerdictAlt},
		{"it flips with the order, so it is position bias, not a judgement", []string{"ANSWER: 1", "ANSWER: 1"}, reconcile.VerdictUnsure},
		{"one order unsure is unsure", []string{"ANSWER: 1", "ANSWER: UNSURE"}, reconcile.VerdictUnsure},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			fc := &fakeCompleter{replies: tt.replies}

			got, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(fc)).Adjudicate(context.Background(), req)

			require.NoError(t, err)
			assert.Equal(t, tt.want, got)
			require.Len(t, fc.users, 2)
			assert.True(t, strings.Contains(fc.users[0], "Reading 1: målade"), "first call shows the base first")
			assert.True(t, strings.Contains(fc.users[1], "Reading 1: mölade"), "second call shows the alternative first")
		})
	}
}

func TestOrderChecked_AnErrorOnEitherCallIsAnError(t *testing.T) {
	_, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(&fakeCompleter{err: errors.New("down")})).Adjudicate(context.Background(), req)

	assert.Error(t, err)
}