From 481321a75caa1c2776e9688edb0f9dad75cbb511 Mon Sep 17 00:00:00 2001 From: Mathias Date: Thu, 1 Oct 2026 23:17:42 +0200 Subject: [PATCH] test(reconcile): the adjudicator sends the measured prompt and maps answers back (red) Readings in the requested order; 1/2 mapped to base/alt under both orders; UNSURE, an unparseable or out-of-range reply is unsure rather than a guess; an empty reading is described, not blank; a model error is an error, distinguishable from 'unsure'. The optional order-checked wrapper asks twice with the order swapped and keeps only agreement. Red: Request, Verdict, NewLLMAdjudicator and NewOrderChecked are undefined. Co-Authored-By: Claude Sonnet 5.5 --- internal/reconcile/adjudicate_test.go | 145 ++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 internal/reconcile/adjudicate_test.go diff --git a/internal/reconcile/adjudicate_test.go b/internal/reconcile/adjudicate_test.go new file mode 100644 index 0000000..beaf539 --- /dev/null +++ b/internal/reconcile/adjudicate_test.go @@ -0,0 +1,145 @@ +package reconcile_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "git.d-ma.be/mathias/tapir/internal/reconcile" +) + +type fakeCompleter struct { + replies []string + err error + systems []string + users []string +} + +func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) { + f.systems = append(f.systems, system) + f.users = append(f.users, user) + if f.err != nil { + return "", f.err + } + i := len(f.users) - 1 + if i >= len(f.replies) { + i = len(f.replies) - 1 + } + return f.replies[i], nil +} + +var req = reconcile.Request{Before: "kunderna ska då", After: "De ska kunna betala", Base: "onboardade", Alt: "omvårdade"} + +func TestLLMAdjudicator_SendsTheMeasuredPromptWithTheReadingsInTheRequestedOrder(t *testing.T) { + fc := &fakeCompleter{replies: []string{"ANSWER: 1"}} + a := reconcile.NewLLMAdjudicator(fc) + + _, err := a.Adjudicate(context.Background(), req) + + require.NoError(t, err) + require.Len(t, fc.users, 1) + assert.Contains(t, fc.systems[0], "two automatic transcriptions of the same Swedish speech") + assert.Contains(t, fc.systems[0], "ANSWER: 1") + u := fc.users[0] + assert.Contains(t, u, "Text before the spot:\nkunderna ska då") + assert.Contains(t, u, "[[ THE SPOT ]]") + assert.Contains(t, u, "Text after the spot:\nDe ska kunna betala") + assert.Contains(t, u, "Reading 1: onboardade\nReading 2: omvårdade", "base first when AltFirst is false") +} + +func TestLLMAdjudicator_AltFirstSwapsWhichReadingIsNumberOne(t *testing.T) { + fc := &fakeCompleter{replies: []string{"ANSWER: 1"}} + r := req + r.AltFirst = true + + _, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r) + + require.NoError(t, err) + assert.Contains(t, fc.users[0], "Reading 1: omvårdade\nReading 2: onboardade") +} + +func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *testing.T) { + tests := []struct { + name string + altFirst bool + reply string + want reconcile.Verdict + }{ + {"1 with base first is base", false, "ANSWER: 1", reconcile.VerdictBase}, + {"2 with base first is alt", false, "ANSWER: 2", reconcile.VerdictAlt}, + {"1 with alt first is alt", true, "ANSWER: 1", reconcile.VerdictAlt}, + {"2 with alt first is base", true, "ANSWER: 2", reconcile.VerdictBase}, + {"unsure is unsure", false, "ANSWER: UNSURE", reconcile.VerdictUnsure}, + {"case and trailing reasoning are tolerated", false, "answer: 2\nThe second reading fits the context.", reconcile.VerdictAlt}, + {"reasoning before the answer line is tolerated", true, "Hmm.\nANSWER: 2", reconcile.VerdictBase}, + {"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure}, + {"an empty reply is unsure", false, "", reconcile.VerdictUnsure}, + {"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := req + r.AltFirst = tt.altFirst + + got, err := reconcile.NewLLMAdjudicator(&fakeCompleter{replies: []string{tt.reply}}).Adjudicate(context.Background(), r) + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestLLMAdjudicator_AnEmptyReadingIsDescribedNotLeftBlank(t *testing.T) { + fc := &fakeCompleter{replies: []string{"ANSWER: 1"}} + r := req + r.Base = "" + + _, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r) + + require.NoError(t, err) + assert.Contains(t, fc.users[0], "Reading 1: (nothing: the words are simply absent)") +} + +func TestLLMAdjudicator_ACompleterErrorIsReturnedNotTurnedIntoAVerdict(t *testing.T) { + boom := errors.New("gateway down") + + _, err := reconcile.NewLLMAdjudicator(&fakeCompleter{err: boom}).Adjudicate(context.Background(), req) + + assert.ErrorIs(t, err, boom, "the caller must be able to tell 'the model is down' from 'the model is unsure'") +} + +func TestOrderChecked_AsksTwiceWithTheOrderSwappedAndKeepsOnlyAgreement(t *testing.T) { + tests := []struct { + name string + replies []string // for [base-first call, alt-first call] + want reconcile.Verdict + }{ + {"both orders pick the base", []string{"ANSWER: 1", "ANSWER: 2"}, reconcile.VerdictBase}, + {"both orders pick the alternative", []string{"ANSWER: 2", "ANSWER: 1"}, reconcile.VerdictAlt}, + {"it flips with the order, so it is position bias, not a judgement", []string{"ANSWER: 1", "ANSWER: 1"}, reconcile.VerdictUnsure}, + {"one order unsure is unsure", []string{"ANSWER: 1", "ANSWER: UNSURE"}, reconcile.VerdictUnsure}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fc := &fakeCompleter{replies: tt.replies} + + got, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(fc)).Adjudicate(context.Background(), req) + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + require.Len(t, fc.users, 2) + assert.True(t, strings.Contains(fc.users[0], "Reading 1: onboardade"), "first call shows the base first") + assert.True(t, strings.Contains(fc.users[1], "Reading 1: omvårdade"), "second call shows the alternative first") + }) + } +} + +func TestOrderChecked_AnErrorOnEitherCallIsAnError(t *testing.T) { + _, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(&fakeCompleter{err: errors.New("down")})).Adjudicate(context.Background(), req) + + assert.Error(t, err) +}