test(reconcile): a number or word that merely starts with 1 or 2 is not an answer
Mutation-testing the adjudicator showed dropping the word boundary went unnoticed: 'ANSWER: 12' would be read as 1. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -79,6 +79,8 @@ func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *te
|
||||
{"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure},
|
||||
{"an empty reply is unsure", false, "", reconcile.VerdictUnsure},
|
||||
{"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure},
|
||||
{"a number that merely starts with 1 is not 1", false, "ANSWER: 12", reconcile.VerdictUnsure},
|
||||
{"a word that merely starts with 2 is not 2", false, "ANSWER: 2nd option", reconcile.VerdictUnsure},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user