tapir/internal/reconcile/adjudicate.go

package reconcile

import (
	"context"
	"fmt"
	"regexp"
	"strings"
)

// Verdict is an adjudicator's decision about one dispute. The zero value is
// VerdictUnsure: with no decision the base reading stands and the spot is flagged.
type Verdict int

const (
	VerdictUnsure Verdict = iota
	VerdictBase
	VerdictAlt
)

// Request is one dispute as the adjudicator sees it: the surrounding words, the two
// readings, and which of them is shown as "Reading 1". Alternating AltFirst between
// disputes averages out position bias.
type Request struct {
	Before, After string
	Base, Alt     string
	AltFirst      bool
}

// Adjudicator decides which reading of a dispute the speaker most likely said.
type Adjudicator interface {
	Adjudicate(ctx context.Context, r Request) (Verdict, error)
}

// Completer is the chat surface the LLM adjudicator needs. *llm.Client satisfies it.
type Completer interface {
	Complete(ctx context.Context, system, user string) (string, error)
}

// systemPrompt is the wording that was MEASURED (docs/research/
// local-model-adjudicator-sweep-2026-10-01.md): gemma4-26b 92% and gemma4-31b 97% against
// the speaker's blind answers. Changing it invalidates those figures.
const systemPrompt = "You compare two automatic transcriptions of the same Swedish speech. A single person is dictating project notes " +
	"(finance and payments jargon, with English terms mixed into Swedish). The two transcriptions differ at ONE marked spot. " +
	"Decide which reading is what the speaker most likely actually said, using the surrounding text and ordinary Swedish. " +
	"Reply with exactly one line, either 'ANSWER: 1', 'ANSWER: 2' or 'ANSWER: UNSURE', then one short sentence of reasoning."

var answerRe = regexp.MustCompile(`(?i)ANSWER:\s*(1|2|UNSURE)\b`)

// LLMAdjudicator asks a chat model. It never invents an answer: a reply it cannot read
// is VerdictUnsure, and a transport error is returned as an error so the caller can tell
// "the model is down" from "the model is unsure".
type LLMAdjudicator struct{ c Completer }

// NewLLMAdjudicator wraps a Completer.
func NewLLMAdjudicator(c Completer) *LLMAdjudicator { return &LLMAdjudicator{c: c} }

// Adjudicate implements Adjudicator.
func (a *LLMAdjudicator) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
	first, second := r.Base, r.Alt
	if r.AltFirst {
		first, second = r.Alt, r.Base
	}
	user := fmt.Sprintf("Text before the spot:\n%s\n\n[[ THE SPOT ]]\n\nText after the spot:\n%s\n\nReading 1: %s\nReading 2: %s\n",
		r.Before, r.After, describe(first), describe(second))
	reply, err := a.c.Complete(ctx, systemPrompt, user)
	if err != nil {
		return VerdictUnsure, fmt.Errorf("reconcile: adjudicate: %w", err)
	}
	m := answerRe.FindStringSubmatch(reply)
	if m == nil {
		return VerdictUnsure, nil
	}
	switch strings.ToUpper(m[1]) {
	case "1":
		if r.AltFirst {
			return VerdictAlt, nil
		}
		return VerdictBase, nil
	case "2":
		if r.AltFirst {
			return VerdictBase, nil
		}
		return VerdictAlt, nil
	}
	return VerdictUnsure, nil
}

func describe(reading string) string {
	if strings.TrimSpace(reading) == "" {
		return "(nothing: the words are simply absent)"
	}
	return reading
}

// OrderChecked asks the inner adjudicator twice with the reading order swapped and
// keeps a verdict only when both agree. A model that flips with the order is showing
// position bias, not judgement, so disagreement is VerdictUnsure. It doubles the time.
type OrderChecked struct{ inner Adjudicator }

// NewOrderChecked wraps an adjudicator.
func NewOrderChecked(inner Adjudicator) *OrderChecked { return &OrderChecked{inner: inner} }

// Adjudicate implements Adjudicator.
func (o *OrderChecked) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
	r.AltFirst = false
	v1, err := o.inner.Adjudicate(ctx, r)
	if err != nil {
		return VerdictUnsure, err
	}
	r.AltFirst = true
	v2, err := o.inner.Adjudicate(ctx, r)
	if err != nil {
		return VerdictUnsure, err
	}
	if v1 == v2 {
		return v1, nil
	}
	return VerdictUnsure, nil
}