internal/reconcile/adjudicate.go

@@ -0,0 +1,119 @@
+package reconcile
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// Verdict is an adjudicator's decision about one dispute. The zero value is
+// VerdictUnsure: with no decision the base reading stands and the spot is flagged.
+type Verdict int
+
+const (
+	VerdictUnsure Verdict = iota
+	VerdictBase
+	VerdictAlt
+)
+
+// Request is one dispute as the adjudicator sees it: the surrounding words, the two
+// readings, and which of them is shown as "Reading 1". Alternating AltFirst between
+// disputes averages out position bias.
+type Request struct {
+	Before, After string
+	Base, Alt     string
+	AltFirst      bool
+}
+
+// Adjudicator decides which reading of a dispute the speaker most likely said.
+type Adjudicator interface {
+	Adjudicate(ctx context.Context, r Request) (Verdict, error)
+}
+
+// Completer is the chat surface the LLM adjudicator needs. *llm.Client satisfies it.
+type Completer interface {
+	Complete(ctx context.Context, system, user string) (string, error)
+}
+
+// systemPrompt is the wording that was MEASURED (docs/research/
+// local-model-adjudicator-sweep-2026-10-01.md): gemma4-26b 92% and gemma4-31b 97% against
+// the speaker's blind answers. Changing it invalidates those figures.
+const systemPrompt = "You compare two automatic transcriptions of the same Swedish speech. A single person is dictating project notes " +
+	"(finance and payments jargon, with English terms mixed into Swedish). The two transcriptions differ at ONE marked spot. " +
+	"Decide which reading is what the speaker most likely actually said, using the surrounding text and ordinary Swedish. " +
+	"Reply with exactly one line, either 'ANSWER: 1', 'ANSWER: 2' or 'ANSWER: UNSURE', then one short sentence of reasoning."
+
+var answerRe = regexp.MustCompile(`(?i)ANSWER:\s*(1|2|UNSURE)\b`)
+
+// LLMAdjudicator asks a chat model. It never invents an answer: a reply it cannot read
+// is VerdictUnsure, and a transport error is returned as an error so the caller can tell
+// "the model is down" from "the model is unsure".
+type LLMAdjudicator struct{ c Completer }
+
+// NewLLMAdjudicator wraps a Completer.
+func NewLLMAdjudicator(c Completer) *LLMAdjudicator { return &LLMAdjudicator{c: c} }
+
+// Adjudicate implements Adjudicator.
+func (a *LLMAdjudicator) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
+	first, second := r.Base, r.Alt
+	if r.AltFirst {
+		first, second = r.Alt, r.Base
+	}
+	user := fmt.Sprintf("Text before the spot:\n%s\n\n[[ THE SPOT ]]\n\nText after the spot:\n%s\n\nReading 1: %s\nReading 2: %s\n",
+		r.Before, r.After, describe(first), describe(second))
+	reply, err := a.c.Complete(ctx, systemPrompt, user)
+	if err != nil {
+		return VerdictUnsure, fmt.Errorf("reconcile: adjudicate: %w", err)
+	}
+	m := answerRe.FindStringSubmatch(reply)
+	if m == nil {
+		return VerdictUnsure, nil
+	}
+	switch strings.ToUpper(m[1]) {
+	case "1":
+		if r.AltFirst {
+			return VerdictAlt, nil
+		}
+		return VerdictBase, nil
+	case "2":
+		if r.AltFirst {
+			return VerdictBase, nil
+		}
+		return VerdictAlt, nil
+	}
+	return VerdictUnsure, nil
+}
+
+func describe(reading string) string {
+	if strings.TrimSpace(reading) == "" {
+		return "(nothing: the words are simply absent)"
+	}
+	return reading
+}
+
+// OrderChecked asks the inner adjudicator twice with the reading order swapped and
+// keeps a verdict only when both agree. A model that flips with the order is showing
+// position bias, not judgement, so disagreement is VerdictUnsure. It doubles the time.
+type OrderChecked struct{ inner Adjudicator }
+
+// NewOrderChecked wraps an adjudicator.
+func NewOrderChecked(inner Adjudicator) *OrderChecked { return &OrderChecked{inner: inner} }
+
+// Adjudicate implements Adjudicator.
+func (o *OrderChecked) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
+	r.AltFirst = false
+	v1, err := o.inner.Adjudicate(ctx, r)
+	if err != nil {
+		return VerdictUnsure, err
+	}
+	r.AltFirst = true
+	v2, err := o.inner.Adjudicate(ctx, r)
+	if err != nil {
+		return VerdictUnsure, err
+	}
+	if v1 == v2 {
+		return v1, nil
+	}
+	return VerdictUnsure, nil
+}