diff --git a/internal/reconcile/adjudicate.go b/internal/reconcile/adjudicate.go new file mode 100644 index 0000000..6959a60 --- /dev/null +++ b/internal/reconcile/adjudicate.go @@ -0,0 +1,119 @@ +package reconcile + +import ( + "context" + "fmt" + "regexp" + "strings" +) + +// Verdict is an adjudicator's decision about one dispute. The zero value is +// VerdictUnsure: with no decision the base reading stands and the spot is flagged. +type Verdict int + +const ( + VerdictUnsure Verdict = iota + VerdictBase + VerdictAlt +) + +// Request is one dispute as the adjudicator sees it: the surrounding words, the two +// readings, and which of them is shown as "Reading 1". Alternating AltFirst between +// disputes averages out position bias. +type Request struct { + Before, After string + Base, Alt string + AltFirst bool +} + +// Adjudicator decides which reading of a dispute the speaker most likely said. +type Adjudicator interface { + Adjudicate(ctx context.Context, r Request) (Verdict, error) +} + +// Completer is the chat surface the LLM adjudicator needs. *llm.Client satisfies it. +type Completer interface { + Complete(ctx context.Context, system, user string) (string, error) +} + +// systemPrompt is the wording that was MEASURED (docs/research/ +// local-model-adjudicator-sweep-2026-10-01.md): gemma4-26b 92% and gemma4-31b 97% against +// the speaker's blind answers. Changing it invalidates those figures. +const systemPrompt = "You compare two automatic transcriptions of the same Swedish speech. A single person is dictating project notes " + + "(finance and payments jargon, with English terms mixed into Swedish). The two transcriptions differ at ONE marked spot. " + + "Decide which reading is what the speaker most likely actually said, using the surrounding text and ordinary Swedish. " + + "Reply with exactly one line, either 'ANSWER: 1', 'ANSWER: 2' or 'ANSWER: UNSURE', then one short sentence of reasoning." + +var answerRe = regexp.MustCompile(`(?i)ANSWER:\s*(1|2|UNSURE)\b`) + +// LLMAdjudicator asks a chat model. It never invents an answer: a reply it cannot read +// is VerdictUnsure, and a transport error is returned as an error so the caller can tell +// "the model is down" from "the model is unsure". +type LLMAdjudicator struct{ c Completer } + +// NewLLMAdjudicator wraps a Completer. +func NewLLMAdjudicator(c Completer) *LLMAdjudicator { return &LLMAdjudicator{c: c} } + +// Adjudicate implements Adjudicator. +func (a *LLMAdjudicator) Adjudicate(ctx context.Context, r Request) (Verdict, error) { + first, second := r.Base, r.Alt + if r.AltFirst { + first, second = r.Alt, r.Base + } + user := fmt.Sprintf("Text before the spot:\n%s\n\n[[ THE SPOT ]]\n\nText after the spot:\n%s\n\nReading 1: %s\nReading 2: %s\n", + r.Before, r.After, describe(first), describe(second)) + reply, err := a.c.Complete(ctx, systemPrompt, user) + if err != nil { + return VerdictUnsure, fmt.Errorf("reconcile: adjudicate: %w", err) + } + m := answerRe.FindStringSubmatch(reply) + if m == nil { + return VerdictUnsure, nil + } + switch strings.ToUpper(m[1]) { + case "1": + if r.AltFirst { + return VerdictAlt, nil + } + return VerdictBase, nil + case "2": + if r.AltFirst { + return VerdictBase, nil + } + return VerdictAlt, nil + } + return VerdictUnsure, nil +} + +func describe(reading string) string { + if strings.TrimSpace(reading) == "" { + return "(nothing: the words are simply absent)" + } + return reading +} + +// OrderChecked asks the inner adjudicator twice with the reading order swapped and +// keeps a verdict only when both agree. A model that flips with the order is showing +// position bias, not judgement, so disagreement is VerdictUnsure. It doubles the time. +type OrderChecked struct{ inner Adjudicator } + +// NewOrderChecked wraps an adjudicator. +func NewOrderChecked(inner Adjudicator) *OrderChecked { return &OrderChecked{inner: inner} } + +// Adjudicate implements Adjudicator. +func (o *OrderChecked) Adjudicate(ctx context.Context, r Request) (Verdict, error) { + r.AltFirst = false + v1, err := o.inner.Adjudicate(ctx, r) + if err != nil { + return VerdictUnsure, err + } + r.AltFirst = true + v2, err := o.inner.Adjudicate(ctx, r) + if err != nil { + return VerdictUnsure, err + } + if v1 == v2 { + return v1, nil + } + return VerdictUnsure, nil +}