Upload a recording (audio, or video with the camera off), get a private Swedish transcript, and chat about it. - ADR-033: RLS-scoped private transcript store; ffprobe/ffmpeg intake; gateway transcription (iguana kb-whisper, berget fallback); in-process single-replica worker; chat over the transcript (local models only). - ADR-034: optional two-engine mode (pianissimo + kb-whisper) reconciled by a LOCAL model; off unless TAPIR_RECONCILE_BASE_MODEL is set. - ADR-035: real progress (phases, running/quiet/lost), phase logging, every judged spot stored, merged segments. - Hardening from four adversarial reviews: errors logged and stored as kinds never text; per-engine deadlines; converted-length check; bounded queue; client hang-ups are not errors; strict local-alias check. - Dockerfile: static ffmpeg/ffprobe 7.0.2 pinned by sha256 (#34). Built and inspected by hand; CI does not build the image on a PR, so its first CI build is this merge. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set in the deployment. Closes #33, #34, #42, #43, #44. Co-authored-by: Mathias <mthbqv@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verdict is an adjudicator's decision about one dispute. The zero value is
|
||||
// VerdictUnsure: with no decision the base reading stands and the spot is flagged.
|
||||
type Verdict int
|
||||
|
||||
const (
|
||||
VerdictUnsure Verdict = iota
|
||||
VerdictBase
|
||||
VerdictAlt
|
||||
)
|
||||
|
||||
// Request is one dispute as the adjudicator sees it: the surrounding words, the two
|
||||
// readings, and which of them is shown as "Reading 1". Alternating AltFirst between
|
||||
// disputes averages out position bias.
|
||||
type Request struct {
|
||||
Before, After string
|
||||
Base, Alt string
|
||||
AltFirst bool
|
||||
}
|
||||
|
||||
// Adjudicator decides which reading of a dispute the speaker most likely said.
|
||||
type Adjudicator interface {
|
||||
Adjudicate(ctx context.Context, r Request) (Verdict, error)
|
||||
}
|
||||
|
||||
// Completer is the chat surface the LLM adjudicator needs. *llm.Client satisfies it.
|
||||
type Completer interface {
|
||||
Complete(ctx context.Context, system, user string) (string, error)
|
||||
}
|
||||
|
||||
// systemPrompt is the wording that was MEASURED (docs/research/
|
||||
// local-model-adjudicator-sweep-2026-10-01.md): gemma4-26b 92% and gemma4-31b 97% against
|
||||
// the speaker's blind answers. Changing it invalidates those figures.
|
||||
const systemPrompt = "You compare two automatic transcriptions of the same Swedish speech. A single person is dictating project notes " +
|
||||
"(finance and payments jargon, with English terms mixed into Swedish). The two transcriptions differ at ONE marked spot. " +
|
||||
"Decide which reading is what the speaker most likely actually said, using the surrounding text and ordinary Swedish. " +
|
||||
"Reply with exactly one line, either 'ANSWER: 1', 'ANSWER: 2' or 'ANSWER: UNSURE', then one short sentence of reasoning."
|
||||
|
||||
var answerRe = regexp.MustCompile(`(?i)ANSWER:\s*(1|2|UNSURE)\b`)
|
||||
|
||||
// LLMAdjudicator asks a chat model. It never invents an answer: a reply it cannot read
|
||||
// is VerdictUnsure, and a transport error is returned as an error so the caller can tell
|
||||
// "the model is down" from "the model is unsure".
|
||||
type LLMAdjudicator struct{ c Completer }
|
||||
|
||||
// NewLLMAdjudicator wraps a Completer.
|
||||
func NewLLMAdjudicator(c Completer) *LLMAdjudicator { return &LLMAdjudicator{c: c} }
|
||||
|
||||
// Adjudicate implements Adjudicator.
|
||||
func (a *LLMAdjudicator) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
|
||||
first, second := r.Base, r.Alt
|
||||
if r.AltFirst {
|
||||
first, second = r.Alt, r.Base
|
||||
}
|
||||
user := fmt.Sprintf("Text before the spot:\n%s\n\n[[ THE SPOT ]]\n\nText after the spot:\n%s\n\nReading 1: %s\nReading 2: %s\n",
|
||||
r.Before, r.After, describe(first), describe(second))
|
||||
reply, err := a.c.Complete(ctx, systemPrompt, user)
|
||||
if err != nil {
|
||||
return VerdictUnsure, fmt.Errorf("reconcile: adjudicate: %w", err)
|
||||
}
|
||||
m := answerRe.FindStringSubmatch(reply)
|
||||
if m == nil {
|
||||
return VerdictUnsure, nil
|
||||
}
|
||||
switch strings.ToUpper(m[1]) {
|
||||
case "1":
|
||||
if r.AltFirst {
|
||||
return VerdictAlt, nil
|
||||
}
|
||||
return VerdictBase, nil
|
||||
case "2":
|
||||
if r.AltFirst {
|
||||
return VerdictBase, nil
|
||||
}
|
||||
return VerdictAlt, nil
|
||||
}
|
||||
return VerdictUnsure, nil
|
||||
}
|
||||
|
||||
func describe(reading string) string {
|
||||
if strings.TrimSpace(reading) == "" {
|
||||
return "(nothing: the words are simply absent)"
|
||||
}
|
||||
return reading
|
||||
}
|
||||
|
||||
// OrderChecked asks the inner adjudicator twice with the reading order swapped and
|
||||
// keeps a verdict only when both agree. A model that flips with the order is showing
|
||||
// position bias, not judgement, so disagreement is VerdictUnsure. It doubles the time.
|
||||
type OrderChecked struct{ inner Adjudicator }
|
||||
|
||||
// NewOrderChecked wraps an adjudicator.
|
||||
func NewOrderChecked(inner Adjudicator) *OrderChecked { return &OrderChecked{inner: inner} }
|
||||
|
||||
// Adjudicate implements Adjudicator.
|
||||
func (o *OrderChecked) Adjudicate(ctx context.Context, r Request) (Verdict, error) {
|
||||
r.AltFirst = false
|
||||
v1, err := o.inner.Adjudicate(ctx, r)
|
||||
if err != nil {
|
||||
return VerdictUnsure, err
|
||||
}
|
||||
r.AltFirst = true
|
||||
v2, err := o.inner.Adjudicate(ctx, r)
|
||||
if err != nil {
|
||||
return VerdictUnsure, err
|
||||
}
|
||||
if v1 == v2 {
|
||||
return v1, nil
|
||||
}
|
||||
return VerdictUnsure, nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
type fakeCompleter struct {
|
||||
replies []string
|
||||
err error
|
||||
systems []string
|
||||
users []string
|
||||
}
|
||||
|
||||
func (f *fakeCompleter) Complete(_ context.Context, system, user string) (string, error) {
|
||||
f.systems = append(f.systems, system)
|
||||
f.users = append(f.users, user)
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
i := len(f.users) - 1
|
||||
if i >= len(f.replies) {
|
||||
i = len(f.replies) - 1
|
||||
}
|
||||
return f.replies[i], nil
|
||||
}
|
||||
|
||||
var req = reconcile.Request{Before: "grannarna ska då", After: "De ska kunna måla", Base: "målade", Alt: "mölade"}
|
||||
|
||||
func TestLLMAdjudicator_SendsTheMeasuredPromptWithTheReadingsInTheRequestedOrder(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
a := reconcile.NewLLMAdjudicator(fc)
|
||||
|
||||
_, err := a.Adjudicate(context.Background(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fc.users, 1)
|
||||
assert.Contains(t, fc.systems[0], "two automatic transcriptions of the same Swedish speech")
|
||||
assert.Contains(t, fc.systems[0], "ANSWER: 1")
|
||||
u := fc.users[0]
|
||||
assert.Contains(t, u, "Text before the spot:\ngrannarna ska då")
|
||||
assert.Contains(t, u, "[[ THE SPOT ]]")
|
||||
assert.Contains(t, u, "Text after the spot:\nDe ska kunna måla")
|
||||
assert.Contains(t, u, "Reading 1: målade\nReading 2: mölade", "base first when AltFirst is false")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_AltFirstSwapsWhichReadingIsNumberOne(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
r := req
|
||||
r.AltFirst = true
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, fc.users[0], "Reading 1: mölade\nReading 2: målade")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_MapsTheAnswerBackToBaseOrAltWhicheverOrderWasShown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
altFirst bool
|
||||
reply string
|
||||
want reconcile.Verdict
|
||||
}{
|
||||
{"1 with base first is base", false, "ANSWER: 1", reconcile.VerdictBase},
|
||||
{"2 with base first is alt", false, "ANSWER: 2", reconcile.VerdictAlt},
|
||||
{"1 with alt first is alt", true, "ANSWER: 1", reconcile.VerdictAlt},
|
||||
{"2 with alt first is base", true, "ANSWER: 2", reconcile.VerdictBase},
|
||||
{"unsure is unsure", false, "ANSWER: UNSURE", reconcile.VerdictUnsure},
|
||||
{"case and trailing reasoning are tolerated", false, "answer: 2\nThe second reading fits the context.", reconcile.VerdictAlt},
|
||||
{"reasoning before the answer line is tolerated", true, "Hmm.\nANSWER: 2", reconcile.VerdictBase},
|
||||
{"no answer line is unsure, not a guess", false, "I think the first one.", reconcile.VerdictUnsure},
|
||||
{"an empty reply is unsure", false, "", reconcile.VerdictUnsure},
|
||||
{"an out of range number is unsure", false, "ANSWER: 3", reconcile.VerdictUnsure},
|
||||
{"a number that merely starts with 1 is not 1", false, "ANSWER: 12", reconcile.VerdictUnsure},
|
||||
{"a word that merely starts with 2 is not 2", false, "ANSWER: 2nd option", reconcile.VerdictUnsure},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := req
|
||||
r.AltFirst = tt.altFirst
|
||||
|
||||
got, err := reconcile.NewLLMAdjudicator(&fakeCompleter{replies: []string{tt.reply}}).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_AnEmptyReadingIsDescribedNotLeftBlank(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: []string{"ANSWER: 1"}}
|
||||
r := req
|
||||
r.Base = ""
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(fc).Adjudicate(context.Background(), r)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, fc.users[0], "Reading 1: (nothing: the words are simply absent)")
|
||||
}
|
||||
|
||||
func TestLLMAdjudicator_ACompleterErrorIsReturnedNotTurnedIntoAVerdict(t *testing.T) {
|
||||
boom := errors.New("gateway down")
|
||||
|
||||
_, err := reconcile.NewLLMAdjudicator(&fakeCompleter{err: boom}).Adjudicate(context.Background(), req)
|
||||
|
||||
assert.ErrorIs(t, err, boom, "the caller must be able to tell 'the model is down' from 'the model is unsure'")
|
||||
}
|
||||
|
||||
func TestOrderChecked_AsksTwiceWithTheOrderSwappedAndKeepsOnlyAgreement(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
replies []string // for [base-first call, alt-first call]
|
||||
want reconcile.Verdict
|
||||
}{
|
||||
{"both orders pick the base", []string{"ANSWER: 1", "ANSWER: 2"}, reconcile.VerdictBase},
|
||||
{"both orders pick the alternative", []string{"ANSWER: 2", "ANSWER: 1"}, reconcile.VerdictAlt},
|
||||
{"it flips with the order, so it is position bias, not a judgement", []string{"ANSWER: 1", "ANSWER: 1"}, reconcile.VerdictUnsure},
|
||||
{"one order unsure is unsure", []string{"ANSWER: 1", "ANSWER: UNSURE"}, reconcile.VerdictUnsure},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fc := &fakeCompleter{replies: tt.replies}
|
||||
|
||||
got, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(fc)).Adjudicate(context.Background(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
require.Len(t, fc.users, 2)
|
||||
assert.True(t, strings.Contains(fc.users[0], "Reading 1: målade"), "first call shows the base first")
|
||||
assert.True(t, strings.Contains(fc.users[1], "Reading 1: mölade"), "second call shows the alternative first")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrderChecked_AnErrorOnEitherCallIsAnError(t *testing.T) {
|
||||
_, err := reconcile.NewOrderChecked(reconcile.NewLLMAdjudicator(&fakeCompleter{err: errors.New("down")})).Adjudicate(context.Background(), req)
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package reconcile
|
||||
|
||||
// OpKind is the kind of one run in an edit script.
|
||||
type OpKind int
|
||||
|
||||
const (
|
||||
// Equal: A[AStart:AEnd] and B[BStart:BEnd] are the same words.
|
||||
Equal OpKind = iota
|
||||
// Replace: different words on both sides.
|
||||
Replace
|
||||
// Delete: words only in A.
|
||||
Delete
|
||||
// Insert: words only in B.
|
||||
Insert
|
||||
)
|
||||
|
||||
// Op is one run of the edit script. The ops of a Diff tile both inputs: each op starts
|
||||
// where the previous one ended, in A and in B.
|
||||
type Op struct {
|
||||
Kind OpKind
|
||||
AStart, AEnd int
|
||||
BStart, BEnd int
|
||||
}
|
||||
|
||||
// maxEdits bounds the Myers search. Memory grows with the square of the edit
|
||||
// distance, so two texts that share almost nothing (one engine produced garbage)
|
||||
// are reported as one Replace instead of spending gigabytes proving it. A real
|
||||
// hour disagrees in roughly 400 places, i.e. about 800 edits, far below this.
|
||||
const maxEdits = 3000
|
||||
|
||||
// Diff returns a minimal edit script from a to b (Myers, O(ND)). Adjacent deletions
|
||||
// and insertions are reported as one Replace.
|
||||
func Diff(a, b []string) []Op {
|
||||
pre := 0
|
||||
for pre < len(a) && pre < len(b) && a[pre] == b[pre] {
|
||||
pre++
|
||||
}
|
||||
suf := 0
|
||||
for suf < len(a)-pre && suf < len(b)-pre && a[len(a)-1-suf] == b[len(b)-1-suf] {
|
||||
suf++
|
||||
}
|
||||
midA, midB := a[pre:len(a)-suf], b[pre:len(b)-suf]
|
||||
|
||||
steps, ok := myers(midA, midB)
|
||||
if !ok {
|
||||
steps = steps[:0]
|
||||
for range midA {
|
||||
steps = append(steps, stepDel)
|
||||
}
|
||||
for range midB {
|
||||
steps = append(steps, stepIns)
|
||||
}
|
||||
}
|
||||
|
||||
var ops []Op
|
||||
ai, bi := 0, 0
|
||||
emit := func(k OpKind, da, db int) {
|
||||
if da == 0 && db == 0 {
|
||||
return
|
||||
}
|
||||
if k == Equal && len(ops) > 0 && ops[len(ops)-1].Kind == Equal {
|
||||
ops[len(ops)-1].AEnd += da
|
||||
ops[len(ops)-1].BEnd += db
|
||||
} else {
|
||||
ops = append(ops, Op{Kind: k, AStart: ai, AEnd: ai + da, BStart: bi, BEnd: bi + db})
|
||||
}
|
||||
ai += da
|
||||
bi += db
|
||||
}
|
||||
emit(Equal, pre, pre)
|
||||
for i := 0; i < len(steps); {
|
||||
if steps[i] == stepEq {
|
||||
j := i
|
||||
for j < len(steps) && steps[j] == stepEq {
|
||||
j++
|
||||
}
|
||||
emit(Equal, j-i, j-i)
|
||||
i = j
|
||||
continue
|
||||
}
|
||||
dels, inss := 0, 0
|
||||
for i < len(steps) && steps[i] != stepEq {
|
||||
if steps[i] == stepDel {
|
||||
dels++
|
||||
} else {
|
||||
inss++
|
||||
}
|
||||
i++
|
||||
}
|
||||
switch {
|
||||
case dels > 0 && inss > 0:
|
||||
emit(Replace, dels, inss)
|
||||
case dels > 0:
|
||||
emit(Delete, dels, 0)
|
||||
default:
|
||||
emit(Insert, 0, inss)
|
||||
}
|
||||
}
|
||||
emit(Equal, suf, suf)
|
||||
return ops
|
||||
}
|
||||
|
||||
type step int8
|
||||
|
||||
const (
|
||||
stepEq step = iota
|
||||
stepDel
|
||||
stepIns
|
||||
)
|
||||
|
||||
// myers returns one step per element of the edit script, in order. ok is false when
|
||||
// the edit distance exceeds maxEdits.
|
||||
func myers(a, b []string) ([]step, bool) {
|
||||
n, m := len(a), len(b)
|
||||
if n == 0 && m == 0 {
|
||||
return nil, true
|
||||
}
|
||||
limit := min(n+m, maxEdits)
|
||||
off := limit + 1
|
||||
v := make([]int32, 2*limit+3)
|
||||
var trace [][]int32 // trace[d] is v as it stood at the start of round d, for k in [-d, d]
|
||||
|
||||
for d := 0; d <= limit; d++ {
|
||||
snap := make([]int32, 2*d+1)
|
||||
for k := -d; k <= d; k++ {
|
||||
snap[k+d] = v[off+k]
|
||||
}
|
||||
trace = append(trace, snap)
|
||||
for k := -d; k <= d; k += 2 {
|
||||
var x int
|
||||
if k == -d || (k != d && v[off+k-1] < v[off+k+1]) {
|
||||
x = int(v[off+k+1])
|
||||
} else {
|
||||
x = int(v[off+k-1]) + 1
|
||||
}
|
||||
y := x - k
|
||||
for x < n && y < m && a[x] == b[y] {
|
||||
x++
|
||||
y++
|
||||
}
|
||||
v[off+k] = int32(x)
|
||||
if x >= n && y >= m {
|
||||
return backtrack(trace, n, m, d), true
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func backtrack(trace [][]int32, n, m, d int) []step {
|
||||
var rev []step
|
||||
x, y := n, m
|
||||
for ; d >= 0; d-- {
|
||||
snap := trace[d]
|
||||
at := func(k int) int { return int(snap[k+d]) }
|
||||
k := x - y
|
||||
var prevK int
|
||||
if k == -d || (k != d && at(k-1) < at(k+1)) {
|
||||
prevK = k + 1
|
||||
} else {
|
||||
prevK = k - 1
|
||||
}
|
||||
prevX := 0
|
||||
if d > 0 {
|
||||
prevX = at(prevK)
|
||||
}
|
||||
prevY := prevX - prevK
|
||||
for x > prevX && y > prevY {
|
||||
rev = append(rev, stepEq)
|
||||
x--
|
||||
y--
|
||||
}
|
||||
if d > 0 {
|
||||
if x == prevX {
|
||||
rev = append(rev, stepIns)
|
||||
} else {
|
||||
rev = append(rev, stepDel)
|
||||
}
|
||||
}
|
||||
x, y = prevX, prevY
|
||||
}
|
||||
out := make([]step, len(rev))
|
||||
for i, s := range rev {
|
||||
out[len(rev)-1-i] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
func words(s string) []string { return strings.Fields(s) }
|
||||
|
||||
// apply rebuilds b from a using the ops, copying equal runs from a and the
|
||||
// replacement/insertion runs from b. If Diff is right this must equal b.
|
||||
func apply(a, b []string, ops []reconcile.Op) []string {
|
||||
var out []string
|
||||
for _, o := range ops {
|
||||
switch o.Kind {
|
||||
case reconcile.Equal:
|
||||
out = append(out, a[o.AStart:o.AEnd]...)
|
||||
default:
|
||||
out = append(out, b[o.BStart:o.BEnd]...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDiff_Cases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a, b string
|
||||
want []reconcile.Op
|
||||
}{
|
||||
{"identical", "a b c", "a b c", []reconcile.Op{{Kind: reconcile.Equal, AStart: 0, AEnd: 3, BStart: 0, BEnd: 3}}},
|
||||
{"both empty", "", "", nil},
|
||||
{"one word replaced", "a b c", "a x c", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Replace, AStart: 1, AEnd: 2, BStart: 1, BEnd: 2},
|
||||
{Kind: reconcile.Equal, AStart: 2, AEnd: 3, BStart: 2, BEnd: 3},
|
||||
}},
|
||||
{"word only in b", "a c", "a b c", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Insert, AStart: 1, AEnd: 1, BStart: 1, BEnd: 2},
|
||||
{Kind: reconcile.Equal, AStart: 1, AEnd: 2, BStart: 2, BEnd: 3},
|
||||
}},
|
||||
{"word only in a", "a b c", "a c", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Delete, AStart: 1, AEnd: 2, BStart: 1, BEnd: 1},
|
||||
{Kind: reconcile.Equal, AStart: 2, AEnd: 3, BStart: 1, BEnd: 2},
|
||||
}},
|
||||
{"nothing in common", "a b", "x y z", []reconcile.Op{
|
||||
{Kind: reconcile.Replace, AStart: 0, AEnd: 2, BStart: 0, BEnd: 3},
|
||||
}},
|
||||
{"a is empty", "", "x y", []reconcile.Op{{Kind: reconcile.Insert, AStart: 0, AEnd: 0, BStart: 0, BEnd: 2}}},
|
||||
{"b is empty", "x y", "", []reconcile.Op{{Kind: reconcile.Delete, AStart: 0, AEnd: 2, BStart: 0, BEnd: 0}}},
|
||||
{"adjacent delete and insert read as one replace", "a b c d", "a x y d", []reconcile.Op{
|
||||
{Kind: reconcile.Equal, AStart: 0, AEnd: 1, BStart: 0, BEnd: 1},
|
||||
{Kind: reconcile.Replace, AStart: 1, AEnd: 3, BStart: 1, BEnd: 3},
|
||||
{Kind: reconcile.Equal, AStart: 3, AEnd: 4, BStart: 3, BEnd: 4},
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, reconcile.Diff(words(tt.a), words(tt.b)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// lcsLen is a plain dynamic-programming reference. Myers must find an edit script
|
||||
// whose equal runs add up to exactly this: a diff that is merely valid but not
|
||||
// minimal would turn real agreement into disputes the adjudicator is then asked about.
|
||||
func lcsLen(a, b []string) int {
|
||||
prev := make([]int, len(b)+1)
|
||||
for i := 1; i <= len(a); i++ {
|
||||
cur := make([]int, len(b)+1)
|
||||
for j := 1; j <= len(b); j++ {
|
||||
if a[i-1] == b[j-1] {
|
||||
cur[j] = prev[j-1] + 1
|
||||
} else {
|
||||
cur[j] = max(prev[j], cur[j-1])
|
||||
}
|
||||
}
|
||||
prev = cur
|
||||
}
|
||||
return prev[len(b)]
|
||||
}
|
||||
|
||||
func TestDiff_RebuildsBAndKeepsEveryCommonWordOnRandomInputs(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(7))
|
||||
alphabet := []string{"a", "b", "c", "d", "e", "f"}
|
||||
gen := func(n int) []string {
|
||||
out := make([]string, n)
|
||||
for i := range out {
|
||||
out[i] = alphabet[rng.Intn(len(alphabet))]
|
||||
}
|
||||
return out
|
||||
}
|
||||
for i := 0; i < 400; i++ {
|
||||
a, b := gen(rng.Intn(25)), gen(rng.Intn(25))
|
||||
|
||||
ops := reconcile.Diff(a, b)
|
||||
|
||||
require.Equal(t, append([]string{}, b...), append([]string{}, apply(a, b, ops)...), "a=%v b=%v ops=%v", a, b, ops)
|
||||
common := 0
|
||||
prevA, prevB := 0, 0
|
||||
for _, o := range ops {
|
||||
require.Equal(t, prevA, o.AStart, "ops must tile a without gaps: %v", ops)
|
||||
require.Equal(t, prevB, o.BStart, "ops must tile b without gaps: %v", ops)
|
||||
prevA, prevB = o.AEnd, o.BEnd
|
||||
if o.Kind == reconcile.Equal {
|
||||
common += o.AEnd - o.AStart
|
||||
require.Equal(t, a[o.AStart:o.AEnd], b[o.BStart:o.BEnd], "an Equal op must really be equal")
|
||||
}
|
||||
}
|
||||
require.Equal(t, len(a), prevA)
|
||||
require.Equal(t, len(b), prevB)
|
||||
require.Equal(t, lcsLen(a, b), common, "the diff must be minimal: a=%v b=%v", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiff_HandlesAnHoursWorthOfWordsFast(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(3))
|
||||
a := make([]string, 6500)
|
||||
for i := range a {
|
||||
a[i] = string(rune('a'+rng.Intn(26))) + string(rune('a'+rng.Intn(26)))
|
||||
}
|
||||
b := append([]string{}, a...)
|
||||
for i := 0; i < 380; i++ { // ~380 disputes, as measured on a real hour
|
||||
b[rng.Intn(len(b))] = "zz" + string(rune('a'+rng.Intn(26)))
|
||||
}
|
||||
|
||||
ops := reconcile.Diff(a, b)
|
||||
|
||||
assert.NotEmpty(t, ops)
|
||||
assert.Equal(t, append([]string{}, b...), append([]string{}, apply(a, b, ops)...))
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package reconcile
|
||||
|
||||
import "strings"
|
||||
|
||||
// Dispute is one place where the two transcripts disagree. BaseStart:BaseEnd and
|
||||
// AltStart:AltEnd are token ranges into the two Tokenize results; an empty range
|
||||
// means that engine heard nothing there. Base and Alt are the two readings as each
|
||||
// engine wrote them, so a replaced span takes the alternative's own spelling.
|
||||
type Dispute struct {
|
||||
BaseStart, BaseEnd int
|
||||
AltStart, AltEnd int
|
||||
Base, Alt string
|
||||
}
|
||||
|
||||
// FindDisputes aligns base against alt word by word and returns the disagreements.
|
||||
// Differences separated by a single shared word are merged into one dispute: for a
|
||||
// reader that is one decision, and asking about "b" and "d" separately in "b c d"
|
||||
// hides that "c" sits between them. Two shared words keep disputes apart.
|
||||
func FindDisputes(base, alt []Token) []Dispute {
|
||||
a, b := keysOf(base), keysOf(alt)
|
||||
ops := Diff(a, b)
|
||||
var out []Dispute
|
||||
for i := 0; i < len(ops); i++ {
|
||||
if ops[i].Kind == Equal {
|
||||
continue
|
||||
}
|
||||
cur := ops[i]
|
||||
for i+2 < len(ops) && ops[i+1].Kind == Equal && ops[i+1].AEnd-ops[i+1].AStart == 1 && ops[i+2].Kind != Equal {
|
||||
cur.AEnd, cur.BEnd = ops[i+2].AEnd, ops[i+2].BEnd
|
||||
i += 2
|
||||
}
|
||||
out = append(out, Dispute{
|
||||
BaseStart: cur.AStart, BaseEnd: cur.AEnd,
|
||||
AltStart: cur.BStart, AltEnd: cur.BEnd,
|
||||
Base: join(base[cur.AStart:cur.AEnd]),
|
||||
Alt: join(alt[cur.BStart:cur.BEnd]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fillers are Swedish discourse particles. A dispute that is only one or two of them
|
||||
// on one side changes no meaning, so it is resolved without asking a model. The list
|
||||
// is deliberately short and leaves out words that can be an answer ("nej") or that
|
||||
// carry content.
|
||||
var fillers = map[string]bool{
|
||||
"ju": true, "då": true, "liksom": true, "alltså": true, "så": true, "ja": true,
|
||||
"eh": true, "ehm": true, "öh": true, "mm": true, "hm": true, "typ": true, "va": true, "asså": true,
|
||||
}
|
||||
|
||||
// maxFillerRun is how many filler words may be waved through together.
|
||||
const maxFillerRun = 2
|
||||
|
||||
// AutoResolve reports whether a dispute needs no model, in which case the base
|
||||
// reading is kept: either the two sides are the same letters once spaces are removed
|
||||
// (a compound written as one word or two), or one side is empty and the other is a
|
||||
// short run of filler words.
|
||||
func AutoResolve(d Dispute) bool {
|
||||
baseKeys, altKeys := keysOfText(d.Base), keysOfText(d.Alt)
|
||||
if strings.Join(baseKeys, "") == strings.Join(altKeys, "") {
|
||||
return true
|
||||
}
|
||||
return (len(baseKeys) == 0 && onlyFillers(altKeys)) || (len(altKeys) == 0 && onlyFillers(baseKeys))
|
||||
}
|
||||
|
||||
func onlyFillers(keys []string) bool {
|
||||
if len(keys) == 0 || len(keys) > maxFillerRun {
|
||||
return false
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !fillers[k] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ContextAround returns up to window words of the base before and after the dispute,
|
||||
// as the base wrote them.
|
||||
func ContextAround(base []Token, d Dispute, window int) (before, after string) {
|
||||
return join(base[max(0, d.BaseStart-window):d.BaseStart]), join(base[d.BaseEnd:min(len(base), d.BaseEnd+window)])
|
||||
}
|
||||
|
||||
func keysOf(ts []Token) []string {
|
||||
out := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = t.Key
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysOfText(s string) []string { return keysOf(Tokenize(s)) }
|
||||
|
||||
func join(ts []Token) string {
|
||||
parts := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
parts[i] = t.Text
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
func disputes(base, alt string) []reconcile.Dispute {
|
||||
return reconcile.FindDisputes(reconcile.Tokenize(base), reconcile.Tokenize(alt))
|
||||
}
|
||||
|
||||
func TestFindDisputes_NoneWhenTheTextsAgreeWordForWord(t *testing.T) {
|
||||
assert.Empty(t, disputes("Vi går igenom läget.", "vi går igenom läget"),
|
||||
"casing and punctuation are not disagreements")
|
||||
}
|
||||
|
||||
func TestFindDisputes_OneReplacedWordIsOneDisputeCarryingBothReadings(t *testing.T) {
|
||||
got := disputes("Vi ska målade grannarna.", "Vi ska mölade grannarna.")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
d := got[0]
|
||||
assert.Equal(t, "målade", d.Base)
|
||||
assert.Equal(t, "mölade", d.Alt)
|
||||
assert.Equal(t, 2, d.BaseStart)
|
||||
assert.Equal(t, 3, d.BaseEnd)
|
||||
assert.Equal(t, 2, d.AltStart)
|
||||
assert.Equal(t, 3, d.AltEnd)
|
||||
}
|
||||
|
||||
func TestFindDisputes_ReadingsKeepTheirOwnSpellingAndPunctuation(t *testing.T) {
|
||||
got := disputes("Det blir A+2, alltså.", "Det blir A 2, alltså.")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "A+2,", got[0].Base)
|
||||
assert.Equal(t, "A 2,", got[0].Alt, "the alternative's words replace the span as the engine wrote them")
|
||||
}
|
||||
|
||||
func TestFindDisputes_AWordOnlyOneEngineHeardHasAnEmptyOtherSide(t *testing.T) {
|
||||
got := disputes("Man får tillbaka ett förslag.", "Man får tillbaka ett coolt förslag.")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "", got[0].Base)
|
||||
assert.Equal(t, "coolt", got[0].Alt)
|
||||
assert.Equal(t, got[0].BaseStart, got[0].BaseEnd, "an insertion has an empty base range at its position")
|
||||
assert.Equal(t, 4, got[0].BaseStart)
|
||||
}
|
||||
|
||||
func TestFindDisputes_TwoDifferencesSeparatedByOneSharedWordAreOneDispute(t *testing.T) {
|
||||
// "x ... y" around a single agreed word is one decision for the reader, not two.
|
||||
got := disputes("a b c d e", "a x c y e")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "b c d", got[0].Base)
|
||||
assert.Equal(t, "x c y", got[0].Alt)
|
||||
}
|
||||
|
||||
func TestFindDisputes_TwoSharedWordsKeepTheDisputesApart(t *testing.T) {
|
||||
got := disputes("a b c d e f", "a x c d y f")
|
||||
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, "b", got[0].Base)
|
||||
assert.Equal(t, "e", got[1].Base)
|
||||
}
|
||||
|
||||
func TestFindDisputes_ChainsOfCloseDifferencesMergeIntoOne(t *testing.T) {
|
||||
got := disputes("a b c d e f g", "a x c y e z g")
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "b c d e f", got[0].Base)
|
||||
}
|
||||
|
||||
func TestFindDisputes_LeavesIndicesThatAddressBothTokenSlices(t *testing.T) {
|
||||
base := reconcile.Tokenize("Hej Anders och Maria")
|
||||
alt := reconcile.Tokenize("Hej Sanders och Maria")
|
||||
|
||||
got := reconcile.FindDisputes(base, alt)
|
||||
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "Anders", base[got[0].BaseStart].Text)
|
||||
assert.Equal(t, "Sanders", alt[got[0].AltStart].Text)
|
||||
}
|
||||
|
||||
func TestAutoResolve(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
base, alt string
|
||||
resolved bool
|
||||
}{
|
||||
{"a compound written as two words or one", "i takrenns delen", "i takrennsdelen", true},
|
||||
{"a compound with a hyphen", "takrenns-delen", "takrenns delen", true},
|
||||
{"a filler only the alternative heard", "vi ska ju betala", "vi ska betala", true},
|
||||
{"a filler only the base heard", "vi ska betala", "vi ska ju betala", true},
|
||||
{"two fillers in a row", "vi ska ja ju betala", "vi ska betala", true},
|
||||
{"a different word is for the model", "vi ska målade", "vi ska mölade", false},
|
||||
{"a different number is for the model", "totalt 1050 kronor", "totalt 2050 kronor", false},
|
||||
{"a content word only one side heard is for the model", "ett förslag", "ett coolt förslag", false},
|
||||
{"three filler words are too many to wave through", "vi ska ju då liksom betala", "vi ska betala", false},
|
||||
{"a filler swapped for a content word is for the model", "vi ska ju betala", "vi ska nu betala", false},
|
||||
{"nej is an answer, not a filler", "men nej det går inte", "men det går inte", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := disputes(tt.base, tt.alt)
|
||||
require.Len(t, got, 1, "the fixture must be exactly one dispute")
|
||||
|
||||
assert.Equal(t, tt.resolved, reconcile.AutoResolve(got[0]))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextAround_TakesWordsFromTheBaseOnBothSidesAndStopsAtTheEdges(t *testing.T) {
|
||||
base := reconcile.Tokenize("ett två tre fyra fem sex sju åtta nio tio")
|
||||
d := reconcile.Dispute{BaseStart: 4, BaseEnd: 6} // "fem sex"
|
||||
|
||||
before, after := reconcile.ContextAround(base, d, 3)
|
||||
|
||||
assert.Equal(t, "två tre fyra", before)
|
||||
assert.Equal(t, "sju åtta nio", after)
|
||||
|
||||
b2, a2 := reconcile.ContextAround(base, reconcile.Dispute{BaseStart: 0, BaseEnd: 1}, 3)
|
||||
assert.Equal(t, "", b2)
|
||||
assert.Equal(t, "två tre fyra", a2)
|
||||
|
||||
b3, a3 := reconcile.ContextAround(base, reconcile.Dispute{BaseStart: 9, BaseEnd: 10}, 3)
|
||||
assert.Equal(t, "sju åtta nio", b3)
|
||||
assert.Equal(t, "", a3)
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
)
|
||||
|
||||
// MaxContested is the default bound on the stored list of judged spots. A 90-minute
|
||||
// recording has a few hundred, so this is a guard against a runaway, not a budget. The
|
||||
// counts on the Reconciliation are always exact; only the list is capped.
|
||||
const MaxContested = 2000
|
||||
|
||||
// beforeWords is how many words of left context a stored spot carries: enough to find
|
||||
// the place again, small enough to stay out of the way.
|
||||
const beforeWords = 8
|
||||
|
||||
// Options configure Reconcile. Zero Window and BreakAfter take their defaults.
|
||||
type Options struct {
|
||||
// Window is how many words of the base to show on each side of a dispute. 100 is
|
||||
// what was measured (docs/research/local-model-adjudicator-sweep-2026-10-01.md).
|
||||
Window int
|
||||
// BreakAfter is how many consecutive model errors stop the reconciliation, so a
|
||||
// model that is down costs three timeouts, not one per dispute.
|
||||
BreakAfter int
|
||||
// MaxContested caps the stored judged spots; zero takes the package default.
|
||||
MaxContested int
|
||||
// BaseEngine, AltEngine and Reconciler are recorded on the result.
|
||||
BaseEngine, AltEngine, Reconciler string
|
||||
}
|
||||
|
||||
// Reconcile merges two transcripts of the same speech. The base text stands except
|
||||
// where the model chooses the alternative's reading of a dispute; trivial disputes
|
||||
// (a compound split, a filler word) are resolved without the model. It never fails a
|
||||
// transcription: if the model cannot be used the base text comes back unreconciled
|
||||
// and the result says why. The only error it returns is a cancelled context, which
|
||||
// is the job stopping, not a verdict on the model.
|
||||
func Reconcile(ctx context.Context, base, alt domain.Transcription, adj Adjudicator, o Options) (domain.Transcription, error) {
|
||||
window, breakAfter, maxSpots := o.Window, o.BreakAfter, o.MaxContested
|
||||
if window <= 0 {
|
||||
window = 100
|
||||
}
|
||||
if breakAfter <= 0 {
|
||||
breakAfter = 3
|
||||
}
|
||||
if maxSpots <= 0 {
|
||||
maxSpots = MaxContested
|
||||
}
|
||||
rep := progress.From(ctx)
|
||||
|
||||
rep.Begin("compare", 0)
|
||||
bt, at := Tokenize(base.Text), Tokenize(alt.Text)
|
||||
ds := FindDisputes(bt, at)
|
||||
rep.Done("compare")
|
||||
|
||||
// The deciding phase counts what the MODEL decides: the trivial disputes are settled
|
||||
// without it, so counting them would leave the bar short of its end.
|
||||
deciding := 0
|
||||
for _, d := range ds {
|
||||
if !AutoResolve(d) {
|
||||
deciding++
|
||||
}
|
||||
}
|
||||
rec := &domain.Reconciliation{
|
||||
Status: domain.ReconcileDone, Base: o.BaseEngine, Alt: o.AltEngine, Reconciler: o.Reconciler, Disputes: len(ds),
|
||||
}
|
||||
seconds := positioner(base, len(bt))
|
||||
spot := func(d Dispute, used string, unsure bool) {
|
||||
before, _ := ContextAround(bt, d, window)
|
||||
if w := strings.Fields(before); len(w) > beforeWords {
|
||||
before = strings.Join(w[len(w)-beforeWords:], " ")
|
||||
}
|
||||
if len(rec.Contested) < maxSpots {
|
||||
rec.Contested = append(rec.Contested, domain.ContestedSpot{
|
||||
Seconds: seconds(d.BaseStart), Base: d.Base, Alt: d.Alt, Used: used, Unsure: unsure, Before: before,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
rep.Begin("decide", deciding)
|
||||
verdicts := make([]Verdict, len(ds))
|
||||
consecutive, handled := 0, 0
|
||||
for i, d := range ds {
|
||||
if AutoResolve(d) {
|
||||
rec.AutoResolved++
|
||||
verdicts[i] = VerdictBase
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Transcription{}, err
|
||||
}
|
||||
before, after := ContextAround(bt, d, window)
|
||||
v, err := adj.Adjudicate(ctx, Request{Before: before, After: after, Base: d.Base, Alt: d.Alt, AltFirst: i%2 == 1})
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return domain.Transcription{}, ctxErr
|
||||
}
|
||||
rec.Errors++
|
||||
consecutive++
|
||||
if consecutive >= breakAfter {
|
||||
rep.Fail("decide", err)
|
||||
return skipped(base, rec, fmt.Sprintf("the model failed %d times in a row (%s)", consecutive, errkind.Of(err))), nil
|
||||
}
|
||||
v = VerdictUnsure
|
||||
} else {
|
||||
consecutive = 0
|
||||
rec.Adjudicated++
|
||||
}
|
||||
verdicts[i] = v
|
||||
switch v {
|
||||
case VerdictAlt:
|
||||
rec.Changed++
|
||||
spot(d, "alt", false)
|
||||
case VerdictUnsure:
|
||||
rec.Unsure++
|
||||
spot(d, "base", true)
|
||||
default:
|
||||
spot(d, "base", false)
|
||||
}
|
||||
handled++
|
||||
rep.Advance("decide", handled)
|
||||
}
|
||||
rep.Done("decide")
|
||||
|
||||
out := base
|
||||
out.Text, out.Segments = merge(base, bt, at, ds, verdicts)
|
||||
out.Reconciliation = rec
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// merge applies the verdicts: the base's words stand except where the alternative's
|
||||
// reading was chosen. It returns the merged text and the base's segments carrying the
|
||||
// merged words (timing is the base's). Segments are rewritten only when their words line
|
||||
// up with the text; a word wrapped across a segment edge makes them differ, and guessing
|
||||
// would put words in the wrong place, so then the base's segments are returned as they
|
||||
// were (ADR-035). The base's own segments are never edited in place.
|
||||
func merge(base domain.Transcription, bt, at []Token, ds []Dispute, verdicts []Verdict) (string, []domain.Segment) {
|
||||
segOf, aligned := segmentIndex(base.Segments, len(bt))
|
||||
var perSeg [][]string
|
||||
var changed map[int]bool
|
||||
if aligned {
|
||||
perSeg, changed = make([][]string, len(base.Segments)), map[int]bool{}
|
||||
}
|
||||
|
||||
var parts []string
|
||||
keep := func(from, to int) {
|
||||
for i := from; i < to; i++ {
|
||||
parts = append(parts, bt[i].Text)
|
||||
if aligned {
|
||||
perSeg[segOf[i]] = append(perSeg[segOf[i]], bt[i].Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
pos := 0
|
||||
for i, d := range ds {
|
||||
keep(pos, d.BaseStart)
|
||||
if verdicts[i] == VerdictAlt {
|
||||
for _, t := range at[d.AltStart:d.AltEnd] {
|
||||
parts = append(parts, t.Text)
|
||||
}
|
||||
if aligned {
|
||||
anchor := min(d.BaseStart, len(bt)-1)
|
||||
for _, t := range at[d.AltStart:d.AltEnd] {
|
||||
perSeg[segOf[anchor]] = append(perSeg[segOf[anchor]], t.Text)
|
||||
}
|
||||
changed[segOf[anchor]] = true
|
||||
for k := d.BaseStart; k < d.BaseEnd; k++ {
|
||||
changed[segOf[k]] = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
keep(d.BaseStart, d.BaseEnd)
|
||||
}
|
||||
pos = d.BaseEnd
|
||||
}
|
||||
keep(pos, len(bt))
|
||||
|
||||
if !aligned {
|
||||
return strings.Join(parts, " "), base.Segments
|
||||
}
|
||||
segs := append([]domain.Segment(nil), base.Segments...)
|
||||
for k := range changed {
|
||||
segs[k].Text = strings.Join(perSeg[k], " ")
|
||||
}
|
||||
return strings.Join(parts, " "), segs
|
||||
}
|
||||
|
||||
// segmentIndex maps each base token to the segment it came from. aligned is false when
|
||||
// there are no segments or their word count is not the text's.
|
||||
func segmentIndex(segs []domain.Segment, tokens int) (segOf []int, aligned bool) {
|
||||
if len(segs) == 0 || tokens == 0 {
|
||||
return nil, false
|
||||
}
|
||||
segOf = make([]int, 0, tokens)
|
||||
for k, s := range segs {
|
||||
for range Tokenize(s.Text) {
|
||||
segOf = append(segOf, k)
|
||||
}
|
||||
}
|
||||
return segOf, len(segOf) == tokens
|
||||
}
|
||||
|
||||
// skipped is the base text, none of the partial changes, and the reason.
|
||||
func skipped(base domain.Transcription, rec *domain.Reconciliation, reason string) domain.Transcription {
|
||||
out := base
|
||||
out.Reconciliation = &domain.Reconciliation{
|
||||
Status: domain.ReconcileSkipped, Reason: reason, Base: rec.Base, Alt: rec.Alt, Reconciler: rec.Reconciler,
|
||||
Disputes: rec.Disputes, AutoResolved: rec.AutoResolved, Errors: rec.Errors,
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// positioner maps a token index of the base to the second it starts at: through the
|
||||
// segments when there are any, proportionally otherwise.
|
||||
func positioner(base domain.Transcription, tokens int) func(int) float64 {
|
||||
if len(base.Segments) == 0 {
|
||||
return func(i int) float64 {
|
||||
if tokens == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(i) / float64(tokens) * base.AudioSeconds
|
||||
}
|
||||
}
|
||||
starts := make([]int, len(base.Segments))
|
||||
n := 0
|
||||
for i, s := range base.Segments {
|
||||
starts[i] = n
|
||||
n += len(Tokenize(s.Text))
|
||||
}
|
||||
return func(i int) float64 {
|
||||
k := 0
|
||||
for k+1 < len(starts) && starts[k+1] <= i {
|
||||
k++
|
||||
}
|
||||
return base.Segments[k].Start
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/progress"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// What Reconcile reports while it works (ADR-035): the comparison, then the deciding
|
||||
// phase with the real number of model decisions as its total.
|
||||
|
||||
type repRec struct {
|
||||
mu sync.Mutex
|
||||
events []string
|
||||
}
|
||||
|
||||
func (r *repRec) add(e string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.events = append(r.events, e)
|
||||
}
|
||||
|
||||
func (r *repRec) Begin(key string, total int) { r.add(fmt.Sprintf("begin:%s:%d", key, total)) }
|
||||
func (r *repRec) Advance(key string, done int) { r.add(fmt.Sprintf("advance:%s:%d", key, done)) }
|
||||
func (r *repRec) Done(key string) { r.add("done:" + key) }
|
||||
func (r *repRec) Fail(key string, _ error) { r.add("fail:" + key) }
|
||||
func (r *repRec) Note(string, ...any) {}
|
||||
func (r *repRec) joined() string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return strings.Join(r.events, ",")
|
||||
}
|
||||
func withRep(r *repRec) context.Context { return progress.With(context.Background(), r) }
|
||||
func reconcileWith(t *testing.T, ctx context.Context, base, alt string, adj reconcile.Adjudicator) error {
|
||||
t.Helper()
|
||||
_, err := reconcile.Reconcile(ctx, tr(base), tr(alt), adj, opts)
|
||||
return err
|
||||
}
|
||||
|
||||
// two compound/filler disputes the model is never asked about, then two it is.
|
||||
const (
|
||||
progBase = "i takrenns delen ska vi ju betala totalt 1015 kronor till Axel idag"
|
||||
progAlt = "i takrennsdelen ska vi betala totalt 1050 kronor till Alex idag"
|
||||
)
|
||||
|
||||
func TestReconcileProgress_TheDecidingPhaseTotalIsTheModelDecisionsNotTheDisputes(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
|
||||
require.NoError(t, reconcileWith(t, withRep(rep), progBase, progAlt, always(reconcile.VerdictAlt)))
|
||||
|
||||
assert.Equal(t,
|
||||
"begin:compare:0,done:compare,begin:decide:2,advance:decide:1,advance:decide:2,done:decide",
|
||||
rep.joined(),
|
||||
"4 disputes, 2 settled without the model: the bar counts the 2 the model decides, so it can reach its end")
|
||||
}
|
||||
|
||||
func TestReconcileProgress_TextsThatAgreeStillClosePhasesSoNothingIsLeftRunning(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
|
||||
require.NoError(t, reconcileWith(t, withRep(rep), "samma text hela vägen", "samma text hela vägen", always(reconcile.VerdictAlt)))
|
||||
|
||||
assert.Equal(t, "begin:compare:0,done:compare,begin:decide:0,done:decide", rep.joined())
|
||||
}
|
||||
|
||||
func TestReconcileProgress_AModelErrorStillCountsAsADecisionTaken(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
adj := &scripted{fn: func(i int, _ reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i == 0 {
|
||||
return reconcile.VerdictBase, errors.New("one timeout")
|
||||
}
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
require.NoError(t, reconcileWith(t, withRep(rep), progBase, progAlt, adj))
|
||||
|
||||
assert.Contains(t, rep.joined(), "advance:decide:1,advance:decide:2", "a spot that errored was still handled; the bar must move past it")
|
||||
}
|
||||
|
||||
func TestReconcileProgress_TheBreakerTrippingShowsTheDecidingPhaseFailedNotDone(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
return reconcile.VerdictBase, errors.New("model down")
|
||||
}}
|
||||
small := opts
|
||||
small.BreakAfter = 2
|
||||
|
||||
_, err := reconcile.Reconcile(withRep(rep), tr(progBase), tr(progAlt), adj, small)
|
||||
|
||||
require.NoError(t, err, "a model that is down degrades the result; it does not fail the job")
|
||||
assert.Contains(t, rep.joined(), "fail:decide")
|
||||
assert.NotContains(t, rep.joined(), "done:decide")
|
||||
}
|
||||
|
||||
func TestReconcileProgress_AShutdownIsNeitherDoneNorFailed(t *testing.T) {
|
||||
rep := &repRec{}
|
||||
ctx, cancel := context.WithCancel(withRep(rep))
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
cancel()
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
err := reconcileWith(t, ctx, progBase, progAlt, adj)
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.NotContains(t, rep.joined(), "fail:decide")
|
||||
assert.NotContains(t, rep.joined(), "done:decide")
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// The reason a reconciliation was skipped is stored and shown on a page about a private
|
||||
// recording, so it carries a kind, never the model's error text, which can echo the
|
||||
// prompt (adversarial review F3).
|
||||
|
||||
func TestReconcile_TheBreakerReasonCarriesAKindNotTheModelsText(t *testing.T) {
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
return reconcile.VerdictBase, errors.New("gateway echoed the prompt: zebrakaka secret words")
|
||||
}}
|
||||
small := opts
|
||||
small.BreakAfter = 2
|
||||
|
||||
out, err := reconcile.Reconcile(context.Background(), tr(progBase), tr(progAlt), adj, small)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, domain.ReconcileSkipped, out.Reconciliation.Status)
|
||||
assert.NotContains(t, out.Reconciliation.Reason, "zebrakaka", "provider text must not be stored or shown")
|
||||
assert.Contains(t, out.Reconciliation.Reason, "2 times in a row", "the reason still says what happened")
|
||||
assert.Contains(t, out.Reconciliation.Reason, "error", "and carries a kind")
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// Every judged spot is stored with a few words of left context, and the saved segments
|
||||
// carry the merged words while keeping the base's timing (ADR-035 decisions 6 and 7).
|
||||
|
||||
func TestReconcileSpots_AKeptSpotAndAChangedSpotAreBothStoredWithTheirVerdict(t *testing.T) {
|
||||
adj := &scripted{fn: func(i int, _ reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i == 0 {
|
||||
return reconcile.VerdictBase, nil
|
||||
}
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
out := merge(t, tr(progBase), tr(progAlt), adj)
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 2)
|
||||
assert.Equal(t, "base", out.Reconciliation.Contested[0].Used)
|
||||
assert.Equal(t, "1015", out.Reconciliation.Contested[0].Base)
|
||||
assert.Equal(t, "alt", out.Reconciliation.Contested[1].Used)
|
||||
assert.Equal(t, 1, out.Reconciliation.Changed, "the exact counts are unchanged by storing more")
|
||||
}
|
||||
|
||||
func TestReconcileSpots_AutoResolvedDisputesAreNotStored(t *testing.T) {
|
||||
out := merge(t, tr("i takrenns delen ska vi ju betala"), tr("i takrennsdelen ska vi betala"), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Empty(t, out.Reconciliation.Contested, "compounds and fillers were never judged; they are counted, not listed")
|
||||
}
|
||||
|
||||
func TestReconcileSpots_ASpotCarriesTheWordsJustBeforeIt(t *testing.T) {
|
||||
out := merge(t, tr("grannen målar ett vanligt staket mot gatan"), tr("grannen målar ett KORT staket mot gatan"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.Equal(t, "grannen målar ett", out.Reconciliation.Contested[0].Before)
|
||||
}
|
||||
|
||||
func TestReconcileSpots_TheContextIsAFewWordsNotTheWholeMeeting(t *testing.T) {
|
||||
long := "ett två tre fyra fem sex sju åtta nio tio elva tolv"
|
||||
out := merge(t, tr(long+" X slut"), tr(long+" Y slut"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.Equal(t, "fem sex sju åtta nio tio elva tolv", out.Reconciliation.Contested[0].Before,
|
||||
"eight words: enough to find the place, small enough to stay out of the way")
|
||||
}
|
||||
|
||||
func segmented(text string, segs ...domain.Segment) domain.Transcription {
|
||||
t := tr(text)
|
||||
t.Segments = segs
|
||||
return t
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AChangedWordReplacesTheBaseWordInItsOwnSegmentOnly(t *testing.T) {
|
||||
base := segmented("ett två tre. fyra fem sex. sju åtta nio.",
|
||||
domain.Segment{Start: 0, End: 10, Text: "ett två tre."},
|
||||
domain.Segment{Start: 10, End: 20, Text: "fyra fem sex."},
|
||||
domain.Segment{Start: 20, End: 30, Text: "sju åtta nio."})
|
||||
|
||||
out := merge(t, base, tr("ett två tre. fyra femm sex. sju åtta nio."), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Segments, 3)
|
||||
assert.Equal(t, "ett två tre.", out.Segments[0].Text, "an untouched segment keeps its exact text")
|
||||
assert.Equal(t, "fyra femm sex.", out.Segments[1].Text)
|
||||
assert.Equal(t, "sju åtta nio.", out.Segments[2].Text)
|
||||
assert.InDelta(t, 10.0, out.Segments[1].Start, 1e-9, "timing stays the base's")
|
||||
assert.InDelta(t, 20.0, out.Segments[1].End, 1e-9)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AWordOnlyTheAlternativeHeardLandsInTheSegmentItBelongsTo(t *testing.T) {
|
||||
base := segmented("a b c. d e f.", domain.Segment{Start: 0, End: 5, Text: "a b c."}, domain.Segment{Start: 5, End: 10, Text: "d e f."})
|
||||
|
||||
out := merge(t, base, tr("a b c. d NY e f."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "a b c.", out.Segments[0].Text)
|
||||
assert.Equal(t, "d NY e f.", out.Segments[1].Text)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AWordOnlyTheBaseHeardIsRemovedFromItsSegment(t *testing.T) {
|
||||
base := segmented("a b c. d X e f.", domain.Segment{Start: 0, End: 5, Text: "a b c."}, domain.Segment{Start: 5, End: 10, Text: "d X e f."})
|
||||
|
||||
out := merge(t, base, tr("a b c. d e f."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "d e f.", out.Segments[1].Text)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_AKeptBaseLeavesEverySegmentAsItWas(t *testing.T) {
|
||||
base := segmented("a x b", domain.Segment{Start: 0, End: 4, Text: "a x b"})
|
||||
|
||||
out := merge(t, base, tr("a y b"), always(reconcile.VerdictBase))
|
||||
|
||||
assert.Equal(t, base.Segments, out.Segments)
|
||||
}
|
||||
|
||||
func TestReconcileSegments_TheCallersSegmentsAreNeverChangedInPlace(t *testing.T) {
|
||||
segs := []domain.Segment{{Start: 0, End: 4, Text: "a x b"}}
|
||||
base := segmented("a x b", segs...)
|
||||
|
||||
_ = merge(t, base, tr("a y b"), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "a x b", segs[0].Text, "Reconcile builds new segments; it does not edit the base transcription's")
|
||||
}
|
||||
|
||||
func TestReconcileSegments_SegmentsThatDoNotLineUpWithTheTextAreLeftAsTheBases(t *testing.T) {
|
||||
// A word wrapped across a segment edge makes the segments' word count differ from
|
||||
// the joined text's. Guessing would put words in the wrong place, so leave them.
|
||||
base := segmented("a x b c", domain.Segment{Start: 0, End: 4, Text: "a x"}, domain.Segment{Start: 4, End: 8, Text: "b c d"})
|
||||
|
||||
out := merge(t, base, tr("a y b c"), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, base.Segments, out.Segments)
|
||||
assert.Equal(t, "a y b c", out.Text, "the merged text is unaffected")
|
||||
}
|
||||
|
||||
func TestReconcileSegments_ASpanThatCrossesASegmentEdgeIsRemovedFromBothSegments(t *testing.T) {
|
||||
base := segmented("a b c d", domain.Segment{Start: 0, End: 5, Text: "a b"}, domain.Segment{Start: 5, End: 10, Text: "c d"})
|
||||
|
||||
out := merge(t, base, tr("a X d"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Equal(t, "a X d", out.Text)
|
||||
assert.Equal(t, "a X", out.Segments[0].Text)
|
||||
assert.Equal(t, "d", out.Segments[1].Text, "'c' was replaced too: it must not linger in the second segment")
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
// scripted is an Adjudicator that answers from a function and records every request.
|
||||
type scripted struct {
|
||||
fn func(i int, r reconcile.Request) (reconcile.Verdict, error)
|
||||
calls []reconcile.Request
|
||||
}
|
||||
|
||||
func (s *scripted) Adjudicate(_ context.Context, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
i := len(s.calls)
|
||||
s.calls = append(s.calls, r)
|
||||
return s.fn(i, r)
|
||||
}
|
||||
|
||||
func always(v reconcile.Verdict) *scripted {
|
||||
return &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return v, nil }}
|
||||
}
|
||||
|
||||
func tr(text string) domain.Transcription {
|
||||
return domain.Transcription{Text: text, AudioSeconds: 100, Endpoint: "e"}
|
||||
}
|
||||
|
||||
var opts = reconcile.Options{Window: 100, BreakAfter: 3, Reconciler: "iguana/gemma4-26b", BaseEngine: "iguana/pianissimo", AltEngine: "iguana/kb-whisper"}
|
||||
|
||||
func merge(t *testing.T, base, alt domain.Transcription, adj reconcile.Adjudicator) domain.Transcription {
|
||||
t.Helper()
|
||||
out, err := reconcile.Reconcile(context.Background(), base, alt, adj, opts)
|
||||
require.NoError(t, err)
|
||||
return out
|
||||
}
|
||||
|
||||
func TestReconcile_IdenticalTextsNeedNoModelAndComeBackUnchanged(t *testing.T) {
|
||||
adj := always(reconcile.VerdictAlt)
|
||||
|
||||
out := merge(t, tr("Vi går igenom läget."), tr("vi går igenom läget"), adj)
|
||||
|
||||
assert.Equal(t, "Vi går igenom läget.", out.Text, "the base's own spelling and punctuation stand")
|
||||
assert.Empty(t, adj.calls)
|
||||
require.NotNil(t, out.Reconciliation)
|
||||
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
||||
assert.Zero(t, out.Reconciliation.Disputes)
|
||||
}
|
||||
|
||||
func TestReconcile_TrivialDisputesAreResolvedWithoutAskingTheModel(t *testing.T) {
|
||||
adj := always(reconcile.VerdictAlt)
|
||||
|
||||
out := merge(t, tr("i takrenns delen ska vi ju betala"), tr("i takrennsdelen ska vi betala"), adj)
|
||||
|
||||
assert.Empty(t, adj.calls, "a compound split and a filler particle are not worth a model call")
|
||||
assert.Equal(t, "i takrenns delen ska vi ju betala", out.Text, "the base reading is kept")
|
||||
assert.Equal(t, 2, out.Reconciliation.Disputes)
|
||||
assert.Equal(t, 2, out.Reconciliation.AutoResolved)
|
||||
assert.Zero(t, out.Reconciliation.Adjudicated)
|
||||
}
|
||||
|
||||
func TestReconcile_AltVerdictReplacesTheSpanWithTheAlternativesOwnWords(t *testing.T) {
|
||||
out := merge(t, tr("Totalt 1015 kronor. Det ska betalas."), tr("Totalt 1050 kronor. Det ska betalas."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "Totalt 1050 kronor. Det ska betalas.", out.Text)
|
||||
assert.Equal(t, 1, out.Reconciliation.Changed)
|
||||
assert.Equal(t, 1, out.Reconciliation.Adjudicated)
|
||||
}
|
||||
|
||||
func TestReconcile_BaseAndUnsureVerdictsKeepTheBaseButOnlyUnsureIsFlagged(t *testing.T) {
|
||||
base := tr("Totalt 1015 kronor till Axel.")
|
||||
alt := tr("Totalt 1050 kronor till Alex.")
|
||||
|
||||
keptBase := merge(t, base, alt, always(reconcile.VerdictBase))
|
||||
unsure := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
||||
|
||||
assert.Equal(t, "Totalt 1015 kronor till Axel.", keptBase.Text)
|
||||
assert.Zero(t, keptBase.Reconciliation.Unsure)
|
||||
require.Len(t, keptBase.Reconciliation.Contested, 2,
|
||||
"every judged spot is stored, kept ones too: without them the reconciler cannot be compared with either engine alone (ADR-035)")
|
||||
for _, c := range keptBase.Reconciliation.Contested {
|
||||
assert.Equal(t, "base", c.Used)
|
||||
assert.False(t, c.Unsure, "a confident 'the base is right' is stored but is not an uncertain spot")
|
||||
}
|
||||
assert.Equal(t, "Totalt 1015 kronor till Axel.", unsure.Text, "unsure keeps the base")
|
||||
assert.Equal(t, 2, unsure.Reconciliation.Unsure)
|
||||
require.Len(t, unsure.Reconciliation.Contested, 2)
|
||||
assert.True(t, unsure.Reconciliation.Contested[0].Unsure)
|
||||
assert.Equal(t, "base", unsure.Reconciliation.Contested[0].Used)
|
||||
}
|
||||
|
||||
func TestReconcile_ChangedSpotsAreRecordedWithBothReadings(t *testing.T) {
|
||||
out := merge(t, tr("det blir A+2 imorgon"), tr("det blir en A imorgon"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
c := out.Reconciliation.Contested[0]
|
||||
assert.Equal(t, "A+2", c.Base)
|
||||
assert.Equal(t, "en A", c.Alt)
|
||||
assert.Equal(t, "alt", c.Used)
|
||||
assert.False(t, c.Unsure)
|
||||
}
|
||||
|
||||
func TestReconcile_AWordOnlyTheAlternativeHeardIsInsertedWhereItBelongs(t *testing.T) {
|
||||
out := merge(t, tr("Man får tillbaka ett förslag."), tr("Man får tillbaka ett coolt förslag."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "Man får tillbaka ett coolt förslag.", out.Text)
|
||||
}
|
||||
|
||||
func TestReconcile_AWordOnlyTheBaseHeardCanBeRemoved(t *testing.T) {
|
||||
out := merge(t, tr("Man får tillbaka ett coolt förslag."), tr("Man får tillbaka ett förslag."), always(reconcile.VerdictAlt))
|
||||
|
||||
assert.Equal(t, "Man får tillbaka ett förslag.", out.Text)
|
||||
}
|
||||
|
||||
func TestReconcile_SeveralDisputesAreAppliedIndependentlyAndInOrder(t *testing.T) {
|
||||
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
if r.Alt == "mölade" || r.Alt == "Alex" {
|
||||
return reconcile.VerdictAlt, nil
|
||||
}
|
||||
return reconcile.VerdictBase, nil
|
||||
}}
|
||||
|
||||
out := merge(t, tr("kunderna målade nu och sen betalas till Axel via ombud med 1050 kronor"),
|
||||
tr("kunderna mölade nu och sen betalas till Alex via ombud med 2050 kronor"), adj)
|
||||
|
||||
assert.Equal(t, "kunderna mölade nu och sen betalas till Alex via ombud med 1050 kronor", out.Text)
|
||||
assert.Equal(t, 3, out.Reconciliation.Disputes)
|
||||
assert.Equal(t, 2, out.Reconciliation.Changed)
|
||||
}
|
||||
|
||||
func TestReconcile_TheReadingOrderAlternatesFromOneDisputeToTheNext(t *testing.T) {
|
||||
adj := always(reconcile.VerdictBase)
|
||||
|
||||
merge(t, tr("a x s1 s2 y t1 t2 z d"), tr("a p s1 s2 q t1 t2 r d"), adj)
|
||||
|
||||
require.Len(t, adj.calls, 3)
|
||||
assert.False(t, adj.calls[0].AltFirst)
|
||||
assert.True(t, adj.calls[1].AltFirst)
|
||||
assert.False(t, adj.calls[2].AltFirst, "alternating averages out position bias across a transcript")
|
||||
}
|
||||
|
||||
func TestReconcile_TheModelGetsTheContextWindowFromTheBase(t *testing.T) {
|
||||
adj := always(reconcile.VerdictBase)
|
||||
o := opts
|
||||
o.Window = 2
|
||||
|
||||
_, err := reconcile.Reconcile(context.Background(), tr("ett två tre fyra fem sex sju åtta nio"), tr("ett två tre fyra FEMM sex sju åtta nio"), adj, o)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, adj.calls, 1)
|
||||
assert.Equal(t, "tre fyra", adj.calls[0].Before)
|
||||
assert.Equal(t, "sex sju", adj.calls[0].After)
|
||||
assert.Equal(t, "fem", adj.calls[0].Base)
|
||||
assert.Equal(t, "FEMM", adj.calls[0].Alt)
|
||||
}
|
||||
|
||||
func TestReconcile_OneModelErrorLeavesThatSpotUnsureAndCarriesOn(t *testing.T) {
|
||||
boom := errors.New("timeout")
|
||||
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i == 0 {
|
||||
return reconcile.VerdictUnsure, boom
|
||||
}
|
||||
return reconcile.VerdictAlt, nil
|
||||
}}
|
||||
|
||||
out := merge(t, tr("a x s1 s2 y c"), tr("a p s1 s2 q c"), adj)
|
||||
|
||||
assert.Equal(t, "a x s1 s2 q c", out.Text, "the first spot stayed as the base, the second was decided")
|
||||
assert.Equal(t, 1, out.Reconciliation.Errors)
|
||||
assert.Equal(t, 1, out.Reconciliation.Unsure)
|
||||
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
||||
}
|
||||
|
||||
func TestReconcile_ConsecutiveModelErrorsStopTheWholeReconciliationInsteadOfWaitingOutEveryTimeout(t *testing.T) {
|
||||
boom := errors.New("gateway down")
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) { return reconcile.VerdictUnsure, boom }}
|
||||
a, b := make([]string, 0, 20), make([]string, 0, 20)
|
||||
for i := 0; i < 10; i++ {
|
||||
a = append(a, fmt.Sprintf("w%d", i), "s", "t")
|
||||
b = append(b, fmt.Sprintf("v%d", i), "s", "t")
|
||||
}
|
||||
|
||||
out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)
|
||||
|
||||
assert.Len(t, adj.calls, 3, "three consecutive failures and it stops asking")
|
||||
assert.Equal(t, strings.Join(a, " "), out.Text, "the base text, with none of the partial changes")
|
||||
assert.Equal(t, domain.ReconcileSkipped, out.Reconciliation.Status)
|
||||
assert.Contains(t, out.Reconciliation.Reason, "3 times in a row")
|
||||
assert.NotContains(t, out.Reconciliation.Reason, "gateway down", "the reason is shown on the page: a kind, not the model's own text")
|
||||
}
|
||||
|
||||
func TestReconcile_ASuccessBetweenFailuresResetsTheBreaker(t *testing.T) {
|
||||
boom := errors.New("blip")
|
||||
adj := &scripted{fn: func(i int, r reconcile.Request) (reconcile.Verdict, error) {
|
||||
if i%3 == 2 {
|
||||
return reconcile.VerdictBase, nil
|
||||
}
|
||||
return reconcile.VerdictUnsure, boom
|
||||
}}
|
||||
a, b := make([]string, 0, 20), make([]string, 0, 20)
|
||||
for i := 0; i < 9; i++ {
|
||||
a = append(a, fmt.Sprintf("w%d", i), "s", "t")
|
||||
b = append(b, fmt.Sprintf("v%d", i), "s", "t")
|
||||
}
|
||||
|
||||
out := merge(t, tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), adj)
|
||||
|
||||
assert.Len(t, adj.calls, 9, "two failures then a success never reach three in a row")
|
||||
assert.Equal(t, domain.ReconcileDone, out.Reconciliation.Status)
|
||||
assert.Equal(t, 6, out.Reconciliation.Errors)
|
||||
}
|
||||
|
||||
func TestReconcile_ACancelledJobIsAnErrorNotASkippedReconciliation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
cancel()
|
||||
return reconcile.VerdictUnsure, ctx.Err()
|
||||
}}
|
||||
|
||||
_, err := reconcile.Reconcile(ctx, tr("a x b"), tr("a y b"), adj, opts)
|
||||
|
||||
assert.ErrorIs(t, err, context.Canceled, "a shutdown must not be recorded as 'the model was unavailable'")
|
||||
}
|
||||
|
||||
func TestReconcile_KeepsTheBaseTimingAndDurationAndNamesTheEngines(t *testing.T) {
|
||||
base := tr("a x b")
|
||||
base.Segments = []domain.Segment{{Start: 0, End: 4, Text: "a x b"}}
|
||||
base.AudioSeconds = 3821.5
|
||||
|
||||
out := merge(t, base, tr("a y b"), always(reconcile.VerdictAlt))
|
||||
|
||||
require.Len(t, out.Segments, 1)
|
||||
assert.InDelta(t, 0.0, out.Segments[0].Start, 1e-9, "timing is the base engine's")
|
||||
assert.InDelta(t, 4.0, out.Segments[0].End, 1e-9)
|
||||
assert.InDelta(t, 3821.5, out.AudioSeconds, 1e-9)
|
||||
assert.Equal(t, "iguana/pianissimo", out.Reconciliation.Base)
|
||||
assert.Equal(t, "iguana/kb-whisper", out.Reconciliation.Alt)
|
||||
assert.Equal(t, "iguana/gemma4-26b", out.Reconciliation.Reconciler)
|
||||
}
|
||||
|
||||
func TestReconcile_ASpotCarriesTheSecondItStartsAt(t *testing.T) {
|
||||
base := tr("ett två tre. fyra fem sex. sju åtta nio.")
|
||||
base.Segments = []domain.Segment{
|
||||
{Start: 0, End: 10, Text: "ett två tre."},
|
||||
{Start: 10, End: 20, Text: "fyra fem sex."},
|
||||
{Start: 20, End: 30, Text: "sju åtta nio."},
|
||||
}
|
||||
alt := tr("ett två tre. fyra femm sex. sju åtta nio.")
|
||||
|
||||
out := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.InDelta(t, 10.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "'fem' is in the second segment, which starts at 10 s")
|
||||
}
|
||||
|
||||
func TestReconcile_WithNoSegmentsASpotIsPlacedProportionally(t *testing.T) {
|
||||
base := tr("a b c d e f g h i j")
|
||||
base.AudioSeconds = 100
|
||||
alt := tr("a b c d e f g h i J2")
|
||||
|
||||
out := merge(t, base, alt, always(reconcile.VerdictUnsure))
|
||||
|
||||
require.Len(t, out.Reconciliation.Contested, 1)
|
||||
assert.InDelta(t, 90.0, out.Reconciliation.Contested[0].Seconds, 1e-9, "token 9 of 10 in 100 s")
|
||||
}
|
||||
|
||||
func TestReconcile_JudgedSpotsAreCappedButTheCountsAreExact(t *testing.T) {
|
||||
a, b := make([]string, 0, 600), make([]string, 0, 600)
|
||||
for i := 0; i < 300; i++ {
|
||||
a = append(a, fmt.Sprintf("w%d", i), "same", "same2")
|
||||
b = append(b, fmt.Sprintf("v%d", i), "same", "same2")
|
||||
}
|
||||
capped := opts
|
||||
capped.MaxContested = 50
|
||||
|
||||
out, err := reconcile.Reconcile(context.Background(), tr(strings.Join(a, " ")), tr(strings.Join(b, " ")), always(reconcile.VerdictUnsure), capped)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 300, out.Reconciliation.Unsure)
|
||||
assert.Equal(t, 300, out.Reconciliation.Disputes)
|
||||
assert.Len(t, out.Reconciliation.Contested, 50, "the stored list is bounded; the counts are not")
|
||||
}
|
||||
|
||||
func TestReconcile_TheDefaultCapIsAFewThousandSoAWholeMeetingIsKept(t *testing.T) {
|
||||
assert.Equal(t, 2000, reconcile.MaxContested, "a 90-minute recording has a few hundred judged spots; the cap is a guard, not a budget")
|
||||
}
|
||||
|
||||
func TestReconcile_AVerdictReturnedAlongsideAnErrorIsIgnored(t *testing.T) {
|
||||
// An adjudicator that returns a verdict AND an error must not have the verdict used:
|
||||
// the contract is that an error means no decision.
|
||||
adj := &scripted{fn: func(int, reconcile.Request) (reconcile.Verdict, error) {
|
||||
return reconcile.VerdictAlt, errors.New("half a reply")
|
||||
}}
|
||||
|
||||
out := merge(t, tr("a x b"), tr("a y b"), adj)
|
||||
|
||||
assert.Equal(t, "a x b", out.Text, "no decision means the base stands")
|
||||
assert.Zero(t, out.Reconciliation.Changed)
|
||||
assert.Equal(t, 1, out.Reconciliation.Unsure)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// Package reconcile merges two transcripts of the same speech: it finds where they
|
||||
// disagree word by word, resolves the trivial disagreements itself, and asks a LOCAL
|
||||
// model to choose between the two readings of the rest (ADR-034). It depends on
|
||||
// nothing outside the standard library and the domain; the model, the engines and
|
||||
// the store are wired in from outside.
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Token is one word of a transcript. Text is the word as written, with its casing
|
||||
// and punctuation, so the merged transcript reads naturally; Key is what two engines
|
||||
// are compared on: lower case, letters and digits only.
|
||||
type Token struct {
|
||||
Text string
|
||||
Key string
|
||||
}
|
||||
|
||||
// Tokenize splits s on whitespace. A "word" that has no letter or digit (a dash, a
|
||||
// lone full stop) is not something two engines can disagree about, so it is joined
|
||||
// to the neighbouring word instead of becoming a token of its own.
|
||||
func Tokenize(s string) []Token {
|
||||
var out []Token
|
||||
pending := ""
|
||||
for _, w := range strings.Fields(s) {
|
||||
k := key(w)
|
||||
if k == "" {
|
||||
if len(out) > 0 {
|
||||
out[len(out)-1].Text += " " + w
|
||||
} else if pending == "" {
|
||||
pending = w
|
||||
} else {
|
||||
pending += " " + w
|
||||
}
|
||||
continue
|
||||
}
|
||||
text := w
|
||||
if pending != "" {
|
||||
text = pending + " " + w
|
||||
pending = ""
|
||||
}
|
||||
out = append(out, Token{Text: text, Key: k})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func key(w string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range w {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(unicode.ToLower(r))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package reconcile_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||
)
|
||||
|
||||
func texts(ts []reconcile.Token) []string {
|
||||
out := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = t.Text
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keys(ts []reconcile.Token) []string {
|
||||
out := make([]string, len(ts))
|
||||
for i, t := range ts {
|
||||
out[i] = t.Key
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTokenize_KeepsTheWordAsWrittenAndKeysItWithoutCaseOrPunctuation(t *testing.T) {
|
||||
got := reconcile.Tokenize("Hej, allihopa! Vi går igenom ÅÄÖ.")
|
||||
|
||||
assert.Equal(t, []string{"Hej,", "allihopa!", "Vi", "går", "igenom", "ÅÄÖ."}, texts(got),
|
||||
"the original spelling and punctuation are kept for the merged text")
|
||||
assert.Equal(t, []string{"hej", "allihopa", "vi", "går", "igenom", "åäö"}, keys(got),
|
||||
"the key is lower case with punctuation removed, Swedish letters lowered correctly")
|
||||
}
|
||||
|
||||
func TestTokenize_KeysIgnoreHyphensAndPlusSoOnlyWordsDecideEquality(t *testing.T) {
|
||||
got := reconcile.Tokenize("end-to-end A+2 5 000 kr")
|
||||
|
||||
assert.Equal(t, []string{"endtoend", "a2", "5", "000", "kr"}, keys(got))
|
||||
}
|
||||
|
||||
func TestTokenize_APunctuationOnlyWordJoinsThePreviousWordInsteadOfBecomingOne(t *testing.T) {
|
||||
got := reconcile.Tokenize("Vi går – sen tillbaka . Slut")
|
||||
|
||||
assert.Equal(t, []string{"Vi", "går –", "sen", "tillbaka .", "Slut"}, texts(got),
|
||||
"a dash or a lone full stop is not a word the engines can disagree about")
|
||||
assert.Equal(t, []string{"vi", "går", "sen", "tillbaka", "slut"}, keys(got))
|
||||
}
|
||||
|
||||
func TestTokenize_LeadingPunctuationOnlyWordJoinsTheNextWord(t *testing.T) {
|
||||
got := reconcile.Tokenize("— Hej då")
|
||||
|
||||
assert.Equal(t, []string{"— Hej", "då"}, texts(got))
|
||||
}
|
||||
|
||||
func TestTokenize_NewlinesAndRunsOfSpaceAreJustSeparators(t *testing.T) {
|
||||
got := reconcile.Tokenize(" Hej\n allihopa \t Vi ")
|
||||
|
||||
assert.Equal(t, []string{"Hej", "allihopa", "Vi"}, texts(got))
|
||||
}
|
||||
|
||||
func TestTokenize_EmptyAndPunctuationOnlyInputGiveNoTokens(t *testing.T) {
|
||||
assert.Empty(t, reconcile.Tokenize(""))
|
||||
assert.Empty(t, reconcile.Tokenize(" \n "))
|
||||
assert.Empty(t, reconcile.Tokenize("— . !"))
|
||||
}
|
||||
Reference in New Issue
Block a user