// These tests translate docs/use-cases/ai_routing.feature. They drive the // Summarizer through a FAKE Completer — never the live LiteLLM gateway — and the // load-bearing assertion is the local-first guarantee: when no BYO provider is // configured, the transcript content reaches the local stack and nowhere else. package summarizer import ( "context" "errors" "strings" "testing" "gitea.d-ma.be/mathias/tapir/internal/domain" "gitea.d-ma.be/mathias/tapir/internal/ports" ) // compile-time check: Summarizer satisfies the port. var _ ports.Summarizer = (*Summarizer)(nil) // fakeClient records every prompt it is asked to complete, so a test can prove // whether content reached it. It returns reply, or err when err != nil. type fakeClient struct { reply string err error calls int lastUser string } func (f *fakeClient) Complete(_ context.Context, _, user string) (string, error) { f.calls++ f.lastUser = user if f.err != nil { return "", f.err } return f.reply, nil } const goodReply = `{"summary":"A talk about Go.","highlights":["ports and adapters"],"takeaways":["copy, don't couple"]}` func testVideo() domain.Video { return domain.Video{ID: "vid-1", UserID: "user-1", Title: "Clean Architecture in Go", URL: "https://x/y"} } func testTranscript() domain.Transcript { return domain.Transcript{VideoID: "vid-1", UserID: "user-1", Source: domain.SourceCaptions, Content: "secret confidential transcript body"} } // Scenario: Local AI produces the summary. func TestSummarize_LocalSucceeds(t *testing.T) { local := &fakeClient{reply: goodReply} byo := &fakeClient{reply: `{"summary":"should not be used"}`} s := New( Endpoint{Client: local, Provider: "local", Model: "iguana/deepseek-r1-14b"}, &Endpoint{Client: byo, Provider: "anthropic", Model: "claude"}, ) sum, err := s.Summarize(context.Background(), testVideo(), testTranscript()) if err != nil { t.Fatalf("Summarize: %v", err) } if sum.AIProvider != "local" { t.Errorf("AIProvider = %q, want local", sum.AIProvider) } if sum.AIModel != "iguana/deepseek-r1-14b" { t.Errorf("AIModel = %q", sum.AIModel) } if sum.FallbackUsed { t.Error("FallbackUsed = true, want false") } if byo.calls != 0 { t.Errorf("BYO called %d times; must not be touched when local succeeds", byo.calls) } if sum.Summary == "" || len(sum.Highlights) != 1 || len(sum.Takeaways) != 1 { t.Errorf("parsed summary wrong: %+v", sum) } } // Scenario: Local AI fails and the user has a BYO provider configured. func TestSummarize_FallsBackToBYO(t *testing.T) { local := &fakeClient{err: errors.New("connection refused")} byo := &fakeClient{reply: goodReply} s := New( Endpoint{Client: local, Provider: "local", Model: "iguana/deepseek-r1-14b"}, &Endpoint{Client: byo, Provider: "anthropic", Model: "claude-opus"}, ) sum, err := s.Summarize(context.Background(), testVideo(), testTranscript()) if err != nil { t.Fatalf("Summarize: %v", err) } if sum.AIProvider != "anthropic" { t.Errorf("AIProvider = %q, want anthropic", sum.AIProvider) } if sum.AIModel != "claude-opus" { t.Errorf("AIModel = %q, want claude-opus", sum.AIModel) } if !sum.FallbackUsed { t.Error("FallbackUsed = false, want true") } if local.calls != 1 || byo.calls != 1 { t.Errorf("calls: local=%d byo=%d, want 1 and 1", local.calls, byo.calls) } } // Scenario: Local AI fails and the user has no BYO provider. // AND: my content is not sent to any third-party model. func TestSummarize_LocalFailsNoBYO_NoExternalSend(t *testing.T) { local := &fakeClient{err: errors.New("connection refused")} s := New(Endpoint{Client: local, Provider: "local", Model: "iguana/deepseek-r1-14b"}, nil) _, err := s.Summarize(context.Background(), testVideo(), testTranscript()) if err == nil { t.Fatal("want error when local fails and no BYO, got nil") } // Local was the only place content could go; with nil fallback there is no // external client to receive it at all. Local saw the content once. if local.calls != 1 { t.Errorf("local calls = %d, want 1", local.calls) } if !strings.Contains(local.lastUser, "secret confidential transcript body") { t.Error("transcript content should have reached the local stack") } } // Scenario: A user without BYO never has content sent externally — even across // repeated summarizations. Asserted structurally: a nil fallback means no // external endpoint exists, so content cannot leave the local stack. func TestSummarize_NoBYO_ContentOnlyLocal(t *testing.T) { local := &fakeClient{reply: goodReply} s := New(Endpoint{Client: local, Provider: "local", Model: "iguana/deepseek-r1-14b"}, nil) if len(s.endpoints) != 1 { t.Fatalf("no BYO configured but chain has %d endpoints, want 1", len(s.endpoints)) } for i := 0; i < 3; i++ { sum, err := s.Summarize(context.Background(), testVideo(), testTranscript()) if err != nil { t.Fatalf("Summarize: %v", err) } if sum.AIProvider != "local" || sum.FallbackUsed { t.Errorf("provider=%q fallbackUsed=%v, want local/false", sum.AIProvider, sum.FallbackUsed) } } if local.calls != 3 { t.Errorf("local calls = %d, want 3", local.calls) } } func TestSummarize_EmptyTranscriptIsError(t *testing.T) { local := &fakeClient{reply: goodReply} s := New(Endpoint{Client: local, Provider: "local", Model: "m"}, nil) none := domain.Transcript{VideoID: "vid-1", Source: domain.SourceNone} if _, err := s.Summarize(context.Background(), testVideo(), none); err == nil { t.Fatal("want error for transcript with no text") } if local.calls != 0 { t.Errorf("local called %d times for empty transcript; must not call the model", local.calls) } } // parse tolerates thinking-model wrapping (reasoning + code fences around JSON). func TestParse_ToleratesFencedThinkingOutput(t *testing.T) { raw := "let me reason...\n```json\n" + goodReply + "\n```" p, err := parse(raw) if err != nil { t.Fatalf("parse: %v", err) } if p.Summary != "A talk about Go." { t.Errorf("summary = %q", p.Summary) } } func TestParse_EmptySummaryRejected(t *testing.T) { if _, err := parse(`{"summary":" ","highlights":[]}`); err == nil { t.Fatal("want error for empty summary (thinking model returned no content)") } } // parse tolerates a small model emitting "highlights" as a bare string instead // of an array — the production koala/phi4-mini quirk that errored with // "cannot unmarshal string into Go struct field ... highlights of type []string". func TestParse_ToleratesStringHighlights(t *testing.T) { p, err := parse(`{"summary":"s","highlights":"one big point","takeaways":["a","b"]}`) if err != nil { t.Fatalf("parse: %v", err) } if len(p.Highlights) != 1 || p.Highlights[0] != "one big point" { t.Errorf("highlights = %v, want [\"one big point\"]", p.Highlights) } if len(p.Takeaways) != 2 { t.Errorf("takeaways = %v, want 2", p.Takeaways) } } // Chain: an endpoint that returns a 200 with unparseable output is treated as a // failure, and the next endpoint in the chain is tried. This is the case the old // primary->fallback shape missed — a parse error short-circuited instead of // falling back. func TestSummarize_FallsBackOnMalformedOutput(t *testing.T) { bad := &fakeClient{reply: `{"summary": not json`} good := &fakeClient{reply: goodReply} s := NewChain([]Endpoint{ {Client: bad, Provider: "local", Model: "koala/phi4-mini"}, {Client: good, Provider: "local", Model: "koala/phi4-14b"}, }, 0) sum, err := s.Summarize(context.Background(), testVideo(), testTranscript()) if err != nil { t.Fatalf("Summarize: %v", err) } if sum.AIModel != "koala/phi4-14b" { t.Errorf("AIModel = %q, want koala/phi4-14b (fell back past malformed primary)", sum.AIModel) } if !sum.FallbackUsed { t.Error("FallbackUsed = false, want true") } if bad.calls != 1 || good.calls != 1 { t.Errorf("calls: bad=%d good=%d, want 1 and 1", bad.calls, good.calls) } } // Chain: when every endpoint fails, no summary is produced and the joined error // names each failure so the engine queues the work for retry. func TestSummarize_ChainAllEndpointsFail(t *testing.T) { a := &fakeClient{err: errors.New("context overflow")} b := &fakeClient{reply: "not even json"} s := NewChain([]Endpoint{ {Client: a, Provider: "local", Model: "m1"}, {Client: b, Provider: "berget", Model: "m2"}, }, 0) if _, err := s.Summarize(context.Background(), testVideo(), testTranscript()); err == nil { t.Fatal("want error when all endpoints fail") } if a.calls != 1 || b.calls != 1 { t.Errorf("calls: a=%d b=%d, want 1 and 1", a.calls, b.calls) } } // A transcript longer than the chain's input budget is truncated before it // reaches any model, so a small-context primary does not overflow its window. func TestSummarize_TruncatesLongTranscript(t *testing.T) { local := &fakeClient{reply: goodReply} const budget = 100 s := NewChain([]Endpoint{{Client: local, Provider: "local", Model: "m"}}, budget) long := domain.Transcript{ VideoID: "vid-1", UserID: "user-1", Source: domain.SourceCaptions, Content: strings.Repeat("word ", 1000), // 5000 bytes, well over budget } if _, err := s.Summarize(context.Background(), testVideo(), long); err != nil { t.Fatalf("Summarize: %v", err) } // The prompt carries title/URL framing plus the truncation marker, so allow // headroom over the raw transcript budget — but it must be far below 5000. if len(local.lastUser) > budget+300 { t.Errorf("prompt length = %d, want <= %d (transcript not truncated)", len(local.lastUser), budget+300) } if !strings.Contains(local.lastUser, "truncated") { t.Error("truncation marker missing from prompt") } } func TestNewChain_PanicsOnEmptyChain(t *testing.T) { defer func() { if recover() == nil { t.Fatal("want panic on empty endpoint chain") } }() NewChain(nil, 0) }