feat(summarizer): resilient endpoint chain with local→cloud fallback (ADR-022)
The first friendly-pilot live run produced zero summaries: koala/phi4-mini hit three silent failure modes — 8k context overflow on long transcripts (HTTP 400), intermittent malformed JSON (highlights as a bare string), and no fallback wired at all (summarizer.New(primary, nil)). Keep phi4-mini as the fast primary and add resilience around it: - Ordered endpoint chain (summarizer.NewChain): phi4-mini → koala/phi4-14b (local) → berget/mistral-small (worst-case external). All reached through the one LiteLLM gateway by alias. - A parse failure now advances the chain like a transport error — the old Primary→Fallback shape returned the parse error without trying anyone else. - Tolerant parse: highlights/takeaways coerce string→[]string, absorbing the common small-model quirk without spending a fallback round-trip. - Transcript truncation (TAPIR_MAX_TRANSCRIPT_CHARS=18000) prevents the overflow rather than recovering from it; validated to fit phi4-mini's 8k window. - Bounded completion budget (TAPIR_SUMMARY_MAX_TOKENS=1500) — the old 8192 budget itself contributed to the overflow. Local-first guarantee preserved by ordering: external endpoint is tried only after every local one fails. TAPIR_CLOUD_FALLBACK_MODEL="" disables it entirely for client/NDA deployments. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -129,8 +129,8 @@ func TestSummarize_NoBYO_ContentOnlyLocal(t *testing.T) {
|
||||
local := &fakeClient{reply: goodReply}
|
||||
s := New(Endpoint{Client: local, Provider: "local", Model: "iguana/deepseek-r1-14b"}, nil)
|
||||
|
||||
if s.fallback != nil {
|
||||
t.Fatal("no BYO configured but fallback endpoint is non-nil")
|
||||
if len(s.endpoints) != 1 {
|
||||
t.Fatalf("no BYO configured but chain has %d endpoints, want 1", len(s.endpoints))
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
sum, err := s.Summarize(context.Background(), testVideo(), testTranscript())
|
||||
@@ -176,3 +176,97 @@ func TestParse_EmptySummaryRejected(t *testing.T) {
|
||||
t.Fatal("want error for empty summary (thinking model returned no content)")
|
||||
}
|
||||
}
|
||||
|
||||
// parse tolerates a small model emitting "highlights" as a bare string instead
|
||||
// of an array — the production koala/phi4-mini quirk that errored with
|
||||
// "cannot unmarshal string into Go struct field ... highlights of type []string".
|
||||
func TestParse_ToleratesStringHighlights(t *testing.T) {
|
||||
p, err := parse(`{"summary":"s","highlights":"one big point","takeaways":["a","b"]}`)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(p.Highlights) != 1 || p.Highlights[0] != "one big point" {
|
||||
t.Errorf("highlights = %v, want [\"one big point\"]", p.Highlights)
|
||||
}
|
||||
if len(p.Takeaways) != 2 {
|
||||
t.Errorf("takeaways = %v, want 2", p.Takeaways)
|
||||
}
|
||||
}
|
||||
|
||||
// Chain: an endpoint that returns a 200 with unparseable output is treated as a
|
||||
// failure, and the next endpoint in the chain is tried. This is the case the old
|
||||
// primary->fallback shape missed — a parse error short-circuited instead of
|
||||
// falling back.
|
||||
func TestSummarize_FallsBackOnMalformedOutput(t *testing.T) {
|
||||
bad := &fakeClient{reply: `{"summary": not json`}
|
||||
good := &fakeClient{reply: goodReply}
|
||||
s := NewChain([]Endpoint{
|
||||
{Client: bad, Provider: "local", Model: "koala/phi4-mini"},
|
||||
{Client: good, Provider: "local", Model: "koala/phi4-14b"},
|
||||
}, 0)
|
||||
|
||||
sum, err := s.Summarize(context.Background(), testVideo(), testTranscript())
|
||||
if err != nil {
|
||||
t.Fatalf("Summarize: %v", err)
|
||||
}
|
||||
if sum.AIModel != "koala/phi4-14b" {
|
||||
t.Errorf("AIModel = %q, want koala/phi4-14b (fell back past malformed primary)", sum.AIModel)
|
||||
}
|
||||
if !sum.FallbackUsed {
|
||||
t.Error("FallbackUsed = false, want true")
|
||||
}
|
||||
if bad.calls != 1 || good.calls != 1 {
|
||||
t.Errorf("calls: bad=%d good=%d, want 1 and 1", bad.calls, good.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Chain: when every endpoint fails, no summary is produced and the joined error
|
||||
// names each failure so the engine queues the work for retry.
|
||||
func TestSummarize_ChainAllEndpointsFail(t *testing.T) {
|
||||
a := &fakeClient{err: errors.New("context overflow")}
|
||||
b := &fakeClient{reply: "not even json"}
|
||||
s := NewChain([]Endpoint{
|
||||
{Client: a, Provider: "local", Model: "m1"},
|
||||
{Client: b, Provider: "berget", Model: "m2"},
|
||||
}, 0)
|
||||
|
||||
if _, err := s.Summarize(context.Background(), testVideo(), testTranscript()); err == nil {
|
||||
t.Fatal("want error when all endpoints fail")
|
||||
}
|
||||
if a.calls != 1 || b.calls != 1 {
|
||||
t.Errorf("calls: a=%d b=%d, want 1 and 1", a.calls, b.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A transcript longer than the chain's input budget is truncated before it
|
||||
// reaches any model, so a small-context primary does not overflow its window.
|
||||
func TestSummarize_TruncatesLongTranscript(t *testing.T) {
|
||||
local := &fakeClient{reply: goodReply}
|
||||
const budget = 100
|
||||
s := NewChain([]Endpoint{{Client: local, Provider: "local", Model: "m"}}, budget)
|
||||
|
||||
long := domain.Transcript{
|
||||
VideoID: "vid-1", UserID: "user-1", Source: domain.SourceCaptions,
|
||||
Content: strings.Repeat("word ", 1000), // 5000 bytes, well over budget
|
||||
}
|
||||
if _, err := s.Summarize(context.Background(), testVideo(), long); err != nil {
|
||||
t.Fatalf("Summarize: %v", err)
|
||||
}
|
||||
// The prompt carries title/URL framing plus the truncation marker, so allow
|
||||
// headroom over the raw transcript budget — but it must be far below 5000.
|
||||
if len(local.lastUser) > budget+300 {
|
||||
t.Errorf("prompt length = %d, want <= %d (transcript not truncated)", len(local.lastUser), budget+300)
|
||||
}
|
||||
if !strings.Contains(local.lastUser, "truncated") {
|
||||
t.Error("truncation marker missing from prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewChain_PanicsOnEmptyChain(t *testing.T) {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("want panic on empty endpoint chain")
|
||||
}
|
||||
}()
|
||||
NewChain(nil, 0)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user