feat(runner): 429 backoff — skip still-throttled videos, persist status

After ProcessNewVideo the runner records transcript_status per outcome:
rate_limited (stamps the backoff clock), none, or fetched. Before fetching, a
video inside the TAPIR_FETCH_BACKOFF window is skipped (SkippedRateLimited) so a
just-429'd caption endpoint is not re-hit; once the window expires it retries.

Backoff/clock injected via variadic Options (WithBackoff, WithClock) so existing
New call sites and the fake-driven loop tests stay valid. Backoff 0 = always retry.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-03 22:56:07 +02:00
co-authored by Claude Opus 4.8
parent ce2fc62ef8
commit 40a614c8d4
3 changed files with 163 additions and 21 deletions
+80 -5
View File
@@ -5,6 +5,7 @@ import (
"io"
"log/slog"
"testing"
"time"
"github.com/stretchr/testify/require"
@@ -43,11 +44,13 @@ func (f *fakeSource) FetchTranscript(_ context.Context, v domain.Video) (domain.
// auto controls the summarization mode; requested is the manual-mode queue keyed
// by store id; cleared records the ids whose queue flag the runner reset.
type fakeStore struct {
seen map[string]bool
upserted []domain.Video
auto bool
requested map[string]bool
cleared []string
seen map[string]bool
upserted []domain.Video
auto bool
requested map[string]bool
cleared []string
rateLimited map[string]time.Time // id -> when 429'd (seeds the backoff window)
statuses map[string]string // id -> last SetTranscriptStatus value
}
func (f *fakeStore) UpsertVideo(_ context.Context, v domain.Video) (string, error) {
@@ -80,6 +83,22 @@ func (f *fakeStore) ClearSummarizeRequested(_ context.Context, _, videoID string
return nil
}
func (f *fakeStore) RateLimitedVideoIDs(_ context.Context, _ string) (map[string]time.Time, error) {
cp := make(map[string]time.Time, len(f.rateLimited))
for k, v := range f.rateLimited {
cp[k] = v
}
return cp, nil
}
func (f *fakeStore) SetTranscriptStatus(_ context.Context, _, videoID, status string) error {
if f.statuses == nil {
f.statuses = map[string]string{}
}
f.statuses[videoID] = status
return nil
}
type fakeSummarizer struct{}
func (fakeSummarizer) Summarize(_ context.Context, v domain.Video, _ domain.Transcript) (domain.Summary, error) {
@@ -207,6 +226,62 @@ func TestRunOnce_ManualMode_ProcessesRequested(t *testing.T) {
require.Equal(t, []string{"id-v1"}, st.cleared, "the queue flag is cleared after summarizing")
}
// noFetchSource fails the test if a transcript fetch happens — used to prove the
// runner skips a rate-limited video before touching the caption endpoint.
type noFetchSource struct{ *fakeSource }
func (noFetchSource) FetchTranscript(context.Context, domain.Video) (domain.Transcript, error) {
panic("FetchTranscript must not be called for a rate-limited video within the backoff window")
}
func TestRunOnce_SkipsRateLimitedWithinBackoff(t *testing.T) {
base := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
src := &fakeSource{
subs: []domain.Subscription{sub("chan1", "Channel One")},
videos: map[string][]domain.Video{"chan1": {vid("v1", "Video 1")}},
}
// v1 was rate-limited 5m ago; backoff is 1h, so it is still inside the window.
st := &fakeStore{
seen: map[string]bool{},
auto: true,
rateLimited: map[string]time.Time{"id-v1": base.Add(-5 * time.Minute)},
}
eng := usecase.NewEngine(noFetchSource{src}, fakeSummarizer{}, &recordingSink{})
r := runner.New(noFetchSource{src}, st, eng, testUser, quietLogger(),
runner.WithBackoff(time.Hour), runner.WithClock(func() time.Time { return base }))
stats, err := r.RunOnce(context.Background())
require.NoError(t, err)
require.Equal(t, 1, stats.SkippedRateLimited, "still throttled -> skipped")
require.Equal(t, 0, stats.Summarized)
require.Empty(t, st.statuses, "no status write: the engine was never invoked")
}
func TestRunOnce_RetriesRateLimitedAfterBackoff(t *testing.T) {
base := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
src := &fakeSource{
subs: []domain.Subscription{sub("chan1", "Channel One")},
videos: map[string][]domain.Video{"chan1": {vid("v1", "Video 1")}},
}
// v1 was rate-limited 2h ago; backoff is 1h, so the window has expired.
st := &fakeStore{
seen: map[string]bool{},
auto: true,
rateLimited: map[string]time.Time{"id-v1": base.Add(-2 * time.Hour)},
}
sink := &recordingSink{}
eng := usecase.NewEngine(src, fakeSummarizer{}, sink)
r := runner.New(src, st, eng, testUser, quietLogger(),
runner.WithBackoff(time.Hour), runner.WithClock(func() time.Time { return base }))
stats, err := r.RunOnce(context.Background())
require.NoError(t, err)
require.Equal(t, 0, stats.SkippedRateLimited, "window expired -> not skipped")
require.Equal(t, 1, stats.Summarized, "the video is retried and summarized")
require.Len(t, sink.delivered, 1)
require.Equal(t, "fetched", st.statuses["id-v1"], "status advances to fetched on success")
}
func TestRunOnce_UpsertsEveryCandidate(t *testing.T) {
src := &fakeSource{
subs: []domain.Subscription{sub("chan1", "Channel One")},