feat(runner): 429 backoff — skip still-throttled videos, persist status
After ProcessNewVideo the runner records transcript_status per outcome: rate_limited (stamps the backoff clock), none, or fetched. Before fetching, a video inside the TAPIR_FETCH_BACKOFF window is skipped (SkippedRateLimited) so a just-429'd caption endpoint is not re-hit; once the window expires it retries. Backoff/clock injected via variadic Options (WithBackoff, WithClock) so existing New call sites and the fake-driven loop tests stay valid. Backoff 0 = always retry. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
@@ -43,11 +44,13 @@ func (f *fakeSource) FetchTranscript(_ context.Context, v domain.Video) (domain.
|
||||
// auto controls the summarization mode; requested is the manual-mode queue keyed
|
||||
// by store id; cleared records the ids whose queue flag the runner reset.
|
||||
type fakeStore struct {
|
||||
seen map[string]bool
|
||||
upserted []domain.Video
|
||||
auto bool
|
||||
requested map[string]bool
|
||||
cleared []string
|
||||
seen map[string]bool
|
||||
upserted []domain.Video
|
||||
auto bool
|
||||
requested map[string]bool
|
||||
cleared []string
|
||||
rateLimited map[string]time.Time // id -> when 429'd (seeds the backoff window)
|
||||
statuses map[string]string // id -> last SetTranscriptStatus value
|
||||
}
|
||||
|
||||
func (f *fakeStore) UpsertVideo(_ context.Context, v domain.Video) (string, error) {
|
||||
@@ -80,6 +83,22 @@ func (f *fakeStore) ClearSummarizeRequested(_ context.Context, _, videoID string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) RateLimitedVideoIDs(_ context.Context, _ string) (map[string]time.Time, error) {
|
||||
cp := make(map[string]time.Time, len(f.rateLimited))
|
||||
for k, v := range f.rateLimited {
|
||||
cp[k] = v
|
||||
}
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) SetTranscriptStatus(_ context.Context, _, videoID, status string) error {
|
||||
if f.statuses == nil {
|
||||
f.statuses = map[string]string{}
|
||||
}
|
||||
f.statuses[videoID] = status
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeSummarizer struct{}
|
||||
|
||||
func (fakeSummarizer) Summarize(_ context.Context, v domain.Video, _ domain.Transcript) (domain.Summary, error) {
|
||||
@@ -207,6 +226,62 @@ func TestRunOnce_ManualMode_ProcessesRequested(t *testing.T) {
|
||||
require.Equal(t, []string{"id-v1"}, st.cleared, "the queue flag is cleared after summarizing")
|
||||
}
|
||||
|
||||
// noFetchSource fails the test if a transcript fetch happens — used to prove the
|
||||
// runner skips a rate-limited video before touching the caption endpoint.
|
||||
type noFetchSource struct{ *fakeSource }
|
||||
|
||||
func (noFetchSource) FetchTranscript(context.Context, domain.Video) (domain.Transcript, error) {
|
||||
panic("FetchTranscript must not be called for a rate-limited video within the backoff window")
|
||||
}
|
||||
|
||||
func TestRunOnce_SkipsRateLimitedWithinBackoff(t *testing.T) {
|
||||
base := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
|
||||
src := &fakeSource{
|
||||
subs: []domain.Subscription{sub("chan1", "Channel One")},
|
||||
videos: map[string][]domain.Video{"chan1": {vid("v1", "Video 1")}},
|
||||
}
|
||||
// v1 was rate-limited 5m ago; backoff is 1h, so it is still inside the window.
|
||||
st := &fakeStore{
|
||||
seen: map[string]bool{},
|
||||
auto: true,
|
||||
rateLimited: map[string]time.Time{"id-v1": base.Add(-5 * time.Minute)},
|
||||
}
|
||||
eng := usecase.NewEngine(noFetchSource{src}, fakeSummarizer{}, &recordingSink{})
|
||||
r := runner.New(noFetchSource{src}, st, eng, testUser, quietLogger(),
|
||||
runner.WithBackoff(time.Hour), runner.WithClock(func() time.Time { return base }))
|
||||
|
||||
stats, err := r.RunOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, stats.SkippedRateLimited, "still throttled -> skipped")
|
||||
require.Equal(t, 0, stats.Summarized)
|
||||
require.Empty(t, st.statuses, "no status write: the engine was never invoked")
|
||||
}
|
||||
|
||||
func TestRunOnce_RetriesRateLimitedAfterBackoff(t *testing.T) {
|
||||
base := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
|
||||
src := &fakeSource{
|
||||
subs: []domain.Subscription{sub("chan1", "Channel One")},
|
||||
videos: map[string][]domain.Video{"chan1": {vid("v1", "Video 1")}},
|
||||
}
|
||||
// v1 was rate-limited 2h ago; backoff is 1h, so the window has expired.
|
||||
st := &fakeStore{
|
||||
seen: map[string]bool{},
|
||||
auto: true,
|
||||
rateLimited: map[string]time.Time{"id-v1": base.Add(-2 * time.Hour)},
|
||||
}
|
||||
sink := &recordingSink{}
|
||||
eng := usecase.NewEngine(src, fakeSummarizer{}, sink)
|
||||
r := runner.New(src, st, eng, testUser, quietLogger(),
|
||||
runner.WithBackoff(time.Hour), runner.WithClock(func() time.Time { return base }))
|
||||
|
||||
stats, err := r.RunOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, stats.SkippedRateLimited, "window expired -> not skipped")
|
||||
require.Equal(t, 1, stats.Summarized, "the video is retried and summarized")
|
||||
require.Len(t, sink.delivered, 1)
|
||||
require.Equal(t, "fetched", st.statuses["id-v1"], "status advances to fetched on success")
|
||||
}
|
||||
|
||||
func TestRunOnce_UpsertsEveryCandidate(t *testing.T) {
|
||||
src := &fakeSource{
|
||||
subs: []domain.Subscription{sub("chan1", "Channel One")},
|
||||
|
||||
Reference in New Issue
Block a user