From 598ba5d34f0ca1803beff1da83ce44d4c6f45f94 Mon Sep 17 00:00:00 2001 From: Mathias Date: Wed, 3 Jun 2026 12:20:12 +0200 Subject: [PATCH] feat(runner): TAPIR_FETCH_DELAY to throttle transcript fetches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis of a live run: caption tracks resolve fine (player + watch-page scrape), but the timedtext baseUrl fetch returns 429 under back-to-back volume — YouTube rate-limits the unauthenticated caption-download endpoint per IP. A per-video delay spaces the fetches. (Follow-up: treat 429 distinctly from genuine no-caption instead of silently degrading to SourceNone; consider Whisper if the endpoint stays hostile at any sustainable rate.) --- internal/runner/runner.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 7df7d31..ef5f57c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "log/slog" + "os" "time" "gitea.d-ma.be/mathias/tapir/internal/domain" @@ -70,6 +71,12 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) { errs []error ) + // Throttle transcript fetches: unauthenticated caption scraping gets + // soft-throttled by YouTube under heavy back-to-back volume (captionTracks + // silently stripped from the player response). A small per-video delay keeps + // a full pass under the radar. TAPIR_FETCH_DELAY (Go duration), 0 = off. + fetchDelay, _ := time.ParseDuration(os.Getenv("TAPIR_FETCH_DELAY")) + seen, err := r.store.SeenVideoIDs(ctx, r.userID) if err != nil { return stats, fmt.Errorf("runner: load seen videos: %w", err) @@ -105,6 +112,9 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) { } seen[id] = true // also guard against the same video within this pass + if fetchDelay > 0 { + time.Sleep(fetchDelay) + } res, err := r.engine.ProcessNewVideo(ctx, v) if err != nil { errs = append(errs, fmt.Errorf("process %q: %w", v.ProviderVideoID, err))