feat(runner): TAPIR_FETCH_DELAY to throttle transcript fetches
CI / Lint / Test / Vet (push) Successful in 10s
CI / Build & Import (push) Successful in 9s
CI / Mirror to GitHub (push) Failing after 3s

Diagnosis of a live run: caption tracks resolve fine (player + watch-page scrape),
but the timedtext baseUrl fetch returns 429 under back-to-back volume — YouTube
rate-limits the unauthenticated caption-download endpoint per IP. A per-video
delay spaces the fetches. (Follow-up: treat 429 distinctly from genuine
no-caption instead of silently degrading to SourceNone; consider Whisper if the
endpoint stays hostile at any sustainable rate.)
This commit is contained in:
2026-06-03 12:20:12 +02:00
parent 7589c09201
commit 598ba5d34f
+10
View File
@@ -14,6 +14,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"os"
"time" "time"
"gitea.d-ma.be/mathias/tapir/internal/domain" "gitea.d-ma.be/mathias/tapir/internal/domain"
@@ -70,6 +71,12 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
errs []error errs []error
) )
// Throttle transcript fetches: unauthenticated caption scraping gets
// soft-throttled by YouTube under heavy back-to-back volume (captionTracks
// silently stripped from the player response). A small per-video delay keeps
// a full pass under the radar. TAPIR_FETCH_DELAY (Go duration), 0 = off.
fetchDelay, _ := time.ParseDuration(os.Getenv("TAPIR_FETCH_DELAY"))
seen, err := r.store.SeenVideoIDs(ctx, r.userID) seen, err := r.store.SeenVideoIDs(ctx, r.userID)
if err != nil { if err != nil {
return stats, fmt.Errorf("runner: load seen videos: %w", err) return stats, fmt.Errorf("runner: load seen videos: %w", err)
@@ -105,6 +112,9 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
} }
seen[id] = true // also guard against the same video within this pass seen[id] = true // also guard against the same video within this pass
if fetchDelay > 0 {
time.Sleep(fetchDelay)
}
res, err := r.engine.ProcessNewVideo(ctx, v) res, err := r.engine.ProcessNewVideo(ctx, v)
if err != nil { if err != nil {
errs = append(errs, fmt.Errorf("process %q: %w", v.ProviderVideoID, err)) errs = append(errs, fmt.Errorf("process %q: %w", v.ProviderVideoID, err))