feat(runner): TAPIR_FETCH_DELAY to throttle transcript fetches
Diagnosis of a live run: caption tracks resolve fine (player + watch-page scrape), but the timedtext baseUrl fetch returns 429 under back-to-back volume — YouTube rate-limits the unauthenticated caption-download endpoint per IP. A per-video delay spaces the fetches. (Follow-up: treat 429 distinctly from genuine no-caption instead of silently degrading to SourceNone; consider Whisper if the endpoint stays hostile at any sustainable rate.)
This commit is contained in:
@@ -14,6 +14,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||
@@ -70,6 +71,12 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
|
||||
errs []error
|
||||
)
|
||||
|
||||
// Throttle transcript fetches: unauthenticated caption scraping gets
|
||||
// soft-throttled by YouTube under heavy back-to-back volume (captionTracks
|
||||
// silently stripped from the player response). A small per-video delay keeps
|
||||
// a full pass under the radar. TAPIR_FETCH_DELAY (Go duration), 0 = off.
|
||||
fetchDelay, _ := time.ParseDuration(os.Getenv("TAPIR_FETCH_DELAY"))
|
||||
|
||||
seen, err := r.store.SeenVideoIDs(ctx, r.userID)
|
||||
if err != nil {
|
||||
return stats, fmt.Errorf("runner: load seen videos: %w", err)
|
||||
@@ -105,6 +112,9 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
|
||||
}
|
||||
seen[id] = true // also guard against the same video within this pass
|
||||
|
||||
if fetchDelay > 0 {
|
||||
time.Sleep(fetchDelay)
|
||||
}
|
||||
res, err := r.engine.ProcessNewVideo(ctx, v)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("process %q: %w", v.ProviderVideoID, err))
|
||||
|
||||
Reference in New Issue
Block a user