feat(runner): TAPIR_FETCH_DELAY to throttle transcript fetches
Diagnosis of a live run: caption tracks resolve fine (player + watch-page scrape), but the timedtext baseUrl fetch returns 429 under back-to-back volume — YouTube rate-limits the unauthenticated caption-download endpoint per IP. A per-video delay spaces the fetches. (Follow-up: treat 429 distinctly from genuine no-caption instead of silently degrading to SourceNone; consider Whisper if the endpoint stays hostile at any sustainable rate.)
This commit is contained in:
@@ -14,6 +14,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||||
@@ -70,6 +71,12 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
|
|||||||
errs []error
|
errs []error
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Throttle transcript fetches: unauthenticated caption scraping gets
|
||||||
|
// soft-throttled by YouTube under heavy back-to-back volume (captionTracks
|
||||||
|
// silently stripped from the player response). A small per-video delay keeps
|
||||||
|
// a full pass under the radar. TAPIR_FETCH_DELAY (Go duration), 0 = off.
|
||||||
|
fetchDelay, _ := time.ParseDuration(os.Getenv("TAPIR_FETCH_DELAY"))
|
||||||
|
|
||||||
seen, err := r.store.SeenVideoIDs(ctx, r.userID)
|
seen, err := r.store.SeenVideoIDs(ctx, r.userID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return stats, fmt.Errorf("runner: load seen videos: %w", err)
|
return stats, fmt.Errorf("runner: load seen videos: %w", err)
|
||||||
@@ -105,6 +112,9 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
|
|||||||
}
|
}
|
||||||
seen[id] = true // also guard against the same video within this pass
|
seen[id] = true // also guard against the same video within this pass
|
||||||
|
|
||||||
|
if fetchDelay > 0 {
|
||||||
|
time.Sleep(fetchDelay)
|
||||||
|
}
|
||||||
res, err := r.engine.ProcessNewVideo(ctx, v)
|
res, err := r.engine.ProcessNewVideo(ctx, v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs = append(errs, fmt.Errorf("process %q: %w", v.ProviderVideoID, err))
|
errs = append(errs, fmt.Errorf("process %q: %w", v.ProviderVideoID, err))
|
||||||
|
|||||||
Reference in New Issue
Block a user