feat(discovery): drop Shorts and livestreams before the caption fetch (ADR-023)
The scarce resource is the per-IP timedtext caption fetch (ADR-014); the pilot's candidate set was mostly Shorts/clips/livestreams, each burning a fetch (a "none" result is a completed fetch — it costs budget even when it yields nothing). NewVideos now enriches candidates with one cheap Data API videos.list call (contentDetails.duration + snippet.liveBroadcastContent — the quota API, a DIFFERENT limit from the timedtext 429) and drops, before returning: videos shorter than TAPIR_MIN_VIDEO_SECONDS (default 60) and any live/upcoming broadcast. Dropped videos are never persisted, so the list declutters too. Degrade-open: MinVideoSeconds=0 disables it (no quota call); a videos.list error returns candidates unfiltered so discovery never breaks on a metadata hiccup. The paste-a-URL path (VideoByID) is not filtered — an explicit request is honoured. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -66,6 +66,13 @@ type Config struct {
|
||||
// poll. Zero means defaultMaxVideos.
|
||||
MaxVideosPerSubscription int
|
||||
|
||||
// MinVideoSeconds drops videos shorter than this from discovery (Shorts/clips,
|
||||
// ADR-023). NewVideos enriches candidates with a single cheap videos.list call
|
||||
// (contentDetails.duration + snippet.liveBroadcastContent) and filters before
|
||||
// returning, so the scarce caption-fetch budget is never spent on them. Live
|
||||
// and upcoming broadcasts are dropped too. Zero disables the filter.
|
||||
MinVideoSeconds int
|
||||
|
||||
// BaseURL overrides the Data API root. Empty means defaultBaseURL.
|
||||
BaseURL string
|
||||
|
||||
@@ -242,7 +249,97 @@ func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]dom
|
||||
break
|
||||
}
|
||||
}
|
||||
return videos, nil
|
||||
|
||||
// Drop Shorts/sub-minute clips and live/upcoming broadcasts before they ever
|
||||
// reach the rate-limited caption path (ADR-023). One cheap videos.list call
|
||||
// (quota API, not the timedtext throttle) supplies duration + live status.
|
||||
return a.filterLowValue(ctx, client, videos), nil
|
||||
}
|
||||
|
||||
// filterLowValue removes videos shorter than cfg.MinVideoSeconds and any live or
|
||||
// upcoming broadcast, using a single videos.list lookup for duration +
|
||||
// liveBroadcastContent. The filter is best-effort: if MinVideoSeconds is 0 (off)
|
||||
// or the lookup fails, the input is returned unfiltered — discovery must not break
|
||||
// because a metadata call hiccuped; the worst case is the pre-ADR-023 behaviour.
|
||||
func (a *Adapter) filterLowValue(ctx context.Context, client *http.Client, videos []domain.Video) []domain.Video {
|
||||
if a.cfg.MinVideoSeconds <= 0 || len(videos) == 0 {
|
||||
return videos
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(videos))
|
||||
for _, v := range videos {
|
||||
ids = append(ids, v.ProviderVideoID)
|
||||
}
|
||||
q := url.Values{
|
||||
"part": {"contentDetails,snippet"},
|
||||
"id": {strings.Join(ids, ",")},
|
||||
}
|
||||
var resp videoListResponse
|
||||
if err := a.getJSON(ctx, client, "/videos", q, &resp); err != nil {
|
||||
// Degrade open: keep the candidates rather than lose discovery.
|
||||
return videos
|
||||
}
|
||||
|
||||
type meta struct {
|
||||
seconds int
|
||||
live string
|
||||
}
|
||||
byID := make(map[string]meta, len(resp.Items))
|
||||
for _, it := range resp.Items {
|
||||
byID[it.ID] = meta{seconds: parseISO8601Seconds(it.ContentDetails.Duration), live: it.Snippet.LiveBroadcastContent}
|
||||
}
|
||||
|
||||
kept := videos[:0]
|
||||
for _, v := range videos {
|
||||
m, ok := byID[v.ProviderVideoID]
|
||||
if !ok {
|
||||
kept = append(kept, v) // unknown metadata: keep, let the fetch decide
|
||||
continue
|
||||
}
|
||||
if m.live != "" && m.live != "none" {
|
||||
continue // live or upcoming broadcast
|
||||
}
|
||||
if m.seconds > 0 && m.seconds < a.cfg.MinVideoSeconds {
|
||||
continue // Short / sub-threshold clip
|
||||
}
|
||||
kept = append(kept, v)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// parseISO8601Seconds parses an ISO 8601 duration as returned by the YouTube Data
|
||||
// API (e.g. "PT1H2M3S", "PT45S", "PT3M") into seconds. Only the hour/minute/second
|
||||
// components YouTube emits are handled; an unparseable or zero value returns 0,
|
||||
// which the caller treats as "unknown" (not filtered on duration).
|
||||
func parseISO8601Seconds(d string) int {
|
||||
if !strings.HasPrefix(d, "PT") {
|
||||
return 0
|
||||
}
|
||||
d = d[2:]
|
||||
total, num := 0, 0
|
||||
seen := false
|
||||
for _, r := range d {
|
||||
switch {
|
||||
case r >= '0' && r <= '9':
|
||||
num = num*10 + int(r-'0')
|
||||
seen = true
|
||||
case r == 'H':
|
||||
total += num * 3600
|
||||
num, seen = 0, false
|
||||
case r == 'M':
|
||||
total += num * 60
|
||||
num, seen = 0, false
|
||||
case r == 'S':
|
||||
total += num
|
||||
num, seen = 0, false
|
||||
default:
|
||||
return 0 // unexpected component (days/weeks) — treat as unknown
|
||||
}
|
||||
}
|
||||
if seen {
|
||||
return 0 // trailing digits without a unit: malformed
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// VideoByID fetches a single video's metadata (videos.list, snippet) for an
|
||||
@@ -377,11 +474,16 @@ type playlistItemListResponse struct {
|
||||
|
||||
type videoListResponse struct {
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
Snippet struct {
|
||||
Title string `json:"title"`
|
||||
ChannelTitle string `json:"channelTitle"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
Title string `json:"title"`
|
||||
ChannelTitle string `json:"channelTitle"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
LiveBroadcastContent string `json:"liveBroadcastContent"`
|
||||
} `json:"snippet"`
|
||||
ContentDetails struct {
|
||||
Duration string `json:"duration"` // ISO 8601, e.g. "PT1M30S"
|
||||
} `json:"contentDetails"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user