fix(youtube): discover via uploads playlist, not search.list (100x cheaper)
NewVideos called search.list at 100 quota units/call. With ~143 channels one discovery pass = 14,300 units > the 10,000/day YouTube Data API cap, exhausting the whole day in a single loop (live quotaExceeded). Switch to playlistItems.list (1 unit/call) against the channel's uploads playlist. For a standard channel id UCxxxx the uploads playlist is UUxxxx, derived at zero API cost (uploadsPlaylistID). Non-standard ids fall back to channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads. Newest- first ordering and MaxVideosPerSubscription cap preserved. Side effect: removing search.list also removes the accountDelegationForbidden error that endpoint threw for one channel — no separate hardening needed. New per-pass quota: /subscriptions (1) + ~1/channel discovery (143) + any channels.list fallbacks ≈ 145 units/day, well under 10k. Caption fetch (ADR-010 timedtext) uses no Data API quota. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -179,6 +179,15 @@ func (a *Adapter) ListSubscriptions(ctx context.Context, userID string) ([]domai
|
||||
// first. The engine dedups within its lifetime and the store dedups durably
|
||||
// (data-model.md), so the adapter returns recent candidates rather than tracking
|
||||
// "seen" state itself.
|
||||
//
|
||||
// Discovery uses playlistItems.list against the channel's "uploads" playlist
|
||||
// (1 quota unit/call) instead of search.list (100 units): with ~143 channels a
|
||||
// single search-based pass exhausted the entire 10,000 unit/day cap. For a
|
||||
// standard channel id "UCxxxx" the uploads playlist is "UUxxxx" — derived with
|
||||
// zero API cost. Non-standard ids that don't start with "UC" fall back to
|
||||
// channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads.
|
||||
// Dropping search.list also removes the accountDelegationForbidden failure mode
|
||||
// that endpoint exhibited for one channel.
|
||||
func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) {
|
||||
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
||||
if err != nil {
|
||||
@@ -190,37 +199,76 @@ func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]dom
|
||||
max = defaultMaxVideos
|
||||
}
|
||||
|
||||
playlistID, ok := uploadsPlaylistID(sub.ChannelID)
|
||||
if !ok {
|
||||
// Non-standard channel id: resolve the uploads playlist explicitly.
|
||||
playlistID, err = a.resolveUploadsPlaylist(ctx, client, sub.ChannelID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
|
||||
}
|
||||
}
|
||||
|
||||
q := url.Values{
|
||||
"part": {"snippet"},
|
||||
"channelId": {sub.ChannelID},
|
||||
"order": {"date"},
|
||||
"type": {"video"},
|
||||
"playlistId": {playlistID},
|
||||
"maxResults": {fmt.Sprintf("%d", max)},
|
||||
}
|
||||
|
||||
var resp searchListResponse
|
||||
if err := a.getJSON(ctx, client, "/search", q, &resp); err != nil {
|
||||
var resp playlistItemListResponse
|
||||
if err := a.getJSON(ctx, client, "/playlistItems", q, &resp); err != nil {
|
||||
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
|
||||
}
|
||||
|
||||
videos := make([]domain.Video, 0, len(resp.Items))
|
||||
for _, item := range resp.Items {
|
||||
if item.ID.VideoID == "" {
|
||||
continue // non-video result; type=video should prevent this, be defensive
|
||||
vid := item.Snippet.ResourceID.VideoID
|
||||
if vid == "" {
|
||||
continue // defensive: skip items without a resolvable video id
|
||||
}
|
||||
videos = append(videos, domain.Video{
|
||||
UserID: sub.UserID,
|
||||
SubscriptionID: sub.ID,
|
||||
Provider: domain.ProviderYouTube,
|
||||
ProviderVideoID: item.ID.VideoID,
|
||||
ProviderVideoID: vid,
|
||||
Title: item.Snippet.Title,
|
||||
URL: "https://www.youtube.com/watch?v=" + item.ID.VideoID,
|
||||
URL: "https://www.youtube.com/watch?v=" + vid,
|
||||
PublishedAt: item.Snippet.PublishedAt,
|
||||
})
|
||||
if len(videos) >= max {
|
||||
break
|
||||
}
|
||||
}
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
// uploadsPlaylistID derives a channel's uploads playlist id at zero API cost:
|
||||
// a standard channel id "UCxxxx" maps to uploads playlist "UUxxxx". Returns
|
||||
// ok=false for ids that don't follow this convention (caller falls back to
|
||||
// channels.list).
|
||||
func uploadsPlaylistID(channelID string) (string, bool) {
|
||||
if !strings.HasPrefix(channelID, "UC") {
|
||||
return "", false
|
||||
}
|
||||
return "UU" + channelID[2:], true
|
||||
}
|
||||
|
||||
// resolveUploadsPlaylist reads contentDetails.relatedPlaylists.uploads via
|
||||
// channels.list (1 quota unit) for channels whose id can't be mapped UC->UU.
|
||||
func (a *Adapter) resolveUploadsPlaylist(ctx context.Context, client *http.Client, channelID string) (string, error) {
|
||||
q := url.Values{
|
||||
"part": {"contentDetails"},
|
||||
"id": {channelID},
|
||||
}
|
||||
var resp channelListResponse
|
||||
if err := a.getJSON(ctx, client, "/channels", q, &resp); err != nil {
|
||||
return "", fmt.Errorf("resolve uploads playlist: %w", err)
|
||||
}
|
||||
if len(resp.Items) == 0 || resp.Items[0].ContentDetails.RelatedPlaylists.Uploads == "" {
|
||||
return "", fmt.Errorf("no uploads playlist for channel %q", channelID)
|
||||
}
|
||||
return resp.Items[0].ContentDetails.RelatedPlaylists.Uploads, nil
|
||||
}
|
||||
|
||||
// getJSON issues a GET and decodes a JSON body into out. A non-200 status is an
|
||||
// error carrying a bounded slice of the response body for diagnosis.
|
||||
func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error {
|
||||
@@ -273,14 +321,24 @@ type subscriptionListResponse struct {
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
type searchListResponse struct {
|
||||
type playlistItemListResponse struct {
|
||||
Items []struct {
|
||||
ID struct {
|
||||
VideoID string `json:"videoId"`
|
||||
} `json:"id"`
|
||||
Snippet struct {
|
||||
Title string `json:"title"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
ResourceID struct {
|
||||
VideoID string `json:"videoId"`
|
||||
} `json:"resourceId"`
|
||||
} `json:"snippet"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
type channelListResponse struct {
|
||||
Items []struct {
|
||||
ContentDetails struct {
|
||||
RelatedPlaylists struct {
|
||||
Uploads string `json:"uploads"`
|
||||
} `json:"relatedPlaylists"`
|
||||
} `json:"contentDetails"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user