fix(youtube): discover via uploads playlist, not search.list (100x cheaper)
CI / Lint / Test / Vet (push) Successful in 5s
CI / Build & Import (push) Failing after 1s
CI / Mirror to GitHub (push) Has been skipped

NewVideos called search.list at 100 quota units/call. With ~143 channels one
discovery pass = 14,300 units > the 10,000/day YouTube Data API cap, exhausting
the whole day in a single loop (live quotaExceeded).

Switch to playlistItems.list (1 unit/call) against the channel's uploads
playlist. For a standard channel id UCxxxx the uploads playlist is UUxxxx,
derived at zero API cost (uploadsPlaylistID). Non-standard ids fall back to
channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads. Newest-
first ordering and MaxVideosPerSubscription cap preserved.

Side effect: removing search.list also removes the accountDelegationForbidden
error that endpoint threw for one channel — no separate hardening needed.

New per-pass quota: /subscriptions (1) + ~1/channel discovery (143) + any
channels.list fallbacks ≈ 145 units/day, well under 10k. Caption fetch (ADR-010
timedtext) uses no Data API quota.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-02 23:15:27 +02:00
co-authored by Claude Opus 4.8
parent 92da25d01e
commit af1c163a11
2 changed files with 163 additions and 19 deletions
+71 -13
View File
@@ -179,6 +179,15 @@ func (a *Adapter) ListSubscriptions(ctx context.Context, userID string) ([]domai
// first. The engine dedups within its lifetime and the store dedups durably
// (data-model.md), so the adapter returns recent candidates rather than tracking
// "seen" state itself.
//
// Discovery uses playlistItems.list against the channel's "uploads" playlist
// (1 quota unit/call) instead of search.list (100 units): with ~143 channels a
// single search-based pass exhausted the entire 10,000 unit/day cap. For a
// standard channel id "UCxxxx" the uploads playlist is "UUxxxx" — derived with
// zero API cost. Non-standard ids that don't start with "UC" fall back to
// channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads.
// Dropping search.list also removes the accountDelegationForbidden failure mode
// that endpoint exhibited for one channel.
func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) {
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
if err != nil {
@@ -190,37 +199,76 @@ func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]dom
max = defaultMaxVideos
}
playlistID, ok := uploadsPlaylistID(sub.ChannelID)
if !ok {
// Non-standard channel id: resolve the uploads playlist explicitly.
playlistID, err = a.resolveUploadsPlaylist(ctx, client, sub.ChannelID)
if err != nil {
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
}
}
q := url.Values{
"part": {"snippet"},
"channelId": {sub.ChannelID},
"order": {"date"},
"type": {"video"},
"playlistId": {playlistID},
"maxResults": {fmt.Sprintf("%d", max)},
}
var resp searchListResponse
if err := a.getJSON(ctx, client, "/search", q, &resp); err != nil {
var resp playlistItemListResponse
if err := a.getJSON(ctx, client, "/playlistItems", q, &resp); err != nil {
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
}
videos := make([]domain.Video, 0, len(resp.Items))
for _, item := range resp.Items {
if item.ID.VideoID == "" {
continue // non-video result; type=video should prevent this, be defensive
vid := item.Snippet.ResourceID.VideoID
if vid == "" {
continue // defensive: skip items without a resolvable video id
}
videos = append(videos, domain.Video{
UserID: sub.UserID,
SubscriptionID: sub.ID,
Provider: domain.ProviderYouTube,
ProviderVideoID: item.ID.VideoID,
ProviderVideoID: vid,
Title: item.Snippet.Title,
URL: "https://www.youtube.com/watch?v=" + item.ID.VideoID,
URL: "https://www.youtube.com/watch?v=" + vid,
PublishedAt: item.Snippet.PublishedAt,
})
if len(videos) >= max {
break
}
}
return videos, nil
}
// uploadsPlaylistID derives a channel's uploads playlist id at zero API cost:
// a standard channel id "UCxxxx" maps to uploads playlist "UUxxxx". Returns
// ok=false for ids that don't follow this convention (caller falls back to
// channels.list).
func uploadsPlaylistID(channelID string) (string, bool) {
if !strings.HasPrefix(channelID, "UC") {
return "", false
}
return "UU" + channelID[2:], true
}
// resolveUploadsPlaylist reads contentDetails.relatedPlaylists.uploads via
// channels.list (1 quota unit) for channels whose id can't be mapped UC->UU.
func (a *Adapter) resolveUploadsPlaylist(ctx context.Context, client *http.Client, channelID string) (string, error) {
q := url.Values{
"part": {"contentDetails"},
"id": {channelID},
}
var resp channelListResponse
if err := a.getJSON(ctx, client, "/channels", q, &resp); err != nil {
return "", fmt.Errorf("resolve uploads playlist: %w", err)
}
if len(resp.Items) == 0 || resp.Items[0].ContentDetails.RelatedPlaylists.Uploads == "" {
return "", fmt.Errorf("no uploads playlist for channel %q", channelID)
}
return resp.Items[0].ContentDetails.RelatedPlaylists.Uploads, nil
}
// getJSON issues a GET and decodes a JSON body into out. A non-200 status is an
// error carrying a bounded slice of the response body for diagnosis.
func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error {
@@ -273,14 +321,24 @@ type subscriptionListResponse struct {
} `json:"items"`
}
type searchListResponse struct {
type playlistItemListResponse struct {
Items []struct {
ID struct {
VideoID string `json:"videoId"`
} `json:"id"`
Snippet struct {
Title string `json:"title"`
PublishedAt time.Time `json:"publishedAt"`
ResourceID struct {
VideoID string `json:"videoId"`
} `json:"resourceId"`
} `json:"snippet"`
} `json:"items"`
}
type channelListResponse struct {
Items []struct {
ContentDetails struct {
RelatedPlaylists struct {
Uploads string `json:"uploads"`
} `json:"relatedPlaylists"`
} `json:"contentDetails"`
} `json:"items"`
}