From af1c163a11a382083d0d23637479cd3a924bbb9d Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 2 Jun 2026 23:15:27 +0200 Subject: [PATCH] fix(youtube): discover via uploads playlist, not search.list (100x cheaper) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NewVideos called search.list at 100 quota units/call. With ~143 channels one discovery pass = 14,300 units > the 10,000/day YouTube Data API cap, exhausting the whole day in a single loop (live quotaExceeded). Switch to playlistItems.list (1 unit/call) against the channel's uploads playlist. For a standard channel id UCxxxx the uploads playlist is UUxxxx, derived at zero API cost (uploadsPlaylistID). Non-standard ids fall back to channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads. Newest- first ordering and MaxVideosPerSubscription cap preserved. Side effect: removing search.list also removes the accountDelegationForbidden error that endpoint threw for one channel — no separate hardening needed. New per-pass quota: /subscriptions (1) + ~1/channel discovery (143) + any channels.list fallbacks ≈ 145 units/day, well under 10k. Caption fetch (ADR-010 timedtext) uses no Data API quota. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/adapters/youtube/youtube.go | 84 ++++++++++++++++--- internal/adapters/youtube/youtube_test.go | 98 +++++++++++++++++++++-- 2 files changed, 163 insertions(+), 19 deletions(-) diff --git a/internal/adapters/youtube/youtube.go b/internal/adapters/youtube/youtube.go index 5ae4c16..21ea9f8 100644 --- a/internal/adapters/youtube/youtube.go +++ b/internal/adapters/youtube/youtube.go @@ -179,6 +179,15 @@ func (a *Adapter) ListSubscriptions(ctx context.Context, userID string) ([]domai // first. The engine dedups within its lifetime and the store dedups durably // (data-model.md), so the adapter returns recent candidates rather than tracking // "seen" state itself. +// +// Discovery uses playlistItems.list against the channel's "uploads" playlist +// (1 quota unit/call) instead of search.list (100 units): with ~143 channels a +// single search-based pass exhausted the entire 10,000 unit/day cap. For a +// standard channel id "UCxxxx" the uploads playlist is "UUxxxx" — derived with +// zero API cost. Non-standard ids that don't start with "UC" fall back to +// channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads. +// Dropping search.list also removes the accountDelegationForbidden failure mode +// that endpoint exhibited for one channel. func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) { client, err := a.httpClient(ctx, a.cfg.TokenSecretRef) if err != nil { @@ -190,37 +199,76 @@ func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]dom max = defaultMaxVideos } + playlistID, ok := uploadsPlaylistID(sub.ChannelID) + if !ok { + // Non-standard channel id: resolve the uploads playlist explicitly. + playlistID, err = a.resolveUploadsPlaylist(ctx, client, sub.ChannelID) + if err != nil { + return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err) + } + } + q := url.Values{ "part": {"snippet"}, - "channelId": {sub.ChannelID}, - "order": {"date"}, - "type": {"video"}, + "playlistId": {playlistID}, "maxResults": {fmt.Sprintf("%d", max)}, } - var resp searchListResponse - if err := a.getJSON(ctx, client, "/search", q, &resp); err != nil { + var resp playlistItemListResponse + if err := a.getJSON(ctx, client, "/playlistItems", q, &resp); err != nil { return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err) } videos := make([]domain.Video, 0, len(resp.Items)) for _, item := range resp.Items { - if item.ID.VideoID == "" { - continue // non-video result; type=video should prevent this, be defensive + vid := item.Snippet.ResourceID.VideoID + if vid == "" { + continue // defensive: skip items without a resolvable video id } videos = append(videos, domain.Video{ UserID: sub.UserID, SubscriptionID: sub.ID, Provider: domain.ProviderYouTube, - ProviderVideoID: item.ID.VideoID, + ProviderVideoID: vid, Title: item.Snippet.Title, - URL: "https://www.youtube.com/watch?v=" + item.ID.VideoID, + URL: "https://www.youtube.com/watch?v=" + vid, PublishedAt: item.Snippet.PublishedAt, }) + if len(videos) >= max { + break + } } return videos, nil } +// uploadsPlaylistID derives a channel's uploads playlist id at zero API cost: +// a standard channel id "UCxxxx" maps to uploads playlist "UUxxxx". Returns +// ok=false for ids that don't follow this convention (caller falls back to +// channels.list). +func uploadsPlaylistID(channelID string) (string, bool) { + if !strings.HasPrefix(channelID, "UC") { + return "", false + } + return "UU" + channelID[2:], true +} + +// resolveUploadsPlaylist reads contentDetails.relatedPlaylists.uploads via +// channels.list (1 quota unit) for channels whose id can't be mapped UC->UU. +func (a *Adapter) resolveUploadsPlaylist(ctx context.Context, client *http.Client, channelID string) (string, error) { + q := url.Values{ + "part": {"contentDetails"}, + "id": {channelID}, + } + var resp channelListResponse + if err := a.getJSON(ctx, client, "/channels", q, &resp); err != nil { + return "", fmt.Errorf("resolve uploads playlist: %w", err) + } + if len(resp.Items) == 0 || resp.Items[0].ContentDetails.RelatedPlaylists.Uploads == "" { + return "", fmt.Errorf("no uploads playlist for channel %q", channelID) + } + return resp.Items[0].ContentDetails.RelatedPlaylists.Uploads, nil +} + // getJSON issues a GET and decodes a JSON body into out. A non-200 status is an // error carrying a bounded slice of the response body for diagnosis. func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error { @@ -273,14 +321,24 @@ type subscriptionListResponse struct { } `json:"items"` } -type searchListResponse struct { +type playlistItemListResponse struct { Items []struct { - ID struct { - VideoID string `json:"videoId"` - } `json:"id"` Snippet struct { Title string `json:"title"` PublishedAt time.Time `json:"publishedAt"` + ResourceID struct { + VideoID string `json:"videoId"` + } `json:"resourceId"` } `json:"snippet"` } `json:"items"` } + +type channelListResponse struct { + Items []struct { + ContentDetails struct { + RelatedPlaylists struct { + Uploads string `json:"uploads"` + } `json:"relatedPlaylists"` + } `json:"contentDetails"` + } `json:"items"` +} diff --git a/internal/adapters/youtube/youtube_test.go b/internal/adapters/youtube/youtube_test.go index 734c906..f0feae4 100644 --- a/internal/adapters/youtube/youtube_test.go +++ b/internal/adapters/youtube/youtube_test.go @@ -112,18 +112,21 @@ func TestListSubscriptionsPaginates(t *testing.T) { // --- new videos ------------------------------------------------------------- +// TestNewVideos: discovery uses playlistItems.list (1 quota unit) against the +// uploads playlist derived from the channel id (UC_acme -> UU_acme, ADR via +// Worker H mission), NOT search.list (100 units). Items map newest-first. func TestNewVideos(t *testing.T) { a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/search" { - t.Errorf("unexpected path %q", r.URL.Path) + if r.URL.Path != "/playlistItems" { + t.Errorf("unexpected path %q (must use playlistItems, not search)", r.URL.Path) } - if got := r.URL.Query().Get("channelId"); got != "UC_acme" { - t.Errorf("expected channelId=UC_acme, got %q", got) + if got := r.URL.Query().Get("playlistId"); got != "UU_acme" { + t.Errorf("expected playlistId=UU_acme (uploads playlist), got %q", got) } _, _ = w.Write([]byte(`{ "items": [ - {"id": {"videoId": "vid1"}, "snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z"}}, - {"id": {"videoId": "vid2"}, "snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z"}} + {"snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z", "resourceId": {"videoId": "vid1"}}}, + {"snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z", "resourceId": {"videoId": "vid2"}}} ] }`)) }) @@ -149,6 +152,89 @@ func TestNewVideos(t *testing.T) { if v.PublishedAt.IsZero() { t.Errorf("expected publishedAt parsed, got zero") } + // Newest-first ordering preserved from the playlist response. + if vids[1].ProviderVideoID != "vid2" { + t.Errorf("expected newest-first ordering, got second=%q", vids[1].ProviderVideoID) + } +} + +// TestNewVideosCapsAtMax: MaxVideosPerSubscription bounds the playlistItems +// page size (maxResults) and the number of returned videos. +func TestNewVideosCapsAtMax(t *testing.T) { + a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("maxResults"); got != "2" { + t.Errorf("expected maxResults=2 from cap, got %q", got) + } + _, _ = w.Write([]byte(`{ + "items": [ + {"snippet": {"title": "A", "publishedAt": "2026-06-03T10:00:00Z", "resourceId": {"videoId": "a"}}}, + {"snippet": {"title": "B", "publishedAt": "2026-06-02T10:00:00Z", "resourceId": {"videoId": "b"}}} + ] + }`)) + }) + a.cfg.MaxVideosPerSubscription = 2 + + vids, err := a.NewVideos(context.Background(), domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "UC_acme"}) + if err != nil { + t.Fatalf("NewVideos: %v", err) + } + if len(vids) != 2 { + t.Fatalf("expected cap of 2 videos, got %d", len(vids)) + } +} + +// TestUploadsPlaylistID covers the zero-cost UC->UU derivation, including +// non-standard ids that must fall through unchanged (handled via fallback). +func TestUploadsPlaylistID(t *testing.T) { + cases := []struct { + channel string + want string + derived bool + }{ + {"UC_acme", "UU_acme", true}, + {"UCabcdef123456", "UUabcdef123456", true}, + {"HC_handle_style", "", false}, + {"", "", false}, + } + for _, c := range cases { + got, ok := uploadsPlaylistID(c.channel) + if ok != c.derived { + t.Errorf("uploadsPlaylistID(%q) derived=%v, want %v", c.channel, ok, c.derived) + } + if got != c.want { + t.Errorf("uploadsPlaylistID(%q)=%q, want %q", c.channel, got, c.want) + } + } +} + +// TestNewVideosFallbackToChannelsList: a non-standard channel id can't be +// mapped UC->UU, so the adapter reads contentDetails.relatedPlaylists.uploads +// via channels.list (1 unit) and then fetches that playlist. +func TestNewVideosFallbackToChannelsList(t *testing.T) { + a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/channels": + if got := r.URL.Query().Get("id"); got != "HC_weird" { + t.Errorf("expected channels id=HC_weird, got %q", got) + } + _, _ = w.Write([]byte(`{"items":[{"contentDetails":{"relatedPlaylists":{"uploads":"UU_resolved"}}}]}`)) + case "/playlistItems": + if got := r.URL.Query().Get("playlistId"); got != "UU_resolved" { + t.Errorf("expected playlistId=UU_resolved, got %q", got) + } + _, _ = w.Write([]byte(`{"items":[{"snippet":{"title":"X","publishedAt":"2026-06-01T10:00:00Z","resourceId":{"videoId":"x"}}}]}`)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + }) + + vids, err := a.NewVideos(context.Background(), domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "HC_weird"}) + if err != nil { + t.Fatalf("NewVideos fallback: %v", err) + } + if len(vids) != 1 || vids[0].ProviderVideoID != "x" { + t.Fatalf("expected 1 video via fallback, got %+v", vids) + } } // --- transcript: captions present (player response + timedtext baseUrl) ------