fix(youtube): discover via uploads playlist, not search.list (100x cheaper)

NewVideos called search.list at 100 quota units/call. With ~143 channels one discovery pass = 14,300 units > the 10,000/day YouTube Data API cap, exhausting the whole day in a single loop (live quotaExceeded). Switch to playlistItems.list (1 unit/call) against the channel's uploads playlist. For a standard channel id UCxxxx the uploads playlist is UUxxxx, derived at zero API cost (uploadsPlaylistID). Non-standard ids fall back to channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads. Newest- first ordering and MaxVideosPerSubscription cap preserved. Side effect: removing search.list also removes the accountDelegationForbidden error that endpoint threw for one channel — no separate hardening needed. New per-pass quota: /subscriptions (1) + ~1/channel discovery (143) + any channels.list fallbacks ≈ 145 units/day, well under 10k. Caption fetch (ADR-010 timedtext) uses no Data API quota. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-02 23:15:27 +02:00
parent 92da25d01e
commit af1c163a11
2 changed files with 163 additions and 19 deletions
@@ -179,6 +179,15 @@ func (a *Adapter) ListSubscriptions(ctx context.Context, userID string) ([]domai
 // first. The engine dedups within its lifetime and the store dedups durably
 // (data-model.md), so the adapter returns recent candidates rather than tracking
 // "seen" state itself.
+//
+// Discovery uses playlistItems.list against the channel's "uploads" playlist
+// (1 quota unit/call) instead of search.list (100 units): with ~143 channels a
+// single search-based pass exhausted the entire 10,000 unit/day cap. For a
+// standard channel id "UCxxxx" the uploads playlist is "UUxxxx" — derived with
+// zero API cost. Non-standard ids that don't start with "UC" fall back to
+// channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads.
+// Dropping search.list also removes the accountDelegationForbidden failure mode
+// that endpoint exhibited for one channel.
 func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) {
 	client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
 	if err != nil {
@@ -190,37 +199,76 @@ func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]dom
 		max = defaultMaxVideos
 	}

+	playlistID, ok := uploadsPlaylistID(sub.ChannelID)
+	if !ok {
+		// Non-standard channel id: resolve the uploads playlist explicitly.
+		playlistID, err = a.resolveUploadsPlaylist(ctx, client, sub.ChannelID)
+		if err != nil {
+			return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
+		}
+	}
+
 	q := url.Values{
 		"part":       {"snippet"},
-		"channelId":  {sub.ChannelID},
-		"order":      {"date"},
-		"type":       {"video"},
+		"playlistId": {playlistID},
 		"maxResults": {fmt.Sprintf("%d", max)},
 	}

-	var resp searchListResponse
-	if err := a.getJSON(ctx, client, "/search", q, &resp); err != nil {
+	var resp playlistItemListResponse
+	if err := a.getJSON(ctx, client, "/playlistItems", q, &resp); err != nil {
 		return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
 	}

 	videos := make([]domain.Video, 0, len(resp.Items))
 	for _, item := range resp.Items {
-		if item.ID.VideoID == "" {
-			continue // non-video result; type=video should prevent this, be defensive
+		vid := item.Snippet.ResourceID.VideoID
+		if vid == "" {
+			continue // defensive: skip items without a resolvable video id
 		}
 		videos = append(videos, domain.Video{
 			UserID:          sub.UserID,
 			SubscriptionID:  sub.ID,
 			Provider:        domain.ProviderYouTube,
-			ProviderVideoID: item.ID.VideoID,
+			ProviderVideoID: vid,
 			Title:           item.Snippet.Title,
-			URL:             "https://www.youtube.com/watch?v=" + item.ID.VideoID,
+			URL:             "https://www.youtube.com/watch?v=" + vid,
 			PublishedAt:     item.Snippet.PublishedAt,
 		})
+		if len(videos) >= max {
+			break
+		}
 	}
 	return videos, nil
 }

+// uploadsPlaylistID derives a channel's uploads playlist id at zero API cost:
+// a standard channel id "UCxxxx" maps to uploads playlist "UUxxxx". Returns
+// ok=false for ids that don't follow this convention (caller falls back to
+// channels.list).
+func uploadsPlaylistID(channelID string) (string, bool) {
+	if !strings.HasPrefix(channelID, "UC") {
+		return "", false
+	}
+	return "UU" + channelID[2:], true
+}
+
+// resolveUploadsPlaylist reads contentDetails.relatedPlaylists.uploads via
+// channels.list (1 quota unit) for channels whose id can't be mapped UC->UU.
+func (a *Adapter) resolveUploadsPlaylist(ctx context.Context, client *http.Client, channelID string) (string, error) {
+	q := url.Values{
+		"part": {"contentDetails"},
+		"id":   {channelID},
+	}
+	var resp channelListResponse
+	if err := a.getJSON(ctx, client, "/channels", q, &resp); err != nil {
+		return "", fmt.Errorf("resolve uploads playlist: %w", err)
+	}
+	if len(resp.Items) == 0 || resp.Items[0].ContentDetails.RelatedPlaylists.Uploads == "" {
+		return "", fmt.Errorf("no uploads playlist for channel %q", channelID)
+	}
+	return resp.Items[0].ContentDetails.RelatedPlaylists.Uploads, nil
+}
+
 // getJSON issues a GET and decodes a JSON body into out. A non-200 status is an
 // error carrying a bounded slice of the response body for diagnosis.
 func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error {
@@ -273,14 +321,24 @@ type subscriptionListResponse struct {
 	} `json:"items"`
 }

-type searchListResponse struct {
+type playlistItemListResponse struct {
 	Items []struct {
-		ID struct {
-			VideoID string `json:"videoId"`
-		} `json:"id"`
 		Snippet struct {
 			Title       string    `json:"title"`
 			PublishedAt time.Time `json:"publishedAt"`
+			ResourceID  struct {
+				VideoID string `json:"videoId"`
+			} `json:"resourceId"`
 		} `json:"snippet"`
 	} `json:"items"`
 }
+
+type channelListResponse struct {
+	Items []struct {
+		ContentDetails struct {
+			RelatedPlaylists struct {
+				Uploads string `json:"uploads"`
+			} `json:"relatedPlaylists"`
+		} `json:"contentDetails"`
+	} `json:"items"`
+}
@@ -112,18 +112,21 @@ func TestListSubscriptionsPaginates(t *testing.T) {

 // --- new videos -------------------------------------------------------------

+// TestNewVideos: discovery uses playlistItems.list (1 quota unit) against the
+// uploads playlist derived from the channel id (UC_acme -> UU_acme, ADR via
+// Worker H mission), NOT search.list (100 units). Items map newest-first.
 func TestNewVideos(t *testing.T) {
 	a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path != "/search" {
-			t.Errorf("unexpected path %q", r.URL.Path)
+		if r.URL.Path != "/playlistItems" {
+			t.Errorf("unexpected path %q (must use playlistItems, not search)", r.URL.Path)
 		}
-		if got := r.URL.Query().Get("channelId"); got != "UC_acme" {
-			t.Errorf("expected channelId=UC_acme, got %q", got)
+		if got := r.URL.Query().Get("playlistId"); got != "UU_acme" {
+			t.Errorf("expected playlistId=UU_acme (uploads playlist), got %q", got)
 		}
 		_, _ = w.Write([]byte(`{
 			"items": [
-				{"id": {"videoId": "vid1"}, "snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z"}},
-				{"id": {"videoId": "vid2"}, "snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z"}}
+				{"snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z", "resourceId": {"videoId": "vid1"}}},
+				{"snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z", "resourceId": {"videoId": "vid2"}}}
 			]
 		}`))
 	})
@@ -149,6 +152,89 @@ func TestNewVideos(t *testing.T) {
 	if v.PublishedAt.IsZero() {
 		t.Errorf("expected publishedAt parsed, got zero")
 	}
+	// Newest-first ordering preserved from the playlist response.
+	if vids[1].ProviderVideoID != "vid2" {
+		t.Errorf("expected newest-first ordering, got second=%q", vids[1].ProviderVideoID)
+	}
+}
+
+// TestNewVideosCapsAtMax: MaxVideosPerSubscription bounds the playlistItems
+// page size (maxResults) and the number of returned videos.
+func TestNewVideosCapsAtMax(t *testing.T) {
+	a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
+		if got := r.URL.Query().Get("maxResults"); got != "2" {
+			t.Errorf("expected maxResults=2 from cap, got %q", got)
+		}
+		_, _ = w.Write([]byte(`{
+			"items": [
+				{"snippet": {"title": "A", "publishedAt": "2026-06-03T10:00:00Z", "resourceId": {"videoId": "a"}}},
+				{"snippet": {"title": "B", "publishedAt": "2026-06-02T10:00:00Z", "resourceId": {"videoId": "b"}}}
+			]
+		}`))
+	})
+	a.cfg.MaxVideosPerSubscription = 2
+
+	vids, err := a.NewVideos(context.Background(), domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "UC_acme"})
+	if err != nil {
+		t.Fatalf("NewVideos: %v", err)
+	}
+	if len(vids) != 2 {
+		t.Fatalf("expected cap of 2 videos, got %d", len(vids))
+	}
+}
+
+// TestUploadsPlaylistID covers the zero-cost UC->UU derivation, including
+// non-standard ids that must fall through unchanged (handled via fallback).
+func TestUploadsPlaylistID(t *testing.T) {
+	cases := []struct {
+		channel string
+		want    string
+		derived bool
+	}{
+		{"UC_acme", "UU_acme", true},
+		{"UCabcdef123456", "UUabcdef123456", true},
+		{"HC_handle_style", "", false},
+		{"", "", false},
+	}
+	for _, c := range cases {
+		got, ok := uploadsPlaylistID(c.channel)
+		if ok != c.derived {
+			t.Errorf("uploadsPlaylistID(%q) derived=%v, want %v", c.channel, ok, c.derived)
+		}
+		if got != c.want {
+			t.Errorf("uploadsPlaylistID(%q)=%q, want %q", c.channel, got, c.want)
+		}
+	}
+}
+
+// TestNewVideosFallbackToChannelsList: a non-standard channel id can't be
+// mapped UC->UU, so the adapter reads contentDetails.relatedPlaylists.uploads
+// via channels.list (1 unit) and then fetches that playlist.
+func TestNewVideosFallbackToChannelsList(t *testing.T) {
+	a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/channels":
+			if got := r.URL.Query().Get("id"); got != "HC_weird" {
+				t.Errorf("expected channels id=HC_weird, got %q", got)
+			}
+			_, _ = w.Write([]byte(`{"items":[{"contentDetails":{"relatedPlaylists":{"uploads":"UU_resolved"}}}]}`))
+		case "/playlistItems":
+			if got := r.URL.Query().Get("playlistId"); got != "UU_resolved" {
+				t.Errorf("expected playlistId=UU_resolved, got %q", got)
+			}
+			_, _ = w.Write([]byte(`{"items":[{"snippet":{"title":"X","publishedAt":"2026-06-01T10:00:00Z","resourceId":{"videoId":"x"}}}]}`))
+		default:
+			t.Errorf("unexpected path %q", r.URL.Path)
+		}
+	})
+
+	vids, err := a.NewVideos(context.Background(), domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "HC_weird"})
+	if err != nil {
+		t.Fatalf("NewVideos fallback: %v", err)
+	}
+	if len(vids) != 1 || vids[0].ProviderVideoID != "x" {
+		t.Fatalf("expected 1 video via fallback, got %+v", vids)
+	}
 }

 // --- transcript: captions present (player response + timedtext baseUrl) ------