fix(youtube): discover via uploads playlist, not search.list (100x cheaper)
NewVideos called search.list at 100 quota units/call. With ~143 channels one discovery pass = 14,300 units > the 10,000/day YouTube Data API cap, exhausting the whole day in a single loop (live quotaExceeded). Switch to playlistItems.list (1 unit/call) against the channel's uploads playlist. For a standard channel id UCxxxx the uploads playlist is UUxxxx, derived at zero API cost (uploadsPlaylistID). Non-standard ids fall back to channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads. Newest- first ordering and MaxVideosPerSubscription cap preserved. Side effect: removing search.list also removes the accountDelegationForbidden error that endpoint threw for one channel — no separate hardening needed. New per-pass quota: /subscriptions (1) + ~1/channel discovery (143) + any channels.list fallbacks ≈ 145 units/day, well under 10k. Caption fetch (ADR-010 timedtext) uses no Data API quota. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -179,6 +179,15 @@ func (a *Adapter) ListSubscriptions(ctx context.Context, userID string) ([]domai
|
|||||||
// first. The engine dedups within its lifetime and the store dedups durably
|
// first. The engine dedups within its lifetime and the store dedups durably
|
||||||
// (data-model.md), so the adapter returns recent candidates rather than tracking
|
// (data-model.md), so the adapter returns recent candidates rather than tracking
|
||||||
// "seen" state itself.
|
// "seen" state itself.
|
||||||
|
//
|
||||||
|
// Discovery uses playlistItems.list against the channel's "uploads" playlist
|
||||||
|
// (1 quota unit/call) instead of search.list (100 units): with ~143 channels a
|
||||||
|
// single search-based pass exhausted the entire 10,000 unit/day cap. For a
|
||||||
|
// standard channel id "UCxxxx" the uploads playlist is "UUxxxx" — derived with
|
||||||
|
// zero API cost. Non-standard ids that don't start with "UC" fall back to
|
||||||
|
// channels.list (1 unit) to read contentDetails.relatedPlaylists.uploads.
|
||||||
|
// Dropping search.list also removes the accountDelegationForbidden failure mode
|
||||||
|
// that endpoint exhibited for one channel.
|
||||||
func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) {
|
func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) {
|
||||||
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -190,37 +199,76 @@ func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]dom
|
|||||||
max = defaultMaxVideos
|
max = defaultMaxVideos
|
||||||
}
|
}
|
||||||
|
|
||||||
|
playlistID, ok := uploadsPlaylistID(sub.ChannelID)
|
||||||
|
if !ok {
|
||||||
|
// Non-standard channel id: resolve the uploads playlist explicitly.
|
||||||
|
playlistID, err = a.resolveUploadsPlaylist(ctx, client, sub.ChannelID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
q := url.Values{
|
q := url.Values{
|
||||||
"part": {"snippet"},
|
"part": {"snippet"},
|
||||||
"channelId": {sub.ChannelID},
|
"playlistId": {playlistID},
|
||||||
"order": {"date"},
|
|
||||||
"type": {"video"},
|
|
||||||
"maxResults": {fmt.Sprintf("%d", max)},
|
"maxResults": {fmt.Sprintf("%d", max)},
|
||||||
}
|
}
|
||||||
|
|
||||||
var resp searchListResponse
|
var resp playlistItemListResponse
|
||||||
if err := a.getJSON(ctx, client, "/search", q, &resp); err != nil {
|
if err := a.getJSON(ctx, client, "/playlistItems", q, &resp); err != nil {
|
||||||
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
|
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
videos := make([]domain.Video, 0, len(resp.Items))
|
videos := make([]domain.Video, 0, len(resp.Items))
|
||||||
for _, item := range resp.Items {
|
for _, item := range resp.Items {
|
||||||
if item.ID.VideoID == "" {
|
vid := item.Snippet.ResourceID.VideoID
|
||||||
continue // non-video result; type=video should prevent this, be defensive
|
if vid == "" {
|
||||||
|
continue // defensive: skip items without a resolvable video id
|
||||||
}
|
}
|
||||||
videos = append(videos, domain.Video{
|
videos = append(videos, domain.Video{
|
||||||
UserID: sub.UserID,
|
UserID: sub.UserID,
|
||||||
SubscriptionID: sub.ID,
|
SubscriptionID: sub.ID,
|
||||||
Provider: domain.ProviderYouTube,
|
Provider: domain.ProviderYouTube,
|
||||||
ProviderVideoID: item.ID.VideoID,
|
ProviderVideoID: vid,
|
||||||
Title: item.Snippet.Title,
|
Title: item.Snippet.Title,
|
||||||
URL: "https://www.youtube.com/watch?v=" + item.ID.VideoID,
|
URL: "https://www.youtube.com/watch?v=" + vid,
|
||||||
PublishedAt: item.Snippet.PublishedAt,
|
PublishedAt: item.Snippet.PublishedAt,
|
||||||
})
|
})
|
||||||
|
if len(videos) >= max {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return videos, nil
|
return videos, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// uploadsPlaylistID derives a channel's uploads playlist id at zero API cost:
|
||||||
|
// a standard channel id "UCxxxx" maps to uploads playlist "UUxxxx". Returns
|
||||||
|
// ok=false for ids that don't follow this convention (caller falls back to
|
||||||
|
// channels.list).
|
||||||
|
func uploadsPlaylistID(channelID string) (string, bool) {
|
||||||
|
if !strings.HasPrefix(channelID, "UC") {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return "UU" + channelID[2:], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveUploadsPlaylist reads contentDetails.relatedPlaylists.uploads via
|
||||||
|
// channels.list (1 quota unit) for channels whose id can't be mapped UC->UU.
|
||||||
|
func (a *Adapter) resolveUploadsPlaylist(ctx context.Context, client *http.Client, channelID string) (string, error) {
|
||||||
|
q := url.Values{
|
||||||
|
"part": {"contentDetails"},
|
||||||
|
"id": {channelID},
|
||||||
|
}
|
||||||
|
var resp channelListResponse
|
||||||
|
if err := a.getJSON(ctx, client, "/channels", q, &resp); err != nil {
|
||||||
|
return "", fmt.Errorf("resolve uploads playlist: %w", err)
|
||||||
|
}
|
||||||
|
if len(resp.Items) == 0 || resp.Items[0].ContentDetails.RelatedPlaylists.Uploads == "" {
|
||||||
|
return "", fmt.Errorf("no uploads playlist for channel %q", channelID)
|
||||||
|
}
|
||||||
|
return resp.Items[0].ContentDetails.RelatedPlaylists.Uploads, nil
|
||||||
|
}
|
||||||
|
|
||||||
// getJSON issues a GET and decodes a JSON body into out. A non-200 status is an
|
// getJSON issues a GET and decodes a JSON body into out. A non-200 status is an
|
||||||
// error carrying a bounded slice of the response body for diagnosis.
|
// error carrying a bounded slice of the response body for diagnosis.
|
||||||
func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error {
|
func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error {
|
||||||
@@ -273,14 +321,24 @@ type subscriptionListResponse struct {
|
|||||||
} `json:"items"`
|
} `json:"items"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type searchListResponse struct {
|
type playlistItemListResponse struct {
|
||||||
Items []struct {
|
Items []struct {
|
||||||
ID struct {
|
|
||||||
VideoID string `json:"videoId"`
|
|
||||||
} `json:"id"`
|
|
||||||
Snippet struct {
|
Snippet struct {
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
PublishedAt time.Time `json:"publishedAt"`
|
PublishedAt time.Time `json:"publishedAt"`
|
||||||
|
ResourceID struct {
|
||||||
|
VideoID string `json:"videoId"`
|
||||||
|
} `json:"resourceId"`
|
||||||
} `json:"snippet"`
|
} `json:"snippet"`
|
||||||
} `json:"items"`
|
} `json:"items"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type channelListResponse struct {
|
||||||
|
Items []struct {
|
||||||
|
ContentDetails struct {
|
||||||
|
RelatedPlaylists struct {
|
||||||
|
Uploads string `json:"uploads"`
|
||||||
|
} `json:"relatedPlaylists"`
|
||||||
|
} `json:"contentDetails"`
|
||||||
|
} `json:"items"`
|
||||||
|
}
|
||||||
|
|||||||
@@ -112,18 +112,21 @@ func TestListSubscriptionsPaginates(t *testing.T) {
|
|||||||
|
|
||||||
// --- new videos -------------------------------------------------------------
|
// --- new videos -------------------------------------------------------------
|
||||||
|
|
||||||
|
// TestNewVideos: discovery uses playlistItems.list (1 quota unit) against the
|
||||||
|
// uploads playlist derived from the channel id (UC_acme -> UU_acme, ADR via
|
||||||
|
// Worker H mission), NOT search.list (100 units). Items map newest-first.
|
||||||
func TestNewVideos(t *testing.T) {
|
func TestNewVideos(t *testing.T) {
|
||||||
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.URL.Path != "/search" {
|
if r.URL.Path != "/playlistItems" {
|
||||||
t.Errorf("unexpected path %q", r.URL.Path)
|
t.Errorf("unexpected path %q (must use playlistItems, not search)", r.URL.Path)
|
||||||
}
|
}
|
||||||
if got := r.URL.Query().Get("channelId"); got != "UC_acme" {
|
if got := r.URL.Query().Get("playlistId"); got != "UU_acme" {
|
||||||
t.Errorf("expected channelId=UC_acme, got %q", got)
|
t.Errorf("expected playlistId=UU_acme (uploads playlist), got %q", got)
|
||||||
}
|
}
|
||||||
_, _ = w.Write([]byte(`{
|
_, _ = w.Write([]byte(`{
|
||||||
"items": [
|
"items": [
|
||||||
{"id": {"videoId": "vid1"}, "snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z"}},
|
{"snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z", "resourceId": {"videoId": "vid1"}}},
|
||||||
{"id": {"videoId": "vid2"}, "snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z"}}
|
{"snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z", "resourceId": {"videoId": "vid2"}}}
|
||||||
]
|
]
|
||||||
}`))
|
}`))
|
||||||
})
|
})
|
||||||
@@ -149,6 +152,89 @@ func TestNewVideos(t *testing.T) {
|
|||||||
if v.PublishedAt.IsZero() {
|
if v.PublishedAt.IsZero() {
|
||||||
t.Errorf("expected publishedAt parsed, got zero")
|
t.Errorf("expected publishedAt parsed, got zero")
|
||||||
}
|
}
|
||||||
|
// Newest-first ordering preserved from the playlist response.
|
||||||
|
if vids[1].ProviderVideoID != "vid2" {
|
||||||
|
t.Errorf("expected newest-first ordering, got second=%q", vids[1].ProviderVideoID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNewVideosCapsAtMax: MaxVideosPerSubscription bounds the playlistItems
|
||||||
|
// page size (maxResults) and the number of returned videos.
|
||||||
|
func TestNewVideosCapsAtMax(t *testing.T) {
|
||||||
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if got := r.URL.Query().Get("maxResults"); got != "2" {
|
||||||
|
t.Errorf("expected maxResults=2 from cap, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"items": [
|
||||||
|
{"snippet": {"title": "A", "publishedAt": "2026-06-03T10:00:00Z", "resourceId": {"videoId": "a"}}},
|
||||||
|
{"snippet": {"title": "B", "publishedAt": "2026-06-02T10:00:00Z", "resourceId": {"videoId": "b"}}}
|
||||||
|
]
|
||||||
|
}`))
|
||||||
|
})
|
||||||
|
a.cfg.MaxVideosPerSubscription = 2
|
||||||
|
|
||||||
|
vids, err := a.NewVideos(context.Background(), domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "UC_acme"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewVideos: %v", err)
|
||||||
|
}
|
||||||
|
if len(vids) != 2 {
|
||||||
|
t.Fatalf("expected cap of 2 videos, got %d", len(vids))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUploadsPlaylistID covers the zero-cost UC->UU derivation, including
|
||||||
|
// non-standard ids that must fall through unchanged (handled via fallback).
|
||||||
|
func TestUploadsPlaylistID(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
channel string
|
||||||
|
want string
|
||||||
|
derived bool
|
||||||
|
}{
|
||||||
|
{"UC_acme", "UU_acme", true},
|
||||||
|
{"UCabcdef123456", "UUabcdef123456", true},
|
||||||
|
{"HC_handle_style", "", false},
|
||||||
|
{"", "", false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got, ok := uploadsPlaylistID(c.channel)
|
||||||
|
if ok != c.derived {
|
||||||
|
t.Errorf("uploadsPlaylistID(%q) derived=%v, want %v", c.channel, ok, c.derived)
|
||||||
|
}
|
||||||
|
if got != c.want {
|
||||||
|
t.Errorf("uploadsPlaylistID(%q)=%q, want %q", c.channel, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNewVideosFallbackToChannelsList: a non-standard channel id can't be
|
||||||
|
// mapped UC->UU, so the adapter reads contentDetails.relatedPlaylists.uploads
|
||||||
|
// via channels.list (1 unit) and then fetches that playlist.
|
||||||
|
func TestNewVideosFallbackToChannelsList(t *testing.T) {
|
||||||
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/channels":
|
||||||
|
if got := r.URL.Query().Get("id"); got != "HC_weird" {
|
||||||
|
t.Errorf("expected channels id=HC_weird, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"items":[{"contentDetails":{"relatedPlaylists":{"uploads":"UU_resolved"}}}]}`))
|
||||||
|
case "/playlistItems":
|
||||||
|
if got := r.URL.Query().Get("playlistId"); got != "UU_resolved" {
|
||||||
|
t.Errorf("expected playlistId=UU_resolved, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"items":[{"snippet":{"title":"X","publishedAt":"2026-06-01T10:00:00Z","resourceId":{"videoId":"x"}}}]}`))
|
||||||
|
default:
|
||||||
|
t.Errorf("unexpected path %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
vids, err := a.NewVideos(context.Background(), domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "HC_weird"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewVideos fallback: %v", err)
|
||||||
|
}
|
||||||
|
if len(vids) != 1 || vids[0].ProviderVideoID != "x" {
|
||||||
|
t.Fatalf("expected 1 video via fallback, got %+v", vids)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- transcript: captions present (player response + timedtext baseUrl) ------
|
// --- transcript: captions present (player response + timedtext baseUrl) ------
|
||||||
|
|||||||
Reference in New Issue
Block a user