The Data API captions.download endpoint is owner-only: every subscription video the user does not own returned HTTP 403, producing 0 summaries and a ~150-line error spew in the first live Stage-0 run. Captions-first (ADR-007) is sound; only the acquisition mechanism was wrong. FetchTranscript now resolves caption tracks from the InnerTube player response (ANDROID client, unauthenticated) and GETs the chosen track's timedtext baseUrl with a plain http.Client — no OAuth token, which can break the endpoint. The srv3 XML, json3, and legacy <transcript> formats all parse; non-asr tracks in a preferred language win. Watch-page ytInitialPlayerResponse scrape is the fallback when InnerTube returns no tracks. Degrade, don't error (explicit quick-fix): no captionTracks, empty baseUrl, a non-200 fetch, or an unparseable body yield Source=none, not an error. Only genuine transport faults error — this kills the spew. OAuth stays on ListSubscriptions/NewVideos (Data API); only transcript fetch goes unauthed. Validated live from koala: the ANDROID client returned working baseUrls and real transcript text for public videos the run identity does not own. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
323 lines
11 KiB
Go
323 lines
11 KiB
Go
package youtube
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
|
)
|
|
|
|
// --- fakes ------------------------------------------------------------------
|
|
|
|
// fakeSecrets records the references it was asked to resolve, proving the
|
|
// adapter resolves OAuth material by reference (never holding the token itself).
|
|
type fakeSecrets struct {
|
|
byRef map[string]string
|
|
requested []string
|
|
}
|
|
|
|
func (f *fakeSecrets) Get(_ context.Context, ref string) (string, error) {
|
|
f.requested = append(f.requested, ref)
|
|
v, ok := f.byRef[ref]
|
|
if !ok {
|
|
return "", fmt.Errorf("no secret for ref %q", ref)
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
// newTestAdapter wires an adapter against an httptest server: BaseURL points at
|
|
// the server and the server's transport is injected so the live OAuth2 exchange
|
|
// is skipped while SecretStore resolution still runs.
|
|
func newTestAdapter(t *testing.T, handler http.HandlerFunc) (*Adapter, *fakeSecrets) {
|
|
t.Helper()
|
|
srv := httptest.NewServer(handler)
|
|
t.Cleanup(srv.Close)
|
|
|
|
secrets := &fakeSecrets{byRef: map[string]string{
|
|
"op://HomeLab/tapir-youtube#refresh": "super-secret-refresh-token",
|
|
}}
|
|
a := New(Config{
|
|
ClientID: "cid",
|
|
ClientSecret: "csecret",
|
|
TokenSecretRef: "op://HomeLab/tapir-youtube#refresh",
|
|
ConnectionID: "conn-1",
|
|
PreferredLanguages: []string{"en"},
|
|
BaseURL: srv.URL,
|
|
PlayerBaseURL: srv.URL,
|
|
}, secrets)
|
|
a.transport = srv.Client().Transport
|
|
return a, secrets
|
|
}
|
|
|
|
// --- subscriptions ----------------------------------------------------------
|
|
|
|
func TestListSubscriptions(t *testing.T) {
|
|
a, secrets := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/subscriptions" {
|
|
t.Errorf("unexpected path %q", r.URL.Path)
|
|
}
|
|
if got := r.URL.Query().Get("mine"); got != "true" {
|
|
t.Errorf("expected mine=true, got %q", got)
|
|
}
|
|
_, _ = w.Write([]byte(`{
|
|
"items": [
|
|
{"id": "sub-a", "snippet": {"title": "Acme Talks", "resourceId": {"channelId": "UC_acme"}}},
|
|
{"id": "sub-b", "snippet": {"title": "Beta Lab", "resourceId": {"channelId": "UC_beta"}}}
|
|
]
|
|
}`))
|
|
})
|
|
|
|
subs, err := a.ListSubscriptions(context.Background(), "u1")
|
|
if err != nil {
|
|
t.Fatalf("ListSubscriptions: %v", err)
|
|
}
|
|
if len(subs) != 2 {
|
|
t.Fatalf("expected 2 subscriptions, got %d", len(subs))
|
|
}
|
|
if subs[0].ChannelID != "UC_acme" || subs[0].ChannelTitle != "Acme Talks" {
|
|
t.Errorf("unexpected first subscription: %+v", subs[0])
|
|
}
|
|
if subs[0].UserID != "u1" || subs[0].ConnectionID != "conn-1" || !subs[0].Active {
|
|
t.Errorf("subscription not wired to user/connection: %+v", subs[0])
|
|
}
|
|
// The refresh token was resolved by reference, never read from config directly.
|
|
if len(secrets.requested) == 0 || secrets.requested[0] != "op://HomeLab/tapir-youtube#refresh" {
|
|
t.Errorf("expected token resolved by ref, got %v", secrets.requested)
|
|
}
|
|
}
|
|
|
|
func TestListSubscriptionsPaginates(t *testing.T) {
|
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Query().Get("pageToken") {
|
|
case "":
|
|
_, _ = w.Write([]byte(`{"nextPageToken":"p2","items":[{"id":"s1","snippet":{"title":"One","resourceId":{"channelId":"c1"}}}]}`))
|
|
case "p2":
|
|
_, _ = w.Write([]byte(`{"items":[{"id":"s2","snippet":{"title":"Two","resourceId":{"channelId":"c2"}}}]}`))
|
|
default:
|
|
t.Errorf("unexpected pageToken %q", r.URL.Query().Get("pageToken"))
|
|
}
|
|
})
|
|
|
|
subs, err := a.ListSubscriptions(context.Background(), "u1")
|
|
if err != nil {
|
|
t.Fatalf("ListSubscriptions: %v", err)
|
|
}
|
|
if len(subs) != 2 {
|
|
t.Fatalf("expected 2 subscriptions across pages, got %d", len(subs))
|
|
}
|
|
}
|
|
|
|
// --- new videos -------------------------------------------------------------
|
|
|
|
func TestNewVideos(t *testing.T) {
|
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/search" {
|
|
t.Errorf("unexpected path %q", r.URL.Path)
|
|
}
|
|
if got := r.URL.Query().Get("channelId"); got != "UC_acme" {
|
|
t.Errorf("expected channelId=UC_acme, got %q", got)
|
|
}
|
|
_, _ = w.Write([]byte(`{
|
|
"items": [
|
|
{"id": {"videoId": "vid1"}, "snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z"}},
|
|
{"id": {"videoId": "vid2"}, "snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z"}}
|
|
]
|
|
}`))
|
|
})
|
|
|
|
sub := domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "UC_acme"}
|
|
vids, err := a.NewVideos(context.Background(), sub)
|
|
if err != nil {
|
|
t.Fatalf("NewVideos: %v", err)
|
|
}
|
|
if len(vids) != 2 {
|
|
t.Fatalf("expected 2 videos, got %d", len(vids))
|
|
}
|
|
v := vids[0]
|
|
if v.ProviderVideoID != "vid1" || v.Title != "Designing for Attention" {
|
|
t.Errorf("unexpected video: %+v", v)
|
|
}
|
|
if v.Provider != domain.ProviderYouTube || v.URL != "https://www.youtube.com/watch?v=vid1" {
|
|
t.Errorf("video not wired correctly: %+v", v)
|
|
}
|
|
if v.UserID != "u1" || v.SubscriptionID != "s1" {
|
|
t.Errorf("video not scoped to user/subscription: %+v", v)
|
|
}
|
|
if v.PublishedAt.IsZero() {
|
|
t.Errorf("expected publishedAt parsed, got zero")
|
|
}
|
|
}
|
|
|
|
// --- transcript: captions present (player response + timedtext baseUrl) ------
|
|
|
|
// captionTracksJSON builds a player-response body whose captionTracks point their
|
|
// baseUrls back at the test server (via the request Host), so the unauthenticated
|
|
// baseUrl GET lands on the same httptest handler.
|
|
func playerBody(host string) string {
|
|
base := "http://" + host
|
|
return `{"captions":{"playerCaptionsTracklistRenderer":{"captionTracks":[
|
|
{"baseUrl":"` + base + `/api/timedtext?lang=sv","languageCode":"sv"},
|
|
{"baseUrl":"` + base + `/api/timedtext?lang=en&kind=asr","languageCode":"en","kind":"asr"},
|
|
{"baseUrl":"` + base + `/api/timedtext?lang=en","languageCode":"en"}
|
|
]}}}`
|
|
}
|
|
|
|
// TestFetchTranscriptWithCaptions: ANDROID player response yields tracks; the en
|
|
// non-asr track is preferred over both the sv track and the en asr track; its
|
|
// srv3 timedtext XML is fetched unauthenticated and reduced to plain text.
|
|
func TestFetchTranscriptWithCaptions(t *testing.T) {
|
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/youtubei/v1/player":
|
|
if r.Method != http.MethodPost {
|
|
t.Errorf("player must be POST, got %s", r.Method)
|
|
}
|
|
if h := r.Header.Get("Authorization"); h != "" {
|
|
t.Errorf("player request must not be authenticated, got Authorization=%q", h)
|
|
}
|
|
_, _ = w.Write([]byte(playerBody(r.Host)))
|
|
case "/api/timedtext":
|
|
if h := r.Header.Get("Authorization"); h != "" {
|
|
t.Errorf("timedtext request must not be authenticated, got Authorization=%q", h)
|
|
}
|
|
switch r.URL.Query().Get("kind") {
|
|
case "asr":
|
|
_, _ = w.Write([]byte(`<timedtext format="3"><body><p t="0" d="1">WRONG asr track</p></body></timedtext>`))
|
|
default:
|
|
if got := r.URL.Query().Get("lang"); got != "en" {
|
|
t.Errorf("expected en track selected, got lang=%q", got)
|
|
}
|
|
_, _ = w.Write([]byte(`<?xml version="1.0" encoding="utf-8" ?><timedtext format="3"><body>` +
|
|
`<p t="0" d="2000">Hello world</p>` +
|
|
`<p t="2000" d="2000">Hello world</p>` +
|
|
`<p t="4000" d="2000">Second line</p>` +
|
|
`</body></timedtext>`))
|
|
}
|
|
default:
|
|
t.Errorf("unexpected path %q", r.URL.Path)
|
|
}
|
|
})
|
|
|
|
v := domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid1"}
|
|
tr, err := a.FetchTranscript(context.Background(), v)
|
|
if err != nil {
|
|
t.Fatalf("FetchTranscript: %v", err)
|
|
}
|
|
if tr.Source != domain.SourceCaptions {
|
|
t.Fatalf("expected SourceCaptions, got %q", tr.Source)
|
|
}
|
|
if tr.Language != "en" {
|
|
t.Errorf("expected preferred language en, got %q", tr.Language)
|
|
}
|
|
// Consecutive duplicate cue collapsed; asr track not chosen.
|
|
want := "Hello world\nSecond line"
|
|
if tr.Content != want {
|
|
t.Errorf("transcript text mismatch:\n got %q\nwant %q", tr.Content, want)
|
|
}
|
|
if !tr.HasText() {
|
|
t.Error("expected HasText() true")
|
|
}
|
|
}
|
|
|
|
// TestFetchTranscriptJSON3: a json3 timedtext body parses to clean text.
|
|
func TestFetchTranscriptJSON3(t *testing.T) {
|
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/youtubei/v1/player":
|
|
base := "http://" + r.Host
|
|
_, _ = w.Write([]byte(`{"captions":{"playerCaptionsTracklistRenderer":{"captionTracks":[` +
|
|
`{"baseUrl":"` + base + `/api/timedtext?lang=en","languageCode":"en"}]}}}`))
|
|
case "/api/timedtext":
|
|
_, _ = w.Write([]byte(`{"events":[` +
|
|
`{"segs":[{"utf8":"Hello "},{"utf8":"world"}]},` +
|
|
`{"segs":[{"utf8":"\n"}]},` +
|
|
`{"segs":[{"utf8":"Second line"}]}]}`))
|
|
default:
|
|
t.Errorf("unexpected path %q", r.URL.Path)
|
|
}
|
|
})
|
|
|
|
tr, err := a.FetchTranscript(context.Background(), domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid1"})
|
|
if err != nil {
|
|
t.Fatalf("FetchTranscript: %v", err)
|
|
}
|
|
if tr.Source != domain.SourceCaptions {
|
|
t.Fatalf("expected SourceCaptions, got %q", tr.Source)
|
|
}
|
|
if want := "Hello world\nSecond line"; tr.Content != want {
|
|
t.Errorf("json3 text mismatch:\n got %q\nwant %q", tr.Content, want)
|
|
}
|
|
}
|
|
|
|
// --- transcript: no captions => SourceNone, no error ------------------------
|
|
|
|
func TestFetchTranscriptNoCaptionsReturnsSourceNone(t *testing.T) {
|
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/api/timedtext" {
|
|
t.Errorf("timedtext must not be fetched when no track exists")
|
|
}
|
|
// Player response with no captions block at all.
|
|
_, _ = w.Write([]byte(`{"videoDetails":{"videoId":"vid-nocaps"}}`))
|
|
})
|
|
|
|
v := domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid-nocaps"}
|
|
tr, err := a.FetchTranscript(context.Background(), v)
|
|
if err != nil {
|
|
t.Fatalf("expected no error for missing captions, got %v", err)
|
|
}
|
|
if tr.Source != domain.SourceNone {
|
|
t.Fatalf("expected SourceNone, got %q", tr.Source)
|
|
}
|
|
if tr.HasText() {
|
|
t.Error("expected HasText() false for SourceNone")
|
|
}
|
|
if tr.Content != "" {
|
|
t.Errorf("expected empty content, got %q", tr.Content)
|
|
}
|
|
}
|
|
|
|
// A 403/empty baseUrl fetch degrades to SourceNone, never an error (the explicit
|
|
// quick-fix: a finicky unofficial endpoint must not produce an error spew).
|
|
func TestFetchTranscriptBaseURLForbiddenDegrades(t *testing.T) {
|
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/youtubei/v1/player":
|
|
base := "http://" + r.Host
|
|
_, _ = w.Write([]byte(`{"captions":{"playerCaptionsTracklistRenderer":{"captionTracks":[` +
|
|
`{"baseUrl":"` + base + `/api/timedtext?lang=en","languageCode":"en"}]}}}`))
|
|
case "/api/timedtext":
|
|
w.WriteHeader(http.StatusForbidden)
|
|
}
|
|
})
|
|
|
|
tr, err := a.FetchTranscript(context.Background(), domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid1"})
|
|
if err != nil {
|
|
t.Fatalf("403 on baseUrl must degrade, not error: %v", err)
|
|
}
|
|
if tr.Source != domain.SourceNone {
|
|
t.Fatalf("expected SourceNone on 403, got %q", tr.Source)
|
|
}
|
|
}
|
|
|
|
// An empty baseUrl on the selected track degrades to SourceNone, never an error.
|
|
func TestFetchTranscriptEmptyBaseURLDegrades(t *testing.T) {
|
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/api/timedtext" {
|
|
t.Errorf("must not fetch an empty baseUrl")
|
|
}
|
|
_, _ = w.Write([]byte(`{"captions":{"playerCaptionsTracklistRenderer":{"captionTracks":[` +
|
|
`{"baseUrl":"","languageCode":"en"}]}}}`))
|
|
})
|
|
|
|
tr, err := a.FetchTranscript(context.Background(), domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid1"})
|
|
if err != nil {
|
|
t.Fatalf("empty baseUrl must degrade, not error: %v", err)
|
|
}
|
|
if tr.Source != domain.SourceNone {
|
|
t.Fatalf("expected SourceNone on empty baseUrl, got %q", tr.Source)
|
|
}
|
|
}
|