package youtube import ( "bytes" "context" "encoding/json" "encoding/xml" "fmt" "io" "net/http" "strings" "gitea.d-ma.be/mathias/tapir/internal/domain" ) // defaultPlayerBaseURL is the InnerTube / watch-page host. Overridable via // Config.PlayerBaseURL (tests point it at an httptest server). const defaultPlayerBaseURL = "https://www.youtube.com" // androidUserAgent identifies the InnerTube ANDROID client. ANDROID historically // returns caption baseUrls without a PoToken requirement (ADR-010). const androidUserAgent = "com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip" // browserUserAgent is required for the watch-page scrape: with an app UA YouTube // serves a page WITHOUT the embedded ytInitialPlayerResponse, so captionTracks // come back empty. A desktop-browser UA returns the player JSON with captions. const browserUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" // maxCaptionBytes bounds a single timedtext/player body read. const maxCaptionBytes = 16 << 20 // 16 MiB // FetchTranscript resolves a transcript captions-first (ADR-007) via the player // response + timedtext baseUrl, unauthenticated (ADR-010 — Data API // captions.download is owner-only and 403s for videos the user does not own): // // - POST the InnerTube player endpoint (ANDROID client) and read // captions.playerCaptionsTracklistRenderer.captionTracks[]. None => SourceNone. // - Select a track (preferred language, non-asr first) and GET its baseUrl with // a plain http.Client (no OAuth token — it can break the timedtext endpoint), // parsing srv3 XML / json3 / legacy XML to plain text. // // Degrade, don't error (ADR-010): no captionTracks, empty baseUrl, a non-200 // fetch, or an unparseable body all yield SourceNone rather than an error. Only // genuine transport (network) faults return an error. Audio download and // speech-to-text remain absent (ADR-007). func (a *Adapter) FetchTranscript(ctx context.Context, v domain.Video) (domain.Transcript, error) { client := a.plainClient() tracks, err := a.captionTracks(ctx, client, v.ProviderVideoID) if err != nil { return domain.Transcript{}, fmt.Errorf("resolve caption tracks for %q: %w", v.ProviderVideoID, err) } track, ok := a.selectTrack(tracks) if !ok || strings.TrimSpace(track.BaseURL) == "" { return noTranscript(v), nil } raw, status, err := a.httpGet(ctx, client, track.BaseURL, nil) if err != nil { return domain.Transcript{}, fmt.Errorf("download caption track for %q: %w", v.ProviderVideoID, err) } if status != http.StatusOK { // Owner-only 403, region/age gate, or transient unavailability: not an error. return noTranscript(v), nil } text := timedtextToText(string(raw)) if text == "" { return noTranscript(v), nil } return domain.Transcript{ VideoID: v.ID, UserID: v.UserID, Source: domain.SourceCaptions, Language: track.LanguageCode, Content: text, }, nil } // noTranscript is the recorded "checked, none usable" result — not an error. func noTranscript(v domain.Video) domain.Transcript { return domain.Transcript{VideoID: v.ID, UserID: v.UserID, Source: domain.SourceNone} } // captionTracks resolves a video's caption tracks from the player response. It // tries the InnerTube ANDROID client first and falls back to scraping // ytInitialPlayerResponse from the watch page. A nil slice (no tracks) is a // degrade, not an error; only transport faults return an error. func (a *Adapter) captionTracks(ctx context.Context, client *http.Client, videoID string) ([]captionTrack, error) { tracks, err := a.playerCaptionTracks(ctx, client, videoID) if err != nil { return nil, err } if len(tracks) > 0 { return tracks, nil } // Fallback: scrape the watch page (no PoToken/InnerTube context needed). return a.scrapeCaptionTracks(ctx, client, videoID) } // playerCaptionTracks POSTs the InnerTube player endpoint with the ANDROID client // context (unauthenticated) and returns its caption tracks. func (a *Adapter) playerCaptionTracks(ctx context.Context, client *http.Client, videoID string) ([]captionTrack, error) { reqBody, err := json.Marshal(playerRequest{ Context: innertubeContext{Client: innertubeClient{ ClientName: "ANDROID", ClientVersion: "20.10.38", AndroidSDKVersion: 30, HL: "en", GL: "US", }}, VideoID: videoID, }) if err != nil { return nil, fmt.Errorf("encode player request: %w", err) } body, status, err := a.httpPost(ctx, client, a.playerBaseURL+"/youtubei/v1/player", reqBody) if err != nil { return nil, err } if status != http.StatusOK { return nil, nil // degrade } var resp playerResponse if err := json.Unmarshal(body, &resp); err != nil { return nil, nil // unparseable => degrade } return resp.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks, nil } // ytInitialMarker locates the embedded player JSON on the watch page. const ytInitialMarker = "ytInitialPlayerResponse" // scrapeCaptionTracks fetches the watch page and extracts caption tracks from the // embedded ytInitialPlayerResponse JSON. Any failure degrades to no tracks. func (a *Adapter) scrapeCaptionTracks(ctx context.Context, client *http.Client, videoID string) ([]captionTrack, error) { body, status, err := a.httpGet(ctx, client, a.playerBaseURL+"/watch?v="+videoID, map[string]string{ "User-Agent": browserUserAgent, }) if err != nil { return nil, err } if status != http.StatusOK { return nil, nil } obj, ok := extractJSONObject(body, ytInitialMarker) if !ok { return nil, nil } var resp playerResponse if err := json.Unmarshal(obj, &resp); err != nil { return nil, nil } return resp.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks, nil } // selectTrack picks the best caption track: a preferred-language non-asr track // first, then a preferred-language asr track, then any non-asr track, else the // first track. Returns ok == false when there are no tracks at all. func (a *Adapter) selectTrack(tracks []captionTrack) (captionTrack, bool) { if len(tracks) == 0 { return captionTrack{}, false } for _, asr := range []bool{false, true} { for _, pref := range a.cfg.PreferredLanguages { for _, t := range tracks { if t.isASR() == asr && matchLang(t.LanguageCode, pref) { return t, true } } } } for _, t := range tracks { if !t.isASR() { return t, true } } return tracks[0], true } // matchLang matches a track language against a preferred code, tolerating region // suffixes (preferred "en" matches "en", "en-US", "en-GB"). func matchLang(code, pref string) bool { if strings.EqualFold(code, pref) { return true } return strings.HasPrefix(strings.ToLower(code), strings.ToLower(pref)+"-") } // --- HTTP (plain, unauthenticated) ------------------------------------------ // plainClient returns an unauthenticated HTTP client. No OAuth token is attached: // the player/timedtext endpoints can reject authenticated requests (ADR-010). // Tests inject the httptest transport via a.transport. func (a *Adapter) plainClient() *http.Client { if a.transport != nil { return &http.Client{Transport: a.transport} } return &http.Client{} } func (a *Adapter) httpGet(ctx context.Context, client *http.Client, url string, headers map[string]string) ([]byte, int, error) { return a.httpDo(ctx, client, http.MethodGet, url, nil, headers) } func (a *Adapter) httpPost(ctx context.Context, client *http.Client, url string, body []byte) ([]byte, int, error) { return a.httpDo(ctx, client, http.MethodPost, url, body, map[string]string{ "Content-Type": "application/json", "User-Agent": androidUserAgent, }) } // httpDo issues a request and returns (body, status, err). A non-nil err is a // genuine transport fault; a non-200 status is returned to the caller to decide // (callers degrade rather than error per ADR-010). func (a *Adapter) httpDo(ctx context.Context, client *http.Client, method, url string, body []byte, headers map[string]string) ([]byte, int, error) { var rdr io.Reader if body != nil { rdr = bytes.NewReader(body) } req, err := http.NewRequestWithContext(ctx, method, url, rdr) if err != nil { return nil, 0, fmt.Errorf("build %s %s: %w", method, url, err) } for k, v := range headers { req.Header.Set(k, v) } resp, err := client.Do(req) if err != nil { return nil, 0, fmt.Errorf("%s %s: %w", method, url, err) } defer func() { _ = resp.Body.Close() }() b, err := io.ReadAll(io.LimitReader(resp.Body, maxCaptionBytes)) if err != nil { return nil, 0, fmt.Errorf("read %s %s body: %w", method, url, err) } return b, resp.StatusCode, nil } // --- player / caption response shapes --------------------------------------- type playerRequest struct { Context innertubeContext `json:"context"` VideoID string `json:"videoId"` } type innertubeContext struct { Client innertubeClient `json:"client"` } type innertubeClient struct { ClientName string `json:"clientName"` ClientVersion string `json:"clientVersion"` AndroidSDKVersion int `json:"androidSdkVersion,omitempty"` HL string `json:"hl"` GL string `json:"gl"` } type playerResponse struct { Captions struct { PlayerCaptionsTracklistRenderer struct { CaptionTracks []captionTrack `json:"captionTracks"` } `json:"playerCaptionsTracklistRenderer"` } `json:"captions"` } type captionTrack struct { BaseURL string `json:"baseUrl"` LanguageCode string `json:"languageCode"` Kind string `json:"kind"` // "asr" for auto-generated } func (t captionTrack) isASR() bool { return strings.EqualFold(t.Kind, "asr") } // --- timedtext -> plain text ------------------------------------------------ // timedtextToText reduces a timedtext caption body to plain transcript text. It // auto-detects the format: json3 (a JSON object), else XML (srv3
cues or the
// legacy