merge: captions-first YouTube VideoSource (Worker C, agent/youtube-source)
This commit is contained in:
@@ -69,6 +69,11 @@ This maps directly onto the copied `llm` package: `Client` is the OpenAI-compati
|
|||||||
- **Exact ref format / vault item naming for Tapir:** **confirm / decide during build.** Follow
|
- **Exact ref format / vault item naming for Tapir:** **confirm / decide during build.** Follow
|
||||||
the pattern existing services use (e.g. how `gitea-mcp` references `GITEA_MCP_DEFAULT_TOKEN`)
|
the pattern existing services use (e.g. how `gitea-mcp` references `GITEA_MCP_DEFAULT_TOKEN`)
|
||||||
rather than inventing a new scheme.
|
rather than inventing a new scheme.
|
||||||
|
- **Adapter status (YouTube):** the YouTube `VideoSource` adapter does **not** assume a scheme.
|
||||||
|
The OAuth refresh-token reference is an opaque `Config.TokenSecretRef` resolved through the
|
||||||
|
`SecretStore` port (`youtube.New(cfg, secrets)`). Pinning the actual vault-item name only
|
||||||
|
changes wiring/config, not the adapter — so this `confirm` does not block the adapter. Decide
|
||||||
|
the name when wiring the live connection and record it here.
|
||||||
|
|
||||||
## Hosts (for reference)
|
## Hosts (for reference)
|
||||||
|
|
||||||
@@ -106,7 +111,17 @@ homelab's actual `GOPROXY`/`GONOSUMCHECK` policy here once known would remove th
|
|||||||
- `koala:4000` (or the NodePort / post-relocation address) — LiteLLM gateway, the Primary.
|
- `koala:4000` (or the NodePort / post-relocation address) — LiteLLM gateway, the Primary.
|
||||||
- brain-mcp host (`*-mcp.d-ma.be`, **confirm**) — only when the brain sink is enabled.
|
- brain-mcp host (`*-mcp.d-ma.be`, **confirm**) — only when the brain sink is enabled.
|
||||||
- `www.googleapis.com` / YouTube Data API + `oauth2.googleapis.com` (token exchange/refresh) —
|
- `www.googleapis.com` / YouTube Data API + `oauth2.googleapis.com` (token exchange/refresh) —
|
||||||
the YouTube provider adapter.
|
the YouTube provider adapter (`internal/adapters/youtube`). OAuth is hand-rolled on
|
||||||
|
`golang.org/x/oauth2` (ADR-006); the Google token endpoint is hardcoded inline to avoid the
|
||||||
|
heavy `golang.org/x/oauth2/google` dependency.
|
||||||
|
- **`confirm` — captions.download ownership limitation.** The Data API `captions.download`
|
||||||
|
endpoint only authorizes the *owner* of the video; for arbitrary subscribed channels it
|
||||||
|
typically returns `403`. The adapter lists tracks via `captions.list` (no track → `SourceNone`,
|
||||||
|
ADR-007) and downloads the selected track as WebVTT; a download failure surfaces as an error
|
||||||
|
(retryable), not a silent skip. If the owner-only limit proves blocking in practice, the
|
||||||
|
fallback is the unofficial `timedtext` endpoint or the deferred STT path (ADR-007) — a new
|
||||||
|
ADR, not a default. Verify against a real non-owned video before relying on live caption
|
||||||
|
download.
|
||||||
- `api.vimeo.com` — the Vimeo provider adapter.
|
- `api.vimeo.com` — the Vimeo provider adapter.
|
||||||
- BYO-AI endpoints, per configured provider and **only when a user opts in**:
|
- BYO-AI endpoints, per configured provider and **only when a user opts in**:
|
||||||
`api.anthropic.com`, `api.openai.com`, `generativelanguage.googleapis.com`.
|
`api.anthropic.com`, `api.openai.com`, `generativelanguage.googleapis.com`.
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
module gitea.d-ma.be/mathias/tapir
|
module gitea.d-ma.be/mathias/tapir
|
||||||
|
|
||||||
go 1.23
|
go 1.23.0
|
||||||
|
|
||||||
|
require golang.org/x/oauth2 v0.30.0
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||||
|
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package youtube
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FetchTranscript resolves a transcript captions-first (ADR-007):
|
||||||
|
//
|
||||||
|
// - List the video's caption tracks. No track => domain.SourceNone, no error.
|
||||||
|
// - Select a track (preferred language first, else the first track) and
|
||||||
|
// download it as WebVTT, stripping cue timing/markup to plain text.
|
||||||
|
//
|
||||||
|
// Audio download and speech-to-text are deliberately absent: no captions means
|
||||||
|
// SourceNone, full stop.
|
||||||
|
func (a *Adapter) FetchTranscript(ctx context.Context, v domain.Video) (domain.Transcript, error) {
|
||||||
|
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcript{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
tracks, err := a.listCaptionTracks(ctx, client, v.ProviderVideoID)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcript{}, fmt.Errorf("list captions for %q: %w", v.ProviderVideoID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
track, ok := a.selectTrack(tracks)
|
||||||
|
if !ok {
|
||||||
|
// No usable caption track: a recorded "checked, none available", not an error.
|
||||||
|
return domain.Transcript{
|
||||||
|
VideoID: v.ID,
|
||||||
|
UserID: v.UserID,
|
||||||
|
Source: domain.SourceNone,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := a.getRaw(ctx, client, "/captions/"+track.ID, url.Values{"tfmt": {"vtt"}})
|
||||||
|
if err != nil {
|
||||||
|
return domain.Transcript{}, fmt.Errorf("download caption track %q: %w", track.ID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
text := vttToText(string(raw))
|
||||||
|
if text == "" {
|
||||||
|
// Track existed but carried no text: treat as no usable transcript.
|
||||||
|
return domain.Transcript{
|
||||||
|
VideoID: v.ID,
|
||||||
|
UserID: v.UserID,
|
||||||
|
Source: domain.SourceNone,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return domain.Transcript{
|
||||||
|
VideoID: v.ID,
|
||||||
|
UserID: v.UserID,
|
||||||
|
Source: domain.SourceCaptions,
|
||||||
|
Language: track.Snippet.Language,
|
||||||
|
Content: text,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) listCaptionTracks(ctx context.Context, client *http.Client, videoID string) ([]captionTrack, error) {
|
||||||
|
q := url.Values{
|
||||||
|
"part": {"snippet"},
|
||||||
|
"videoId": {videoID},
|
||||||
|
}
|
||||||
|
var resp captionListResponse
|
||||||
|
if err := a.getJSON(ctx, client, "/captions", q, &resp); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return resp.Items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// selectTrack picks the best caption track: the first track whose language
|
||||||
|
// matches a configured preferred language, else the first track. Returns ok ==
|
||||||
|
// false when there are no tracks at all.
|
||||||
|
func (a *Adapter) selectTrack(tracks []captionTrack) (captionTrack, bool) {
|
||||||
|
if len(tracks) == 0 {
|
||||||
|
return captionTrack{}, false
|
||||||
|
}
|
||||||
|
for _, pref := range a.cfg.PreferredLanguages {
|
||||||
|
for _, t := range tracks {
|
||||||
|
if strings.EqualFold(t.Snippet.Language, pref) {
|
||||||
|
return t, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tracks[0], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- caption response shapes ------------------------------------------------
|
||||||
|
|
||||||
|
type captionListResponse struct {
|
||||||
|
Items []captionTrack `json:"items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type captionTrack struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Snippet struct {
|
||||||
|
Language string `json:"language"`
|
||||||
|
TrackKind string `json:"trackKind"` // "standard" | "ASR" | "forced"
|
||||||
|
Name string `json:"name"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
} `json:"snippet"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- WebVTT -> plain text ---------------------------------------------------
|
||||||
|
|
||||||
|
var (
|
||||||
|
vttCueTime = regexp.MustCompile(`-->`)
|
||||||
|
vttTag = regexp.MustCompile(`<[^>]+>`) // inline <00:00:01.000>, <c> markup
|
||||||
|
vttIndex = regexp.MustCompile(`^\d+$`) // SRT-style numeric cue index
|
||||||
|
vttSetting = regexp.MustCompile(`^(NOTE|STYLE|REGION)`) // VTT block headers
|
||||||
|
)
|
||||||
|
|
||||||
|
// vttToText reduces a WebVTT (or SRT-ish) caption file to plain transcript text:
|
||||||
|
// the WEBVTT header, NOTE/STYLE blocks, cue-timing lines, numeric indices, and
|
||||||
|
// inline markup are dropped; consecutive duplicate lines (common in rolling
|
||||||
|
// auto-captions) are collapsed.
|
||||||
|
func vttToText(raw string) string {
|
||||||
|
raw = strings.ReplaceAll(raw, "\r\n", "\n")
|
||||||
|
var out []string
|
||||||
|
var prev string
|
||||||
|
for _, line := range strings.Split(raw, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(line, "WEBVTT") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if vttSetting.MatchString(line) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if vttCueTime.MatchString(line) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if vttIndex.MatchString(line) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
line = strings.TrimSpace(vttTag.ReplaceAllString(line, ""))
|
||||||
|
if line == "" || line == prev {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, line)
|
||||||
|
prev = line
|
||||||
|
}
|
||||||
|
return strings.Join(out, "\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
// Package youtube is a ports.VideoSource adapter backed by the YouTube Data API
|
||||||
|
// v3. It lists a user's subscriptions, detects recent videos per subscription,
|
||||||
|
// and resolves transcripts captions-first (ADR-007): when no usable caption
|
||||||
|
// track exists it returns a domain.Transcript with Source == SourceNone, never
|
||||||
|
// an error, and it never downloads audio or runs speech-to-text.
|
||||||
|
//
|
||||||
|
// OAuth is written fresh against golang.org/x/oauth2 (ADR-006 — the ingestion
|
||||||
|
// repo's oauth package is inbound MCP auth, unrelated to outbound provider
|
||||||
|
// OAuth). The per-user refresh token is resolved through the SecretStore port
|
||||||
|
// from an opaque reference; the token material is never stored on the adapter,
|
||||||
|
// logged, or returned.
|
||||||
|
//
|
||||||
|
// Live runtime egress: www.googleapis.com (Data API) and oauth2.googleapis.com
|
||||||
|
// (token refresh). Tests inject an httptest transport and a fake SecretStore so
|
||||||
|
// neither host is contacted.
|
||||||
|
package youtube
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/oauth2"
|
||||||
|
|
||||||
|
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"gitea.d-ma.be/mathias/tapir/internal/ports"
|
||||||
|
)
|
||||||
|
|
||||||
|
// defaultBaseURL is the YouTube Data API v3 root. Overridable via Config.BaseURL
|
||||||
|
// (tests point it at an httptest server).
|
||||||
|
const defaultBaseURL = "https://www.googleapis.com/youtube/v3"
|
||||||
|
|
||||||
|
// googleEndpoint is the OAuth2 token endpoint for Google. Defined inline to avoid
|
||||||
|
// pulling the heavy golang.org/x/oauth2/google dependency for a single URL.
|
||||||
|
var googleEndpoint = oauth2.Endpoint{
|
||||||
|
AuthURL: "https://accounts.google.com/o/oauth2/auth",
|
||||||
|
TokenURL: "https://oauth2.googleapis.com/token",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config wires the adapter. ClientID/ClientSecret are the registered app
|
||||||
|
// credentials. TokenSecretRef is the opaque SecretStore reference that resolves
|
||||||
|
// to the connection's OAuth refresh token — the exact ref/vault-item naming for
|
||||||
|
// Tapir is still `confirm` (see docs/homelab-integration.md); the adapter treats
|
||||||
|
// it as an opaque parameter and never assumes a scheme.
|
||||||
|
//
|
||||||
|
// At Stage 0 there is a single connection, so one TokenSecretRef/ConnectionID
|
||||||
|
// covers every call. Mapping a Subscription back to its connection's ref is a
|
||||||
|
// Stage 1 concern (per-user isolation, data-model.md) and is deliberately not
|
||||||
|
// modelled here.
|
||||||
|
type Config struct {
|
||||||
|
ClientID string
|
||||||
|
ClientSecret string
|
||||||
|
TokenSecretRef string
|
||||||
|
ConnectionID string
|
||||||
|
|
||||||
|
// PreferredLanguages orders caption-track selection (e.g. {"en"}). The first
|
||||||
|
// track matching a preferred language wins; otherwise the first track is used.
|
||||||
|
PreferredLanguages []string
|
||||||
|
|
||||||
|
// MaxVideosPerSubscription caps how many recent videos NewVideos returns per
|
||||||
|
// poll. Zero means defaultMaxVideos.
|
||||||
|
MaxVideosPerSubscription int
|
||||||
|
|
||||||
|
// BaseURL overrides the Data API root. Empty means defaultBaseURL.
|
||||||
|
BaseURL string
|
||||||
|
}
|
||||||
|
|
||||||
|
const defaultMaxVideos = 10
|
||||||
|
|
||||||
|
// Adapter implements ports.VideoSource for YouTube.
|
||||||
|
type Adapter struct {
|
||||||
|
cfg Config
|
||||||
|
secrets ports.SecretStore
|
||||||
|
baseURL string
|
||||||
|
|
||||||
|
// transport, when non-nil, replaces the live OAuth2 transport for API calls.
|
||||||
|
// Production leaves it nil and an oauth2-authorized client is built per call.
|
||||||
|
// Tests inject an httptest transport so no Google host is contacted while the
|
||||||
|
// SecretStore resolution path still runs.
|
||||||
|
transport http.RoundTripper
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds a YouTube adapter. secrets must be non-nil: every authorized call
|
||||||
|
// resolves the refresh token through it by reference.
|
||||||
|
func New(cfg Config, secrets ports.SecretStore) *Adapter {
|
||||||
|
base := cfg.BaseURL
|
||||||
|
if base == "" {
|
||||||
|
base = defaultBaseURL
|
||||||
|
}
|
||||||
|
return &Adapter{
|
||||||
|
cfg: cfg,
|
||||||
|
secrets: secrets,
|
||||||
|
baseURL: strings.TrimRight(base, "/"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// httpClient returns an HTTP client authorized for the connection's token ref.
|
||||||
|
// The refresh token is resolved through SecretStore on every call and is never
|
||||||
|
// retained or logged; oauth2 exchanges it for a short-lived access token against
|
||||||
|
// googleEndpoint.TokenURL.
|
||||||
|
func (a *Adapter) httpClient(ctx context.Context, tokenRef string) (*http.Client, error) {
|
||||||
|
refresh, err := a.secrets.Get(ctx, tokenRef)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve youtube oauth token: %w", err)
|
||||||
|
}
|
||||||
|
if a.transport != nil {
|
||||||
|
// Test seam: the SecretStore lookup above has run (proving by-reference
|
||||||
|
// resolution); skip the live token exchange and use the injected transport.
|
||||||
|
return &http.Client{Transport: a.transport}, nil
|
||||||
|
}
|
||||||
|
conf := &oauth2.Config{
|
||||||
|
ClientID: a.cfg.ClientID,
|
||||||
|
ClientSecret: a.cfg.ClientSecret,
|
||||||
|
Endpoint: googleEndpoint,
|
||||||
|
}
|
||||||
|
ts := conf.TokenSource(ctx, &oauth2.Token{RefreshToken: refresh})
|
||||||
|
return oauth2.NewClient(ctx, ts), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListSubscriptions returns the channels the user subscribes to, paging through
|
||||||
|
// the Data API until exhausted.
|
||||||
|
func (a *Adapter) ListSubscriptions(ctx context.Context, userID string) ([]domain.Subscription, error) {
|
||||||
|
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var subs []domain.Subscription
|
||||||
|
pageToken := ""
|
||||||
|
for {
|
||||||
|
q := url.Values{
|
||||||
|
"part": {"snippet"},
|
||||||
|
"mine": {"true"},
|
||||||
|
"maxResults": {"50"},
|
||||||
|
}
|
||||||
|
if pageToken != "" {
|
||||||
|
q.Set("pageToken", pageToken)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp subscriptionListResponse
|
||||||
|
if err := a.getJSON(ctx, client, "/subscriptions", q, &resp); err != nil {
|
||||||
|
return nil, fmt.Errorf("list subscriptions: %w", err)
|
||||||
|
}
|
||||||
|
for _, item := range resp.Items {
|
||||||
|
subs = append(subs, domain.Subscription{
|
||||||
|
ID: item.ID,
|
||||||
|
UserID: userID,
|
||||||
|
ConnectionID: a.cfg.ConnectionID,
|
||||||
|
ChannelID: item.Snippet.ResourceID.ChannelID,
|
||||||
|
ChannelTitle: item.Snippet.Title,
|
||||||
|
Active: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if resp.NextPageToken == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
pageToken = resp.NextPageToken
|
||||||
|
}
|
||||||
|
return subs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewVideos returns the most recent videos for a subscription's channel, newest
|
||||||
|
// first. The engine dedups within its lifetime and the store dedups durably
|
||||||
|
// (data-model.md), so the adapter returns recent candidates rather than tracking
|
||||||
|
// "seen" state itself.
|
||||||
|
func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) {
|
||||||
|
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
max := a.cfg.MaxVideosPerSubscription
|
||||||
|
if max <= 0 {
|
||||||
|
max = defaultMaxVideos
|
||||||
|
}
|
||||||
|
|
||||||
|
q := url.Values{
|
||||||
|
"part": {"snippet"},
|
||||||
|
"channelId": {sub.ChannelID},
|
||||||
|
"order": {"date"},
|
||||||
|
"type": {"video"},
|
||||||
|
"maxResults": {fmt.Sprintf("%d", max)},
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp searchListResponse
|
||||||
|
if err := a.getJSON(ctx, client, "/search", q, &resp); err != nil {
|
||||||
|
return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
videos := make([]domain.Video, 0, len(resp.Items))
|
||||||
|
for _, item := range resp.Items {
|
||||||
|
if item.ID.VideoID == "" {
|
||||||
|
continue // non-video result; type=video should prevent this, be defensive
|
||||||
|
}
|
||||||
|
videos = append(videos, domain.Video{
|
||||||
|
UserID: sub.UserID,
|
||||||
|
SubscriptionID: sub.ID,
|
||||||
|
Provider: domain.ProviderYouTube,
|
||||||
|
ProviderVideoID: item.ID.VideoID,
|
||||||
|
Title: item.Snippet.Title,
|
||||||
|
URL: "https://www.youtube.com/watch?v=" + item.ID.VideoID,
|
||||||
|
PublishedAt: item.Snippet.PublishedAt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return videos, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getJSON issues a GET and decodes a JSON body into out. A non-200 status is an
|
||||||
|
// error carrying a bounded slice of the response body for diagnosis.
|
||||||
|
func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error {
|
||||||
|
body, err := a.getRaw(ctx, client, path, q)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, out); err != nil {
|
||||||
|
return fmt.Errorf("decode %s response: %w", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getRaw issues a GET and returns the raw response body (used for caption
|
||||||
|
// downloads, which are not JSON).
|
||||||
|
func (a *Adapter) getRaw(ctx context.Context, client *http.Client, path string, q url.Values) ([]byte, error) {
|
||||||
|
u := a.baseURL + path
|
||||||
|
if len(q) > 0 {
|
||||||
|
u += "?" + q.Encode()
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("build request %s: %w", path, err)
|
||||||
|
}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("request %s: %w", path, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||||
|
return nil, fmt.Errorf("youtube api %s: status %d: %s", path, resp.StatusCode, strings.TrimSpace(string(snippet)))
|
||||||
|
}
|
||||||
|
return io.ReadAll(resp.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Data API response shapes (only the fields used) -----------------------
|
||||||
|
|
||||||
|
type subscriptionListResponse struct {
|
||||||
|
NextPageToken string `json:"nextPageToken"`
|
||||||
|
Items []struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Snippet struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
ResourceID struct {
|
||||||
|
ChannelID string `json:"channelId"`
|
||||||
|
} `json:"resourceId"`
|
||||||
|
} `json:"snippet"`
|
||||||
|
} `json:"items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type searchListResponse struct {
|
||||||
|
Items []struct {
|
||||||
|
ID struct {
|
||||||
|
VideoID string `json:"videoId"`
|
||||||
|
} `json:"id"`
|
||||||
|
Snippet struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
PublishedAt time.Time `json:"publishedAt"`
|
||||||
|
} `json:"snippet"`
|
||||||
|
} `json:"items"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
package youtube
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// --- fakes ------------------------------------------------------------------
|
||||||
|
|
||||||
|
// fakeSecrets records the references it was asked to resolve, proving the
|
||||||
|
// adapter resolves OAuth material by reference (never holding the token itself).
|
||||||
|
type fakeSecrets struct {
|
||||||
|
byRef map[string]string
|
||||||
|
requested []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeSecrets) Get(_ context.Context, ref string) (string, error) {
|
||||||
|
f.requested = append(f.requested, ref)
|
||||||
|
v, ok := f.byRef[ref]
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("no secret for ref %q", ref)
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newTestAdapter wires an adapter against an httptest server: BaseURL points at
|
||||||
|
// the server and the server's transport is injected so the live OAuth2 exchange
|
||||||
|
// is skipped while SecretStore resolution still runs.
|
||||||
|
func newTestAdapter(t *testing.T, handler http.HandlerFunc) (*Adapter, *fakeSecrets) {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(handler)
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
secrets := &fakeSecrets{byRef: map[string]string{
|
||||||
|
"op://HomeLab/tapir-youtube#refresh": "super-secret-refresh-token",
|
||||||
|
}}
|
||||||
|
a := New(Config{
|
||||||
|
ClientID: "cid",
|
||||||
|
ClientSecret: "csecret",
|
||||||
|
TokenSecretRef: "op://HomeLab/tapir-youtube#refresh",
|
||||||
|
ConnectionID: "conn-1",
|
||||||
|
PreferredLanguages: []string{"en"},
|
||||||
|
BaseURL: srv.URL,
|
||||||
|
}, secrets)
|
||||||
|
a.transport = srv.Client().Transport
|
||||||
|
return a, secrets
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- subscriptions ----------------------------------------------------------
|
||||||
|
|
||||||
|
func TestListSubscriptions(t *testing.T) {
|
||||||
|
a, secrets := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/subscriptions" {
|
||||||
|
t.Errorf("unexpected path %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
if got := r.URL.Query().Get("mine"); got != "true" {
|
||||||
|
t.Errorf("expected mine=true, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"items": [
|
||||||
|
{"id": "sub-a", "snippet": {"title": "Acme Talks", "resourceId": {"channelId": "UC_acme"}}},
|
||||||
|
{"id": "sub-b", "snippet": {"title": "Beta Lab", "resourceId": {"channelId": "UC_beta"}}}
|
||||||
|
]
|
||||||
|
}`))
|
||||||
|
})
|
||||||
|
|
||||||
|
subs, err := a.ListSubscriptions(context.Background(), "u1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListSubscriptions: %v", err)
|
||||||
|
}
|
||||||
|
if len(subs) != 2 {
|
||||||
|
t.Fatalf("expected 2 subscriptions, got %d", len(subs))
|
||||||
|
}
|
||||||
|
if subs[0].ChannelID != "UC_acme" || subs[0].ChannelTitle != "Acme Talks" {
|
||||||
|
t.Errorf("unexpected first subscription: %+v", subs[0])
|
||||||
|
}
|
||||||
|
if subs[0].UserID != "u1" || subs[0].ConnectionID != "conn-1" || !subs[0].Active {
|
||||||
|
t.Errorf("subscription not wired to user/connection: %+v", subs[0])
|
||||||
|
}
|
||||||
|
// The refresh token was resolved by reference, never read from config directly.
|
||||||
|
if len(secrets.requested) == 0 || secrets.requested[0] != "op://HomeLab/tapir-youtube#refresh" {
|
||||||
|
t.Errorf("expected token resolved by ref, got %v", secrets.requested)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListSubscriptionsPaginates(t *testing.T) {
|
||||||
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Query().Get("pageToken") {
|
||||||
|
case "":
|
||||||
|
_, _ = w.Write([]byte(`{"nextPageToken":"p2","items":[{"id":"s1","snippet":{"title":"One","resourceId":{"channelId":"c1"}}}]}`))
|
||||||
|
case "p2":
|
||||||
|
_, _ = w.Write([]byte(`{"items":[{"id":"s2","snippet":{"title":"Two","resourceId":{"channelId":"c2"}}}]}`))
|
||||||
|
default:
|
||||||
|
t.Errorf("unexpected pageToken %q", r.URL.Query().Get("pageToken"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
subs, err := a.ListSubscriptions(context.Background(), "u1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListSubscriptions: %v", err)
|
||||||
|
}
|
||||||
|
if len(subs) != 2 {
|
||||||
|
t.Fatalf("expected 2 subscriptions across pages, got %d", len(subs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- new videos -------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestNewVideos(t *testing.T) {
|
||||||
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/search" {
|
||||||
|
t.Errorf("unexpected path %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
if got := r.URL.Query().Get("channelId"); got != "UC_acme" {
|
||||||
|
t.Errorf("expected channelId=UC_acme, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"items": [
|
||||||
|
{"id": {"videoId": "vid1"}, "snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z"}},
|
||||||
|
{"id": {"videoId": "vid2"}, "snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z"}}
|
||||||
|
]
|
||||||
|
}`))
|
||||||
|
})
|
||||||
|
|
||||||
|
sub := domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "UC_acme"}
|
||||||
|
vids, err := a.NewVideos(context.Background(), sub)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewVideos: %v", err)
|
||||||
|
}
|
||||||
|
if len(vids) != 2 {
|
||||||
|
t.Fatalf("expected 2 videos, got %d", len(vids))
|
||||||
|
}
|
||||||
|
v := vids[0]
|
||||||
|
if v.ProviderVideoID != "vid1" || v.Title != "Designing for Attention" {
|
||||||
|
t.Errorf("unexpected video: %+v", v)
|
||||||
|
}
|
||||||
|
if v.Provider != domain.ProviderYouTube || v.URL != "https://www.youtube.com/watch?v=vid1" {
|
||||||
|
t.Errorf("video not wired correctly: %+v", v)
|
||||||
|
}
|
||||||
|
if v.UserID != "u1" || v.SubscriptionID != "s1" {
|
||||||
|
t.Errorf("video not scoped to user/subscription: %+v", v)
|
||||||
|
}
|
||||||
|
if v.PublishedAt.IsZero() {
|
||||||
|
t.Errorf("expected publishedAt parsed, got zero")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- transcript: captions present ------------------------------------------
|
||||||
|
|
||||||
|
func TestFetchTranscriptWithCaptions(t *testing.T) {
|
||||||
|
a, secrets := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/captions":
|
||||||
|
if got := r.URL.Query().Get("videoId"); got != "vid1" {
|
||||||
|
t.Errorf("expected videoId=vid1, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"items":[
|
||||||
|
{"id":"cap-sv","snippet":{"language":"sv","trackKind":"standard","status":"serving"}},
|
||||||
|
{"id":"cap-en","snippet":{"language":"en","trackKind":"standard","status":"serving"}}
|
||||||
|
]}`))
|
||||||
|
case "/captions/cap-en":
|
||||||
|
if got := r.URL.Query().Get("tfmt"); got != "vtt" {
|
||||||
|
t.Errorf("expected tfmt=vtt, got %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte("WEBVTT\n\n1\n00:00:00.000 --> 00:00:02.000\nHello <c>world</c>\n\n2\n00:00:02.000 --> 00:00:04.000\nHello world\n\n3\n00:00:04.000 --> 00:00:06.000\nSecond line\n"))
|
||||||
|
default:
|
||||||
|
t.Errorf("unexpected path %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
v := domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid1"}
|
||||||
|
tr, err := a.FetchTranscript(context.Background(), v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FetchTranscript: %v", err)
|
||||||
|
}
|
||||||
|
if tr.Source != domain.SourceCaptions {
|
||||||
|
t.Fatalf("expected SourceCaptions, got %q", tr.Source)
|
||||||
|
}
|
||||||
|
if tr.Language != "en" {
|
||||||
|
t.Errorf("expected preferred language en, got %q", tr.Language)
|
||||||
|
}
|
||||||
|
// Markup stripped, timestamps/indices dropped, consecutive duplicate collapsed.
|
||||||
|
want := "Hello world\nSecond line"
|
||||||
|
if tr.Content != want {
|
||||||
|
t.Errorf("transcript text mismatch:\n got %q\nwant %q", tr.Content, want)
|
||||||
|
}
|
||||||
|
if !tr.HasText() {
|
||||||
|
t.Error("expected HasText() true")
|
||||||
|
}
|
||||||
|
if len(secrets.requested) == 0 {
|
||||||
|
t.Error("expected SecretStore consulted for OAuth token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- transcript: no captions => SourceNone, no error ------------------------
|
||||||
|
|
||||||
|
func TestFetchTranscriptNoCaptionsReturnsSourceNone(t *testing.T) {
|
||||||
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/captions" {
|
||||||
|
t.Errorf("download must not be attempted when no track exists; got path %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"items":[]}`))
|
||||||
|
})
|
||||||
|
|
||||||
|
v := domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid-nocaps"}
|
||||||
|
tr, err := a.FetchTranscript(context.Background(), v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error for missing captions, got %v", err)
|
||||||
|
}
|
||||||
|
if tr.Source != domain.SourceNone {
|
||||||
|
t.Fatalf("expected SourceNone, got %q", tr.Source)
|
||||||
|
}
|
||||||
|
if tr.HasText() {
|
||||||
|
t.Error("expected HasText() false for SourceNone")
|
||||||
|
}
|
||||||
|
if tr.Content != "" {
|
||||||
|
t.Errorf("expected empty content, got %q", tr.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A missing-token reference surfaces as an error rather than a silent skip.
|
||||||
|
func TestFetchTranscriptSecretResolutionError(t *testing.T) {
|
||||||
|
a, _ := newTestAdapter(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{"items":[]}`))
|
||||||
|
})
|
||||||
|
a.cfg.TokenSecretRef = "op://HomeLab/missing#refresh"
|
||||||
|
|
||||||
|
_, err := a.FetchTranscript(context.Background(), domain.Video{ID: "v1", ProviderVideoID: "x"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error when secret ref cannot be resolved")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user