diff --git a/go.mod b/go.mod index 481107e..68341d2 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module gitea.d-ma.be/mathias/tapir -go 1.23 +go 1.23.0 + +require golang.org/x/oauth2 v0.30.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c05b468 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= diff --git a/internal/adapters/youtube/captions.go b/internal/adapters/youtube/captions.go new file mode 100644 index 0000000..d63af30 --- /dev/null +++ b/internal/adapters/youtube/captions.go @@ -0,0 +1,154 @@ +package youtube + +import ( + "context" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" + + "gitea.d-ma.be/mathias/tapir/internal/domain" +) + +// FetchTranscript resolves a transcript captions-first (ADR-007): +// +// - List the video's caption tracks. No track => domain.SourceNone, no error. +// - Select a track (preferred language first, else the first track) and +// download it as WebVTT, stripping cue timing/markup to plain text. +// +// Audio download and speech-to-text are deliberately absent: no captions means +// SourceNone, full stop. +func (a *Adapter) FetchTranscript(ctx context.Context, v domain.Video) (domain.Transcript, error) { + client, err := a.httpClient(ctx, a.cfg.TokenSecretRef) + if err != nil { + return domain.Transcript{}, err + } + + tracks, err := a.listCaptionTracks(ctx, client, v.ProviderVideoID) + if err != nil { + return domain.Transcript{}, fmt.Errorf("list captions for %q: %w", v.ProviderVideoID, err) + } + + track, ok := a.selectTrack(tracks) + if !ok { + // No usable caption track: a recorded "checked, none available", not an error. + return domain.Transcript{ + VideoID: v.ID, + UserID: v.UserID, + Source: domain.SourceNone, + }, nil + } + + raw, err := a.getRaw(ctx, client, "/captions/"+track.ID, url.Values{"tfmt": {"vtt"}}) + if err != nil { + return domain.Transcript{}, fmt.Errorf("download caption track %q: %w", track.ID, err) + } + + text := vttToText(string(raw)) + if text == "" { + // Track existed but carried no text: treat as no usable transcript. + return domain.Transcript{ + VideoID: v.ID, + UserID: v.UserID, + Source: domain.SourceNone, + }, nil + } + + return domain.Transcript{ + VideoID: v.ID, + UserID: v.UserID, + Source: domain.SourceCaptions, + Language: track.Snippet.Language, + Content: text, + }, nil +} + +func (a *Adapter) listCaptionTracks(ctx context.Context, client *http.Client, videoID string) ([]captionTrack, error) { + q := url.Values{ + "part": {"snippet"}, + "videoId": {videoID}, + } + var resp captionListResponse + if err := a.getJSON(ctx, client, "/captions", q, &resp); err != nil { + return nil, err + } + return resp.Items, nil +} + +// selectTrack picks the best caption track: the first track whose language +// matches a configured preferred language, else the first track. Returns ok == +// false when there are no tracks at all. +func (a *Adapter) selectTrack(tracks []captionTrack) (captionTrack, bool) { + if len(tracks) == 0 { + return captionTrack{}, false + } + for _, pref := range a.cfg.PreferredLanguages { + for _, t := range tracks { + if strings.EqualFold(t.Snippet.Language, pref) { + return t, true + } + } + } + return tracks[0], true +} + +// --- caption response shapes ------------------------------------------------ + +type captionListResponse struct { + Items []captionTrack `json:"items"` +} + +type captionTrack struct { + ID string `json:"id"` + Snippet struct { + Language string `json:"language"` + TrackKind string `json:"trackKind"` // "standard" | "ASR" | "forced" + Name string `json:"name"` + Status string `json:"status"` + } `json:"snippet"` +} + +// --- WebVTT -> plain text --------------------------------------------------- + +var ( + vttCueTime = regexp.MustCompile(`-->`) + vttTag = regexp.MustCompile(`<[^>]+>`) // inline <00:00:01.000>, markup + vttIndex = regexp.MustCompile(`^\d+$`) // SRT-style numeric cue index + vttSetting = regexp.MustCompile(`^(NOTE|STYLE|REGION)`) // VTT block headers +) + +// vttToText reduces a WebVTT (or SRT-ish) caption file to plain transcript text: +// the WEBVTT header, NOTE/STYLE blocks, cue-timing lines, numeric indices, and +// inline markup are dropped; consecutive duplicate lines (common in rolling +// auto-captions) are collapsed. +func vttToText(raw string) string { + raw = strings.ReplaceAll(raw, "\r\n", "\n") + var out []string + var prev string + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "WEBVTT") { + continue + } + if vttSetting.MatchString(line) { + continue + } + if vttCueTime.MatchString(line) { + continue + } + if vttIndex.MatchString(line) { + continue + } + line = strings.TrimSpace(vttTag.ReplaceAllString(line, "")) + if line == "" || line == prev { + continue + } + out = append(out, line) + prev = line + } + return strings.Join(out, "\n") +} diff --git a/internal/adapters/youtube/youtube.go b/internal/adapters/youtube/youtube.go new file mode 100644 index 0000000..c315ed1 --- /dev/null +++ b/internal/adapters/youtube/youtube.go @@ -0,0 +1,275 @@ +// Package youtube is a ports.VideoSource adapter backed by the YouTube Data API +// v3. It lists a user's subscriptions, detects recent videos per subscription, +// and resolves transcripts captions-first (ADR-007): when no usable caption +// track exists it returns a domain.Transcript with Source == SourceNone, never +// an error, and it never downloads audio or runs speech-to-text. +// +// OAuth is written fresh against golang.org/x/oauth2 (ADR-006 — the ingestion +// repo's oauth package is inbound MCP auth, unrelated to outbound provider +// OAuth). The per-user refresh token is resolved through the SecretStore port +// from an opaque reference; the token material is never stored on the adapter, +// logged, or returned. +// +// Live runtime egress: www.googleapis.com (Data API) and oauth2.googleapis.com +// (token refresh). Tests inject an httptest transport and a fake SecretStore so +// neither host is contacted. +package youtube + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/oauth2" + + "gitea.d-ma.be/mathias/tapir/internal/domain" + "gitea.d-ma.be/mathias/tapir/internal/ports" +) + +// defaultBaseURL is the YouTube Data API v3 root. Overridable via Config.BaseURL +// (tests point it at an httptest server). +const defaultBaseURL = "https://www.googleapis.com/youtube/v3" + +// googleEndpoint is the OAuth2 token endpoint for Google. Defined inline to avoid +// pulling the heavy golang.org/x/oauth2/google dependency for a single URL. +var googleEndpoint = oauth2.Endpoint{ + AuthURL: "https://accounts.google.com/o/oauth2/auth", + TokenURL: "https://oauth2.googleapis.com/token", +} + +// Config wires the adapter. ClientID/ClientSecret are the registered app +// credentials. TokenSecretRef is the opaque SecretStore reference that resolves +// to the connection's OAuth refresh token — the exact ref/vault-item naming for +// Tapir is still `confirm` (see docs/homelab-integration.md); the adapter treats +// it as an opaque parameter and never assumes a scheme. +// +// At Stage 0 there is a single connection, so one TokenSecretRef/ConnectionID +// covers every call. Mapping a Subscription back to its connection's ref is a +// Stage 1 concern (per-user isolation, data-model.md) and is deliberately not +// modelled here. +type Config struct { + ClientID string + ClientSecret string + TokenSecretRef string + ConnectionID string + + // PreferredLanguages orders caption-track selection (e.g. {"en"}). The first + // track matching a preferred language wins; otherwise the first track is used. + PreferredLanguages []string + + // MaxVideosPerSubscription caps how many recent videos NewVideos returns per + // poll. Zero means defaultMaxVideos. + MaxVideosPerSubscription int + + // BaseURL overrides the Data API root. Empty means defaultBaseURL. + BaseURL string +} + +const defaultMaxVideos = 10 + +// Adapter implements ports.VideoSource for YouTube. +type Adapter struct { + cfg Config + secrets ports.SecretStore + baseURL string + + // transport, when non-nil, replaces the live OAuth2 transport for API calls. + // Production leaves it nil and an oauth2-authorized client is built per call. + // Tests inject an httptest transport so no Google host is contacted while the + // SecretStore resolution path still runs. + transport http.RoundTripper +} + +// New builds a YouTube adapter. secrets must be non-nil: every authorized call +// resolves the refresh token through it by reference. +func New(cfg Config, secrets ports.SecretStore) *Adapter { + base := cfg.BaseURL + if base == "" { + base = defaultBaseURL + } + return &Adapter{ + cfg: cfg, + secrets: secrets, + baseURL: strings.TrimRight(base, "/"), + } +} + +// httpClient returns an HTTP client authorized for the connection's token ref. +// The refresh token is resolved through SecretStore on every call and is never +// retained or logged; oauth2 exchanges it for a short-lived access token against +// googleEndpoint.TokenURL. +func (a *Adapter) httpClient(ctx context.Context, tokenRef string) (*http.Client, error) { + refresh, err := a.secrets.Get(ctx, tokenRef) + if err != nil { + return nil, fmt.Errorf("resolve youtube oauth token: %w", err) + } + if a.transport != nil { + // Test seam: the SecretStore lookup above has run (proving by-reference + // resolution); skip the live token exchange and use the injected transport. + return &http.Client{Transport: a.transport}, nil + } + conf := &oauth2.Config{ + ClientID: a.cfg.ClientID, + ClientSecret: a.cfg.ClientSecret, + Endpoint: googleEndpoint, + } + ts := conf.TokenSource(ctx, &oauth2.Token{RefreshToken: refresh}) + return oauth2.NewClient(ctx, ts), nil +} + +// ListSubscriptions returns the channels the user subscribes to, paging through +// the Data API until exhausted. +func (a *Adapter) ListSubscriptions(ctx context.Context, userID string) ([]domain.Subscription, error) { + client, err := a.httpClient(ctx, a.cfg.TokenSecretRef) + if err != nil { + return nil, err + } + + var subs []domain.Subscription + pageToken := "" + for { + q := url.Values{ + "part": {"snippet"}, + "mine": {"true"}, + "maxResults": {"50"}, + } + if pageToken != "" { + q.Set("pageToken", pageToken) + } + + var resp subscriptionListResponse + if err := a.getJSON(ctx, client, "/subscriptions", q, &resp); err != nil { + return nil, fmt.Errorf("list subscriptions: %w", err) + } + for _, item := range resp.Items { + subs = append(subs, domain.Subscription{ + ID: item.ID, + UserID: userID, + ConnectionID: a.cfg.ConnectionID, + ChannelID: item.Snippet.ResourceID.ChannelID, + ChannelTitle: item.Snippet.Title, + Active: true, + }) + } + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + return subs, nil +} + +// NewVideos returns the most recent videos for a subscription's channel, newest +// first. The engine dedups within its lifetime and the store dedups durably +// (data-model.md), so the adapter returns recent candidates rather than tracking +// "seen" state itself. +func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]domain.Video, error) { + client, err := a.httpClient(ctx, a.cfg.TokenSecretRef) + if err != nil { + return nil, err + } + + max := a.cfg.MaxVideosPerSubscription + if max <= 0 { + max = defaultMaxVideos + } + + q := url.Values{ + "part": {"snippet"}, + "channelId": {sub.ChannelID}, + "order": {"date"}, + "type": {"video"}, + "maxResults": {fmt.Sprintf("%d", max)}, + } + + var resp searchListResponse + if err := a.getJSON(ctx, client, "/search", q, &resp); err != nil { + return nil, fmt.Errorf("new videos for channel %q: %w", sub.ChannelID, err) + } + + videos := make([]domain.Video, 0, len(resp.Items)) + for _, item := range resp.Items { + if item.ID.VideoID == "" { + continue // non-video result; type=video should prevent this, be defensive + } + videos = append(videos, domain.Video{ + UserID: sub.UserID, + SubscriptionID: sub.ID, + Provider: domain.ProviderYouTube, + ProviderVideoID: item.ID.VideoID, + Title: item.Snippet.Title, + URL: "https://www.youtube.com/watch?v=" + item.ID.VideoID, + PublishedAt: item.Snippet.PublishedAt, + }) + } + return videos, nil +} + +// getJSON issues a GET and decodes a JSON body into out. A non-200 status is an +// error carrying a bounded slice of the response body for diagnosis. +func (a *Adapter) getJSON(ctx context.Context, client *http.Client, path string, q url.Values, out any) error { + body, err := a.getRaw(ctx, client, path, q) + if err != nil { + return err + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("decode %s response: %w", path, err) + } + return nil +} + +// getRaw issues a GET and returns the raw response body (used for caption +// downloads, which are not JSON). +func (a *Adapter) getRaw(ctx context.Context, client *http.Client, path string, q url.Values) ([]byte, error) { + u := a.baseURL + path + if len(q) > 0 { + u += "?" + q.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, fmt.Errorf("build request %s: %w", path, err) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("request %s: %w", path, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return nil, fmt.Errorf("youtube api %s: status %d: %s", path, resp.StatusCode, strings.TrimSpace(string(snippet))) + } + return io.ReadAll(resp.Body) +} + +// --- Data API response shapes (only the fields used) ----------------------- + +type subscriptionListResponse struct { + NextPageToken string `json:"nextPageToken"` + Items []struct { + ID string `json:"id"` + Snippet struct { + Title string `json:"title"` + ResourceID struct { + ChannelID string `json:"channelId"` + } `json:"resourceId"` + } `json:"snippet"` + } `json:"items"` +} + +type searchListResponse struct { + Items []struct { + ID struct { + VideoID string `json:"videoId"` + } `json:"id"` + Snippet struct { + Title string `json:"title"` + PublishedAt time.Time `json:"publishedAt"` + } `json:"snippet"` + } `json:"items"` +} diff --git a/internal/adapters/youtube/youtube_test.go b/internal/adapters/youtube/youtube_test.go new file mode 100644 index 0000000..6eb0abe --- /dev/null +++ b/internal/adapters/youtube/youtube_test.go @@ -0,0 +1,237 @@ +package youtube + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "gitea.d-ma.be/mathias/tapir/internal/domain" +) + +// --- fakes ------------------------------------------------------------------ + +// fakeSecrets records the references it was asked to resolve, proving the +// adapter resolves OAuth material by reference (never holding the token itself). +type fakeSecrets struct { + byRef map[string]string + requested []string +} + +func (f *fakeSecrets) Get(_ context.Context, ref string) (string, error) { + f.requested = append(f.requested, ref) + v, ok := f.byRef[ref] + if !ok { + return "", fmt.Errorf("no secret for ref %q", ref) + } + return v, nil +} + +// newTestAdapter wires an adapter against an httptest server: BaseURL points at +// the server and the server's transport is injected so the live OAuth2 exchange +// is skipped while SecretStore resolution still runs. +func newTestAdapter(t *testing.T, handler http.HandlerFunc) (*Adapter, *fakeSecrets) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + secrets := &fakeSecrets{byRef: map[string]string{ + "op://HomeLab/tapir-youtube#refresh": "super-secret-refresh-token", + }} + a := New(Config{ + ClientID: "cid", + ClientSecret: "csecret", + TokenSecretRef: "op://HomeLab/tapir-youtube#refresh", + ConnectionID: "conn-1", + PreferredLanguages: []string{"en"}, + BaseURL: srv.URL, + }, secrets) + a.transport = srv.Client().Transport + return a, secrets +} + +// --- subscriptions ---------------------------------------------------------- + +func TestListSubscriptions(t *testing.T) { + a, secrets := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/subscriptions" { + t.Errorf("unexpected path %q", r.URL.Path) + } + if got := r.URL.Query().Get("mine"); got != "true" { + t.Errorf("expected mine=true, got %q", got) + } + _, _ = w.Write([]byte(`{ + "items": [ + {"id": "sub-a", "snippet": {"title": "Acme Talks", "resourceId": {"channelId": "UC_acme"}}}, + {"id": "sub-b", "snippet": {"title": "Beta Lab", "resourceId": {"channelId": "UC_beta"}}} + ] + }`)) + }) + + subs, err := a.ListSubscriptions(context.Background(), "u1") + if err != nil { + t.Fatalf("ListSubscriptions: %v", err) + } + if len(subs) != 2 { + t.Fatalf("expected 2 subscriptions, got %d", len(subs)) + } + if subs[0].ChannelID != "UC_acme" || subs[0].ChannelTitle != "Acme Talks" { + t.Errorf("unexpected first subscription: %+v", subs[0]) + } + if subs[0].UserID != "u1" || subs[0].ConnectionID != "conn-1" || !subs[0].Active { + t.Errorf("subscription not wired to user/connection: %+v", subs[0]) + } + // The refresh token was resolved by reference, never read from config directly. + if len(secrets.requested) == 0 || secrets.requested[0] != "op://HomeLab/tapir-youtube#refresh" { + t.Errorf("expected token resolved by ref, got %v", secrets.requested) + } +} + +func TestListSubscriptionsPaginates(t *testing.T) { + a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Query().Get("pageToken") { + case "": + _, _ = w.Write([]byte(`{"nextPageToken":"p2","items":[{"id":"s1","snippet":{"title":"One","resourceId":{"channelId":"c1"}}}]}`)) + case "p2": + _, _ = w.Write([]byte(`{"items":[{"id":"s2","snippet":{"title":"Two","resourceId":{"channelId":"c2"}}}]}`)) + default: + t.Errorf("unexpected pageToken %q", r.URL.Query().Get("pageToken")) + } + }) + + subs, err := a.ListSubscriptions(context.Background(), "u1") + if err != nil { + t.Fatalf("ListSubscriptions: %v", err) + } + if len(subs) != 2 { + t.Fatalf("expected 2 subscriptions across pages, got %d", len(subs)) + } +} + +// --- new videos ------------------------------------------------------------- + +func TestNewVideos(t *testing.T) { + a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/search" { + t.Errorf("unexpected path %q", r.URL.Path) + } + if got := r.URL.Query().Get("channelId"); got != "UC_acme" { + t.Errorf("expected channelId=UC_acme, got %q", got) + } + _, _ = w.Write([]byte(`{ + "items": [ + {"id": {"videoId": "vid1"}, "snippet": {"title": "Designing for Attention", "publishedAt": "2026-06-01T10:00:00Z"}}, + {"id": {"videoId": "vid2"}, "snippet": {"title": "Second", "publishedAt": "2026-05-31T10:00:00Z"}} + ] + }`)) + }) + + sub := domain.Subscription{ID: "s1", UserID: "u1", ChannelID: "UC_acme"} + vids, err := a.NewVideos(context.Background(), sub) + if err != nil { + t.Fatalf("NewVideos: %v", err) + } + if len(vids) != 2 { + t.Fatalf("expected 2 videos, got %d", len(vids)) + } + v := vids[0] + if v.ProviderVideoID != "vid1" || v.Title != "Designing for Attention" { + t.Errorf("unexpected video: %+v", v) + } + if v.Provider != domain.ProviderYouTube || v.URL != "https://www.youtube.com/watch?v=vid1" { + t.Errorf("video not wired correctly: %+v", v) + } + if v.UserID != "u1" || v.SubscriptionID != "s1" { + t.Errorf("video not scoped to user/subscription: %+v", v) + } + if v.PublishedAt.IsZero() { + t.Errorf("expected publishedAt parsed, got zero") + } +} + +// --- transcript: captions present ------------------------------------------ + +func TestFetchTranscriptWithCaptions(t *testing.T) { + a, secrets := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/captions": + if got := r.URL.Query().Get("videoId"); got != "vid1" { + t.Errorf("expected videoId=vid1, got %q", got) + } + _, _ = w.Write([]byte(`{"items":[ + {"id":"cap-sv","snippet":{"language":"sv","trackKind":"standard","status":"serving"}}, + {"id":"cap-en","snippet":{"language":"en","trackKind":"standard","status":"serving"}} + ]}`)) + case "/captions/cap-en": + if got := r.URL.Query().Get("tfmt"); got != "vtt" { + t.Errorf("expected tfmt=vtt, got %q", got) + } + _, _ = w.Write([]byte("WEBVTT\n\n1\n00:00:00.000 --> 00:00:02.000\nHello world\n\n2\n00:00:02.000 --> 00:00:04.000\nHello world\n\n3\n00:00:04.000 --> 00:00:06.000\nSecond line\n")) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + }) + + v := domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid1"} + tr, err := a.FetchTranscript(context.Background(), v) + if err != nil { + t.Fatalf("FetchTranscript: %v", err) + } + if tr.Source != domain.SourceCaptions { + t.Fatalf("expected SourceCaptions, got %q", tr.Source) + } + if tr.Language != "en" { + t.Errorf("expected preferred language en, got %q", tr.Language) + } + // Markup stripped, timestamps/indices dropped, consecutive duplicate collapsed. + want := "Hello world\nSecond line" + if tr.Content != want { + t.Errorf("transcript text mismatch:\n got %q\nwant %q", tr.Content, want) + } + if !tr.HasText() { + t.Error("expected HasText() true") + } + if len(secrets.requested) == 0 { + t.Error("expected SecretStore consulted for OAuth token") + } +} + +// --- transcript: no captions => SourceNone, no error ------------------------ + +func TestFetchTranscriptNoCaptionsReturnsSourceNone(t *testing.T) { + a, _ := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/captions" { + t.Errorf("download must not be attempted when no track exists; got path %q", r.URL.Path) + } + _, _ = w.Write([]byte(`{"items":[]}`)) + }) + + v := domain.Video{ID: "v1", UserID: "u1", ProviderVideoID: "vid-nocaps"} + tr, err := a.FetchTranscript(context.Background(), v) + if err != nil { + t.Fatalf("expected no error for missing captions, got %v", err) + } + if tr.Source != domain.SourceNone { + t.Fatalf("expected SourceNone, got %q", tr.Source) + } + if tr.HasText() { + t.Error("expected HasText() false for SourceNone") + } + if tr.Content != "" { + t.Errorf("expected empty content, got %q", tr.Content) + } +} + +// A missing-token reference surfaces as an error rather than a silent skip. +func TestFetchTranscriptSecretResolutionError(t *testing.T) { + a, _ := newTestAdapter(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"items":[]}`)) + }) + a.cfg.TokenSecretRef = "op://HomeLab/missing#refresh" + + _, err := a.FetchTranscript(context.Background(), domain.Video{ID: "v1", ProviderVideoID: "x"}) + if err == nil { + t.Fatal("expected error when secret ref cannot be resolved") + } +}