feat(adapters): add captions-first YouTube VideoSource
Implements ports.VideoSource against the YouTube Data API v3: ListSubscriptions (paginated), NewVideos (recent per channel), and captions-first FetchTranscript — an absent caption track yields domain.SourceNone (not an error) per ADR-007, with no audio download or speech-to-text. OAuth is written fresh on golang.org/x/oauth2 (ADR-006, distinct from ingestion's inbound MCP auth); the Google token endpoint is inlined to avoid the heavy x/oauth2/google dep. The per-connection refresh token is resolved through the SecretStore port from an opaque TokenSecretRef and is never stored on the adapter or logged. Unit-tested against an httptest server + fake SecretStore (no live googleapis egress): subscriptions list/pagination, new-video detection, captions present -> Source set, captions absent -> SourceNone no error, and secret-ref resolution failure surfacing as an error. oauth2 pinned to v0.30.0 to keep the go directive at 1.23.x (koala runner), not the v0.36 line that requires a newer toolchain. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
package youtube
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// FetchTranscript resolves a transcript captions-first (ADR-007):
|
||||
//
|
||||
// - List the video's caption tracks. No track => domain.SourceNone, no error.
|
||||
// - Select a track (preferred language first, else the first track) and
|
||||
// download it as WebVTT, stripping cue timing/markup to plain text.
|
||||
//
|
||||
// Audio download and speech-to-text are deliberately absent: no captions means
|
||||
// SourceNone, full stop.
|
||||
func (a *Adapter) FetchTranscript(ctx context.Context, v domain.Video) (domain.Transcript, error) {
|
||||
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
||||
if err != nil {
|
||||
return domain.Transcript{}, err
|
||||
}
|
||||
|
||||
tracks, err := a.listCaptionTracks(ctx, client, v.ProviderVideoID)
|
||||
if err != nil {
|
||||
return domain.Transcript{}, fmt.Errorf("list captions for %q: %w", v.ProviderVideoID, err)
|
||||
}
|
||||
|
||||
track, ok := a.selectTrack(tracks)
|
||||
if !ok {
|
||||
// No usable caption track: a recorded "checked, none available", not an error.
|
||||
return domain.Transcript{
|
||||
VideoID: v.ID,
|
||||
UserID: v.UserID,
|
||||
Source: domain.SourceNone,
|
||||
}, nil
|
||||
}
|
||||
|
||||
raw, err := a.getRaw(ctx, client, "/captions/"+track.ID, url.Values{"tfmt": {"vtt"}})
|
||||
if err != nil {
|
||||
return domain.Transcript{}, fmt.Errorf("download caption track %q: %w", track.ID, err)
|
||||
}
|
||||
|
||||
text := vttToText(string(raw))
|
||||
if text == "" {
|
||||
// Track existed but carried no text: treat as no usable transcript.
|
||||
return domain.Transcript{
|
||||
VideoID: v.ID,
|
||||
UserID: v.UserID,
|
||||
Source: domain.SourceNone,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return domain.Transcript{
|
||||
VideoID: v.ID,
|
||||
UserID: v.UserID,
|
||||
Source: domain.SourceCaptions,
|
||||
Language: track.Snippet.Language,
|
||||
Content: text,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *Adapter) listCaptionTracks(ctx context.Context, client *http.Client, videoID string) ([]captionTrack, error) {
|
||||
q := url.Values{
|
||||
"part": {"snippet"},
|
||||
"videoId": {videoID},
|
||||
}
|
||||
var resp captionListResponse
|
||||
if err := a.getJSON(ctx, client, "/captions", q, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.Items, nil
|
||||
}
|
||||
|
||||
// selectTrack picks the best caption track: the first track whose language
|
||||
// matches a configured preferred language, else the first track. Returns ok ==
|
||||
// false when there are no tracks at all.
|
||||
func (a *Adapter) selectTrack(tracks []captionTrack) (captionTrack, bool) {
|
||||
if len(tracks) == 0 {
|
||||
return captionTrack{}, false
|
||||
}
|
||||
for _, pref := range a.cfg.PreferredLanguages {
|
||||
for _, t := range tracks {
|
||||
if strings.EqualFold(t.Snippet.Language, pref) {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return tracks[0], true
|
||||
}
|
||||
|
||||
// --- caption response shapes ------------------------------------------------
|
||||
|
||||
type captionListResponse struct {
|
||||
Items []captionTrack `json:"items"`
|
||||
}
|
||||
|
||||
type captionTrack struct {
|
||||
ID string `json:"id"`
|
||||
Snippet struct {
|
||||
Language string `json:"language"`
|
||||
TrackKind string `json:"trackKind"` // "standard" | "ASR" | "forced"
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
} `json:"snippet"`
|
||||
}
|
||||
|
||||
// --- WebVTT -> plain text ---------------------------------------------------
|
||||
|
||||
var (
|
||||
vttCueTime = regexp.MustCompile(`-->`)
|
||||
vttTag = regexp.MustCompile(`<[^>]+>`) // inline <00:00:01.000>, <c> markup
|
||||
vttIndex = regexp.MustCompile(`^\d+$`) // SRT-style numeric cue index
|
||||
vttSetting = regexp.MustCompile(`^(NOTE|STYLE|REGION)`) // VTT block headers
|
||||
)
|
||||
|
||||
// vttToText reduces a WebVTT (or SRT-ish) caption file to plain transcript text:
|
||||
// the WEBVTT header, NOTE/STYLE blocks, cue-timing lines, numeric indices, and
|
||||
// inline markup are dropped; consecutive duplicate lines (common in rolling
|
||||
// auto-captions) are collapsed.
|
||||
func vttToText(raw string) string {
|
||||
raw = strings.ReplaceAll(raw, "\r\n", "\n")
|
||||
var out []string
|
||||
var prev string
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "WEBVTT") {
|
||||
continue
|
||||
}
|
||||
if vttSetting.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
if vttCueTime.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
if vttIndex.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
line = strings.TrimSpace(vttTag.ReplaceAllString(line, ""))
|
||||
if line == "" || line == prev {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
prev = line
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
Reference in New Issue
Block a user