feat(discovery): per-channel caption-availability memory (ADR-024)

After transcript caching (ADR-021) and the Shorts filter (ADR-023), the remaining caption waste is the first fetch on every new video of a channel that never has English captions — each costs one rate-limited fetch to resolve to "none", and on a throttled IP churns the backoff machinery first. Remember, per (user, channel), a streak of consecutive no-caption outcomes (channel_caption_state, migration 016, RLS-scoped). Once it reaches TAPIR_CHANNEL_CAPTIONLESS_THRESHOLD (default 5) the channel is suppressed — videos discovered/listed but not caption-fetched — for TAPIR_CHANNEL_CAPTIONLESS_WINDOW (default 14d), then one is re-probed (auto-recovery). A successful fetch resets the streak; a 429 does not count; an explicit manual request bypasses suppression. threshold=0 disables. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-10 20:27:25 +02:00
parent 9db06d8a63
commit 5219561a91
12 changed files with 403 additions and 26 deletions
@@ -27,8 +27,9 @@ import (
 // passCandidate is a video that passed all pre-filters (seen/manual/backoff)
 // and is queued for transcript fetch + summarization in this pass.
 type passCandidate struct {
-	v   domain.Video
-	pos int // discovery position — used as a stable tiebreak when published_at ties
+	v         domain.Video
+	channelID string // owning channel — keys the caption-availability memory (ADR-024)
+	pos       int    // discovery position — used as a stable tiebreak when published_at ties
 }

 // compareNewestFirst orders candidates by published_at descending, NULLS LAST,
@@ -74,6 +75,14 @@ type VideoStore interface {
 	// Called when NewVideos returns domain.ErrChannelUnavailable; best-effort, errors
 	// are logged and never abort the pass.
 	UpsertChannelError(ctx context.Context, userID, channelID, channelTitle string) error
+	// CaptionlessChannels returns channel ids currently suppressed because their
+	// recent videos all yielded no captions (ADR-024). The loop skips caption
+	// fetches for these channels' (non-requested) videos.
+	CaptionlessChannels(ctx context.Context, userID string) (map[string]bool, error)
+	// RecordChannelCaptionOutcome updates a channel's caption memory after a fetch:
+	// hadCaptions resets it, otherwise the no-caption streak grows and the channel
+	// is suppressed for window once it reaches threshold. A no-op when threshold<=0.
+	RecordChannelCaptionOutcome(ctx context.Context, userID, channelID string, hadCaptions bool, threshold int, window time.Duration) error
 }

 // Processor runs the core use case for a single video. *usecase.Engine
@@ -93,6 +102,9 @@ type Runner struct {
 	backoff    time.Duration    // rate-limit retry window; 0 = always retry
 	autoWindow time.Duration    // recency bound for auto-summarize; 0 = no bound
 	now        func() time.Time // injectable clock (tests); defaults to time.Now
+
+	captionThreshold int           // consecutive no-caption results before a channel is suppressed; 0 = feature off
+	captionWindow    time.Duration // how long a caption-less channel stays suppressed before re-probe
 }

 // Option configures a Runner at construction. Variadic so existing call sites
@@ -114,6 +126,14 @@ func WithClock(now func() time.Time) Option { return func(r *Runner) { r.now = n
 // bypasses the bound. 0 (the default) disables it (summarize every unseen video).
 func WithAutoWindow(d time.Duration) Option { return func(r *Runner) { r.autoWindow = d } }

+// WithCaptionMemory enables per-channel caption-availability suppression
+// (ADR-024): after threshold consecutive no-caption results a channel's videos
+// are skipped (no caption fetch) for window, then one is re-probed. threshold<=0
+// (the default) disables the feature entirely.
+func WithCaptionMemory(threshold int, window time.Duration) Option {
+	return func(r *Runner) { r.captionThreshold = threshold; r.captionWindow = window }
+}
+
 // New builds a Runner. A nil logger falls back to slog.Default.
 func New(src ports.VideoSource, store VideoStore, engine Processor, userID string, log *slog.Logger, opts ...Option) *Runner {
 	if log == nil {
@@ -131,15 +151,16 @@ func New(src ports.VideoSource, store VideoStore, engine Processor, userID strin

 // Stats summarizes one RunOnce pass.
 type Stats struct {
-	Candidates         int
-	Summarized         int
-	SkippedSeen        int
-	SkippedNoText      int
-	SkippedManual      int // discovered but not queued, in manual mode
-	SkippedTooOld      int // auto mode: published outside the recency window (not requested)
-	SkippedRateLimited int // 429'd previously and still inside the backoff window
-	Errors             int
-	ChannelUnavailable int // channels that returned HTTP 404 (deleted/private)
+	Candidates              int
+	Summarized              int
+	SkippedSeen             int
+	SkippedNoText           int
+	SkippedManual           int // discovered but not queued, in manual mode
+	SkippedTooOld           int // auto mode: published outside the recency window (not requested)
+	SkippedRateLimited      int // 429'd previously and still inside the backoff window
+	SkippedNoCaptionChannel int // channel suppressed as caption-less (ADR-024)
+	Errors                  int
+	ChannelUnavailable      int // channels that returned HTTP 404 (deleted/private)
 }

 // tooOld reports whether a video published at publishedAt falls outside the
@@ -213,6 +234,17 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
 		}
 	}

+	// Per-channel caption memory (ADR-024): channels whose recent videos all
+	// yielded no captions are suppressed so their new videos don't burn the scarce
+	// fetch budget. Loaded only when the feature is enabled (threshold > 0).
+	var captionless map[string]bool
+	if r.captionThreshold > 0 {
+		captionless, err = r.store.CaptionlessChannels(ctx, r.userID)
+		if err != nil {
+			return stats, fmt.Errorf("runner: load caption-less channels: %w", err)
+		}
+	}
+
 	subs, err := r.src.ListSubscriptions(ctx, r.userID)
 	if err != nil {
 		return stats, fmt.Errorf("runner: list subscriptions: %w", err)
@@ -273,6 +305,14 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
 				continue
 			}

+			// Caption-less channel (ADR-024): its recent videos all returned no
+			// captions, so skip the fetch entirely. The video is still listed
+			// (UpsertVideo above); an explicit manual request bypasses the skip.
+			if !requested[id] && captionless[sub.ChannelID] {
+				stats.SkippedNoCaptionChannel++
+				continue
+			}
+
 			// Still inside the rate-limit backoff window: skip without fetching.
 			if at, ok := rateLimited[id]; ok && r.now().Sub(at) < r.backoff {
 				stats.SkippedRateLimited++
@@ -280,7 +320,7 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
 				continue
 			}

-			candidates = append(candidates, passCandidate{v: v, pos: pos})
+			candidates = append(candidates, passCandidate{v: v, channelID: sub.ChannelID, pos: pos})
 			pos++
 		}
 	}
@@ -320,6 +360,11 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
 				errs = append(errs, fmt.Errorf("set none status %q: %w", c.v.ProviderVideoID, err))
 				stats.Errors++
 			}
+			// No captions: grow this channel's no-caption streak (ADR-024).
+			if err := r.store.RecordChannelCaptionOutcome(ctx, r.userID, c.channelID, false, r.captionThreshold, r.captionWindow); err != nil {
+				errs = append(errs, fmt.Errorf("record no-caption %q: %w", c.v.ProviderVideoID, err))
+				stats.Errors++
+			}
 			r.log.Info("skipped video (no transcript)", "video", c.v.ProviderVideoID, "title", c.v.Title)
 		case res.Summary != nil:
 			stats.Summarized++
@@ -327,6 +372,11 @@ func (r *Runner) RunOnce(ctx context.Context) (Stats, error) {
 				errs = append(errs, fmt.Errorf("set fetched status %q: %w", c.v.ProviderVideoID, err))
 				stats.Errors++
 			}
+			// Captions present: reset this channel's caption memory (ADR-024).
+			if err := r.store.RecordChannelCaptionOutcome(ctx, r.userID, c.channelID, true, r.captionThreshold, r.captionWindow); err != nil {
+				errs = append(errs, fmt.Errorf("record has-caption %q: %w", c.v.ProviderVideoID, err))
+				stats.Errors++
+			}
 			// In manual mode the video was explicitly queued; clear the flag so
 			// it is not re-summarized and the UI drops the "Queued" chip.
 			if !auto {
@@ -354,6 +404,7 @@ func (r *Runner) Loop(ctx context.Context, interval time.Duration) error {
 			"skipped_seen", stats.SkippedSeen, "skipped_no_text", stats.SkippedNoText,
 			"skipped_manual", stats.SkippedManual, "skipped_too_old", stats.SkippedTooOld,
 			"skipped_rate_limited", stats.SkippedRateLimited,
+			"skipped_no_caption_channel", stats.SkippedNoCaptionChannel,
 			"channel_unavailable", stats.ChannelUnavailable, "errors", stats.Errors)
 		if err != nil {
 			r.log.Warn("run pass had errors", "err", err)
@@ -51,6 +51,13 @@ type fakeStore struct {
 	cleared     []string
 	rateLimited map[string]time.Time // id -> when 429'd (seeds the backoff window)
 	statuses    map[string]string    // id -> last SetTranscriptStatus value
+	captionless map[string]bool      // channel ids currently suppressed (ADR-024)
+	captionRecs []captionRec         // RecordChannelCaptionOutcome calls, in order
+}
+
+type captionRec struct {
+	channelID string
+	had       bool
 }

 func (f *fakeStore) UpsertVideo(_ context.Context, v domain.Video) (string, error) {
@@ -93,6 +100,22 @@ func (f *fakeStore) RateLimitedVideoIDs(_ context.Context, _ string) (map[string

 func (f *fakeStore) UpsertChannelError(_ context.Context, _, _, _ string) error { return nil }

+func (f *fakeStore) CaptionlessChannels(_ context.Context, _ string) (map[string]bool, error) {
+	cp := make(map[string]bool, len(f.captionless))
+	for k, v := range f.captionless {
+		cp[k] = v
+	}
+	return cp, nil
+}
+
+func (f *fakeStore) RecordChannelCaptionOutcome(_ context.Context, _, channelID string, hadCaptions bool, threshold int, _ time.Duration) error {
+	if threshold <= 0 {
+		return nil
+	}
+	f.captionRecs = append(f.captionRecs, captionRec{channelID: channelID, had: hadCaptions})
+	return nil
+}
+
 func (f *fakeStore) SetTranscriptStatus(_ context.Context, _, videoID, status string) error {
 	if f.statuses == nil {
 		f.statuses = map[string]string{}
@@ -284,6 +307,55 @@ func TestRunOnce_AutoMode_OldVideoRequestedBypassesWindow(t *testing.T) {
 	require.Len(t, sink.delivered, 1)
 }

+// TestRunOnce_CaptionlessChannelSkipped: a channel flagged caption-less (ADR-024)
+// has its videos skipped from fetching but still discovered/listed, while a
+// normal channel's video is summarized.
+func TestRunOnce_CaptionlessChannelSkipped(t *testing.T) {
+	src := &fakeSource{
+		subs: []domain.Subscription{sub("dead", "Dead Channel"), sub("live", "Live Channel")},
+		videos: map[string][]domain.Video{
+			"dead": {vid("d1", "Dead One")},
+			"live": {vid("l1", "Live One")},
+		},
+	}
+	st := &fakeStore{seen: map[string]bool{}, auto: true, captionless: map[string]bool{"dead": true}}
+	sink := &recordingSink{}
+	eng := usecase.NewEngine(src, fakeSummarizer{}, sink)
+	r := runner.New(src, st, eng, testUser, quietLogger(),
+		runner.WithCaptionMemory(5, 14*24*time.Hour))
+
+	stats, err := r.RunOnce(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, 1, stats.SkippedNoCaptionChannel, "dead channel's video skipped from fetch")
+	require.Equal(t, 1, stats.Summarized, "live channel's video still summarized")
+	require.Len(t, st.upserted, 2, "both videos are still discovered and listed")
+}
+
+// TestRunOnce_RecordsCaptionOutcomes: a no-caption result grows the channel's
+// streak (had=false); a successful summary resets it (had=true).
+func TestRunOnce_RecordsCaptionOutcomes(t *testing.T) {
+	src := &fakeSource{
+		subs: []domain.Subscription{sub("c1", "Has Caps"), sub("c2", "No Caps")},
+		videos: map[string][]domain.Video{
+			"c1": {vid("good", "Good")},
+			"c2": {vid("bad", "Bad")},
+		},
+		transcripts: map[string]domain.Transcript{
+			"bad": {Source: domain.SourceNone}, // no usable text → engine skips
+		},
+	}
+	st := &fakeStore{seen: map[string]bool{}, auto: true}
+	sink := &recordingSink{}
+	eng := usecase.NewEngine(src, fakeSummarizer{}, sink)
+	r := runner.New(src, st, eng, testUser, quietLogger(),
+		runner.WithCaptionMemory(5, 14*24*time.Hour))
+
+	_, err := r.RunOnce(context.Background())
+	require.NoError(t, err)
+	require.Contains(t, st.captionRecs, captionRec{channelID: "c1", had: true}, "captioned channel reset")
+	require.Contains(t, st.captionRecs, captionRec{channelID: "c2", had: false}, "no-caption channel streak grown")
+}
+
 // TestRunOnce_AutoWindowZero_SummarizesOld: a zero window disables the bound —
 // the pre-recency behaviour (summarize every unseen video) is preserved.
 func TestRunOnce_AutoWindowZero_SummarizesOld(t *testing.T) {