Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d5b2c6365 | ||
|
|
62acfee2ed | ||
|
|
2907801aca | ||
|
|
59050c4db6 | ||
|
|
c320ed88aa | ||
|
|
bddd75d92e | ||
|
|
e4c701c6f1 | ||
|
|
b527db9739 | ||
|
|
c5f556d1d6 | ||
|
|
a884e7e9c5 | ||
|
|
27fd33c99c | ||
|
|
2c96926ff7 |
@@ -60,8 +60,14 @@ These caused real mistakes that were caught and corrected; the corrections are l
|
||||
stores, and sinks are adapters. Adding a video provider or a sink = a new adapter implementing
|
||||
the interface, nothing in the engine changes. This is what keeps "standalone vs homelab" a
|
||||
wiring choice (ADR-003).
|
||||
- **BDD.** The `docs/use-cases/*.feature` files are the behavior spec. New behavior gets a
|
||||
scenario; the use-case core is tested through fake adapters, not live YouTube/brain.
|
||||
- **BDD.** The `docs/use-cases/*.feature` files are the behavior spec (design records — there is
|
||||
no godog runner). New behavior gets a scenario; the use-case core is tested through fake
|
||||
adapters, not live YouTube/brain. A name-coverage gate (`test/acceptance/scenario_coverage_test.go`,
|
||||
`TestScenarioCoverage`) keeps the two from drifting: every non-`@pending` scenario must be
|
||||
mapped to an existing Go test in `scenarioCoverage`. When you add a scenario, either map it to
|
||||
its covering test or tag it `@pending` in the `.feature` with a one-line reason. It checks the
|
||||
*link*, not that the test exercises the scenario — that's the deliberate trade for not running
|
||||
godog (see issue #5 / the BDD-runner decision).
|
||||
|
||||
## Skills (engineering discipline)
|
||||
|
||||
@@ -79,7 +85,7 @@ Skills live in the canonical library `mathias/skills` and are wired into this re
|
||||
|
||||
## Current build state (start here for the first task)
|
||||
|
||||
The repo is **green and shipping** — last tag `v0.4.0`. `task check` passes (fmt, vet, lint,
|
||||
The repo is **green and shipping** — last tag `v0.9.0`. `task check` passes (fmt, vet, lint,
|
||||
`go test -p 1 ./...`). Go is `1.26.1` (see `go.mod`).
|
||||
|
||||
- Clean Architecture core is implemented: `internal/domain` (entities), `internal/ports`
|
||||
|
||||
@@ -75,8 +75,10 @@ password or Google); a new Dex subject is routed to `/register` to create a Tapi
|
||||
Stage 1 is multi-user: each user connects their own YouTube account from the browser and
|
||||
manages their own summaries under DB-enforced RLS isolation. When `TAPIR_DISCOVERY_INTERVAL`
|
||||
is set (e.g. `2h`), the serve process runs a scheduled discovery pass for every registered
|
||||
user automatically — no CronJob required. See `docs/homelab-integration.md` for the full
|
||||
config reference.
|
||||
user automatically — no CronJob required. In auto mode only videos published within
|
||||
`TAPIR_AUTO_SUMMARIZE_WINDOW` (default ~7d, ADR-020) are summarised automatically; older videos
|
||||
are listed and summarised on demand, so a large back-catalogue doesn't keep re-driving the
|
||||
caption rate gate. See `docs/homelab-integration.md` for the full config reference.
|
||||
|
||||
### Headless on koala
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/runner"
|
||||
)
|
||||
|
||||
// discoveryRunner runs one user's discovery pass.
|
||||
type discoveryRunner func(ctx context.Context, userID string) (runner.Stats, error)
|
||||
|
||||
// serialize wraps run so calls never overlap: every discovery pass — scheduled
|
||||
// or connect-triggered (#6) — acquires the same lock, preserving the
|
||||
// one-fetcher-at-a-time invariant the scheduler relies on (ADR-018, the
|
||||
// single-replica assumption). Locking is per-user, so a connect-triggered pass
|
||||
// interleaves between the scheduler's users instead of waiting for a whole pass.
|
||||
func serialize(mu *sync.Mutex, run discoveryRunner) discoveryRunner {
|
||||
return func(ctx context.Context, userID string) (runner.Stats, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return run(ctx, userID)
|
||||
}
|
||||
}
|
||||
|
||||
// discoveryTrigger fires an out-of-band discovery pass for one user without
|
||||
// blocking the caller (the connect HTTP handler). The pass runs on the server's
|
||||
// long-lived ctx — not the request ctx — so it survives the post-connect
|
||||
// redirect. run is the serialized runner, so a trigger never overlaps the
|
||||
// scheduler. Satisfies web.DiscoveryTrigger.
|
||||
type discoveryTrigger struct {
|
||||
ctx context.Context
|
||||
run discoveryRunner
|
||||
// onboard, when set, runs after the discovery pass to summarize a capped number
|
||||
// of the user's newest videos (Feature 1). Optional.
|
||||
onboard func(ctx context.Context, userID string)
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func (t *discoveryTrigger) Enqueue(userID string) {
|
||||
go func() {
|
||||
if _, err := t.run(t.ctx, userID); err != nil {
|
||||
t.log.Warn("discovery: connect-triggered pass had errors", "user", userID, "err", err)
|
||||
}
|
||||
if t.onboard != nil {
|
||||
t.onboard(t.ctx, userID)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/runner"
|
||||
)
|
||||
|
||||
// serialize must guarantee at most one discovery pass runs at a time, so a
|
||||
// connect-triggered pass never fetches concurrently with the scheduler.
|
||||
func TestSerializeRunsOneAtATime(t *testing.T) {
|
||||
var active, maxActive int32
|
||||
run := func(_ context.Context, _ string) (runner.Stats, error) {
|
||||
n := atomic.AddInt32(&active, 1)
|
||||
for { // record the high-water mark of concurrent runs
|
||||
m := atomic.LoadInt32(&maxActive)
|
||||
if n <= m || atomic.CompareAndSwapInt32(&maxActive, m, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
atomic.AddInt32(&active, -1)
|
||||
return runner.Stats{}, nil
|
||||
}
|
||||
|
||||
s := serialize(&sync.Mutex{}, run)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) { defer wg.Done(); _, _ = s(context.Background(), fmt.Sprintf("u%d", i)) }(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
require.Equal(t, int32(1), atomic.LoadInt32(&maxActive),
|
||||
"serialize must run at most one pass at a time")
|
||||
}
|
||||
|
||||
// Enqueue runs the user's pass out-of-band (non-blocking) on the trigger's ctx.
|
||||
func TestDiscoveryTriggerEnqueueRunsUser(t *testing.T) {
|
||||
done := make(chan string, 1)
|
||||
run := func(_ context.Context, userID string) (runner.Stats, error) {
|
||||
done <- userID
|
||||
return runner.Stats{}, nil
|
||||
}
|
||||
tr := &discoveryTrigger{ctx: context.Background(), run: run, log: quietLog()}
|
||||
|
||||
tr.Enqueue("u1")
|
||||
|
||||
select {
|
||||
case got := <-done:
|
||||
require.Equal(t, "u1", got)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Enqueue did not run the user's pass")
|
||||
}
|
||||
}
|
||||
+37
-2
@@ -20,6 +20,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/secrets"
|
||||
@@ -209,7 +210,10 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
|
||||
ClientSecret: cfg.YTClientSecret,
|
||||
RedirectURL: cfg.YTConnectRedirectURL,
|
||||
}, secretStore, st, log)
|
||||
log.Info("web youtube connect enabled", "redirect", cfg.YTConnectRedirectURL)
|
||||
// Paste-a-URL (Feature 2): same YouTube credentials, per-user adapter built
|
||||
// per request. Mounting the /paste route keys off app.Fetcher being set.
|
||||
app.Fetcher = videoFetcher{cfg: cfg, secrets: secretStore}
|
||||
log.Info("web youtube connect + paste enabled", "redirect", cfg.YTConnectRedirectURL)
|
||||
}
|
||||
|
||||
// Immediate summarization for the web "Summarize" button. When the engine can
|
||||
@@ -238,13 +242,44 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
|
||||
if cfg.DiscoveryInterval > 0 {
|
||||
log.Info("scheduled discovery enabled", "interval", cfg.DiscoveryInterval, "fetch_rate", cfg.FetchRate)
|
||||
log.Warn("scheduled discovery assumes a SINGLE replica — running serve at >1 replica double-runs discovery (ADR-018)")
|
||||
runUser := func(ctx context.Context, userID string) (runner.Stats, error) {
|
||||
rawRunUser := func(ctx context.Context, userID string) (runner.Stats, error) {
|
||||
r, err := buildUserRunner(cfg, st, secretStore, userID, log)
|
||||
if err != nil {
|
||||
return runner.Stats{}, err
|
||||
}
|
||||
return r.RunOnce(ctx)
|
||||
}
|
||||
// One lock shared by the scheduler and connect-triggered passes (#6) so
|
||||
// they never fetch concurrently — the single-fetcher invariant (ADR-018).
|
||||
runUser := serialize(&sync.Mutex{}, rawRunUser)
|
||||
// Onboarding burst (Feature 1): after the connect-triggered discovery pass,
|
||||
// summarize up to OnboardSummarizeCount of the user's NEWEST unsummarized
|
||||
// videos so a fresh account gets real summaries in its first session. Hard
|
||||
// cap; explicit, so it bypasses the recency window — but every fetch still
|
||||
// goes through globalFetchGate via the Processor. No-op when disabled
|
||||
// (count 0) or queue-only (no Processor).
|
||||
onboard := func(ctx context.Context, userID string) {
|
||||
if cfg.OnboardSummarizeCount <= 0 || app.Processor == nil {
|
||||
return
|
||||
}
|
||||
ids, err := st.NewestUnsummarizedVideoIDs(ctx, userID, cfg.OnboardSummarizeCount)
|
||||
if err != nil {
|
||||
log.Warn("onboarding: list newest unsummarized", "user", userID, "err", err)
|
||||
return
|
||||
}
|
||||
for _, id := range ids {
|
||||
if err := app.Processor.ProcessVideo(ctx, userID, id); err != nil {
|
||||
log.Warn("onboarding: summarize", "user", userID, "video", id, "err", err)
|
||||
}
|
||||
}
|
||||
if len(ids) > 0 {
|
||||
log.Info("onboarding burst complete", "user", userID, "summarized", len(ids), "cap", cfg.OnboardSummarizeCount)
|
||||
}
|
||||
}
|
||||
if app.Connect != nil {
|
||||
app.Connect.Discovery = &discoveryTrigger{ctx: ctx, run: runUser, onboard: onboard, log: log}
|
||||
log.Info("connect-triggered discovery enabled", "onboard_cap", cfg.OnboardSummarizeCount)
|
||||
}
|
||||
go runScheduler(ctx, cfg.DiscoveryInterval, st, runUser, log)
|
||||
} else {
|
||||
log.Info("scheduled discovery disabled (TAPIR_DISCOVERY_INTERVAL unset or 0)")
|
||||
|
||||
@@ -11,9 +11,29 @@ import (
|
||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/youtube"
|
||||
"gitea.d-ma.be/mathias/tapir/internal/config"
|
||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||
"gitea.d-ma.be/mathias/tapir/internal/ports"
|
||||
"gitea.d-ma.be/mathias/tapir/internal/usecase"
|
||||
"gitea.d-ma.be/mathias/tapir/internal/web"
|
||||
)
|
||||
|
||||
// videoFetcher adapts the YouTube adapter to web.VideoFetcher for the paste flow
|
||||
// (Feature 2). It builds a per-user adapter bound to that user's token ref and
|
||||
// resolves a single video's metadata via the Data API — ungated; only the later
|
||||
// transcript fetch goes through globalFetchGate.
|
||||
type videoFetcher struct {
|
||||
cfg config.Config
|
||||
secrets ports.SecretStore
|
||||
}
|
||||
|
||||
func (f videoFetcher) FetchVideo(ctx context.Context, userID, videoID string) (domain.Video, error) {
|
||||
a := youtube.New(youtube.Config{
|
||||
ClientID: f.cfg.YTClientID,
|
||||
ClientSecret: f.cfg.YTClientSecret,
|
||||
TokenSecretRef: web.YouTubeTokenRef(userID),
|
||||
}, f.secrets)
|
||||
return a.VideoByID(ctx, userID, videoID)
|
||||
}
|
||||
|
||||
// buildProcessor wires the summarization engine — YouTube source (captions-first),
|
||||
// AI-router summarizer, store sink — shared by `tapir run` and the web
|
||||
// "Summarize now" path so the wiring lives in one place. It returns (nil, nil) —
|
||||
|
||||
+16
-2
@@ -49,10 +49,12 @@ func buildUserRunner(cfg config.Config, st *store.Store, secretStore ports.Secre
|
||||
runner.WithAutoWindow(cfg.AutoSummarizeWindow)), nil
|
||||
}
|
||||
|
||||
// userLister enumerates every registered user. *store.Store satisfies it via
|
||||
// ListAllUsers. A small local interface keeps the scheduler testable with a fake.
|
||||
// userLister enumerates every registered user and reports a user's video
|
||||
// connections. *store.Store satisfies it via ListAllUsers + ConnectionsForUser.
|
||||
// A small local interface keeps the scheduler testable with a fake.
|
||||
type userLister interface {
|
||||
ListAllUsers(ctx context.Context) ([]store.UserIdentity, error)
|
||||
ConnectionsForUser(ctx context.Context, userID string) ([]store.Connection, error)
|
||||
}
|
||||
|
||||
// runDiscoveryPass runs one discovery pass for every user. runUser performs a
|
||||
@@ -78,6 +80,18 @@ func runDiscoveryPass(
|
||||
if ctx.Err() != nil {
|
||||
break // shutting down: stop enumerating
|
||||
}
|
||||
// Skip users with no video connection. A discovery pass for them only
|
||||
// attempts to resolve a token that was never minted, logging a spurious
|
||||
// "ref not found" every tick (e.g. stale Dex-era orphan identities).
|
||||
conns, err := lister.ConnectionsForUser(ctx, u.UserID)
|
||||
if err != nil {
|
||||
log.Warn("scheduler: list connections failed", "user", u.UserID, "err", err)
|
||||
continue
|
||||
}
|
||||
if len(conns) == 0 {
|
||||
log.Debug("scheduler: skipping user with no video connections", "user", u.UserID)
|
||||
continue
|
||||
}
|
||||
stats, err := runUser(ctx, u.UserID)
|
||||
total = sumStats(total, stats)
|
||||
if err != nil {
|
||||
|
||||
@@ -21,14 +21,25 @@ func quietLog() *slog.Logger {
|
||||
|
||||
// fakeLister returns a fixed user set (or an error) for the scheduler under test.
|
||||
type fakeLister struct {
|
||||
users []store.UserIdentity
|
||||
err error
|
||||
users []store.UserIdentity
|
||||
err error
|
||||
noConn map[string]bool // users that have NOT connected a video source
|
||||
}
|
||||
|
||||
func (f fakeLister) ListAllUsers(context.Context) ([]store.UserIdentity, error) {
|
||||
return f.users, f.err
|
||||
}
|
||||
|
||||
// ConnectionsForUser reports a single youtube connection for every user except
|
||||
// those in noConn, which return zero — the connection-less case the scheduler
|
||||
// must skip instead of running (and failing to resolve a token for).
|
||||
func (f fakeLister) ConnectionsForUser(_ context.Context, userID string) ([]store.Connection, error) {
|
||||
if f.noConn[userID] {
|
||||
return nil, nil
|
||||
}
|
||||
return []store.Connection{{Provider: "youtube"}}, nil
|
||||
}
|
||||
|
||||
// countingRunUser records how many passes each user got, optionally failing for
|
||||
// specific users, under a mutex so it is safe across the scheduler goroutine.
|
||||
type countingRunUser struct {
|
||||
@@ -91,6 +102,21 @@ func TestDiscoveryPassRunsEveryUserOnce(t *testing.T) {
|
||||
require.Equal(t, 3, stats.Summarized, "stats are summed across users")
|
||||
}
|
||||
|
||||
func TestDiscoveryPassSkipsUsersWithoutConnections(t *testing.T) {
|
||||
// b never connected a video source (e.g. a stale Dex-era orphan identity).
|
||||
// It must be skipped silently — not run and logged as a token error every pass.
|
||||
lister := fakeLister{users: usersN("a", "b", "c"), noConn: map[string]bool{"b": true}}
|
||||
rc := newCountingRunUser()
|
||||
|
||||
stats := runDiscoveryPass(context.Background(), lister, rc.run, quietLog())
|
||||
|
||||
require.Equal(t, 1, rc.count("a"))
|
||||
require.Equal(t, 0, rc.count("b"), "a user with no connection must be skipped, not run")
|
||||
require.Equal(t, 1, rc.count("c"))
|
||||
require.Equal(t, 2, stats.Summarized, "only connected users contribute")
|
||||
require.Equal(t, 0, stats.Errors, "skipping is silent — no spurious error stat")
|
||||
}
|
||||
|
||||
func TestDiscoveryPassOneUserFailureDoesNotStopOthers(t *testing.T) {
|
||||
lister := fakeLister{users: usersN("a", "b", "c")}
|
||||
rc := newCountingRunUser("b") // user b's pass errors
|
||||
|
||||
@@ -147,9 +147,17 @@ graph TB
|
||||
rate-limiting — ADR-014).
|
||||
- **Summarization mode** — `users.auto_summarize` (migration 006). Default is **true** for new
|
||||
users (migration 011, ADR-018); all existing rows were back-filled via migration 012. Auto:
|
||||
every new video is summarized. Manual: new videos appear unsummarized; the button sets
|
||||
`videos.summarize_requested`, which the next `tapir run` processes and clears. Both the click
|
||||
path and the batch `tapir run` drive the same unchanged engine.
|
||||
new videos **published within the recency window** (`TAPIR_AUTO_SUMMARIZE_WINDOW`, default ~7d,
|
||||
ADR-020) are summarized automatically; older videos are discovered and listed but wait for an
|
||||
explicit "Summarize". Manual: new videos appear unsummarized; the button sets
|
||||
`videos.summarize_requested`, which the next `tapir run` processes and clears. A manual request
|
||||
bypasses the recency bound. Both the click path and the batch `tapir run` drive the same
|
||||
unchanged engine.
|
||||
- **List surface (ADR-020)** — the list reads `ListVideos` ordered summarized-first, then
|
||||
`published_at DESC NULLS LAST`. The web layer collapses the noise so summaries are not buried:
|
||||
un-summarized videos older than the recency window fold into one "Show N older videos"
|
||||
disclosure, and caption-less videos collapse to a single count line. Copy surfaces scarcity
|
||||
honestly (queue counts, gradual-fill note) — it never implies the feed is fuller than it is.
|
||||
|
||||
The engine, ports, and sink adapters are **untouched** by all of the above — the web surface only
|
||||
reads the store and triggers the existing engine. Adding it changed wiring, not the core (ADR-003).
|
||||
@@ -174,6 +182,7 @@ sequenceDiagram
|
||||
loop per user
|
||||
S->>DB: GetAutoSummarize(userID)
|
||||
S->>YT: ListSubscriptions + NewVideos
|
||||
Note over S,DB: auto: skip videos published before<br/>TAPIR_AUTO_SUMMARIZE_WINDOW (ADR-020);<br/>older ones listed, await manual request
|
||||
Note over S,YT: WaitFetchGate(ctx) throttles<br/>all fetches to TAPIR_FETCH_RATE
|
||||
alt transcript available
|
||||
S->>LLM: Summarize
|
||||
@@ -214,19 +223,22 @@ Both paths share `globalFetchGate` — rate limiting is **respected in both**, n
|
||||
|
||||
| Path | Trigger | Order | Rationale |
|
||||
|------|---------|-------|-----------|
|
||||
| **Foreground** | User clicks "Summarize now" on any non-summarized card (`POST /v/{id}/retry-now` for rate-limited; `POST /v/{id}/summarize` for pending) | Single chosen video | On-demand value: user picks a specific video to read now |
|
||||
| **Background batch** | Scheduled discovery pass every `TAPIR_DISCOVERY_INTERVAL` | **Newest-first across all channels** (see below) | Onboarding prioritisation: most recent, relevant videos surface first |
|
||||
| **Foreground** | User clicks "Summarize" on any non-summarized card (`POST /v/{id}/retry-now` for rate-limited; `POST /v/{id}/summarize` for pending) | Single chosen video | On-demand value: user picks a specific video to read now — bypasses the recency bound |
|
||||
| **Background batch** | Scheduled discovery pass every `TAPIR_DISCOVERY_INTERVAL` | **Newest-first across all channels** (see below), **bounded to the recency window** (ADR-020) | Onboarding prioritisation within bounded load: recent videos auto-fill; the older back-catalogue stays on-demand |
|
||||
|
||||
The rationale for both paths is **onboarding prioritisation** — a new user should get summaries
|
||||
of their most recent, relevant videos quickly while the older back-catalogue fills in behind,
|
||||
all within the honest shared rate limit.
|
||||
The rationale for both paths is **onboarding prioritisation under an honest, bounded load** — a
|
||||
new user gets summaries of their most recent videos automatically, while the older back-catalogue
|
||||
is listed but summarised only on demand, so it never re-drives the shared rate gate every cycle.
|
||||
|
||||
### Newest-first batch ordering (ADR-018)
|
||||
|
||||
Within each scheduled pass, `RunOnce` uses a three-phase structure:
|
||||
|
||||
1. **Discover + persist**: walk all channels, `UpsertVideo` every candidate (so it appears in
|
||||
the list), apply pre-filters (seen/manual/backoff), collect surviving candidates.
|
||||
the list), apply pre-filters (seen/manual/backoff/**recency**), collect surviving candidates.
|
||||
The recency pre-filter (ADR-020) drops auto-mode videos published before
|
||||
`now - TAPIR_AUTO_SUMMARIZE_WINDOW` unless they are explicitly requested; an undated video is
|
||||
never aged out. They remain persisted/listed — only auto-summarisation is skipped.
|
||||
2. **Sort**: order candidates `published_at DESC, NULLS LAST, discovery_pos ASC`. Videos with
|
||||
no publish date (schema 001: nullable) sort after all dated content. The sort is in-memory
|
||||
(`slices.SortStableFunc`) — at current scale this is fine.
|
||||
@@ -235,7 +247,8 @@ Within each scheduled pass, `RunOnce` uses a three-phase structure:
|
||||
Before (per-channel inline): `[chanA-old, chanA-mid, chanB-new, chanB-null]`
|
||||
After (newest-first): `[chanB-new, chanA-mid, chanA-old, chanB-null]`
|
||||
|
||||
The set of processed videos is identical; only the order within a pass changes.
|
||||
The set of *processed* videos now also excludes auto-mode back-catalogue beyond the recency
|
||||
window (those stay listed, summarised on demand); within the processed set, only order changes.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+4
-3
@@ -147,9 +147,10 @@ mechanism.
|
||||
|
||||
- **USER** — one row per registered user (Stage 1, ADR-012; no longer single-row). The Tapir-side
|
||||
profile; the Dex identity is held separately in `USER_IDENTITY`, not on this row. `auto_summarize`
|
||||
(migration 006) is the per-user mode flag: `TRUE` = auto-summarize every new video. Default is
|
||||
**true** for new users (migration 011, ADR-018); existing rows were back-filled via migration 012
|
||||
with RLS bypass.
|
||||
(migration 006) is the per-user mode flag: `TRUE` = auto-summarize new videos **published within
|
||||
the recency window** (`TAPIR_AUTO_SUMMARIZE_WINDOW`, default ~7d, ADR-020); older videos are
|
||||
listed but summarised on demand. Default is **true** for new users (migration 011, ADR-018);
|
||||
existing rows were back-filled via migration 012 with RLS bypass.
|
||||
- **USER_IDENTITY** (migration 004) — the `dex_subject → user_id` map. `dex_subject` is the PK,
|
||||
`user_id` a `UNIQUE` FK to `users` with `ON DELETE CASCADE`. This is the bridge resolved at login
|
||||
*before* a `user_id` is known, so it is **deliberately not RLS-enabled** (it holds no user data;
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# Spec — Newest-first batch ordering + honest "Try now" / prioritisation docs
|
||||
|
||||
> **Extended by ADR-020 (2026-06-08).** This spec covers the *batch processing* order within a
|
||||
> pass. ADR-020 adds (a) a recency pre-filter — auto mode skips videos published before
|
||||
> `TAPIR_AUTO_SUMMARIZE_WINDOW`, listed but summarised on demand — and (b) the same
|
||||
> `published_at DESC NULLS LAST` ordering on the **list read** (`ListVideos`), which previously
|
||||
> sorted by `seen_at`. See `DECISIONS.md` ADR-020.
|
||||
|
||||
**Repo:** tapir · **Size:** small · **Solo session.**
|
||||
|
||||
**Why.** Product intent (maintainer, 2026-06-06): a new user should get summaries of their
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# Spec — In-process scheduled discovery + auto-summarize + rate-gate finish
|
||||
|
||||
> **Extended by ADR-020 (2026-06-08).** Auto-summarize is no longer "every unseen video": the
|
||||
> scheduler now skips videos published before `TAPIR_AUTO_SUMMARIZE_WINDOW` (default ~7d) unless
|
||||
> explicitly requested, so a back-catalogue does not re-drive the rate gate every cycle. See
|
||||
> `DECISIONS.md` ADR-020.
|
||||
|
||||
**Repo:** tapir · **Size:** medium · **Solo session** (not a swarm).
|
||||
|
||||
**Why this exists.** The Stage-0 gate ("me or a friend returns and reads/acts in ≥2 separate
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
# Spec — Unify video-card states: one "Summarize now" verb, honest no-captions state
|
||||
|
||||
> **Superseded in part by ADR-020 (2026-06-08).** The five card states still hold, but the copy
|
||||
> changed: the nudge verb is now **"Summarize"** (not "Summarize now"), the rate-limited state
|
||||
> reads **"In queue"** (not "Fetching soon…"), and the queued state reads **"summarizing
|
||||
> shortly"** (not "waiting for the next run"). The list also now collapses older un-summarized
|
||||
> and caption-less videos. See `DECISIONS.md` ADR-020 and `views.templ` (`VideoCard`) for the
|
||||
> current copy; this doc is kept as the original design record.
|
||||
|
||||
**Repo:** tapir · **Size:** small, **view-layer only** (`views.templ` + a little CSS in
|
||||
`view.go`; regenerate `views_templ.go`). No handler, store, or DB change. The two existing
|
||||
handlers (`/summarize`, `/retry-now`) stay exactly as they are — only what the card *shows*
|
||||
|
||||
@@ -179,3 +179,4 @@ distinguishable.
|
||||
| **"Summarize now" foreground path** | Unified quiet nudge button on actionable non-summarized cards. Five explicit card states — (1) summarized: chip + no button; (2) no captions (`transcript_status = 'none'`): "No transcript available", no button; (3) queued: "Queued" chip, no button; (4) rate-limited: "Fetching soon…" + "Summarize now" → `POST /v/{id}/retry-now` (clears `rate_limited_at`, triggers engine); (5) pending: "Not summarized" + "Summarize now" → `POST /v/{id}/summarize` (queues + triggers engine). One verb, one style (`.btn-quiet`); backend difference invisible to user. Both handlers call `ProcessVideo` through `globalFetchGate`. Rate gate respected, not bypassed — this is onboarding prioritisation. | Fast onboarding value; honest dead-end for no-captions videos (no button that fails). | `internal/web/handlers.go` (`handleRetryNow`, `handleRequestSummarize`); `internal/web/views.templ` (`VideoCard`) |
|
||||
| **Pipeline stats bar** | A one-line status bar above the video list: `N summarized · M fetching soon · K no captions`. Computed from the unfiltered row set; hidden when all videos are summarized. Gives the user a clear read on pipeline state without any interaction. | Replaces the "why is nothing happening?" confusion when most videos are pending or rate-limited. | `internal/web/view.go` (`PipelineStats`, `pipelineStats`) |
|
||||
| **Unavailable channels (account page)** | The `/account` page shows a "Unavailable channels" section when any channels returned HTTP 404 on the last discovery pass. Lists channel name, an "unavailable" badge, and the first-seen date. Data sourced from the `channel_errors` table (migration 013). | Surfaces silent failures so users know why some subscribed channels produce no new videos. | migration 013; `internal/web/account.go`; `internal/adapters/youtube/youtube.go` (`domain.ErrChannelUnavailable`) |
|
||||
| **Recency window + sparse-state honesty (ADR-020)** | Supersedes the copy/sort in the rows above. Auto-summarize is bounded to videos published within `TAPIR_AUTO_SUMMARIZE_WINDOW` (~7d); older un-summarized videos collapse behind a single "Show N older videos — summarize on demand" disclosure, and caption-less videos collapse to a one-line count (not N cards). List order is now `summarized-first, published_at DESC NULLS LAST`. Copy reframed for honest scarcity: pipeline bar reads "N ready · M in queue · K no captions" (no "fetching soon"); a gradual-fill note explains the rate limit; the nudge verb is "Summarize" (not "Summarize now"); the queued card says "summarizing shortly"; the empty-connected state drops the impossible `tapir run` instruction. Detail leads with Takeaways. Filters slimmed (no date pickers; hidden when empty); watched/skipped segmented; back link on detail; empty terms checkbox removed. | Make the sparse reality legible and honest instead of implying abundance/imminence; bound auto load so the back-catalogue doesn't re-drive the caption gate. Never fetch harder — scarcity is surfaced, not engineered around. | ADR-020; `2384c47`, `3df0459`, `40b703e`, `a1a5217`, `4a0a56e`, `9bf1c31`, `980638d`, `12fb031`, `f775441`, `51aa5d9` |
|
||||
|
||||
@@ -10,12 +10,20 @@ Feature: Connect and manage video accounts
|
||||
And my refresh token is stored only as a secret reference
|
||||
And my subscriptions are synced
|
||||
|
||||
@pending
|
||||
# Vimeo connect is not built yet (provider label exists; no connect flow or test).
|
||||
Scenario: Connect a Vimeo account
|
||||
Given I have no connected video accounts
|
||||
When I connect my Vimeo account
|
||||
Then the connection is stored with status "active"
|
||||
And my subscriptions are synced
|
||||
|
||||
Scenario: Connecting an account discovers videos immediately
|
||||
Given I have no connected video accounts
|
||||
When I connect my YouTube account
|
||||
Then a discovery pass for my account is triggered right away
|
||||
And I do not have to wait for the next scheduled pass to see my videos
|
||||
|
||||
Scenario: Tokens are never stored in the clear
|
||||
When I connect any video account
|
||||
Then no OAuth token value is stored in the database
|
||||
@@ -28,6 +36,9 @@ Feature: Connect and manage video accounts
|
||||
And no new videos are watched for that connection
|
||||
And my existing summaries remain readable
|
||||
|
||||
@pending
|
||||
# Per-provider BYO credential config is not built as a web flow yet (the summarizer
|
||||
# supports a fallback endpoint, but there is no user-facing BYO setup + its test).
|
||||
Scenario Outline: BYO AI credential is optional and per-provider
|
||||
When I configure a BYO provider "<provider>"
|
||||
Then the credential is stored only as a secret reference
|
||||
|
||||
@@ -19,6 +19,9 @@ Feature: Public landing page
|
||||
Then I see a link to my summaries
|
||||
And I see a way to log out
|
||||
|
||||
@pending
|
||||
# Behaviour ships (logout redirects to /welcome) but is not unit-tested: logout lives in
|
||||
# the OIDC Auth impl and StubAuth has no routes to exercise it cheaply.
|
||||
Scenario: Logging out returns to the welcome page
|
||||
Given I am logged in
|
||||
When I log out
|
||||
|
||||
@@ -34,6 +34,9 @@ Feature: Register and manage a multi-user account
|
||||
And the other user's data remains intact
|
||||
And my Dex identity is left intact
|
||||
|
||||
@pending
|
||||
# Re-registration after delete is supported by design (delete leaves the Dex identity,
|
||||
# ADR-013) but has no dedicated end-to-end test yet.
|
||||
Scenario: A deleted user can register again as a fresh account
|
||||
Given I deleted my Tapir account but my Dex identity still exists
|
||||
When I sign in again
|
||||
|
||||
@@ -6,15 +6,25 @@ Feature: Choose how new videos get summarized
|
||||
Background:
|
||||
Given I am a registered user with a connected video account
|
||||
|
||||
Scenario: Auto mode summarizes every new video
|
||||
Scenario: Auto mode summarizes recent new videos automatically
|
||||
Given my summarization mode is "auto"
|
||||
When a subscribed channel posts a new video with captions
|
||||
When a subscribed channel posts a new video with captions within the recency window
|
||||
Then Tapir summarizes it without my asking
|
||||
And the summary appears in my list
|
||||
|
||||
Scenario: Manual mode is the default and leaves new videos unsummarized
|
||||
Given I have not changed my summarization mode
|
||||
Then my mode is "manual"
|
||||
Scenario: Auto mode lists older videos without summarizing them
|
||||
Given my summarization mode is "auto"
|
||||
When discovery finds a video published before the recency window
|
||||
Then the video appears in my list with no summary
|
||||
And it is not summarized automatically
|
||||
And I can still summarize it on demand with "Summarize"
|
||||
|
||||
Scenario: Automatic is the default for a new user
|
||||
Given I have just registered
|
||||
Then my summarization mode is "auto"
|
||||
|
||||
Scenario: Manual mode leaves new videos unsummarized
|
||||
Given my summarization mode is "manual"
|
||||
When a subscribed channel posts a new video with captions
|
||||
Then the video appears in my list with no summary
|
||||
And nothing is summarized until I request it
|
||||
@@ -30,3 +40,8 @@ Feature: Choose how new videos get summarized
|
||||
# auto_summarize is a per-user setting and summarize_requested is a per-video queue
|
||||
# flag (migration 006). The web button sets the flag; `tapir run` processes both the
|
||||
# auto videos and the manually queued ones, then clears the flag.
|
||||
#
|
||||
# Recency bound (ADR-020): in auto mode the scheduler only summarizes videos published
|
||||
# within TAPIR_AUTO_SUMMARIZE_WINDOW (default ~7d); older videos are discovered and
|
||||
# listed but wait for an explicit "Summarize" — so a back-catalogue does not re-drive
|
||||
# the per-IP caption gate (ADR-014) every cycle. A manual request bypasses the bound.
|
||||
|
||||
@@ -72,3 +72,41 @@ func nullTime(t time.Time) *time.Time {
|
||||
}
|
||||
return &t
|
||||
}
|
||||
|
||||
// NewestUnsummarizedVideoIDs returns up to limit of the user's videos that have
|
||||
// no summary yet, newest first (published_at DESC, NULLS LAST). It caps the
|
||||
// connect-time onboarding burst (Feature 1) at a fixed count: the caller marks
|
||||
// these for summarization through the shared rate gate. RLS-scoped via withUser,
|
||||
// so it only ever sees the requesting user's rows. limit <= 0 returns nil.
|
||||
func (s *Store) NewestUnsummarizedVideoIDs(ctx context.Context, userID string, limit int) ([]string, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var ids []string
|
||||
if err := s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||
rows, err := tx.Query(ctx,
|
||||
`SELECT v.id
|
||||
FROM videos v
|
||||
WHERE v.user_id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM summaries su
|
||||
WHERE su.user_id = v.user_id AND su.video_id = v.id)
|
||||
ORDER BY v.published_at DESC NULLS LAST, v.seen_at DESC
|
||||
LIMIT $2`, userID, limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store: newest unsummarized: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return fmt.Errorf("store: scan newest unsummarized: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return rows.Err()
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
@@ -81,3 +81,35 @@ func TestUpsertVideo_PerUserIsolation(t *testing.T) {
|
||||
|
||||
require.NotEqual(t, idA, idB, "same provider video for two users must be two distinct rows")
|
||||
}
|
||||
|
||||
func TestNewestUnsummarizedVideoIDs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newStore(t)
|
||||
resetDB(t, rawPool(t))
|
||||
|
||||
mk := func(user, pid string, day int) string {
|
||||
v := ytVideo(user, pid, pid)
|
||||
v.PublishedAt = time.Date(2026, 6, day, 12, 0, 0, 0, time.UTC)
|
||||
id, err := s.UpsertVideo(ctx, v)
|
||||
require.NoError(t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
_ = mk(userA, "a1vid000001", 1)
|
||||
id2 := mk(userA, "a2vid000002", 2)
|
||||
id3 := mk(userA, "a3vid000003", 3)
|
||||
id4 := mk(userA, "a4vid000004", 4)
|
||||
mk(userB, "b1vid000009", 9) // userB's newest — must never leak via RLS
|
||||
|
||||
// The newest (v4) is summarized, so it's excluded from "unsummarized".
|
||||
require.NoError(t, s.Deliver(ctx, summary(userA, id4, "done")))
|
||||
|
||||
// Cap 2, newest-first unsummarized: v3 then v2 (v4 excluded; userB excluded).
|
||||
got, err := s.NewestUnsummarizedVideoIDs(ctx, userA, 2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{id3, id2}, got)
|
||||
|
||||
none, err := s.NewestUnsummarizedVideoIDs(ctx, userA, 0)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, none, "limit 0 returns nothing")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package youtube
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
func TestVideoByID(t *testing.T) {
|
||||
const id = "dQw4w9WgXcQ"
|
||||
a, secrets := newTestAdapter(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/videos" {
|
||||
t.Errorf("unexpected path %q (must use videos.list)", r.URL.Path)
|
||||
}
|
||||
if got := r.URL.Query().Get("id"); got != id {
|
||||
t.Errorf("expected id=%s, got %q", id, got)
|
||||
}
|
||||
if got := r.URL.Query().Get("part"); got != "snippet" {
|
||||
t.Errorf("expected part=snippet, got %q", got)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"items":[{"snippet":{"title":"Never Gonna Give You Up","publishedAt":"2026-05-20T09:00:00Z"}}]}`))
|
||||
})
|
||||
|
||||
v, err := a.VideoByID(context.Background(), "u1", id)
|
||||
if err != nil {
|
||||
t.Fatalf("VideoByID: %v", err)
|
||||
}
|
||||
if v.UserID != "u1" {
|
||||
t.Errorf("UserID = %q, want u1", v.UserID)
|
||||
}
|
||||
if v.ProviderVideoID != id || v.Title != "Never Gonna Give You Up" {
|
||||
t.Errorf("unexpected video: %+v", v)
|
||||
}
|
||||
if v.Provider != domain.ProviderYouTube || v.URL != "https://www.youtube.com/watch?v="+id {
|
||||
t.Errorf("video not wired correctly: %+v", v)
|
||||
}
|
||||
if v.PublishedAt.IsZero() {
|
||||
t.Errorf("expected publishedAt parsed, got zero")
|
||||
}
|
||||
if v.SubscriptionID != "" {
|
||||
t.Errorf("a pasted video must have no subscription, got %q", v.SubscriptionID)
|
||||
}
|
||||
if secrets.byRef == nil {
|
||||
t.Errorf("token must be resolved by reference through the SecretStore")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoByIDNotFound(t *testing.T) {
|
||||
a, _ := newTestAdapter(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"items":[]}`))
|
||||
})
|
||||
_, err := a.VideoByID(context.Background(), "u1", "missingvid0")
|
||||
if !errors.Is(err, domain.ErrVideoNotFound) {
|
||||
t.Fatalf("VideoByID for missing id = %v, want domain.ErrVideoNotFound", err)
|
||||
}
|
||||
}
|
||||
@@ -244,6 +244,37 @@ func (a *Adapter) NewVideos(ctx context.Context, sub domain.Subscription) ([]dom
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
// VideoByID fetches a single video's metadata (videos.list, snippet) for an
|
||||
// arbitrary video id — including channels the user does not follow (paste-a-URL,
|
||||
// Feature 2). This is a Data API call (1 quota unit), NOT the rate-limited
|
||||
// caption path, so it is not gated: only the later transcript fetch goes through
|
||||
// globalFetchGate. UserID is set on the result and SubscriptionID is left empty
|
||||
// (a pasted video has no subscription parent). Returns ErrVideoNotFound when the
|
||||
// id resolves to no video.
|
||||
func (a *Adapter) VideoByID(ctx context.Context, userID, videoID string) (domain.Video, error) {
|
||||
client, err := a.httpClient(ctx, a.cfg.TokenSecretRef)
|
||||
if err != nil {
|
||||
return domain.Video{}, err
|
||||
}
|
||||
q := url.Values{"part": {"snippet"}, "id": {videoID}}
|
||||
var resp videoListResponse
|
||||
if err := a.getJSON(ctx, client, "/videos", q, &resp); err != nil {
|
||||
return domain.Video{}, fmt.Errorf("video by id %q: %w", videoID, err)
|
||||
}
|
||||
if len(resp.Items) == 0 {
|
||||
return domain.Video{}, fmt.Errorf("video %q: %w", videoID, domain.ErrVideoNotFound)
|
||||
}
|
||||
it := resp.Items[0]
|
||||
return domain.Video{
|
||||
UserID: userID,
|
||||
Provider: domain.ProviderYouTube,
|
||||
ProviderVideoID: videoID,
|
||||
Title: it.Snippet.Title,
|
||||
URL: "https://www.youtube.com/watch?v=" + videoID,
|
||||
PublishedAt: it.Snippet.PublishedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// uploadsPlaylistID derives a channel's uploads playlist id at zero API cost:
|
||||
// a standard channel id "UCxxxx" maps to uploads playlist "UUxxxx". Returns
|
||||
// ok=false for ids that don't follow this convention (caller falls back to
|
||||
@@ -342,6 +373,15 @@ type playlistItemListResponse struct {
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
type videoListResponse struct {
|
||||
Items []struct {
|
||||
Snippet struct {
|
||||
Title string `json:"title"`
|
||||
PublishedAt time.Time `json:"publishedAt"`
|
||||
} `json:"snippet"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
type channelListResponse struct {
|
||||
Items []struct {
|
||||
ContentDetails struct {
|
||||
|
||||
+45
-11
@@ -13,6 +13,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -79,6 +80,13 @@ type Config struct {
|
||||
// pre-recency behaviour). Default ~7 days.
|
||||
AutoSummarizeWindow time.Duration
|
||||
|
||||
// OnboardSummarizeCount caps how many of a freshly-connected user's newest
|
||||
// videos are summarized immediately on connect (the onboarding "it works"
|
||||
// burst). HARD-capped at maxOnboardSummarizeCount so onboarding can never
|
||||
// bulk-fetch; 0 disables the burst. Every fetch still flows through the shared
|
||||
// caption rate gate (ADR-014) — the cap bounds count, never the pacing. Default 3.
|
||||
OnboardSummarizeCount int
|
||||
|
||||
// DiscoveryInterval, when > 0, makes `serve` run in-process scheduled discovery
|
||||
// for ALL users on that cadence (ADR-018). Zero/unset = disabled, so dev and
|
||||
// tests never auto-fetch. Single-replica assumption — see cmdServe.
|
||||
@@ -108,17 +116,19 @@ func (c Config) DexConfigured() bool { return strings.TrimSpace(c.OIDCIssuer) !=
|
||||
|
||||
// Defaults (see docs/homelab-integration.md). All overridable via env.
|
||||
const (
|
||||
defaultGatewayURL = "http://koala:30401/v1"
|
||||
defaultSummarizerModel = "koala/phi4-mini"
|
||||
defaultSummarizerTimeout = 5 * time.Minute
|
||||
defaultYTTokenRef = "youtube/refresh_token"
|
||||
defaultYTConnectRedirectURL = "https://tapir.d-ma.be/oauth/youtube/callback"
|
||||
defaultOAuthRedirectAddr = "localhost:8080"
|
||||
defaultHTTPAddr = ":8080"
|
||||
defaultFetchBackoff = time.Hour
|
||||
defaultFetchRate = 2 * time.Second
|
||||
defaultPublicURL = "https://tapir.d-ma.be"
|
||||
defaultAutoSummarizeWindow = 7 * 24 * time.Hour
|
||||
defaultGatewayURL = "http://koala:30401/v1"
|
||||
defaultSummarizerModel = "koala/phi4-mini"
|
||||
defaultSummarizerTimeout = 5 * time.Minute
|
||||
defaultYTTokenRef = "youtube/refresh_token"
|
||||
defaultYTConnectRedirectURL = "https://tapir.d-ma.be/oauth/youtube/callback"
|
||||
defaultOAuthRedirectAddr = "localhost:8080"
|
||||
defaultHTTPAddr = ":8080"
|
||||
defaultFetchBackoff = time.Hour
|
||||
defaultFetchRate = 2 * time.Second
|
||||
defaultPublicURL = "https://tapir.d-ma.be"
|
||||
defaultAutoSummarizeWindow = 7 * 24 * time.Hour
|
||||
defaultOnboardSummarizeCount = 3
|
||||
maxOnboardSummarizeCount = 5
|
||||
)
|
||||
|
||||
// Load reads the environment into a Config, applying defaults. It does not
|
||||
@@ -183,6 +193,18 @@ func Load() (Config, error) {
|
||||
}
|
||||
c.AutoSummarizeWindow = autoWindow
|
||||
|
||||
onboard, err := intOr("TAPIR_ONBOARD_SUMMARIZE_COUNT", defaultOnboardSummarizeCount)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if onboard < 0 {
|
||||
onboard = 0
|
||||
}
|
||||
if onboard > maxOnboardSummarizeCount {
|
||||
onboard = maxOnboardSummarizeCount
|
||||
}
|
||||
c.OnboardSummarizeCount = onboard
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@@ -247,6 +269,18 @@ func envOr(key, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
func intOr(key string, fallback int) (int, error) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("config: %s=%q: %w", key, v, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func durationOr(key string, fallback time.Duration) (time.Duration, error) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
|
||||
@@ -53,6 +53,38 @@ func TestLoad_AppliesDefaults(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_OnboardSummarizeCount(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, env string
|
||||
want int
|
||||
}{
|
||||
{"default", "", defaultOnboardSummarizeCount},
|
||||
{"explicit", "4", 4},
|
||||
{"zero disables", "0", 0},
|
||||
{"clamped to hard cap", "50", maxOnboardSummarizeCount},
|
||||
{"negative clamps to zero", "-3", 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
setEnv(t, map[string]string{"TAPIR_ONBOARD_SUMMARIZE_COUNT": c.env})
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.OnboardSummarizeCount != c.want {
|
||||
t.Fatalf("OnboardSummarizeCount = %d, want %d", cfg.OnboardSummarizeCount, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_OnboardSummarizeCountInvalid(t *testing.T) {
|
||||
setEnv(t, map[string]string{"TAPIR_ONBOARD_SUMMARIZE_COUNT": "three"})
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatal("Load: want error for non-numeric TAPIR_ONBOARD_SUMMARIZE_COUNT")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_ParsesValues(t *testing.T) {
|
||||
setEnv(t, map[string]string{
|
||||
"TAPIR_USER_ID": "11111111-1111-1111-1111-111111111111",
|
||||
|
||||
@@ -3,10 +3,16 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrVideoNotFound is returned when a video id resolves to no video (deleted,
|
||||
// private, or a typo'd paste). Defined in domain so adapters and the web layer
|
||||
// share one sentinel without coupling to each other.
|
||||
var ErrVideoNotFound = errors.New("video not found")
|
||||
|
||||
// ErrChannelUnavailable is returned by a VideoSource when a channel's upload
|
||||
// playlist returns HTTP 404 — the channel was deleted or made private. The runner
|
||||
// stores these so the account page can surface them to the user.
|
||||
|
||||
@@ -23,6 +23,15 @@ type Connections interface {
|
||||
UpsertConnection(ctx context.Context, userID string, c store.Connection) error
|
||||
}
|
||||
|
||||
// DiscoveryTrigger requests an out-of-band discovery pass for a user. The connect
|
||||
// flow fires it the moment a YouTube account is linked so videos appear promptly
|
||||
// instead of waiting for the next scheduled pass (#6). Enqueue must be
|
||||
// non-blocking and safe to call from the request goroutine; the implementation
|
||||
// owns serialization with the scheduler (one pass at a time). nil = no trigger.
|
||||
type DiscoveryTrigger interface {
|
||||
Enqueue(userID string)
|
||||
}
|
||||
|
||||
// connectStateTTL bounds how long a generated CSRF state is valid between the
|
||||
// connect redirect and the provider callback.
|
||||
const connectStateTTL = 10 * time.Minute
|
||||
@@ -43,6 +52,10 @@ type ConnectHandler struct {
|
||||
Conns Connections
|
||||
Log *slog.Logger
|
||||
|
||||
// Discovery, when set, is fired after a successful connect so the new
|
||||
// connection's videos are discovered immediately (#6). Optional.
|
||||
Discovery DiscoveryTrigger
|
||||
|
||||
states *connectStateStore
|
||||
now func() time.Time
|
||||
}
|
||||
@@ -134,6 +147,12 @@ func (h *ConnectHandler) handleCallback(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
// Discover this user's videos now rather than waiting for the next scheduled
|
||||
// pass (#6). Non-blocking; the trigger serializes with the scheduler.
|
||||
if h.Discovery != nil {
|
||||
h.Discovery.Enqueue(userID)
|
||||
}
|
||||
|
||||
setFlash(w, flashConnected)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
@@ -45,6 +45,24 @@ func (c *fakeConns) UpsertConnection(_ context.Context, userID string, conn stor
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeTrigger records Enqueue calls so a test can assert connect fired discovery.
|
||||
type fakeTrigger struct {
|
||||
mu sync.Mutex
|
||||
users []string
|
||||
}
|
||||
|
||||
func (f *fakeTrigger) Enqueue(userID string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.users = append(f.users, userID)
|
||||
}
|
||||
|
||||
func (f *fakeTrigger) seen() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]string(nil), f.users...)
|
||||
}
|
||||
|
||||
// tokenServer fakes Google's token endpoint, returning body for any POST.
|
||||
func tokenServer(t *testing.T, body string) *httptest.Server {
|
||||
t.Helper()
|
||||
@@ -126,6 +144,36 @@ func TestCallbackExchangesAndRecordsConnection(t *testing.T) {
|
||||
require.Equal(t, wantRef, conns.conn.TokenRef)
|
||||
}
|
||||
|
||||
func TestCallbackTriggersDiscovery(t *testing.T) {
|
||||
srv := tokenServer(t,
|
||||
`{"access_token":"at","refresh_token":"rt-secret","token_type":"Bearer","expires_in":3600}`)
|
||||
app := newConnectApp(t, srv.URL, &fakeWriter{}, &fakeConns{})
|
||||
trig := &fakeTrigger{}
|
||||
app.Connect.Discovery = trig
|
||||
|
||||
state := connectState(t, app)
|
||||
rec := do(t, app, httptest.NewRequest(http.MethodGet,
|
||||
"/oauth/youtube/callback?state="+state+"&code=the-code", nil))
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, rec.Code)
|
||||
require.Equal(t, []string{userID}, trig.seen(),
|
||||
"a successful connect must trigger discovery for the connecting user")
|
||||
}
|
||||
|
||||
func TestCallbackNoDiscoveryOnFailedConnect(t *testing.T) {
|
||||
srv := tokenServer(t,
|
||||
`{"access_token":"at","refresh_token":"rt","token_type":"Bearer","expires_in":3600}`)
|
||||
app := newConnectApp(t, srv.URL, &fakeWriter{}, &fakeConns{})
|
||||
trig := &fakeTrigger{}
|
||||
app.Connect.Discovery = trig
|
||||
|
||||
// No state → CSRF reject → nothing connected, so no discovery.
|
||||
rec := do(t, app, httptest.NewRequest(http.MethodGet,
|
||||
"/oauth/youtube/callback?code=the-code", nil))
|
||||
require.Equal(t, http.StatusBadRequest, rec.Code)
|
||||
require.Empty(t, trig.seen(), "a failed connect must not trigger discovery")
|
||||
}
|
||||
|
||||
func TestCallbackRejectsMissingState(t *testing.T) {
|
||||
srv := tokenServer(t,
|
||||
`{"access_token":"at","refresh_token":"rt","token_type":"Bearer","expires_in":3600}`)
|
||||
|
||||
@@ -3,6 +3,8 @@ package web
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"html/template"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
@@ -10,6 +12,7 @@ import (
|
||||
"github.com/a-h/templ"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// Store is the read/write surface the web handlers depend on — a narrow port over
|
||||
@@ -30,6 +33,10 @@ type Store interface {
|
||||
SetAutoSummarize(ctx context.Context, userID string, enabled bool) error
|
||||
RequestSummarize(ctx context.Context, userID, videoID string) error
|
||||
|
||||
// UpsertVideo persists a pasted video (idempotent on user+provider+video id,
|
||||
// so it also dedups) and returns its durable store id.
|
||||
UpsertVideo(ctx context.Context, v domain.Video) (string, error)
|
||||
|
||||
// Account management (the /account page, disconnect, delete-account).
|
||||
ConnectionsForUser(ctx context.Context, userID string) ([]store.Connection, error)
|
||||
DeleteConnection(ctx context.Context, userID, provider string) error
|
||||
@@ -78,6 +85,9 @@ type App struct {
|
||||
// background goroutine (the "Summarize" button kicks it off). Nil = queue-only:
|
||||
// the button flips the DB flag and the next `tapir run` does the work.
|
||||
Processor Processor
|
||||
// Fetcher, when non-nil, resolves an arbitrary YouTube video id to metadata for
|
||||
// the paste-a-URL flow (Feature 2). Nil = the /paste route is not mounted.
|
||||
Fetcher VideoFetcher
|
||||
// Processing tracks in-flight immediate summarizations so the status endpoint
|
||||
// shows the animation until the summary lands. The zero value is ready to use.
|
||||
Processing ProcessingSet
|
||||
@@ -132,6 +142,9 @@ func (a *App) Router() http.Handler {
|
||||
app.HandleFunc("POST /v/{videoId}/action", a.handleAction)
|
||||
app.HandleFunc("POST /v/{videoId}/summarize", a.handleRequestSummarize)
|
||||
app.HandleFunc("POST /v/{videoId}/retry-now", a.handleRetryNow)
|
||||
if a.Fetcher != nil {
|
||||
app.HandleFunc("POST /paste", a.handlePaste)
|
||||
}
|
||||
app.HandleFunc("GET /v/{videoId}/status", a.handleStatus)
|
||||
app.HandleFunc("GET /register", a.handleRegisterForm)
|
||||
app.HandleFunc("POST /register", a.handleRegister)
|
||||
@@ -324,6 +337,75 @@ func (a *App) handleRequestSummarize(w http.ResponseWriter, r *http.Request) {
|
||||
a.render(w, r, VideoCard(*row))
|
||||
}
|
||||
|
||||
// handlePaste handles "paste a YouTube URL" (Feature 2). It parses the video id,
|
||||
// fetches metadata (Data API — ungated), upserts a subscription-less video row
|
||||
// scoped to the user (idempotent, so it also dedups), and — if the video isn't
|
||||
// already summarized — requests a summary and kicks off immediate processing
|
||||
// through the SAME rate gate as the Summarize button. An explicit paste is a
|
||||
// manual request, so it summarizes regardless of the recency window. A video that
|
||||
// turns out to have no captions resolves to the honest "no transcript" terminal
|
||||
// state via the engine (ADR-010), not an error here.
|
||||
func (a *App) handlePaste(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := a.currentUserID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
videoID, err := parseYouTubeVideoID(r.FormValue("url"))
|
||||
if err != nil {
|
||||
a.pasteFailure(w, http.StatusBadRequest, "That doesn't look like a YouTube video link.")
|
||||
return
|
||||
}
|
||||
|
||||
v, err := a.Fetcher.FetchVideo(r.Context(), userID, videoID)
|
||||
if errors.Is(err, domain.ErrVideoNotFound) {
|
||||
a.pasteFailure(w, http.StatusNotFound, "That video couldn't be found — it may be private or removed.")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
a.serverError(w, r, "paste fetch", err)
|
||||
return
|
||||
}
|
||||
|
||||
id, err := a.Store.UpsertVideo(r.Context(), v)
|
||||
if err != nil {
|
||||
a.serverError(w, r, "paste upsert", err)
|
||||
return
|
||||
}
|
||||
row, err := a.Store.GetVideoRow(r.Context(), userID, id)
|
||||
if err != nil {
|
||||
a.serverError(w, r, "paste get video", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Dedup: already in the feed with a summary — surface the existing entry,
|
||||
// don't re-summarize.
|
||||
if row.Summarized {
|
||||
a.render(w, r, VideoCard(*row))
|
||||
return
|
||||
}
|
||||
|
||||
// New or unsummarized: queue + (if a Processor is wired) summarize now, through
|
||||
// the shared gate. RequestSummarize makes it durable even if the process dies.
|
||||
if err := a.Store.RequestSummarize(r.Context(), userID, id); err != nil {
|
||||
a.serverError(w, r, "paste request summarize", err)
|
||||
return
|
||||
}
|
||||
if a.Processor != nil {
|
||||
a.startProcessing(userID, id)
|
||||
a.render(w, r, processingCard(*row))
|
||||
return
|
||||
}
|
||||
a.render(w, r, VideoCard(*row))
|
||||
}
|
||||
|
||||
// pasteFailure renders a minimal inline error fragment for the paste form (HTMX
|
||||
// swaps it in). No templ dependency so it renders even on a bad-input fast path.
|
||||
func (a *App) pasteFailure(w http.ResponseWriter, status int, msg string) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, `<p class="paste-error" role="alert">`+template.HTMLEscapeString(msg)+`</p>`)
|
||||
}
|
||||
|
||||
// handleRetryNow handles the "Try now" button on rate-limited video cards. It
|
||||
// clears the rate_limited_at backoff so the scheduler won't skip the video, then
|
||||
// triggers an immediate ProcessVideo — same background path as handleRequestSummarize.
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// youtubeVideoID matches a canonical YouTube video id: exactly 11 URL-safe chars.
|
||||
var youtubeVideoID = regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`)
|
||||
|
||||
// parseYouTubeVideoID extracts the 11-character video id from a pasted YouTube
|
||||
// URL (watch?v=, youtu.be/, shorts/, embed/) or a bare id. It rejects non-YouTube
|
||||
// hosts and anything that doesn't yield a valid id, so the paste flow never tries
|
||||
// to fetch a video that can't exist (Feature 2).
|
||||
func parseYouTubeVideoID(raw string) (string, error) {
|
||||
s := strings.TrimSpace(raw)
|
||||
if s == "" {
|
||||
return "", fmt.Errorf("empty input")
|
||||
}
|
||||
|
||||
// Bare id (no URL) — accept directly.
|
||||
if youtubeVideoID.MatchString(s) {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Accept scheme-less URLs (youtube.com/watch?v=...) by giving url.Parse a host.
|
||||
if !strings.Contains(s, "://") {
|
||||
s = "https://" + s
|
||||
}
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("not a URL: %w", err)
|
||||
}
|
||||
|
||||
host := strings.ToLower(u.Hostname())
|
||||
isYouTube := host == "youtu.be" || host == "youtube.com" || strings.HasSuffix(host, ".youtube.com")
|
||||
if !isYouTube {
|
||||
return "", fmt.Errorf("not a YouTube URL: %q", host)
|
||||
}
|
||||
|
||||
var id string
|
||||
switch {
|
||||
case host == "youtu.be":
|
||||
// youtu.be/<id>
|
||||
id = strings.Trim(u.Path, "/")
|
||||
case u.Path == "/watch":
|
||||
id = u.Query().Get("v")
|
||||
default:
|
||||
// /shorts/<id>, /embed/<id>
|
||||
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
if len(parts) == 2 && (parts[0] == "shorts" || parts[0] == "embed") {
|
||||
id = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
if !youtubeVideoID.MatchString(id) {
|
||||
return "", fmt.Errorf("no YouTube video id in %q", raw)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package web_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// fakeFetcher is a web.VideoFetcher returning a fixed video (or an error),
|
||||
// scoped to whatever (userID, videoID) the handler asks for.
|
||||
type fakeFetcher struct {
|
||||
title string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeFetcher) FetchVideo(_ context.Context, userID, videoID string) (domain.Video, error) {
|
||||
f.calls++
|
||||
if f.err != nil {
|
||||
return domain.Video{}, f.err
|
||||
}
|
||||
return domain.Video{
|
||||
UserID: userID,
|
||||
Provider: domain.ProviderYouTube,
|
||||
ProviderVideoID: videoID,
|
||||
Title: f.title,
|
||||
URL: "https://www.youtube.com/watch?v=" + videoID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func pasteReq(rawURL string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, "/paste",
|
||||
strings.NewReader("url="+url.QueryEscape(rawURL)))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return req
|
||||
}
|
||||
|
||||
func TestPasteValidURLAddsAndRequests(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
app := newApp(t)
|
||||
resetDB(t, rawPool(t))
|
||||
app.Fetcher = &fakeFetcher{title: "Pasted Talk"}
|
||||
p := rawPool(t)
|
||||
|
||||
rec := do(t, app, pasteReq("https://youtu.be/dQw4w9WgXcQ"))
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
|
||||
var (
|
||||
count, requested int
|
||||
title string
|
||||
)
|
||||
require.NoError(t, p.QueryRow(ctx,
|
||||
`SELECT count(*), coalesce(max(title),'') FROM videos
|
||||
WHERE user_id=$1 AND provider_video_id='dQw4w9WgXcQ'`, userID).Scan(&count, &title))
|
||||
require.Equal(t, 1, count, "pasted video added once, scoped to the user")
|
||||
require.Equal(t, "Pasted Talk", title)
|
||||
|
||||
require.NoError(t, p.QueryRow(ctx,
|
||||
`SELECT count(*) FROM videos
|
||||
WHERE user_id=$1 AND provider_video_id='dQw4w9WgXcQ' AND summarize_requested`,
|
||||
userID).Scan(&requested))
|
||||
require.Equal(t, 1, requested, "pasted video is queued for summarization (through the gate)")
|
||||
}
|
||||
|
||||
func TestPasteInvalidURLRejected(t *testing.T) {
|
||||
app := newApp(t)
|
||||
resetDB(t, rawPool(t))
|
||||
app.Fetcher = &fakeFetcher{title: "x"}
|
||||
|
||||
rec := do(t, app, pasteReq("definitely not a url"))
|
||||
require.Equal(t, http.StatusBadRequest, rec.Code)
|
||||
|
||||
var count int
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT count(*) FROM videos WHERE user_id=$1`, userID).Scan(&count))
|
||||
require.Equal(t, 0, count, "invalid input adds nothing")
|
||||
}
|
||||
|
||||
func TestPasteVideoNotFound(t *testing.T) {
|
||||
app := newApp(t)
|
||||
resetDB(t, rawPool(t))
|
||||
app.Fetcher = &fakeFetcher{err: domain.ErrVideoNotFound}
|
||||
|
||||
rec := do(t, app, pasteReq("https://youtu.be/dQw4w9WgXcQ"))
|
||||
require.Equal(t, http.StatusNotFound, rec.Code)
|
||||
|
||||
var count int
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT count(*) FROM videos WHERE user_id=$1`, userID).Scan(&count))
|
||||
require.Equal(t, 0, count, "a not-found video adds nothing")
|
||||
}
|
||||
|
||||
func TestPasteDedupNoDuplicate(t *testing.T) {
|
||||
app := newApp(t)
|
||||
resetDB(t, rawPool(t))
|
||||
app.Fetcher = &fakeFetcher{title: "Pasted Talk"}
|
||||
|
||||
require.Equal(t, http.StatusOK, do(t, app, pasteReq("https://youtu.be/dQw4w9WgXcQ")).Code)
|
||||
require.Equal(t, http.StatusOK, do(t, app, pasteReq("https://www.youtube.com/watch?v=dQw4w9WgXcQ")).Code)
|
||||
|
||||
var count int
|
||||
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||
`SELECT count(*) FROM videos WHERE user_id=$1 AND provider_video_id='dQw4w9WgXcQ'`,
|
||||
userID).Scan(&count))
|
||||
require.Equal(t, 1, count, "pasting the same video twice must not duplicate the row")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package web
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseYouTubeVideoID(t *testing.T) {
|
||||
const id = "dQw4w9WgXcQ"
|
||||
ok := []struct {
|
||||
name, in string
|
||||
}{
|
||||
{"watch", "https://www.youtube.com/watch?v=" + id},
|
||||
{"watch no www", "https://youtube.com/watch?v=" + id},
|
||||
{"watch m", "https://m.youtube.com/watch?v=" + id},
|
||||
{"watch extra params", "https://www.youtube.com/watch?v=" + id + "&t=42s&list=PLxyz"},
|
||||
{"watch param after", "https://www.youtube.com/watch?list=PLxyz&v=" + id},
|
||||
{"short link", "https://youtu.be/" + id},
|
||||
{"short link param", "https://youtu.be/" + id + "?si=abcd&t=1"},
|
||||
{"shorts", "https://www.youtube.com/shorts/" + id},
|
||||
{"embed", "https://www.youtube.com/embed/" + id},
|
||||
{"bare id", id},
|
||||
{"http scheme", "http://youtube.com/watch?v=" + id},
|
||||
{"no scheme", "youtube.com/watch?v=" + id},
|
||||
{"trailing space", " https://youtu.be/" + id + " "},
|
||||
}
|
||||
for _, c := range ok {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got, err := parseYouTubeVideoID(c.in)
|
||||
if err != nil {
|
||||
t.Fatalf("parseYouTubeVideoID(%q) error: %v", c.in, err)
|
||||
}
|
||||
if got != id {
|
||||
t.Fatalf("parseYouTubeVideoID(%q) = %q, want %q", c.in, got, id)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
bad := []struct {
|
||||
name, in string
|
||||
}{
|
||||
{"empty", ""},
|
||||
{"blank", " "},
|
||||
{"vimeo", "https://vimeo.com/123456789"},
|
||||
{"other host", "https://example.com/watch?v=" + id},
|
||||
{"watch no id", "https://www.youtube.com/watch?v="},
|
||||
{"short id", "https://youtu.be/abc"},
|
||||
{"long id", "https://youtu.be/" + id + "extra"},
|
||||
{"bad chars", "https://youtu.be/dQw4w9Wg!cQ"},
|
||||
{"not a url", "just some text"},
|
||||
{"channel url", "https://www.youtube.com/@somechannel"},
|
||||
}
|
||||
for _, c := range bad {
|
||||
t.Run("reject "+c.name, func(t *testing.T) {
|
||||
if got, err := parseYouTubeVideoID(c.in); err == nil {
|
||||
t.Fatalf("parseYouTubeVideoID(%q) = %q, want error", c.in, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package web
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// Processor runs the core summarization use case for a single already-discovered
|
||||
@@ -14,6 +16,14 @@ type Processor interface {
|
||||
ProcessVideo(ctx context.Context, userID, videoID string) error
|
||||
}
|
||||
|
||||
// VideoFetcher resolves an arbitrary YouTube video id to its metadata for the
|
||||
// paste-a-URL flow (Feature 2). It is a Data API call, NOT the rate-limited
|
||||
// caption path. Returns domain.ErrVideoNotFound for a deleted/private/typo'd id.
|
||||
// cmd/tapir wires a per-user YouTube adapter; nil disables the paste route.
|
||||
type VideoFetcher interface {
|
||||
FetchVideo(ctx context.Context, userID, videoID string) (domain.Video, error)
|
||||
}
|
||||
|
||||
// ProcessingSet tracks the (user, video) ids currently being summarized in-process
|
||||
// so the status endpoint can show the animation until the summary lands. It is
|
||||
// ephemeral (single-instance Stage-1): a restart drops it, and the DB holds the
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
package acceptance
|
||||
|
||||
// This is the name-coverage gate for the BDD spec (see docs/use-cases/*.feature).
|
||||
// There is no godog runner — the .feature files are design records, and the real
|
||||
// behaviour is covered by the hand-written Go tests across the module. This test
|
||||
// keeps the two from drifting in the cheapest honest way: every non-@pending
|
||||
// Scenario must have an entry in scenarioCoverage pointing at a Go test that
|
||||
// actually exists. It does NOT prove the test exercises the scenario (only godog
|
||||
// could); it catches the common drift — "added a scenario, forgot the test", a
|
||||
// renamed/deleted covering test, or a scenario removed without cleaning the map.
|
||||
//
|
||||
// When you add a Scenario: either map it here to its covering test, or tag it
|
||||
// @pending in the .feature with a one-line reason for why it has no test yet.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// scenarioCoverage maps each non-@pending Scenario name to the Go test that
|
||||
// covers it. Keep it in sync with docs/use-cases/*.feature — the test below
|
||||
// fails if a scenario is unmapped, a mapped test is missing, or an entry no
|
||||
// longer matches a real non-pending scenario.
|
||||
var scenarioCoverage = map[string]string{
|
||||
// ai_routing.feature
|
||||
"Local AI produces the summary": "TestSummarize_LocalSucceeds",
|
||||
"Local AI fails and the user has a BYO provider configured": "TestSummarize_FallsBackToBYO",
|
||||
"Local AI fails and the user has no BYO provider": "TestSummarize_LocalFailsNoBYO_NoExternalSend",
|
||||
"A user without BYO never has content sent externally": "TestSummarize_NoBYO_ContentOnlyLocal",
|
||||
|
||||
// landing_page.feature
|
||||
"An unauthenticated visit to the root is sent to the welcome page": "TestUnauthenticatedRootRedirectsToWelcome",
|
||||
"The welcome page invites an unauthenticated visitor to start": "TestWelcomeLoggedOut",
|
||||
"An authenticated user on the welcome page sees their way in and out": "TestWelcomeLoggedIn",
|
||||
|
||||
// connect_account.feature
|
||||
"Connect a YouTube account": "TestCallbackExchangesAndRecordsConnection",
|
||||
"Connecting an account discovers videos immediately": "TestCallbackTriggersDiscovery",
|
||||
"Tokens are never stored in the clear": "TestCallbackExchangesAndRecordsConnection",
|
||||
"Revoking a connection stops watching but keeps history": "TestDisconnectRemovesTokenAndConnectionKeepsAccount",
|
||||
|
||||
// summarize_mode.feature
|
||||
"Auto mode summarizes recent new videos automatically": "TestRunOnce_AutoMode_SkipsOldVideos",
|
||||
"Auto mode lists older videos without summarizing them": "TestRunOnce_AutoMode_SkipsOldVideos",
|
||||
"Automatic is the default for a new user": "TestRegisteredUserDefaultsAutoSummarizeOn",
|
||||
"Manual mode leaves new videos unsummarized": "TestRunOnce_ManualMode_SkipsUnrequested",
|
||||
"Requesting a summary in manual mode queues it for the next run": "TestRunOnce_ManualMode_ProcessesRequested",
|
||||
|
||||
// registration.feature
|
||||
"A new Dex subject is routed to registration": "TestUnregisteredSubjectRedirectedToRegister",
|
||||
"Registering creates the account and its identity mapping": "TestRegisterCreatesExactlyOneUserAndIdentity",
|
||||
"A returning subject passes straight through": "TestRegisteredSubjectPassesThrough",
|
||||
"Deleting an account removes only my data and leaves other users untouched": "TestDeleteAccountWipesDataAndSecretsAndLogsOut",
|
||||
|
||||
// summarize_new_video.feature
|
||||
"A subscribed channel posts a video that has captions": "TestSubscribedVideoWithCaptionsIsSummarizedAndDelivered",
|
||||
"A subscribed channel posts a video with no usable transcript": "TestVideoWithNoTranscriptIsSkipped",
|
||||
"A channel I am not subscribed to posts a video": "TestUnsubscribedChannelVideoIsNotProcessed",
|
||||
"The same video is not summarized twice": "TestAlreadySummarizedVideoIsNotReprocessed",
|
||||
}
|
||||
|
||||
var (
|
||||
scenarioRe = regexp.MustCompile(`^\s*Scenario(?: Outline)?:\s*(.+?)\s*$`)
|
||||
testFuncRe = regexp.MustCompile(`^func (Test\w+)\(`)
|
||||
)
|
||||
|
||||
// scenario is one parsed Gherkin scenario and whether it is @pending.
|
||||
type scenario struct {
|
||||
name string
|
||||
pending bool
|
||||
}
|
||||
|
||||
func TestScenarioCoverage(t *testing.T) {
|
||||
root := moduleRoot(t)
|
||||
|
||||
scenarios := parseScenarios(t, filepath.Join(root, "docs", "use-cases"))
|
||||
if len(scenarios) == 0 {
|
||||
t.Fatal("no scenarios parsed from docs/use-cases — wrong path?")
|
||||
}
|
||||
tests := allTestFuncNames(t, root)
|
||||
|
||||
// Index scenario names for the reverse (stale-entry) check.
|
||||
active := map[string]bool{} // non-pending scenario names
|
||||
var pending []string
|
||||
for _, s := range scenarios {
|
||||
if s.pending {
|
||||
pending = append(pending, s.name)
|
||||
continue
|
||||
}
|
||||
active[s.name] = true
|
||||
|
||||
// 1. Every non-pending scenario must be mapped.
|
||||
fn, ok := scenarioCoverage[s.name]
|
||||
if !ok {
|
||||
t.Errorf("scenario %q has no coverage entry — map it in scenarioCoverage to a covering test, or tag it @pending in the .feature", s.name)
|
||||
continue
|
||||
}
|
||||
// 2. The mapped test must actually exist.
|
||||
if !tests[fn] {
|
||||
t.Errorf("scenario %q maps to %q, which is not a Test function anywhere in the module", s.name, fn)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. No stale entries: every map key must be a real, non-pending scenario.
|
||||
for name := range scenarioCoverage {
|
||||
if !active[name] {
|
||||
t.Errorf("scenarioCoverage has entry %q, which is not a current non-pending scenario (renamed, removed, or now @pending?)", name)
|
||||
}
|
||||
}
|
||||
|
||||
if len(pending) > 0 {
|
||||
t.Logf("%d @pending scenario(s) without a test (tracked, not required): %s",
|
||||
len(pending), strings.Join(pending, "; "))
|
||||
}
|
||||
}
|
||||
|
||||
// parseScenarios reads every *.feature under dir and returns its scenarios with
|
||||
// their @pending status. A scenario is @pending when a `@pending` tag line
|
||||
// precedes it (tags survive intervening comment lines, the layout these files
|
||||
// use); the flag is consumed at the Scenario line and reset afterward.
|
||||
func parseScenarios(t *testing.T, dir string) []scenario {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read use-cases dir: %v", err)
|
||||
}
|
||||
var out []scenario
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".feature") {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", e.Name(), err)
|
||||
}
|
||||
pending := false
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "@") {
|
||||
if strings.Contains(trimmed, "@pending") {
|
||||
pending = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if m := scenarioRe.FindStringSubmatch(line); m != nil {
|
||||
out = append(out, scenario{name: m[1], pending: pending})
|
||||
pending = false
|
||||
}
|
||||
// comment (#) and step lines leave a set @pending intact until the
|
||||
// scenario consumes it; a blank line between scenarios is harmless.
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// allTestFuncNames walks the module for `func TestXxx(` declarations, excluding
|
||||
// this file (whose regex literal would otherwise look like a definition).
|
||||
func allTestFuncNames(t *testing.T, root string) map[string]bool {
|
||||
t.Helper()
|
||||
self := "scenario_coverage_test.go"
|
||||
names := map[string]bool{}
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if d.Name() == ".git" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, "_test.go") || filepath.Base(path) == self {
|
||||
return nil
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
if m := testFuncRe.FindStringSubmatch(line); m != nil {
|
||||
names[m[1]] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("walk module: %v", err)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// moduleRoot walks up from the working directory to the dir containing go.mod.
|
||||
func moduleRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
||||
return dir
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
t.Fatal("go.mod not found walking up from cwd")
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user