After transcript caching (ADR-021) and the Shorts filter (ADR-023), the remaining caption waste is the first fetch on every new video of a channel that never has English captions — each costs one rate-limited fetch to resolve to "none", and on a throttled IP churns the backoff machinery first. Remember, per (user, channel), a streak of consecutive no-caption outcomes (channel_caption_state, migration 016, RLS-scoped). Once it reaches TAPIR_CHANNEL_CAPTIONLESS_THRESHOLD (default 5) the channel is suppressed — videos discovered/listed but not caption-fetched — for TAPIR_CHANNEL_CAPTIONLESS_WINDOW (default 14d), then one is re-probed (auto-recovery). A successful fetch resets the streak; a 429 does not count; an explicit manual request bypasses suppression. threshold=0 disables. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
155 lines
7.4 KiB
Go
155 lines
7.4 KiB
Go
package store_test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"os"
|
|
"testing"
|
|
|
|
"github.com/golang-migrate/migrate/v4"
|
|
migratepgx "github.com/golang-migrate/migrate/v4/database/pgx/v5"
|
|
"github.com/golang-migrate/migrate/v4/source/iofs"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
_ "github.com/jackc/pgx/v5/stdlib" // register the "pgx" database/sql driver
|
|
)
|
|
|
|
// fileMigrator builds a golang-migrate instance from the on-disk migration files
|
|
// (not the embedded FS the production Migrate uses), so a test can step the schema
|
|
// up and down. os.DirFS(".") is rooted at the package dir; the SQL lives under
|
|
// "migrations". Mirrors store.Migrate's construction otherwise.
|
|
func fileMigrator(t *testing.T) *migrate.Migrate {
|
|
t.Helper()
|
|
db, err := sql.Open("pgx", dsn)
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
|
|
drv, err := migratepgx.WithInstance(db, &migratepgx.Config{})
|
|
require.NoError(t, err)
|
|
src, err := iofs.New(os.DirFS("."), "migrations")
|
|
require.NoError(t, err)
|
|
m, err := migrate.NewWithInstance("iofs", src, "pgx", drv)
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { _, _ = m.Close() })
|
|
return m
|
|
}
|
|
|
|
// loginEventsExists reports whether the login_events relation is present.
|
|
func loginEventsExists(t *testing.T) bool {
|
|
t.Helper()
|
|
var reg *string
|
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
|
`SELECT to_regclass('public.login_events')::text`).Scan(®))
|
|
return reg != nil
|
|
}
|
|
|
|
// TestMigration010LoginEventsUpDown proves migration 010 is reversible: the down
|
|
// migration drops login_events cleanly and the up migration recreates it. A rotten
|
|
// down migration (forgotten DROP, dangling policy) would fail here rather than in
|
|
// production during a rollback. The test restores the schema to latest before
|
|
// returning so the shared embedded-postgres stays at HEAD for sibling tests.
|
|
func TestMigration010LoginEventsUpDown(t *testing.T) {
|
|
newStore(t) // ensure the schema is migrated to latest (011 applied)
|
|
require.True(t, loginEventsExists(t), "login_events must exist at latest migration")
|
|
|
|
m := fileMigrator(t)
|
|
// 011..016 sit above 010; step them down first so 010 is exercised in isolation.
|
|
require.NoError(t, m.Steps(-1), "down 016 drops channel_caption_state, login_events intact")
|
|
require.True(t, loginEventsExists(t), "016 down leaves login_events intact")
|
|
require.NoError(t, m.Steps(-1), "down 015 reshapes transcripts, login_events intact")
|
|
require.True(t, loginEventsExists(t), "015 down leaves login_events intact")
|
|
require.NoError(t, m.Steps(-1), "down 014 drops channel_title, login_events intact")
|
|
require.True(t, loginEventsExists(t), "014 down leaves login_events intact")
|
|
require.NoError(t, m.Steps(-1), "down 013 drops channel_errors, login_events intact")
|
|
require.True(t, loginEventsExists(t), "013 down leaves login_events intact")
|
|
require.NoError(t, m.Steps(-1), "down 012 is a no-op, login_events intact")
|
|
require.True(t, loginEventsExists(t), "012 down leaves login_events intact")
|
|
require.NoError(t, m.Steps(-1), "down 011 must not touch login_events")
|
|
require.True(t, loginEventsExists(t), "011 down leaves login_events intact")
|
|
|
|
require.NoError(t, m.Steps(-1), "down 010 must drop login_events")
|
|
require.False(t, loginEventsExists(t), "login_events must be gone after the down migration")
|
|
|
|
require.NoError(t, m.Steps(7), "up must recreate 010 then re-apply 011..016")
|
|
require.True(t, loginEventsExists(t), "login_events must be restored after the up migration")
|
|
}
|
|
|
|
// autoSummarizeDefault reads the users.auto_summarize column default as text
|
|
// ("true"/"false"), so the migration's default flip is verifiable directly.
|
|
func autoSummarizeDefault(t *testing.T) string {
|
|
t.Helper()
|
|
var def string
|
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
|
`SELECT column_default FROM information_schema.columns
|
|
WHERE table_name = 'users' AND column_name = 'auto_summarize'`).Scan(&def))
|
|
return def
|
|
}
|
|
|
|
// TestMigration011AutoSummarizeDefaultUpDown proves migration 011 is reversible:
|
|
// up sets the auto_summarize column default to TRUE (ADR-018), down restores
|
|
// FALSE. The down intentionally does not revert existing rows — only the default.
|
|
func TestMigration011AutoSummarizeDefaultUpDown(t *testing.T) {
|
|
newStore(t) // latest (013 applied)
|
|
require.Equal(t, "true", autoSummarizeDefault(t), "011 sets the default to TRUE")
|
|
|
|
m := fileMigrator(t)
|
|
require.NoError(t, m.Steps(-1), "down 016 drops channel_caption_state")
|
|
require.NoError(t, m.Steps(-1), "down 015 reshapes transcripts")
|
|
require.NoError(t, m.Steps(-1), "down 014 drops channel_title")
|
|
require.NoError(t, m.Steps(-1), "down 013 drops channel_errors")
|
|
require.NoError(t, m.Steps(-1), "down 012 is a no-op")
|
|
require.NoError(t, m.Steps(-1), "down 011 reverts the column default")
|
|
require.Equal(t, "false", autoSummarizeDefault(t), "default is FALSE after the down migration")
|
|
|
|
require.NoError(t, m.Steps(1), "up 011 re-applies the TRUE default")
|
|
require.Equal(t, "true", autoSummarizeDefault(t))
|
|
require.NoError(t, m.Steps(1), "up 012 runs clean (no FORCE RLS on fresh schema)")
|
|
require.NoError(t, m.Steps(1), "up 013 creates channel_errors")
|
|
require.NoError(t, m.Steps(1), "up 014 recreates channel_title")
|
|
require.NoError(t, m.Steps(1), "up 015 reshapes transcripts to shared")
|
|
require.NoError(t, m.Steps(1), "up 016 recreates channel_caption_state (HEAD)")
|
|
}
|
|
|
|
// channelTitleExists reports whether videos.channel_title is present.
|
|
func channelTitleExists(t *testing.T) bool {
|
|
t.Helper()
|
|
var exists bool
|
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
|
`SELECT EXISTS (SELECT 1 FROM information_schema.columns
|
|
WHERE table_name = 'videos' AND column_name = 'channel_title')`).Scan(&exists))
|
|
return exists
|
|
}
|
|
|
|
// TestMigration014VideoChannelTitleUpDown proves 014 is reversible: down drops
|
|
// videos.channel_title, up recreates it.
|
|
func TestMigration014VideoChannelTitleUpDown(t *testing.T) {
|
|
newStore(t) // latest (014 applied)
|
|
require.True(t, channelTitleExists(t), "channel_title exists at latest migration")
|
|
|
|
m := fileMigrator(t)
|
|
require.NoError(t, m.Steps(-1), "down 016 drops channel_caption_state, channel_title intact")
|
|
require.True(t, channelTitleExists(t), "016 down leaves channel_title intact")
|
|
require.NoError(t, m.Steps(-1), "down 015 reshapes transcripts, channel_title intact")
|
|
require.True(t, channelTitleExists(t), "015 down leaves channel_title intact")
|
|
require.NoError(t, m.Steps(-1), "down 014 must drop channel_title")
|
|
require.False(t, channelTitleExists(t), "channel_title must be gone after the down migration")
|
|
|
|
require.NoError(t, m.Steps(1), "up 014 must recreate channel_title")
|
|
require.True(t, channelTitleExists(t), "channel_title must be restored after the up migration")
|
|
require.NoError(t, m.Steps(1), "up 015 restores the shared transcripts shape")
|
|
require.NoError(t, m.Steps(1), "up 016 recreates channel_caption_state (HEAD)")
|
|
}
|
|
|
|
// TestMigration012FixAutoSummarizeRLS proves 012 runs cleanly and flips any
|
|
// remaining auto_summarize=FALSE rows to TRUE (the back-fill blocked by RLS in 011).
|
|
func TestMigration012FixAutoSummarizeRLS(t *testing.T) {
|
|
newStore(t) // apply all migrations including 012
|
|
require.Equal(t, "true", autoSummarizeDefault(t), "column default is TRUE after 012")
|
|
|
|
// Round-trip: down 012, then up 012 — must be idempotent.
|
|
m := fileMigrator(t)
|
|
require.NoError(t, m.Steps(-1), "down 012 must not error")
|
|
require.NoError(t, m.Steps(1), "up 012 must re-apply cleanly")
|
|
require.Equal(t, "true", autoSummarizeDefault(t), "default still TRUE after 012 re-applied")
|
|
}
|