In automatic mode the scheduler now only summarizes videos published within TAPIR_AUTO_SUMMARIZE_WINDOW (default ~7d). Older videos are still discovered and listed — they keep the manual "Summarize" affordance — but are not auto-processed, so a large back-catalogue (the maintainer's ~256-deep queue) stops self-inflicting 429s against the per-IP caption gate each cycle (UX review B1, recency design). - runner.WithAutoWindow + Stats.SkippedTooOld; tooOld() treats a zero window as disabled and an undated video as never-aged-out (processed, not stranded). - An explicit manual request bypasses the bound even in auto mode (requested videos are loaded in auto mode when a window is active). - Wired through cmdRun, the scheduler's per-user runner, sumStats, and pass logging. config: TAPIR_AUTO_SUMMARIZE_WINDOW (default 168h), .env.example. - Account copy (A7) updated to match: "Automatic summarizes new videos from about the last week; older videos stay browsable — summarize on demand." The rate gate is untouched; the manual path still serialises through it. This bounds auto LOAD, it does not fetch harder. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
271 lines
9.9 KiB
Go
271 lines
9.9 KiB
Go
// Package config parses Tapir's runtime configuration from environment
|
|
// variables into a typed struct. No secrets are baked into code: the gateway
|
|
// key and OAuth client secret are read from the environment (later resolved via
|
|
// op/ESO), and the OAuth refresh token is never held here — it lives behind the
|
|
// SecretStore port, addressed by the opaque TokenRef.
|
|
//
|
|
// Defaults target the homelab snapshot in docs/homelab-integration.md; every
|
|
// value is overridable so the same binary runs standalone or in-cluster.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Config is Tapir's fully-resolved runtime configuration.
|
|
type Config struct {
|
|
// UserID is the Tapir user the run operates as. Stage 0 has exactly one.
|
|
// It must be a UUID: it keys the UUID user_id/video_id columns (data-model).
|
|
UserID string
|
|
|
|
// GatewayURL is the OpenAI-compatible LiteLLM base URL (".../v1").
|
|
GatewayURL string
|
|
// GatewayKey authorizes the gateway. Read from env, never committed.
|
|
GatewayKey string
|
|
// SummarizerModel is the alias in host/name form, e.g. "koala/phi4-mini".
|
|
SummarizerModel string
|
|
// SummarizerTimeout bounds a single completion call. Thinking models are
|
|
// slow, so the default is generous.
|
|
SummarizerTimeout time.Duration
|
|
|
|
// DBDSN is the Postgres DSN for the store sink.
|
|
DBDSN string
|
|
|
|
// YouTube OAuth app credentials (the registered client), read from env.
|
|
YTClientID string
|
|
YTClientSecret string
|
|
// YTTokenRef is the opaque SecretStore reference under which the YouTube
|
|
// refresh token is persisted/resolved. Not the token itself.
|
|
YTTokenRef string
|
|
// YTConnectRedirectURL is the public callback URL the web connect flow
|
|
// registers with Google, e.g. "https://tapir.d-ma.be/oauth/youtube/callback".
|
|
// Must be in the OAuth client's authorized redirects. Distinct from the CLI
|
|
// auth command's localhost listener and from the Dex OIDC redirect.
|
|
YTConnectRedirectURL string
|
|
|
|
// SecretsFile is the path to the local file-backed SecretStore (0600). A
|
|
// Stage-0 stand-in for op/ESO, swappable behind the SecretStore port.
|
|
SecretsFile string
|
|
|
|
// OAuthRedirectAddr is the host:port the `auth` command's local listener
|
|
// binds for the OAuth redirect, e.g. "localhost:8080".
|
|
OAuthRedirectAddr string
|
|
|
|
// PollInterval, when > 0, makes `run` loop on that cadence; 0 means run once.
|
|
PollInterval time.Duration
|
|
|
|
// FetchBackoff is how long the run loop waits before re-fetching a transcript
|
|
// that previously returned HTTP 429 (rate_limited). Inside the window the video
|
|
// is skipped without hitting the caption endpoint, saving requests; after it
|
|
// expires the video is retried. Zero means "always retry" (no backoff).
|
|
FetchBackoff time.Duration
|
|
|
|
// FetchRate is the minimum interval between outbound caption fetches across the
|
|
// whole process — the shared per-egress-IP rate gate (ADR-014 item 2). It is
|
|
// the gate that makes auto-summarize-on-a-schedule safe: scheduler runners and
|
|
// the web click-path serialise through it. Zero = unlimited (dev/tests).
|
|
FetchRate time.Duration
|
|
|
|
// AutoSummarizeWindow bounds auto-summarization to recent videos: in automatic
|
|
// mode the scheduler only summarizes videos published within this window of now.
|
|
// Older videos are still discovered and listed, but wait for an explicit manual
|
|
// "Summarize" — so a large back-catalogue does not self-inflict 429s against the
|
|
// caption rate gate. Zero disables the bound (summarize every unseen video, the
|
|
// pre-recency behaviour). Default ~7 days.
|
|
AutoSummarizeWindow time.Duration
|
|
|
|
// DiscoveryInterval, when > 0, makes `serve` run in-process scheduled discovery
|
|
// for ALL users on that cadence (ADR-018). Zero/unset = disabled, so dev and
|
|
// tests never auto-fetch. Single-replica assumption — see cmdServe.
|
|
DiscoveryInterval time.Duration
|
|
|
|
// HTTPAddr is the listen address for `tapir serve` (the Stage-0 web UI).
|
|
HTTPAddr string
|
|
|
|
// PublicURL is the externally-reachable base URL of the deployed service,
|
|
// e.g. "https://tapir.d-ma.be". Used to build absolute links handed to humans
|
|
// (the `tapir invite` URL). No trailing slash is assumed — callers trim it.
|
|
PublicURL string
|
|
|
|
// Dex OIDC (web login, ADR-011/012). When OIDCIssuer is empty, `serve` falls
|
|
// back to the allow-all StubAuth (local dev). When set, serve uses Dex: any
|
|
// Dex-authenticated subject may sign in, then registers a tapir user (ADR-012).
|
|
OIDCIssuer string
|
|
DexClientID string
|
|
DexClientSecret string
|
|
OIDCRedirectURL string
|
|
SessionSecret string
|
|
}
|
|
|
|
// DexConfigured reports whether Dex OIDC login is wired (issuer present). When
|
|
// false, `serve` uses StubAuth (dev only).
|
|
func (c Config) DexConfigured() bool { return strings.TrimSpace(c.OIDCIssuer) != "" }
|
|
|
|
// Defaults (see docs/homelab-integration.md). All overridable via env.
|
|
const (
|
|
defaultGatewayURL = "http://koala:30401/v1"
|
|
defaultSummarizerModel = "koala/phi4-mini"
|
|
defaultSummarizerTimeout = 5 * time.Minute
|
|
defaultYTTokenRef = "youtube/refresh_token"
|
|
defaultYTConnectRedirectURL = "https://tapir.d-ma.be/oauth/youtube/callback"
|
|
defaultOAuthRedirectAddr = "localhost:8080"
|
|
defaultHTTPAddr = ":8080"
|
|
defaultFetchBackoff = time.Hour
|
|
defaultFetchRate = 2 * time.Second
|
|
defaultPublicURL = "https://tapir.d-ma.be"
|
|
defaultAutoSummarizeWindow = 7 * 24 * time.Hour
|
|
)
|
|
|
|
// Load reads the environment into a Config, applying defaults. It does not
|
|
// validate that required fields are present — call ValidateForAuth or
|
|
// ValidateForRun for the command being run, so each command demands only what
|
|
// it needs.
|
|
func Load() (Config, error) {
|
|
c := Config{
|
|
UserID: os.Getenv("TAPIR_USER_ID"),
|
|
GatewayURL: envOr("TAPIR_GATEWAY_URL", defaultGatewayURL),
|
|
GatewayKey: os.Getenv("TAPIR_GATEWAY_KEY"),
|
|
SummarizerModel: envOr("TAPIR_SUMMARIZER_MODEL", defaultSummarizerModel),
|
|
DBDSN: os.Getenv("TAPIR_DB_DSN"),
|
|
YTClientID: os.Getenv("TAPIR_YT_CLIENT_ID"),
|
|
YTClientSecret: os.Getenv("TAPIR_YT_CLIENT_SECRET"),
|
|
YTTokenRef: envOr("TAPIR_YT_TOKEN_REF", defaultYTTokenRef),
|
|
YTConnectRedirectURL: envOr("TAPIR_YT_CONNECT_REDIRECT_URL", defaultYTConnectRedirectURL),
|
|
SecretsFile: envOr("TAPIR_SECRETS_FILE", defaultSecretsFile()),
|
|
OAuthRedirectAddr: envOr("TAPIR_OAUTH_REDIRECT_ADDR", defaultOAuthRedirectAddr),
|
|
HTTPAddr: envOr("TAPIR_HTTP_ADDR", defaultHTTPAddr),
|
|
PublicURL: envOr("TAPIR_PUBLIC_URL", defaultPublicURL),
|
|
OIDCIssuer: os.Getenv("TAPIR_OIDC_ISSUER"),
|
|
DexClientID: os.Getenv("TAPIR_DEX_CLIENT_ID"),
|
|
DexClientSecret: os.Getenv("TAPIR_DEX_CLIENT_SECRET"),
|
|
OIDCRedirectURL: os.Getenv("TAPIR_OIDC_REDIRECT_URL"),
|
|
SessionSecret: os.Getenv("TAPIR_SESSION_SECRET"),
|
|
}
|
|
|
|
timeout, err := durationOr("TAPIR_SUMMARIZER_TIMEOUT", defaultSummarizerTimeout)
|
|
if err != nil {
|
|
return Config{}, err
|
|
}
|
|
c.SummarizerTimeout = timeout
|
|
|
|
interval, err := durationOr("TAPIR_POLL_INTERVAL", 0)
|
|
if err != nil {
|
|
return Config{}, err
|
|
}
|
|
c.PollInterval = interval
|
|
|
|
backoff, err := durationOr("TAPIR_FETCH_BACKOFF", defaultFetchBackoff)
|
|
if err != nil {
|
|
return Config{}, err
|
|
}
|
|
c.FetchBackoff = backoff
|
|
|
|
fetchRate, err := durationOr("TAPIR_FETCH_RATE", defaultFetchRate)
|
|
if err != nil {
|
|
return Config{}, err
|
|
}
|
|
c.FetchRate = fetchRate
|
|
|
|
discovery, err := durationOr("TAPIR_DISCOVERY_INTERVAL", 0)
|
|
if err != nil {
|
|
return Config{}, err
|
|
}
|
|
c.DiscoveryInterval = discovery
|
|
|
|
autoWindow, err := durationOr("TAPIR_AUTO_SUMMARIZE_WINDOW", defaultAutoSummarizeWindow)
|
|
if err != nil {
|
|
return Config{}, err
|
|
}
|
|
c.AutoSummarizeWindow = autoWindow
|
|
|
|
return c, nil
|
|
}
|
|
|
|
// ValidateForAuth checks the fields the `auth` command needs: the YouTube OAuth
|
|
// app credentials, a place to persist the token, and the redirect listener.
|
|
func (c Config) ValidateForAuth() error {
|
|
return c.require(map[string]string{
|
|
"TAPIR_YT_CLIENT_ID": c.YTClientID,
|
|
"TAPIR_YT_CLIENT_SECRET": c.YTClientSecret,
|
|
"TAPIR_YT_TOKEN_REF": c.YTTokenRef,
|
|
"TAPIR_SECRETS_FILE": c.SecretsFile,
|
|
})
|
|
}
|
|
|
|
// ValidateForRun checks the fields the `run` command needs end to end.
|
|
func (c Config) ValidateForRun() error {
|
|
return c.require(map[string]string{
|
|
"TAPIR_USER_ID": c.UserID,
|
|
"TAPIR_GATEWAY_URL": c.GatewayURL,
|
|
"TAPIR_SUMMARIZER_MODEL": c.SummarizerModel,
|
|
"TAPIR_DB_DSN": c.DBDSN,
|
|
"TAPIR_YT_CLIENT_ID": c.YTClientID,
|
|
"TAPIR_YT_CLIENT_SECRET": c.YTClientSecret,
|
|
"TAPIR_YT_TOKEN_REF": c.YTTokenRef,
|
|
"TAPIR_SECRETS_FILE": c.SecretsFile,
|
|
})
|
|
}
|
|
|
|
// ValidateForServe checks the fields the `serve` command needs: a user to act
|
|
// as, the store DSN, and a listen address. Auth (Dex) config is validated by the
|
|
// auth layer the Conductor wires in, not here.
|
|
func (c Config) ValidateForServe() error {
|
|
return c.require(map[string]string{
|
|
"TAPIR_USER_ID": c.UserID,
|
|
"TAPIR_DB_DSN": c.DBDSN,
|
|
"TAPIR_HTTP_ADDR": c.HTTPAddr,
|
|
})
|
|
}
|
|
|
|
func (c Config) require(fields map[string]string) error {
|
|
var missing []string
|
|
for name, val := range fields {
|
|
if strings.TrimSpace(val) == "" {
|
|
missing = append(missing, name)
|
|
}
|
|
}
|
|
if len(missing) > 0 {
|
|
return fmt.Errorf("missing required config: %s", strings.Join(sortedMissing(missing), ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sortedMissing(xs []string) []string {
|
|
sort.Strings(xs)
|
|
return xs
|
|
}
|
|
|
|
func envOr(key, fallback string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func durationOr(key string, fallback time.Duration) (time.Duration, error) {
|
|
v := os.Getenv(key)
|
|
if v == "" {
|
|
return fallback, nil
|
|
}
|
|
d, err := time.ParseDuration(v)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("config: %s=%q: %w", key, v, err)
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// defaultSecretsFile resolves to <user-config-dir>/tapir/secrets.json, falling
|
|
// back to a cwd-relative path when the config dir is unavailable.
|
|
func defaultSecretsFile() string {
|
|
dir, err := os.UserConfigDir()
|
|
if err != nil {
|
|
return "tapir-secrets.json"
|
|
}
|
|
return filepath.Join(dir, "tapir", "secrets.json")
|
|
}
|