// Package startupretry retries a startup dependency that may not be serving // yet, instead of exiting on its first failure. // // It exists for the cold-boot ordering race (#83): on a cluster restart the // ingestion pod can reach OIDC discovery before dex is answering, gets a 502 // from ingress, and dies — after having already connected postgres, backfilled // the graph and ingested session batches, all of which the restart redoes. // Nothing is misconfigured in that scenario; the pod simply started first. // // The budget is bounded on purpose. Retrying forever would turn a genuine // misconfiguration (wrong issuer URL, bad audience) into a process that looks // healthy and never serves, which is harder to diagnose than a crash loop. package startupretry import ( "context" "fmt" "time" ) // Policy bounds the retry: at most Attempts calls, with the delay doubling // from Base and capped at Max. type Policy struct { Attempts int // total calls, including the first; < 1 means 1 Base time.Duration // delay before the second attempt Max time.Duration // ceiling for the doubling delay } // Do calls fn until it succeeds, the attempt budget is spent, or ctx ends. // // The returned error wraps the last failure, so callers keep the diagnosis // they would have had without the retry. func Do[T any](ctx context.Context, p Policy, fn func(context.Context) (T, error)) (T, error) { var zero T attempts := p.Attempts if attempts < 1 { attempts = 1 } delay := p.Base var lastErr error for i := range attempts { if err := ctx.Err(); err != nil { return zero, fmt.Errorf("startupretry: %w", err) } v, err := fn(ctx) if err == nil { return v, nil } lastErr = err if i == attempts-1 { break } if delay <= 0 { delay = time.Second } if p.Max > 0 && delay > p.Max { delay = p.Max } select { case <-ctx.Done(): return zero, fmt.Errorf("startupretry: %w", ctx.Err()) case <-time.After(delay): } delay *= 2 } return zero, fmt.Errorf("startupretry: gave up after %d attempts: %w", attempts, lastErr) }