fix(ingestion): retry OIDC discovery instead of exiting on cold-boot 502 (#83)

On a cluster restart this pod can beat dex to serving, gets a 502 from
ingress, and exits — after connecting postgres, backfilling 794 graph
nodes and ingesting three session batches, all of which the restart
redoes. Nothing was misconfigured; it started first.

The validator now retries with bounded exponential backoff (8 attempts,
1s doubling to 30s) and is built immediately after ctx, before the
watcher and the backfill. That second part is the other half of the
issue: a genuine config error now fails in seconds rather than after all
that work.

Bounded on purpose. Retrying forever would convert a wrong issuer URL
into a process that looks alive and never serves, which is harder to
diagnose than a crash loop.

startupretry.Do is generic and tested directly: transient-then-success,
budget exhaustion surfacing the last error via errors.Is, prompt return
on context cancellation, growing-and-capped delays, and a zero-value
policy still making one attempt rather than silently succeeding.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BeAp5LTscnz2W7Ubt4eJ6m

ingestion/cmd/server/main.go

@@ -32,6 +32,7 @@ import (
 	"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/search"
+	"github.com/mathiasbq/hyperguild/ingestion/internal/startupretry"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/watcher"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/webhook"
@@ -314,6 +315,36 @@ func main() {
 	}
 
 	ctx := context.Background()
+
+	// Built before the watcher, the graph backfill and session ingestion, and
+	// retried rather than fatal (#83). Two separate reasons:
+	//
+	//   - On a cold cluster boot this pod can beat dex to serving and gets a
+	//     502 from ingress. Nothing is misconfigured; it simply started first.
+	//     Exiting there meant CrashLoopBackOff redoing the backfill on every
+	//     attempt, with exponential backoff stretching the brain outage.
+	//   - Constructing it first means a genuine config error (wrong issuer,
+	//     bad audience) fails in seconds instead of after all that work.
+	//
+	// The budget stays bounded: retrying forever would turn a misconfiguration
+	// into a process that looks alive and never serves.
+	jwtValidator, err := startupretry.Do(ctx,
+		startupretry.Policy{Attempts: 8, Base: time.Second, Max: 30 * time.Second},
+		func(ctx context.Context) (*chassisauth.JWTValidator, error) {
+			v, err := chassisauth.NewJWTValidator(ctx, os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE"))
+			if err != nil {
+				logger.Warn("oidc discovery not ready, retrying", "err", err)
+			}
+			return v, err
+		})
+	if err != nil {
+		logger.Error("build jwt validator", "err", err)
+		os.Exit(1)
+	}
+	if jwtValidator != nil {
+		logger.Info("jwt auth enabled", "issuer", os.Getenv("DEX_ISSUER_URL"))
+	}
+
 	if watchInterval > 0 {
 		watcher.Start(ctx, watcher.Config{
 			BrainDir: brainDir,
@@ -418,15 +449,6 @@ func main() {
 	if webhookHandler != nil {
 		mux.Handle("POST /webhooks/brain-sync", webhookHandler)
 	}
-	jwtValidator, err := chassisauth.NewJWTValidator(ctx, os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE"))
-	if err != nil {
-		logger.Error("build jwt validator", "err", err)
-		os.Exit(1)
-	}
-	if jwtValidator != nil {
-		logger.Info("jwt auth enabled", "issuer", os.Getenv("DEX_ISSUER_URL"))
-	}
-
 	// Resource-metadata URL is only emitted on 401 when Dex OAuth is
 	// configured. Static-Bearer-only deployments leave this empty so
 	// clients never see an OAuth challenge.