On a cluster restart this pod can beat dex to serving, gets a 502 from ingress, and exits — after connecting postgres, backfilling 794 graph nodes and ingesting three session batches, all of which the restart redoes. Nothing was misconfigured; it started first. The validator now retries with bounded exponential backoff (8 attempts, 1s doubling to 30s) and is built immediately after ctx, before the watcher and the backfill. That second part is the other half of the issue: a genuine config error now fails in seconds rather than after all that work. Bounded on purpose. Retrying forever would convert a wrong issuer URL into a process that looks alive and never serves, which is harder to diagnose than a crash loop. startupretry.Do is generic and tested directly: transient-then-success, budget exhaustion surfacing the last error via errors.Is, prompt return on context cancellation, growing-and-capped delays, and a zero-value policy still making one attempt rather than silently succeeding. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BeAp5LTscnz2W7Ubt4eJ6m
145 lines
3.7 KiB
Go
145 lines
3.7 KiB
Go
// ingestion/internal/startupretry/startupretry_test.go
|
|
package startupretry
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func fastPolicy(attempts int) Policy {
|
|
return Policy{Attempts: attempts, Base: time.Millisecond, Max: 2 * time.Millisecond}
|
|
}
|
|
|
|
func TestDo_ReturnsImmediatelyOnFirstSuccess(t *testing.T) {
|
|
calls := 0
|
|
got, err := Do(context.Background(), fastPolicy(5), func(context.Context) (string, error) {
|
|
calls++
|
|
return "ready", nil
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatalf("Do: %v", err)
|
|
}
|
|
if got != "ready" {
|
|
t.Errorf("got %q, want %q", got, "ready")
|
|
}
|
|
if calls != 1 {
|
|
t.Errorf("calls = %d, want 1", calls)
|
|
}
|
|
}
|
|
|
|
func TestDo_SurvivesTransientFailuresThenSucceeds(t *testing.T) {
|
|
// The observed cold-boot race: dex is not serving yet, ingress answers
|
|
// 502 for the first few attempts, then discovery works.
|
|
calls := 0
|
|
got, err := Do(context.Background(), fastPolicy(5), func(context.Context) (string, error) {
|
|
calls++
|
|
if calls < 4 {
|
|
return "", errors.New("oidc discovery: status 502")
|
|
}
|
|
return "ready", nil
|
|
})
|
|
|
|
if err != nil {
|
|
t.Fatalf("Do returned %v — a transient 502 must not be fatal", err)
|
|
}
|
|
if got != "ready" {
|
|
t.Errorf("got %q, want %q", got, "ready")
|
|
}
|
|
if calls != 4 {
|
|
t.Errorf("calls = %d, want 4", calls)
|
|
}
|
|
}
|
|
|
|
func TestDo_GivesUpAfterTheAttemptBudgetAndReportsTheLastError(t *testing.T) {
|
|
// A genuinely broken issuer must still fail — retrying forever would turn
|
|
// a config error into a silent hang, which is worse than crash-looping.
|
|
last := errors.New("oidc discovery: status 404")
|
|
calls := 0
|
|
|
|
_, err := Do(context.Background(), fastPolicy(3), func(context.Context) (string, error) {
|
|
calls++
|
|
return "", last
|
|
})
|
|
|
|
if !errors.Is(err, last) {
|
|
t.Fatalf("err = %v, want it to wrap the last failure", err)
|
|
}
|
|
if calls != 3 {
|
|
t.Errorf("calls = %d, want 3 (the attempt budget)", calls)
|
|
}
|
|
}
|
|
|
|
func TestDo_StopsPromptlyWhenTheContextIsCancelled(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
calls := 0
|
|
|
|
done := make(chan struct{})
|
|
var err error
|
|
go func() {
|
|
defer close(done)
|
|
_, err = Do(ctx, Policy{Attempts: 100, Base: 50 * time.Millisecond, Max: time.Second},
|
|
func(context.Context) (string, error) {
|
|
calls++
|
|
if calls == 1 {
|
|
cancel()
|
|
}
|
|
return "", errors.New("still down")
|
|
})
|
|
}()
|
|
|
|
select {
|
|
case <-done:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("Do did not return after context cancellation")
|
|
}
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Errorf("err = %v, want context.Canceled", err)
|
|
}
|
|
}
|
|
|
|
func TestDo_BacksOffAndIsCappedByMax(t *testing.T) {
|
|
var delays []time.Duration
|
|
p := Policy{Attempts: 5, Base: 10 * time.Millisecond, Max: 25 * time.Millisecond}
|
|
|
|
last := time.Now()
|
|
_, _ = Do(context.Background(), p, func(context.Context) (string, error) {
|
|
now := time.Now()
|
|
delays = append(delays, now.Sub(last))
|
|
last = now
|
|
return "", errors.New("down")
|
|
})
|
|
|
|
// delays[0] is the first call (no wait); the rest are the sleeps.
|
|
if len(delays) != 5 {
|
|
t.Fatalf("attempts = %d, want 5", len(delays))
|
|
}
|
|
if delays[1] >= delays[2] {
|
|
t.Errorf("backoff is not growing: %v then %v", delays[1], delays[2])
|
|
}
|
|
for i, d := range delays[1:] {
|
|
if d > 2*p.Max {
|
|
t.Errorf("delay %d = %v, exceeds the Max cap %v with slack", i+1, d, p.Max)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPolicy_ZeroValueStillMakesOneAttempt(t *testing.T) {
|
|
// A caller that forgets to configure the policy must not silently get a
|
|
// no-op that returns a nil validator and a nil error.
|
|
calls := 0
|
|
_, err := Do(context.Background(), Policy{}, func(context.Context) (string, error) {
|
|
calls++
|
|
return "", errors.New("down")
|
|
})
|
|
|
|
if calls != 1 {
|
|
t.Errorf("calls = %d, want 1", calls)
|
|
}
|
|
if err == nil {
|
|
t.Error("err = nil, want the failure surfaced")
|
|
}
|
|
}
|