From debf723ef5aeeab699c5189c3b5a6fee0d247be5 Mon Sep 17 00:00:00 2001 From: Leon Hubrich Date: Sat, 1 Aug 2026 16:06:12 +0000 Subject: [PATCH] Retry OIDC provisioner initialization instead of disabling it An OIDC provisioner gets the openid-configuration and the JWK key set of the identity provider on Init. When the provider could not be reached, for example because step-ca started before the network was up, the provisioner was replaced by an Uninitialized one and never recovered: OIDC authentication kept failing until step-ca was restarted. Init now marks a failure to reach the identity provider as temporary, and the provisioner gets the configuration again when it is used, using an exponential backoff between attempts. Any other initialization error, e.g. an invalid configuration, still disables the provisioner. Fixes #2724 Co-Authored-By: Claude Opus 5 --- authority/authority.go | 14 ++- authority/authority_test.go | 42 ++++++++ authority/provisioner/oidc.go | 91 ++++++++++++++++-- authority/provisioner/oidc_test.go | 137 +++++++++++++++++++++++++++ authority/provisioner/provisioner.go | 26 +++++ 5 files changed, 298 insertions(+), 12 deletions(-) diff --git a/authority/authority.go b/authority/authority.go index 8dd6a0b8e..e3f4c7bcb 100644 --- a/authority/authority.go +++ b/authority/authority.go @@ -273,9 +273,17 @@ func (a *Authority) ReloadAdminResources(ctx context.Context) error { provClxn := provisioner.NewCollection(provisionerConfig.Audiences) for _, p := range provList { if err := p.Init(provisionerConfig); err != nil { - log.Printf("failed to initialize %s provisioner %q: %v\n", p.GetType(), p.GetName(), err) - p = provisioner.Uninitialized{ - Interface: p, Reason: err, + // Provisioners that failed for a temporary reason, e.g. an OIDC + // provisioner that cannot reach its identity provider, stay enabled + // and initialize themselves when they are used. + if errors.Is(err, provisioner.ErrRetryInit) { + log.Printf("failed to initialize %s provisioner %q, it will be retried when the provisioner is used: %v\n", + p.GetType(), p.GetName(), err) + } else { + log.Printf("failed to initialize %s provisioner %q: %v\n", p.GetType(), p.GetName(), err) + p = provisioner.Uninitialized{ + Interface: p, Reason: err, + } } } if err := provClxn.Store(p); err != nil { diff --git a/authority/authority_test.go b/authority/authority_test.go index dca6cf14f..07e161901 100644 --- a/authority/authority_test.go +++ b/authority/authority_test.go @@ -200,6 +200,48 @@ func TestAuthorityNew(t *testing.T) { } } +func TestAuthorityNew_retryInit(t *testing.T) { + c := &Config{ + Address: "127.0.0.1:443", + Root: []string{"testdata/certs/root_ca.crt"}, + IntermediateCert: "testdata/certs/intermediate_ca.crt", + IntermediateKey: "testdata/secrets/intermediate_ca_key", + DNSNames: []string{"example.com"}, + Password: "pass", + AuthorityConfig: &AuthConfig{ + Provisioners: provisioner.List{ + // Nothing listens on port 1, so the identity provider cannot be + // reached and the initialization is retried. + &provisioner.OIDC{ + Name: "oidc", + Type: "OIDC", + ClientID: "client-id", + ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration", + }, + // A provisioner without a client ID can never be initialized. + &provisioner.OIDC{ + Name: "uninitialized", + Type: "OIDC", + ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration", + }, + }, + }, + } + + auth, err := New(c) + assert.FatalError(t, err) + + p, ok := auth.provisioners.LoadByName("oidc") + assert.True(t, ok) + _, isUninitialized := p.(provisioner.Uninitialized) + assert.False(t, isUninitialized) + + p, ok = auth.provisioners.LoadByName("uninitialized") + assert.True(t, ok) + _, isUninitialized = p.(provisioner.Uninitialized) + assert.True(t, isUninitialized) +} + func TestAuthorityNew_bundles(t *testing.T) { ca0, err := minica.New() if err != nil { diff --git a/authority/provisioner/oidc.go b/authority/provisioner/oidc.go index 044971bf6..e9fc7a013 100644 --- a/authority/provisioner/oidc.go +++ b/authority/provisioner/oidc.go @@ -9,6 +9,7 @@ import ( "net/url" "path" "strings" + "sync" "time" "github.com/pkg/errors" @@ -21,6 +22,17 @@ import ( "github.com/smallstep/certificates/errs" ) +const ( + // oidcInitRetryInterval is the time to wait before the second attempt to + // get the OpenID configuration of a provisioner that failed to initialize. + // It doubles on every failure up to oidcMaxInitRetryInterval. + oidcInitRetryInterval = 5 * time.Second + + // oidcMaxInitRetryInterval is the maximum time between two attempts to get + // the OpenID configuration of a provisioner that failed to initialize. + oidcMaxInitRetryInterval = 5 * time.Minute +) + // openIDConfiguration contains the necessary properties in the // `/.well-known/openid-configuration` document. type openIDConfiguration struct { @@ -95,9 +107,17 @@ type OIDC struct { Options *Options `json:"options,omitempty"` Scopes []string `json:"scopes,omitempty"` AuthParams []string `json:"authParams,omitempty"` - configuration openIDConfiguration - keyStore *keyStore + wellKnownEndpoint string ctl *Controller + + // initMutex guards the fields below it, which are only read after a + // successful ensureInitialized. + initMutex sync.Mutex + initError error + initRetryAfter time.Time + initRetryInterval time.Duration + configuration openIDConfiguration + keyStore *keyStore } func sanitizeEmail(email string) string { @@ -156,7 +176,9 @@ func (o *OIDC) GetEncryptedKey() (kid, key string, ok bool) { return "", "", false } -// Init validates and initializes the OIDC provider. +// Init validates and initializes the OIDC provider. If the identity provider +// cannot be reached, the returned error matches ErrRetryInit and the +// provisioner initializes itself the next time that it is used. func (o *OIDC) Init(config Config) (err error) { switch { case o.Type == "": @@ -184,6 +206,7 @@ func (o *OIDC) Init(config Config) (err error) { if !strings.Contains(u.Path, "/.well-known/openid-configuration") { u.Path = path.Join(u.Path, "/.well-known/openid-configuration") } + o.wellKnownEndpoint = u.String() // Initialize the common provisioner controller o.ctl, err = NewController(o, o.Claims, config, o.Options) @@ -191,23 +214,68 @@ func (o *OIDC) Init(config Config) (err error) { return err } - // Decode and validate openid-configuration + // Get the openid-configuration and the JWK key set. Not being able to reach + // the identity provider is usually temporary, e.g. when step-ca starts + // before the network is up, so the provisioner retries when it is used. + o.initMutex.Lock() + defer o.initMutex.Unlock() + if o.initError = o.initialize(); o.initError != nil { + o.initRetryInterval = oidcInitRetryInterval + return retryInit(o.initError) + } + return nil +} + +// initialize gets and validates the openid-configuration and the JWK key set of +// the identity provider. Callers must hold o.initMutex. +func (o *OIDC) initialize() error { httpClient := o.ctl.GetHTTPClient() - if err := getAndDecode(httpClient, u.String(), &o.configuration); err != nil { + + // Decode and validate openid-configuration + var configuration openIDConfiguration + if err := getAndDecode(httpClient, o.wellKnownEndpoint, &configuration); err != nil { return err } - if err := o.configuration.Validate(); err != nil { + if err := configuration.Validate(); err != nil { return errors.Wrapf(err, "error parsing %s", o.ConfigurationEndpoint) } // Replace {tenantid} with the configured one if o.TenantID != "" { - o.configuration.Issuer = strings.ReplaceAll(o.configuration.Issuer, "{tenantid}", o.TenantID) + configuration.Issuer = strings.ReplaceAll(configuration.Issuer, "{tenantid}", o.TenantID) } // Get JWK key set - o.keyStore, err = newKeyStore(httpClient, o.configuration.JWKSetURI) - return + keyStore, err := newKeyStore(httpClient, configuration.JWKSetURI) + if err != nil { + return err + } + + o.configuration = configuration + o.keyStore = keyStore + return nil +} + +// ensureInitialized initializes the provisioner if a previous attempt failed, +// e.g. because the identity provider was down when step-ca started. Attempts +// are spaced using an exponential backoff, so that requests are not blocked on +// an identity provider that is still unavailable. +func (o *OIDC) ensureInitialized() error { + o.initMutex.Lock() + defer o.initMutex.Unlock() + + if o.initError == nil { + return nil + } + now := time.Now() + if now.Before(o.initRetryAfter) { + return o.initError + } + if o.initError = o.initialize(); o.initError != nil { + o.initRetryAfter = now.Add(o.initRetryInterval) + o.initRetryInterval = min(2*o.initRetryInterval, oidcMaxInitRetryInterval) + } + return o.initError } // ValidatePayload validates the given token payload. @@ -264,6 +332,11 @@ func (o *OIDC) ValidatePayload(p openIDPayload) error { // authorizeToken applies the most common provisioner authorization claims, // leaving the rest to context specific methods. func (o *OIDC) authorizeToken(token string) (*openIDPayload, error) { + if err := o.ensureInitialized(); err != nil { + return nil, errs.Wrapf(http.StatusServiceUnavailable, err, + "oidc.AuthorizeToken; oidc provisioner '%s' is not initialized", o.GetName()) + } + jwt, err := jose.ParseSigned(token) if err != nil { return nil, errs.Wrap(http.StatusUnauthorized, err, diff --git a/authority/provisioner/oidc_test.go b/authority/provisioner/oidc_test.go index a26b87db5..279b7751a 100644 --- a/authority/provisioner/oidc_test.go +++ b/authority/provisioner/oidc_test.go @@ -9,8 +9,11 @@ import ( "errors" "fmt" "net/http" + "net/http/httptest" "net/url" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -156,6 +159,140 @@ func TestOIDC_Init(t *testing.T) { } } +func TestOIDC_Init_unreachableProvider(t *testing.T) { + var reachable atomic.Bool + + srv := httptest.NewUnstartedServer(nil) + handler := generateJWKServerHandler(2, srv) + srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Drop the connection to simulate an identity provider that cannot be + // reached, e.g. one behind a network that is not up yet. + if !reachable.Load() { + if conn, _, err := w.(http.Hijacker).Hijack(); err == nil { + conn.Close() + } + return + } + handler.ServeHTTP(w, r) + }) + srv.Start() + defer srv.Close() + + p := &OIDC{ + Type: "oidc", + Name: "name", + ClientID: "client-id", + ConfigurationEndpoint: srv.URL + "/.well-known/openid-configuration", + } + + // The provisioner cannot be initialized, but the failure is temporary, so + // it is not disabled. + err := p.Init(Config{Claims: globalProvisionerClaims, HTTPClient: srv.Client()}) + require.Error(t, err) + require.ErrorIs(t, err, ErrRetryInit) + + // Tokens cannot be authorized while the identity provider is down. This + // also consumes the first retry. + _, err = p.authorizeToken("token") + require.ErrorContains(t, err, "oidc provisioner 'name' is not initialized") + assert.Equals(t, 2*oidcInitRetryInterval, p.initRetryInterval) + + // The next retries are delayed by the backoff. + reachable.Store(true) + require.Error(t, p.ensureInitialized()) + + // The provisioner initializes itself once the backoff expires. + p.initRetryAfter = time.Now() + require.NoError(t, p.ensureInitialized()) + assert.Len(t, 2, p.keyStore.keySet.Keys) + assert.Equals(t, openIDConfiguration{ + Issuer: "the-issuer", + JWKSetURI: srv.URL + "/jwks_uri", + }, p.configuration) + + // An initialized provisioner does not contact the identity provider again. + reachable.Store(false) + require.NoError(t, p.ensureInitialized()) +} + +func TestOIDC_ensureInitialized_backoff(t *testing.T) { + p := &OIDC{ + Type: "oidc", + Name: "name", + ClientID: "client-id", + // Nothing listens on port 1, so every attempt fails. + ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration", + } + require.ErrorIs(t, p.Init(Config{Claims: globalProvisionerClaims}), ErrRetryInit) + + want := oidcInitRetryInterval + for range 10 { + require.Error(t, p.ensureInitialized()) + want = min(2*want, oidcMaxInitRetryInterval) + assert.Equals(t, want, p.initRetryInterval) + p.initRetryAfter = time.Now() + } + assert.Equals(t, oidcMaxInitRetryInterval, p.initRetryInterval) +} + +func TestOIDC_authorizeToken_concurrentInit(t *testing.T) { + reachable := atomic.Bool{} + reachable.Store(true) + + srv := httptest.NewUnstartedServer(nil) + handler := generateJWKServerHandler(1, srv) + srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !reachable.Load() { + if conn, _, err := w.(http.Hijacker).Hijack(); err == nil { + conn.Close() + } + return + } + handler.ServeHTTP(w, r) + }) + srv.Start() + defer srv.Close() + + var keys jose.JSONWebKeySet + assert.FatalError(t, getAndDecode(srv.Client(), srv.URL+"/private", &keys)) + + p := &OIDC{ + Type: "oidc", + Name: "name", + ClientID: "client-id", + ConfigurationEndpoint: srv.URL + "/.well-known/openid-configuration", + } + token, err := generateSimpleToken("the-issuer", p.ClientID, &keys.Keys[0]) + assert.FatalError(t, err) + + // The identity provider is down when the provisioner is initialized. + reachable.Store(false) + require.ErrorIs(t, p.Init(Config{Claims: globalProvisionerClaims, HTTPClient: srv.Client()}), ErrRetryInit) + + // Authorize tokens while the identity provider comes back, to check that + // the initialization does not race with the requests using its results. + var wg sync.WaitGroup + for range 10 { + wg.Add(1) + go func() { + defer wg.Done() + for range 20 { + // Skip the backoff, so that every request retries. + p.initMutex.Lock() + p.initRetryAfter = time.Time{} + p.initMutex.Unlock() + _, _ = p.authorizeToken(token) + } + }() + } + time.Sleep(10 * time.Millisecond) + reachable.Store(true) + wg.Wait() + + _, err = p.authorizeToken(token) + require.NoError(t, err) +} + func TestOIDC_authorizeToken(t *testing.T) { srv := generateJWKServer(3) defer srv.Close() diff --git a/authority/provisioner/provisioner.go b/authority/provisioner/provisioner.go index ae6e0f978..ef9e2a45a 100644 --- a/authority/provisioner/provisioner.go +++ b/authority/provisioner/provisioner.go @@ -85,6 +85,32 @@ var ErrTokenFlowNotSupported = stderrors.New("token flow is not supported") // ErrNotImplemented is an error returned when one method is not implemented. var ErrNotImplemented = stderrors.New("not implemented") +// ErrRetryInit is an error that the errors returned by Init match when the +// initialization failed for a temporary reason, e.g. an OIDC provisioner that +// cannot reach its identity provider. Provisioners failing with this error +// initialize themselves when they are used, so they are not disabled. +var ErrRetryInit = stderrors.New("retry initialization") + +// retryInitError wraps an initialization error to mark it as temporary. It +// keeps the message of the wrapped error and matches ErrRetryInit. +type retryInitError struct { + err error +} + +func (e *retryInitError) Error() string { return e.err.Error() } + +func (e *retryInitError) Unwrap() error { return e.err } + +func (*retryInitError) Is(target error) bool { return target == ErrRetryInit } + +// retryInit marks an initialization error as temporary. +func retryInit(err error) error { + if err == nil { + return nil + } + return &retryInitError{err: err} +} + // Audiences stores all supported audiences by request type. type Audiences struct { Sign []string