Retry OIDC provisioner initialization instead of disabling it

authority/authority.go

@@ -273,9 +273,17 @@ func (a *Authority) ReloadAdminResources(ctx context.Context) error {
 	provClxn := provisioner.NewCollection(provisionerConfig.Audiences)
 	for _, p := range provList {
 		if err := p.Init(provisionerConfig); err != nil {
-			log.Printf("failed to initialize %s provisioner %q: %v\n", p.GetType(), p.GetName(), err)
-			p = provisioner.Uninitialized{
-				Interface: p, Reason: err,
+			// Provisioners that failed for a temporary reason, e.g. an OIDC
+			// provisioner that cannot reach its identity provider, stay enabled
+			// and initialize themselves when they are used.
+			if errors.Is(err, provisioner.ErrRetryInit) {
+				log.Printf("failed to initialize %s provisioner %q, it will be retried when the provisioner is used: %v\n",
+					p.GetType(), p.GetName(), err)
+			} else {
+				log.Printf("failed to initialize %s provisioner %q: %v\n", p.GetType(), p.GetName(), err)
+				p = provisioner.Uninitialized{
+					Interface: p, Reason: err,
+				}
 			}
 		}
 		if err := provClxn.Store(p); err != nil {

authority/authority_test.go

@@ -200,6 +200,48 @@ func TestAuthorityNew(t *testing.T) {
 	}
 }
 
+func TestAuthorityNew_retryInit(t *testing.T) {
+	c := &Config{
+		Address:          "127.0.0.1:443",
+		Root:             []string{"testdata/certs/root_ca.crt"},
+		IntermediateCert: "testdata/certs/intermediate_ca.crt",
+		IntermediateKey:  "testdata/secrets/intermediate_ca_key",
+		DNSNames:         []string{"example.com"},
+		Password:         "pass",
+		AuthorityConfig: &AuthConfig{
+			Provisioners: provisioner.List{
+				// Nothing listens on port 1, so the identity provider cannot be
+				// reached and the initialization is retried.
+				&provisioner.OIDC{
+					Name:                  "oidc",
+					Type:                  "OIDC",
+					ClientID:              "client-id",
+					ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration",
+				},
+				// A provisioner without a client ID can never be initialized.
+				&provisioner.OIDC{
+					Name:                  "uninitialized",
+					Type:                  "OIDC",
+					ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration",
+				},
+			},
+		},
+	}
+
+	auth, err := New(c)
+	assert.FatalError(t, err)
+
+	p, ok := auth.provisioners.LoadByName("oidc")
+	assert.True(t, ok)
+	_, isUninitialized := p.(provisioner.Uninitialized)
+	assert.False(t, isUninitialized)
+
+	p, ok = auth.provisioners.LoadByName("uninitialized")
+	assert.True(t, ok)
+	_, isUninitialized = p.(provisioner.Uninitialized)
+	assert.True(t, isUninitialized)
+}
+
 func TestAuthorityNew_bundles(t *testing.T) {
 	ca0, err := minica.New()
 	if err != nil {

authority/provisioner/oidc.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"path"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -21,6 +22,17 @@ import (
 	"github.com/smallstep/certificates/errs"
 )
 
+const (
+	// oidcInitRetryInterval is the time to wait before the second attempt to
+	// get the OpenID configuration of a provisioner that failed to initialize.
+	// It doubles on every failure up to oidcMaxInitRetryInterval.
+	oidcInitRetryInterval = 5 * time.Second
+
+	// oidcMaxInitRetryInterval is the maximum time between two attempts to get
+	// the OpenID configuration of a provisioner that failed to initialize.
+	oidcMaxInitRetryInterval = 5 * time.Minute
+)
+
 // openIDConfiguration contains the necessary properties in the
 // `/.well-known/openid-configuration` document.
 type openIDConfiguration struct {
@@ -95,9 +107,17 @@ type OIDC struct {
 	Options               *Options `json:"options,omitempty"`
 	Scopes                []string `json:"scopes,omitempty"`
 	AuthParams            []string `json:"authParams,omitempty"`
-	configuration         openIDConfiguration
-	keyStore              *keyStore
+	wellKnownEndpoint     string
 	ctl                   *Controller
+
+	// initMutex guards the fields below it, which are only read after a
+	// successful ensureInitialized.
+	initMutex         sync.Mutex
+	initError         error
+	initRetryAfter    time.Time
+	initRetryInterval time.Duration
+	configuration     openIDConfiguration
+	keyStore          *keyStore
 }
 
 func sanitizeEmail(email string) string {
@@ -156,7 +176,9 @@ func (o *OIDC) GetEncryptedKey() (kid, key string, ok bool) {
 	return "", "", false
 }
 
-// Init validates and initializes the OIDC provider.
+// Init validates and initializes the OIDC provider. If the identity provider
+// cannot be reached, the returned error matches ErrRetryInit and the
+// provisioner initializes itself the next time that it is used.
 func (o *OIDC) Init(config Config) (err error) {
 	switch {
 	case o.Type == "":
@@ -184,30 +206,76 @@ func (o *OIDC) Init(config Config) (err error) {
 	if !strings.Contains(u.Path, "/.well-known/openid-configuration") {
 		u.Path = path.Join(u.Path, "/.well-known/openid-configuration")
 	}
+	o.wellKnownEndpoint = u.String()
 
 	// Initialize the common provisioner controller
 	o.ctl, err = NewController(o, o.Claims, config, o.Options)
 	if err != nil {
 		return err
 	}
 
-	// Decode and validate openid-configuration
+	// Get the openid-configuration and the JWK key set. Not being able to reach
+	// the identity provider is usually temporary, e.g. when step-ca starts
+	// before the network is up, so the provisioner retries when it is used.
+	o.initMutex.Lock()
+	defer o.initMutex.Unlock()
+	if o.initError = o.initialize(); o.initError != nil {
+		o.initRetryInterval = oidcInitRetryInterval
+		return retryInit(o.initError)
+	}
+	return nil
+}
+
+// initialize gets and validates the openid-configuration and the JWK key set of
+// the identity provider. Callers must hold o.initMutex.
+func (o *OIDC) initialize() error {
 	httpClient := o.ctl.GetHTTPClient()
-	if err := getAndDecode(httpClient, u.String(), &o.configuration); err != nil {
+
+	// Decode and validate openid-configuration
+	var configuration openIDConfiguration
+	if err := getAndDecode(httpClient, o.wellKnownEndpoint, &configuration); err != nil {
 		return err
 	}
-	if err := o.configuration.Validate(); err != nil {
+	if err := configuration.Validate(); err != nil {
 		return errors.Wrapf(err, "error parsing %s", o.ConfigurationEndpoint)
 	}
 
 	// Replace {tenantid} with the configured one
 	if o.TenantID != "" {
-		o.configuration.Issuer = strings.ReplaceAll(o.configuration.Issuer, "{tenantid}", o.TenantID)
+		configuration.Issuer = strings.ReplaceAll(configuration.Issuer, "{tenantid}", o.TenantID)
 	}
 
 	// Get JWK key set
-	o.keyStore, err = newKeyStore(httpClient, o.configuration.JWKSetURI)
-	return
+	keyStore, err := newKeyStore(httpClient, configuration.JWKSetURI)
+	if err != nil {
+		return err
+	}
+
+	o.configuration = configuration
+	o.keyStore = keyStore
+	return nil
+}
+
+// ensureInitialized initializes the provisioner if a previous attempt failed,
+// e.g. because the identity provider was down when step-ca started. Attempts
+// are spaced using an exponential backoff, so that requests are not blocked on
+// an identity provider that is still unavailable.
+func (o *OIDC) ensureInitialized() error {
+	o.initMutex.Lock()
+	defer o.initMutex.Unlock()
+
+	if o.initError == nil {
+		return nil
+	}
+	now := time.Now()
+	if now.Before(o.initRetryAfter) {
+		return o.initError
+	}
+	if o.initError = o.initialize(); o.initError != nil {
+		o.initRetryAfter = now.Add(o.initRetryInterval)
+		o.initRetryInterval = min(2*o.initRetryInterval, oidcMaxInitRetryInterval)
+	}
+	return o.initError
 }
 
 // ValidatePayload validates the given token payload.
@@ -264,6 +332,11 @@ func (o *OIDC) ValidatePayload(p openIDPayload) error {
 // authorizeToken applies the most common provisioner authorization claims,
 // leaving the rest to context specific methods.
 func (o *OIDC) authorizeToken(token string) (*openIDPayload, error) {
+	if err := o.ensureInitialized(); err != nil {
+		return nil, errs.Wrapf(http.StatusServiceUnavailable, err,
+			"oidc.AuthorizeToken; oidc provisioner '%s' is not initialized", o.GetName())
+	}
+
 	jwt, err := jose.ParseSigned(token)
 	if err != nil {
 		return nil, errs.Wrap(http.StatusUnauthorized, err,

authority/provisioner/oidc_test.go

@@ -9,8 +9,11 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -156,6 +159,140 @@ func TestOIDC_Init(t *testing.T) {
 	}
 }
 
+func TestOIDC_Init_unreachableProvider(t *testing.T) {
+	var reachable atomic.Bool
+
+	srv := httptest.NewUnstartedServer(nil)
+	handler := generateJWKServerHandler(2, srv)
+	srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Drop the connection to simulate an identity provider that cannot be
+		// reached, e.g. one behind a network that is not up yet.
+		if !reachable.Load() {
+			if conn, _, err := w.(http.Hijacker).Hijack(); err == nil {
+				conn.Close()
+			}
+			return
+		}
+		handler.ServeHTTP(w, r)
+	})
+	srv.Start()
+	defer srv.Close()
+
+	p := &OIDC{
+		Type:                  "oidc",
+		Name:                  "name",
+		ClientID:              "client-id",
+		ConfigurationEndpoint: srv.URL + "/.well-known/openid-configuration",
+	}
+
+	// The provisioner cannot be initialized, but the failure is temporary, so
+	// it is not disabled.
+	err := p.Init(Config{Claims: globalProvisionerClaims, HTTPClient: srv.Client()})
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrRetryInit)
+
+	// Tokens cannot be authorized while the identity provider is down. This
+	// also consumes the first retry.
+	_, err = p.authorizeToken("token")
+	require.ErrorContains(t, err, "oidc provisioner 'name' is not initialized")
+	assert.Equals(t, 2*oidcInitRetryInterval, p.initRetryInterval)
+
+	// The next retries are delayed by the backoff.
+	reachable.Store(true)
+	require.Error(t, p.ensureInitialized())
+
+	// The provisioner initializes itself once the backoff expires.
+	p.initRetryAfter = time.Now()
+	require.NoError(t, p.ensureInitialized())
+	assert.Len(t, 2, p.keyStore.keySet.Keys)
+	assert.Equals(t, openIDConfiguration{
+		Issuer:    "the-issuer",
+		JWKSetURI: srv.URL + "/jwks_uri",
+	}, p.configuration)
+
+	// An initialized provisioner does not contact the identity provider again.
+	reachable.Store(false)
+	require.NoError(t, p.ensureInitialized())
+}
+
+func TestOIDC_ensureInitialized_backoff(t *testing.T) {
+	p := &OIDC{
+		Type:     "oidc",
+		Name:     "name",
+		ClientID: "client-id",
+		// Nothing listens on port 1, so every attempt fails.
+		ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration",
+	}
+	require.ErrorIs(t, p.Init(Config{Claims: globalProvisionerClaims}), ErrRetryInit)
+
+	want := oidcInitRetryInterval
+	for range 10 {
+		require.Error(t, p.ensureInitialized())
+		want = min(2*want, oidcMaxInitRetryInterval)
+		assert.Equals(t, want, p.initRetryInterval)
+		p.initRetryAfter = time.Now()
+	}
+	assert.Equals(t, oidcMaxInitRetryInterval, p.initRetryInterval)
+}
+
+func TestOIDC_authorizeToken_concurrentInit(t *testing.T) {
+	reachable := atomic.Bool{}
+	reachable.Store(true)
+
+	srv := httptest.NewUnstartedServer(nil)
+	handler := generateJWKServerHandler(1, srv)
+	srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !reachable.Load() {
+			if conn, _, err := w.(http.Hijacker).Hijack(); err == nil {
+				conn.Close()
+			}
+			return
+		}
+		handler.ServeHTTP(w, r)
+	})
+	srv.Start()
+	defer srv.Close()
+
+	var keys jose.JSONWebKeySet
+	assert.FatalError(t, getAndDecode(srv.Client(), srv.URL+"/private", &keys))
+
+	p := &OIDC{
+		Type:                  "oidc",
+		Name:                  "name",
+		ClientID:              "client-id",
+		ConfigurationEndpoint: srv.URL + "/.well-known/openid-configuration",
+	}
+	token, err := generateSimpleToken("the-issuer", p.ClientID, &keys.Keys[0])
+	assert.FatalError(t, err)
+
+	// The identity provider is down when the provisioner is initialized.
+	reachable.Store(false)
+	require.ErrorIs(t, p.Init(Config{Claims: globalProvisionerClaims, HTTPClient: srv.Client()}), ErrRetryInit)
+
+	// Authorize tokens while the identity provider comes back, to check that
+	// the initialization does not race with the requests using its results.
+	var wg sync.WaitGroup
+	for range 10 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for range 20 {
+				// Skip the backoff, so that every request retries.
+				p.initMutex.Lock()
+				p.initRetryAfter = time.Time{}
+				p.initMutex.Unlock()
+				_, _ = p.authorizeToken(token)
+			}
+		}()
+	}
+	time.Sleep(10 * time.Millisecond)
+	reachable.Store(true)
+	wg.Wait()
+
+	_, err = p.authorizeToken(token)
+	require.NoError(t, err)
+}
+
 func TestOIDC_authorizeToken(t *testing.T) {
 	srv := generateJWKServer(3)
 	defer srv.Close()