diff --git a/authority/authority.go b/authority/authority.go index 8dd6a0b8e..e3f4c7bcb 100644 --- a/authority/authority.go +++ b/authority/authority.go @@ -273,9 +273,17 @@ func (a *Authority) ReloadAdminResources(ctx context.Context) error { provClxn := provisioner.NewCollection(provisionerConfig.Audiences) for _, p := range provList { if err := p.Init(provisionerConfig); err != nil { - log.Printf("failed to initialize %s provisioner %q: %v\n", p.GetType(), p.GetName(), err) - p = provisioner.Uninitialized{ - Interface: p, Reason: err, + // Provisioners that failed for a temporary reason, e.g. an OIDC + // provisioner that cannot reach its identity provider, stay enabled + // and initialize themselves when they are used. + if errors.Is(err, provisioner.ErrRetryInit) { + log.Printf("failed to initialize %s provisioner %q, it will be retried when the provisioner is used: %v\n", + p.GetType(), p.GetName(), err) + } else { + log.Printf("failed to initialize %s provisioner %q: %v\n", p.GetType(), p.GetName(), err) + p = provisioner.Uninitialized{ + Interface: p, Reason: err, + } } } if err := provClxn.Store(p); err != nil { diff --git a/authority/authority_test.go b/authority/authority_test.go index dca6cf14f..07e161901 100644 --- a/authority/authority_test.go +++ b/authority/authority_test.go @@ -200,6 +200,48 @@ func TestAuthorityNew(t *testing.T) { } } +func TestAuthorityNew_retryInit(t *testing.T) { + c := &Config{ + Address: "127.0.0.1:443", + Root: []string{"testdata/certs/root_ca.crt"}, + IntermediateCert: "testdata/certs/intermediate_ca.crt", + IntermediateKey: "testdata/secrets/intermediate_ca_key", + DNSNames: []string{"example.com"}, + Password: "pass", + AuthorityConfig: &AuthConfig{ + Provisioners: provisioner.List{ + // Nothing listens on port 1, so the identity provider cannot be + // reached and the initialization is retried. + &provisioner.OIDC{ + Name: "oidc", + Type: "OIDC", + ClientID: "client-id", + ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration", + }, + // A provisioner without a client ID can never be initialized. + &provisioner.OIDC{ + Name: "uninitialized", + Type: "OIDC", + ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration", + }, + }, + }, + } + + auth, err := New(c) + assert.FatalError(t, err) + + p, ok := auth.provisioners.LoadByName("oidc") + assert.True(t, ok) + _, isUninitialized := p.(provisioner.Uninitialized) + assert.False(t, isUninitialized) + + p, ok = auth.provisioners.LoadByName("uninitialized") + assert.True(t, ok) + _, isUninitialized = p.(provisioner.Uninitialized) + assert.True(t, isUninitialized) +} + func TestAuthorityNew_bundles(t *testing.T) { ca0, err := minica.New() if err != nil { diff --git a/authority/provisioner/oidc.go b/authority/provisioner/oidc.go index 044971bf6..e9fc7a013 100644 --- a/authority/provisioner/oidc.go +++ b/authority/provisioner/oidc.go @@ -9,6 +9,7 @@ import ( "net/url" "path" "strings" + "sync" "time" "github.com/pkg/errors" @@ -21,6 +22,17 @@ import ( "github.com/smallstep/certificates/errs" ) +const ( + // oidcInitRetryInterval is the time to wait before the second attempt to + // get the OpenID configuration of a provisioner that failed to initialize. + // It doubles on every failure up to oidcMaxInitRetryInterval. + oidcInitRetryInterval = 5 * time.Second + + // oidcMaxInitRetryInterval is the maximum time between two attempts to get + // the OpenID configuration of a provisioner that failed to initialize. + oidcMaxInitRetryInterval = 5 * time.Minute +) + // openIDConfiguration contains the necessary properties in the // `/.well-known/openid-configuration` document. type openIDConfiguration struct { @@ -95,9 +107,17 @@ type OIDC struct { Options *Options `json:"options,omitempty"` Scopes []string `json:"scopes,omitempty"` AuthParams []string `json:"authParams,omitempty"` - configuration openIDConfiguration - keyStore *keyStore + wellKnownEndpoint string ctl *Controller + + // initMutex guards the fields below it, which are only read after a + // successful ensureInitialized. + initMutex sync.Mutex + initError error + initRetryAfter time.Time + initRetryInterval time.Duration + configuration openIDConfiguration + keyStore *keyStore } func sanitizeEmail(email string) string { @@ -156,7 +176,9 @@ func (o *OIDC) GetEncryptedKey() (kid, key string, ok bool) { return "", "", false } -// Init validates and initializes the OIDC provider. +// Init validates and initializes the OIDC provider. If the identity provider +// cannot be reached, the returned error matches ErrRetryInit and the +// provisioner initializes itself the next time that it is used. func (o *OIDC) Init(config Config) (err error) { switch { case o.Type == "": @@ -184,6 +206,7 @@ func (o *OIDC) Init(config Config) (err error) { if !strings.Contains(u.Path, "/.well-known/openid-configuration") { u.Path = path.Join(u.Path, "/.well-known/openid-configuration") } + o.wellKnownEndpoint = u.String() // Initialize the common provisioner controller o.ctl, err = NewController(o, o.Claims, config, o.Options) @@ -191,23 +214,68 @@ func (o *OIDC) Init(config Config) (err error) { return err } - // Decode and validate openid-configuration + // Get the openid-configuration and the JWK key set. Not being able to reach + // the identity provider is usually temporary, e.g. when step-ca starts + // before the network is up, so the provisioner retries when it is used. + o.initMutex.Lock() + defer o.initMutex.Unlock() + if o.initError = o.initialize(); o.initError != nil { + o.initRetryInterval = oidcInitRetryInterval + return retryInit(o.initError) + } + return nil +} + +// initialize gets and validates the openid-configuration and the JWK key set of +// the identity provider. Callers must hold o.initMutex. +func (o *OIDC) initialize() error { httpClient := o.ctl.GetHTTPClient() - if err := getAndDecode(httpClient, u.String(), &o.configuration); err != nil { + + // Decode and validate openid-configuration + var configuration openIDConfiguration + if err := getAndDecode(httpClient, o.wellKnownEndpoint, &configuration); err != nil { return err } - if err := o.configuration.Validate(); err != nil { + if err := configuration.Validate(); err != nil { return errors.Wrapf(err, "error parsing %s", o.ConfigurationEndpoint) } // Replace {tenantid} with the configured one if o.TenantID != "" { - o.configuration.Issuer = strings.ReplaceAll(o.configuration.Issuer, "{tenantid}", o.TenantID) + configuration.Issuer = strings.ReplaceAll(configuration.Issuer, "{tenantid}", o.TenantID) } // Get JWK key set - o.keyStore, err = newKeyStore(httpClient, o.configuration.JWKSetURI) - return + keyStore, err := newKeyStore(httpClient, configuration.JWKSetURI) + if err != nil { + return err + } + + o.configuration = configuration + o.keyStore = keyStore + return nil +} + +// ensureInitialized initializes the provisioner if a previous attempt failed, +// e.g. because the identity provider was down when step-ca started. Attempts +// are spaced using an exponential backoff, so that requests are not blocked on +// an identity provider that is still unavailable. +func (o *OIDC) ensureInitialized() error { + o.initMutex.Lock() + defer o.initMutex.Unlock() + + if o.initError == nil { + return nil + } + now := time.Now() + if now.Before(o.initRetryAfter) { + return o.initError + } + if o.initError = o.initialize(); o.initError != nil { + o.initRetryAfter = now.Add(o.initRetryInterval) + o.initRetryInterval = min(2*o.initRetryInterval, oidcMaxInitRetryInterval) + } + return o.initError } // ValidatePayload validates the given token payload. @@ -264,6 +332,11 @@ func (o *OIDC) ValidatePayload(p openIDPayload) error { // authorizeToken applies the most common provisioner authorization claims, // leaving the rest to context specific methods. func (o *OIDC) authorizeToken(token string) (*openIDPayload, error) { + if err := o.ensureInitialized(); err != nil { + return nil, errs.Wrapf(http.StatusServiceUnavailable, err, + "oidc.AuthorizeToken; oidc provisioner '%s' is not initialized", o.GetName()) + } + jwt, err := jose.ParseSigned(token) if err != nil { return nil, errs.Wrap(http.StatusUnauthorized, err, diff --git a/authority/provisioner/oidc_test.go b/authority/provisioner/oidc_test.go index a26b87db5..279b7751a 100644 --- a/authority/provisioner/oidc_test.go +++ b/authority/provisioner/oidc_test.go @@ -9,8 +9,11 @@ import ( "errors" "fmt" "net/http" + "net/http/httptest" "net/url" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -156,6 +159,140 @@ func TestOIDC_Init(t *testing.T) { } } +func TestOIDC_Init_unreachableProvider(t *testing.T) { + var reachable atomic.Bool + + srv := httptest.NewUnstartedServer(nil) + handler := generateJWKServerHandler(2, srv) + srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Drop the connection to simulate an identity provider that cannot be + // reached, e.g. one behind a network that is not up yet. + if !reachable.Load() { + if conn, _, err := w.(http.Hijacker).Hijack(); err == nil { + conn.Close() + } + return + } + handler.ServeHTTP(w, r) + }) + srv.Start() + defer srv.Close() + + p := &OIDC{ + Type: "oidc", + Name: "name", + ClientID: "client-id", + ConfigurationEndpoint: srv.URL + "/.well-known/openid-configuration", + } + + // The provisioner cannot be initialized, but the failure is temporary, so + // it is not disabled. + err := p.Init(Config{Claims: globalProvisionerClaims, HTTPClient: srv.Client()}) + require.Error(t, err) + require.ErrorIs(t, err, ErrRetryInit) + + // Tokens cannot be authorized while the identity provider is down. This + // also consumes the first retry. + _, err = p.authorizeToken("token") + require.ErrorContains(t, err, "oidc provisioner 'name' is not initialized") + assert.Equals(t, 2*oidcInitRetryInterval, p.initRetryInterval) + + // The next retries are delayed by the backoff. + reachable.Store(true) + require.Error(t, p.ensureInitialized()) + + // The provisioner initializes itself once the backoff expires. + p.initRetryAfter = time.Now() + require.NoError(t, p.ensureInitialized()) + assert.Len(t, 2, p.keyStore.keySet.Keys) + assert.Equals(t, openIDConfiguration{ + Issuer: "the-issuer", + JWKSetURI: srv.URL + "/jwks_uri", + }, p.configuration) + + // An initialized provisioner does not contact the identity provider again. + reachable.Store(false) + require.NoError(t, p.ensureInitialized()) +} + +func TestOIDC_ensureInitialized_backoff(t *testing.T) { + p := &OIDC{ + Type: "oidc", + Name: "name", + ClientID: "client-id", + // Nothing listens on port 1, so every attempt fails. + ConfigurationEndpoint: "http://127.0.0.1:1/.well-known/openid-configuration", + } + require.ErrorIs(t, p.Init(Config{Claims: globalProvisionerClaims}), ErrRetryInit) + + want := oidcInitRetryInterval + for range 10 { + require.Error(t, p.ensureInitialized()) + want = min(2*want, oidcMaxInitRetryInterval) + assert.Equals(t, want, p.initRetryInterval) + p.initRetryAfter = time.Now() + } + assert.Equals(t, oidcMaxInitRetryInterval, p.initRetryInterval) +} + +func TestOIDC_authorizeToken_concurrentInit(t *testing.T) { + reachable := atomic.Bool{} + reachable.Store(true) + + srv := httptest.NewUnstartedServer(nil) + handler := generateJWKServerHandler(1, srv) + srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !reachable.Load() { + if conn, _, err := w.(http.Hijacker).Hijack(); err == nil { + conn.Close() + } + return + } + handler.ServeHTTP(w, r) + }) + srv.Start() + defer srv.Close() + + var keys jose.JSONWebKeySet + assert.FatalError(t, getAndDecode(srv.Client(), srv.URL+"/private", &keys)) + + p := &OIDC{ + Type: "oidc", + Name: "name", + ClientID: "client-id", + ConfigurationEndpoint: srv.URL + "/.well-known/openid-configuration", + } + token, err := generateSimpleToken("the-issuer", p.ClientID, &keys.Keys[0]) + assert.FatalError(t, err) + + // The identity provider is down when the provisioner is initialized. + reachable.Store(false) + require.ErrorIs(t, p.Init(Config{Claims: globalProvisionerClaims, HTTPClient: srv.Client()}), ErrRetryInit) + + // Authorize tokens while the identity provider comes back, to check that + // the initialization does not race with the requests using its results. + var wg sync.WaitGroup + for range 10 { + wg.Add(1) + go func() { + defer wg.Done() + for range 20 { + // Skip the backoff, so that every request retries. + p.initMutex.Lock() + p.initRetryAfter = time.Time{} + p.initMutex.Unlock() + _, _ = p.authorizeToken(token) + } + }() + } + time.Sleep(10 * time.Millisecond) + reachable.Store(true) + wg.Wait() + + _, err = p.authorizeToken(token) + require.NoError(t, err) +} + func TestOIDC_authorizeToken(t *testing.T) { srv := generateJWKServer(3) defer srv.Close() diff --git a/authority/provisioner/provisioner.go b/authority/provisioner/provisioner.go index ae6e0f978..ef9e2a45a 100644 --- a/authority/provisioner/provisioner.go +++ b/authority/provisioner/provisioner.go @@ -85,6 +85,32 @@ var ErrTokenFlowNotSupported = stderrors.New("token flow is not supported") // ErrNotImplemented is an error returned when one method is not implemented. var ErrNotImplemented = stderrors.New("not implemented") +// ErrRetryInit is an error that the errors returned by Init match when the +// initialization failed for a temporary reason, e.g. an OIDC provisioner that +// cannot reach its identity provider. Provisioners failing with this error +// initialize themselves when they are used, so they are not disabled. +var ErrRetryInit = stderrors.New("retry initialization") + +// retryInitError wraps an initialization error to mark it as temporary. It +// keeps the message of the wrapped error and matches ErrRetryInit. +type retryInitError struct { + err error +} + +func (e *retryInitError) Error() string { return e.err.Error() } + +func (e *retryInitError) Unwrap() error { return e.err } + +func (*retryInitError) Is(target error) bool { return target == ErrRetryInit } + +// retryInit marks an initialization error as temporary. +func retryInit(err error) error { + if err == nil { + return nil + } + return &retryInitError{err: err} +} + // Audiences stores all supported audiences by request type. type Audiences struct { Sign []string