Fix race condition between service account availability and webhook invocation

Jan Roehrich · Jan Roehrich · commit 79fcca149d7d · 2024-08-13T22:40:14.000+02:00
diff --git a/main.go b/main.go
@@ -86,6 +86,8 @@ func main() {
 
 	debug := flag.Bool("enable-debugging-handlers", false, "Enable debugging handlers. Currently /debug/alpha/cache is supported")
 
+	saLookupGracePeriod := flag.Duration("service-account-lookup-grace-period", 100*time.Millisecond, "The grace period for service account to be available in cache before not mutating a pod. Defaults to 100ms. Set to 0 to deactivate waiting. Carefully use higher values as it may have significant impact on Kubernetes' pod scheduling performance.")
+
 	klog.InitFlags(goflag.CommandLine)
 	// Add klog CommandLine flags to pflag CommandLine
 	goflag.CommandLine.VisitAll(func(f *goflag.Flag) {
@@ -208,6 +210,7 @@ func main() {
 		handler.WithServiceAccountCache(saCache),
 		handler.WithContainerCredentialsConfig(containerCredentialsConfig),
 		handler.WithRegion(*region),
+		handler.WithSALookupGraceTime(*saLookupGracePeriod),
 	)
 
 	addr := fmt.Sprintf(":%d", *port)
diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go
@@ -42,7 +42,8 @@ type CacheResponse struct {
 
 type ServiceAccountCache interface {
 	Start(stop chan struct{})
-	Get(name, namespace string) (role, aud string, useRegionalSTS bool, tokenExpiration int64)
+	Get(name, namespace string) (role, aud string, useRegionalSTS bool, tokenExpiration int64, found bool)
+	GetOrNotify(name, namespace string, handler chan any) (role, aud string, useRegionalSTS bool, tokenExpiration int64, found bool)
 	GetCommonConfigurations(name, namespace string) (useRegionalSTS bool, tokenExpiration int64)
 	// ToJSON returns cache contents as JSON string
 	ToJSON() string
@@ -60,6 +61,7 @@ type serviceAccountCache struct {
 	composeRoleArn         ComposeRoleArn
 	defaultTokenExpiration int64
 	webhookUsage           prometheus.Gauge
+	notificationHandlers   map[string]chan any // type of channel doesn't matter. It's just for being notified
 }
 
 type ComposeRoleArn struct {
@@ -87,41 +89,51 @@ func init() {
 // Get will return the cached configuration of the given ServiceAccount.
 // It will first look at the set of ServiceAccounts configured using annotations. If none are found, it will look for any
 // ServiceAccount configured through the pod-identity-webhook ConfigMap.
-func (c *serviceAccountCache) Get(name, namespace string) (role, aud string, useRegionalSTS bool, tokenExpiration int64) {
+func (c *serviceAccountCache) Get(name, namespace string) (role, aud string, useRegionalSTS bool, tokenExpiration int64, found bool) {
+	return c.GetOrNotify(name, namespace, nil)
+}
+
+// GetOrNotify will return the cached configuration of the given ServiceAccount.
+// It will first look at the set of ServiceAccounts configured using annotations. If none is found, it will register
+// handler to be notified as soon as a ServiceAccount with given key is populated to the cache. Afterwards it will check
+// for a ServiceAccount configured through the pod-identity-webhook ConfigMap.
+func (c *serviceAccountCache) GetOrNotify(name, namespace string, handler chan any) (role, aud string, useRegionalSTS bool, tokenExpiration int64, found bool) {
 	klog.V(5).Infof("Fetching sa %s/%s from cache", namespace, name)
 	{
-		resp := c.getSA(name, namespace)
+		resp := c.getSAorNotify(name, namespace, handler)
 		if resp != nil && resp.RoleARN != "" {
-			return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration
+			return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration, true
 		}
 	}
 	{
 		resp := c.getCM(name, namespace)
 		if resp != nil {
-			return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration
+			return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration, true
 		}
 	}
 	klog.V(5).Infof("Service account %s/%s not found in cache", namespace, name)
-	return "", "", false, pkg.DefaultTokenExpiration
+	return "", "", false, pkg.DefaultTokenExpiration, false
 }
 
 // GetCommonConfigurations returns the common configurations that also applies to the new mutation method(i.e Container Credentials).
 // The config file for the container credentials does not contain "TokenExpiration" or "UseRegionalSTS". For backward compatibility,
 // Use these fields if they are set in the sa annotations or config map.
 func (c *serviceAccountCache) GetCommonConfigurations(name, namespace string) (useRegionalSTS bool, tokenExpiration int64) {
-	if resp := c.getSA(name, namespace); resp != nil {
+	if resp := c.getSAorNotify(name, namespace, nil); resp != nil {
 		return resp.UseRegionalSTS, resp.TokenExpiration
 	} else if resp := c.getCM(name, namespace); resp != nil {
 		return resp.UseRegionalSTS, resp.TokenExpiration
 	}
 	return false, pkg.DefaultTokenExpiration
 }
 
-func (c *serviceAccountCache) getSA(name, namespace string) *CacheResponse {
+func (c *serviceAccountCache) getSAorNotify(name, namespace string, handler chan any) *CacheResponse {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 	resp, ok := c.saCache[namespace+"/"+name]
-	if !ok {
+	if !ok && handler != nil {
+		klog.V(5).Infof("Service Account %s/%s not found in cache, adding notification handler", namespace, name)
+		c.notificationHandlers[namespace+"/"+name] = handler
 		return nil
 	}
 	return resp
@@ -212,8 +224,16 @@ func (c *serviceAccountCache) addSA(sa *v1.ServiceAccount) {
 func (c *serviceAccountCache) setSA(name, namespace string, resp *CacheResponse) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	klog.V(5).Infof("Adding SA %s/%s to SA cache: %+v", namespace, name, resp)
+
+	key := namespace + "/" + name
+	klog.V(5).Infof("Adding SA %q to SA cache: %+v", key, resp)
 	c.saCache[namespace+"/"+name] = resp
+
+	if handler, found := c.notificationHandlers[key]; found {
+		klog.V(5).Infof("Notifying handler for %q", key)
+		handler <- 1
+		delete(c.notificationHandlers, key)
+	}
 }
 
 func (c *serviceAccountCache) setCM(name, namespace string, resp *CacheResponse) {
@@ -242,6 +262,7 @@ func New(defaultAudience, prefix string, defaultRegionalSTS bool, defaultTokenEx
 		defaultTokenExpiration: defaultTokenExpiration,
 		hasSynced:              hasSynced,
 		webhookUsage:           webhookUsage,
+		notificationHandlers:   map[string]chan any{},
 	}
 
 	saInformer.Informer().AddEventHandler(
diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go
@@ -36,16 +36,18 @@ func TestSaCache(t *testing.T) {
 		webhookUsage:     prometheus.NewGauge(prometheus.GaugeOpts{}),
 	}
 
-	role, aud, useRegionalSTS, tokenExpiration := cache.Get("default", "default")
+	role, aud, useRegionalSTS, tokenExpiration, found := cache.Get("default", "default")
 
+	assert.False(t, found, "Expected no cache entry to be found")
 	if role != "" || aud != "" {
 		t.Errorf("Expected role and aud to be empty, got %s, %s, %t, %d", role, aud, useRegionalSTS, tokenExpiration)
 	}
 
 	cache.addSA(testSA)
 
-	role, aud, useRegionalSTS, tokenExpiration = cache.Get("default", "default")
+	role, aud, useRegionalSTS, tokenExpiration, found = cache.Get("default", "default")
 
+	assert.True(t, found, "Expected cache entry to be found")
 	assert.Equal(t, roleArn, role, "Expected role to be %s, got %s", roleArn, role)
 	assert.Equal(t, "sts.amazonaws.com", aud, "Expected aud to be sts.amzonaws.com, got %s", aud)
 	assert.True(t, useRegionalSTS, "Expected regional STS to be true, got false")
@@ -157,7 +159,8 @@ func TestNonRegionalSTS(t *testing.T) {
 				t.Fatalf("cache never called addSA: %v", err)
 			}
 
-			gotRoleArn, gotAudience, useRegionalSTS, gotTokenExpiration := cache.Get("default", "default")
+			gotRoleArn, gotAudience, useRegionalSTS, gotTokenExpiration, found := cache.Get("default", "default")
+			assert.True(t, found, "Expected cache entry to be found")
 			if gotRoleArn != roleArn {
 				t.Errorf("got roleArn %v, expected %v", gotRoleArn, roleArn)
 			}
@@ -202,7 +205,7 @@ func TestPopulateCacheFromCM(t *testing.T) {
 			t.Errorf("failed to build cache: %v", err)
 		}
 
-		role, _, _, _ := c.Get("mysa2", "myns2")
+		role, _, _, _, _ := c.Get("mysa2", "myns2")
 		if role == "" {
 			t.Errorf("cloud not find entry that should have been added")
 		}
@@ -214,7 +217,7 @@ func TestPopulateCacheFromCM(t *testing.T) {
 			t.Errorf("failed to build cache: %v", err)
 		}
 
-		role, _, _, _ := c.Get("mysa2", "myns2")
+		role, _, _, _, _ := c.Get("mysa2", "myns2")
 		if role == "" {
 			t.Errorf("cloud not find entry that should have been added")
 		}
@@ -226,7 +229,7 @@ func TestPopulateCacheFromCM(t *testing.T) {
 			t.Errorf("failed to build cache: %v", err)
 		}
 
-		role, _, _, _ := c.Get("mysa2", "myns2")
+		role, _, _, _, _ := c.Get("mysa2", "myns2")
 		if role != "" {
 			t.Errorf("found entry that should have been removed")
 		}
@@ -256,7 +259,7 @@ func TestSAAnnotationRemoval(t *testing.T) {
 	c.addSA(oldSA)
 
 	{
-		gotRoleArn, _, _, _ := c.Get("default", "default")
+		gotRoleArn, _, _, _, _ := c.Get("default", "default")
 		if gotRoleArn != roleArn {
 			t.Errorf("got roleArn %q, expected %q", gotRoleArn, roleArn)
 		}
@@ -268,7 +271,7 @@ func TestSAAnnotationRemoval(t *testing.T) {
 	c.addSA(newSA)
 
 	{
-		gotRoleArn, _, _, _ := c.Get("default", "default")
+		gotRoleArn, _, _, _, _ := c.Get("default", "default")
 		if gotRoleArn != "" {
 			t.Errorf("got roleArn %v, expected %q", gotRoleArn, "")
 		}
@@ -323,7 +326,7 @@ func TestCachePrecedence(t *testing.T) {
 			t.Errorf("failed to build cache: %v", err)
 		}
 
-		role, _, _, exp := c.Get("mysa2", "myns2")
+		role, _, _, exp, _ := c.Get("mysa2", "myns2")
 		if role == "" {
 			t.Errorf("could not find entry that should have been added")
 		}
@@ -340,7 +343,7 @@ func TestCachePrecedence(t *testing.T) {
 		}
 
 		// Removing sa2 from CM, but SA still exists
-		role, _, _, exp := c.Get("mysa2", "myns2")
+		role, _, _, exp, _ := c.Get("mysa2", "myns2")
 		if role == "" {
 			t.Errorf("could not find entry that should still exist")
 		}
@@ -356,7 +359,7 @@ func TestCachePrecedence(t *testing.T) {
 		c.addSA(sa2)
 
 		// Neither cache should return any hits now
-		role, _, _, _ := c.Get("myns2", "mysa2")
+		role, _, _, _, _ := c.Get("myns2", "mysa2")
 		if role != "" {
 			t.Errorf("found entry that should not exist")
 		}
@@ -370,7 +373,7 @@ func TestCachePrecedence(t *testing.T) {
 			t.Errorf("failed to build cache: %v", err)
 		}
 
-		role, _, _, exp := c.Get("mysa2", "myns2")
+		role, _, _, exp, _ := c.Get("mysa2", "myns2")
 		if role == "" {
 			t.Errorf("cloud not find entry that should have been added")
 		}
@@ -422,7 +425,7 @@ func TestRoleArnComposition(t *testing.T) {
 
 	var roleArn string
 	err := wait.ExponentialBackoff(wait.Backoff{Duration: 10 * time.Millisecond, Factor: 1.0, Steps: 3}, func() (bool, error) {
-		roleArn, _, _, _ = cache.Get("default", "default")
+		roleArn, _, _, _, _ = cache.Get("default", "default")
 		return roleArn != "", nil
 	})
 	if err != nil {
diff --git a/pkg/cache/fake.go b/pkg/cache/fake.go
@@ -44,14 +44,25 @@ var _ ServiceAccountCache = &FakeServiceAccountCache{}
 func (f *FakeServiceAccountCache) Start(chan struct{}) {}
 
 // Get gets a service account from the cache
-func (f *FakeServiceAccountCache) Get(name, namespace string) (role, aud string, useRegionalSTS bool, tokenExpiration int64) {
+func (f *FakeServiceAccountCache) Get(name, namespace string) (role, aud string, useRegionalSTS bool, tokenExpiration int64, found bool) {
 	f.mu.RLock()
 	defer f.mu.RUnlock()
 	resp, ok := f.cache[namespace+"/"+name]
 	if !ok {
-		return "", "", false, pkg.DefaultTokenExpiration
+		return "", "", false, pkg.DefaultTokenExpiration, false
 	}
-	return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration
+	return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration, true
+}
+
+// GetOrNotify gets a service account from the cache
+func (f *FakeServiceAccountCache) GetOrNotify(name, namespace string, handler chan any) (role, aud string, useRegionalSTS bool, tokenExpiration int64, found bool) {
+	f.mu.RLock()
+	defer f.mu.RUnlock()
+	resp, ok := f.cache[namespace+"/"+name]
+	if !ok {
+		return "", "", false, pkg.DefaultTokenExpiration, false
+	}
+	return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration, true
 }
 
 func (f *FakeServiceAccountCache) GetCommonConfigurations(name, namespace string) (useRegionalSTS bool, tokenExpiration int64) {
diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go
@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/aws/amazon-eks-pod-identity-webhook/pkg/containercredentials"
 
@@ -77,6 +78,12 @@ func WithAnnotationDomain(domain string) ModifierOpt {
 	return func(m *Modifier) { m.AnnotationDomain = domain }
 }
 
+// WithSALookupGraceTime sets the grace time to wait for service accounts to appear in cache
+func WithSALookupGraceTime(saLookupGraceTime time.Duration) ModifierOpt {
+	return func(m *Modifier) { m.saLookupGraceTime = saLookupGraceTime }
+
+}
+
 // NewModifier returns a Modifier with default values
 func NewModifier(opts ...ModifierOpt) *Modifier {
 	mod := &Modifier{
@@ -101,6 +108,7 @@ type Modifier struct {
 	ContainerCredentialsConfig containercredentials.Config
 	volName                    string
 	tokenName                  string
+	saLookupGraceTime          time.Duration
 }
 
 type patchOperation struct {
@@ -425,7 +433,24 @@ func (m *Modifier) buildPodPatchConfig(pod *corev1.Pod) *podPatchConfig {
 	}
 
 	// Use the STS WebIdentity method if set
-	roleArn, audience, regionalSTS, tokenExpiration := m.Cache.Get(pod.Spec.ServiceAccountName, pod.Namespace)
+	handler := make(chan any, 1)
+	roleArn, audience, regionalSTS, tokenExpiration, found := m.Cache.GetOrNotify(pod.Spec.ServiceAccountName, pod.Namespace, handler)
+	key := pod.Namespace + "/" + pod.Spec.ServiceAccountName
+	if !found && m.saLookupGraceTime > 0 {
+		klog.Warningf("Service account %q not found in the cache. Waiting up to %s to be notified", key, m.saLookupGraceTime)
+		select {
+		case <-handler:
+			roleArn, audience, regionalSTS, tokenExpiration, found = m.Cache.Get(pod.Spec.ServiceAccountName, pod.Namespace)
+			if !found {
+				klog.Warningf("Service account %q not found in the cache after being notified. Not mutating.", key)
+				return nil
+			}
+		case <-time.After(m.saLookupGraceTime):
+			klog.Warningf("Service account %q not found in the cache after %s. Not mutating.", key, m.saLookupGraceTime)
+			return nil
+		}
+	}
+	klog.V(5).Infof("Value of roleArn after after cache retrieval for service account %q: %s", key, roleArn)
 	if roleArn != "" {
 		tokenExpiration, containersToSkip := m.parsePodAnnotations(pod, tokenExpiration)
 

Original file line number	Diff line number	Diff line change
`@@ -36,16 +36,18 @@ func TestSaCache(t *testing.T) {`
`36`	`36`	`webhookUsage: prometheus.NewGauge(prometheus.GaugeOpts{}),`
`37`	`37`	`}`
`38`	`38`
`39`		`- role, aud, useRegionalSTS, tokenExpiration := cache.Get("default", "default")`
	`39`	`+ role, aud, useRegionalSTS, tokenExpiration, found := cache.Get("default", "default")`
`40`	`40`
	`41`	`+ assert.False(t, found, "Expected no cache entry to be found")`
`41`	`42`	`if role != "" \|\| aud != "" {`
`42`	`43`	`t.Errorf("Expected role and aud to be empty, got %s, %s, %t, %d", role, aud, useRegionalSTS, tokenExpiration)`
`43`	`44`	`}`
`44`	`45`
`45`	`46`	`cache.addSA(testSA)`
`46`	`47`
`47`		`- role, aud, useRegionalSTS, tokenExpiration = cache.Get("default", "default")`
	`48`	`+ role, aud, useRegionalSTS, tokenExpiration, found = cache.Get("default", "default")`
`48`	`49`
	`50`	`+ assert.True(t, found, "Expected cache entry to be found")`
`49`	`51`	`assert.Equal(t, roleArn, role, "Expected role to be %s, got %s", roleArn, role)`
`50`	`52`	`assert.Equal(t, "sts.amazonaws.com", aud, "Expected aud to be sts.amzonaws.com, got %s", aud)`
`51`	`53`	`assert.True(t, useRegionalSTS, "Expected regional STS to be true, got false")`
`@@ -157,7 +159,8 @@ func TestNonRegionalSTS(t *testing.T) {`
`157`	`159`	`t.Fatalf("cache never called addSA: %v", err)`
`158`	`160`	`}`
`159`	`161`
`160`		`- gotRoleArn, gotAudience, useRegionalSTS, gotTokenExpiration := cache.Get("default", "default")`
	`162`	`+ gotRoleArn, gotAudience, useRegionalSTS, gotTokenExpiration, found := cache.Get("default", "default")`
	`163`	`+ assert.True(t, found, "Expected cache entry to be found")`
`161`	`164`	`if gotRoleArn != roleArn {`
`162`	`165`	`t.Errorf("got roleArn %v, expected %v", gotRoleArn, roleArn)`
`163`	`166`	`}`
`@@ -202,7 +205,7 @@ func TestPopulateCacheFromCM(t *testing.T) {`
`202`	`205`	`t.Errorf("failed to build cache: %v", err)`
`203`	`206`	`}`
`204`	`207`
`205`		`- role, _, _, _ := c.Get("mysa2", "myns2")`
	`208`	`+ role, _, _, _, _ := c.Get("mysa2", "myns2")`
`206`	`209`	`if role == "" {`
`207`	`210`	`t.Errorf("cloud not find entry that should have been added")`
`208`	`211`	`}`
`@@ -214,7 +217,7 @@ func TestPopulateCacheFromCM(t *testing.T) {`
`214`	`217`	`t.Errorf("failed to build cache: %v", err)`
`215`	`218`	`}`
`216`	`219`
`217`		`- role, _, _, _ := c.Get("mysa2", "myns2")`
	`220`	`+ role, _, _, _, _ := c.Get("mysa2", "myns2")`
`218`	`221`	`if role == "" {`
`219`	`222`	`t.Errorf("cloud not find entry that should have been added")`
`220`	`223`	`}`
`@@ -226,7 +229,7 @@ func TestPopulateCacheFromCM(t *testing.T) {`
`226`	`229`	`t.Errorf("failed to build cache: %v", err)`
`227`	`230`	`}`
`228`	`231`
`229`		`- role, _, _, _ := c.Get("mysa2", "myns2")`
	`232`	`+ role, _, _, _, _ := c.Get("mysa2", "myns2")`
`230`	`233`	`if role != "" {`
`231`	`234`	`t.Errorf("found entry that should have been removed")`
`232`	`235`	`}`
`@@ -256,7 +259,7 @@ func TestSAAnnotationRemoval(t *testing.T) {`
`256`	`259`	`c.addSA(oldSA)`
`257`	`260`
`258`	`261`	`{`
`259`		`- gotRoleArn, _, _, _ := c.Get("default", "default")`
	`262`	`+ gotRoleArn, _, _, _, _ := c.Get("default", "default")`
`260`	`263`	`if gotRoleArn != roleArn {`
`261`	`264`	`t.Errorf("got roleArn %q, expected %q", gotRoleArn, roleArn)`
`262`	`265`	`}`
`@@ -268,7 +271,7 @@ func TestSAAnnotationRemoval(t *testing.T) {`
`268`	`271`	`c.addSA(newSA)`
`269`	`272`
`270`	`273`	`{`
`271`		`- gotRoleArn, _, _, _ := c.Get("default", "default")`
	`274`	`+ gotRoleArn, _, _, _, _ := c.Get("default", "default")`
`272`	`275`	`if gotRoleArn != "" {`
`273`	`276`	`t.Errorf("got roleArn %v, expected %q", gotRoleArn, "")`
`274`	`277`	`}`
`@@ -323,7 +326,7 @@ func TestCachePrecedence(t *testing.T) {`
`323`	`326`	`t.Errorf("failed to build cache: %v", err)`
`324`	`327`	`}`
`325`	`328`
`326`		`- role, _, _, exp := c.Get("mysa2", "myns2")`
	`329`	`+ role, _, _, exp, _ := c.Get("mysa2", "myns2")`
`327`	`330`	`if role == "" {`
`328`	`331`	`t.Errorf("could not find entry that should have been added")`
`329`	`332`	`}`
`@@ -340,7 +343,7 @@ func TestCachePrecedence(t *testing.T) {`
`340`	`343`	`}`
`341`	`344`
`342`	`345`	`// Removing sa2 from CM, but SA still exists`
`343`		`- role, _, _, exp := c.Get("mysa2", "myns2")`
	`346`	`+ role, _, _, exp, _ := c.Get("mysa2", "myns2")`
`344`	`347`	`if role == "" {`
`345`	`348`	`t.Errorf("could not find entry that should still exist")`
`346`	`349`	`}`
`@@ -356,7 +359,7 @@ func TestCachePrecedence(t *testing.T) {`
`356`	`359`	`c.addSA(sa2)`
`357`	`360`
`358`	`361`	`// Neither cache should return any hits now`
`359`		`- role, _, _, _ := c.Get("myns2", "mysa2")`
	`362`	`+ role, _, _, _, _ := c.Get("myns2", "mysa2")`
`360`	`363`	`if role != "" {`
`361`	`364`	`t.Errorf("found entry that should not exist")`
`362`	`365`	`}`
`@@ -370,7 +373,7 @@ func TestCachePrecedence(t *testing.T) {`
`370`	`373`	`t.Errorf("failed to build cache: %v", err)`
`371`	`374`	`}`
`372`	`375`
`373`		`- role, _, _, exp := c.Get("mysa2", "myns2")`
	`376`	`+ role, _, _, exp, _ := c.Get("mysa2", "myns2")`
`374`	`377`	`if role == "" {`
`375`	`378`	`t.Errorf("cloud not find entry that should have been added")`
`376`	`379`	`}`
`@@ -422,7 +425,7 @@ func TestRoleArnComposition(t *testing.T) {`
`422`	`425`
`423`	`426`	`var roleArn string`
`424`	`427`	`err := wait.ExponentialBackoff(wait.Backoff{Duration: 10 * time.Millisecond, Factor: 1.0, Steps: 3}, func() (bool, error) {`
`425`		`- roleArn, _, _, _ = cache.Get("default", "default")`
	`428`	`+ roleArn, _, _, _, _ = cache.Get("default", "default")`
`426`	`429`	`return roleArn != "", nil`
`427`	`430`	`})`
`428`	`431`	`if err != nil {`