Kubernetes: ensure ARM operations have appropriate timeout value (#3646)

Author: jackfrancis

cmd/deploy.go

@@ -286,7 +286,8 @@ func autofillApimodel(dc *deployCmd) error {
 		dc.containerService.Properties.LinuxProfile.SSH.PublicKeys = []api.PublicKey{{KeyData: publicKey}}
 	}
 
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	_, err = dc.client.EnsureResourceGroup(ctx, dc.resourceGroup, dc.location, nil)
 	if err != nil {
 		return err
@@ -414,9 +415,11 @@ func (dc *deployCmd) run() error {
 	}
 
 	deploymentSuffix := dc.random.Int31()
+	cx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 
 	if res, err := dc.client.DeployTemplate(
-		context.Background(),
+		cx,
 		dc.resourceGroup,
 		fmt.Sprintf("%s-%d", dc.resourceGroup, deploymentSuffix),
 		templateJSON,

cmd/scale.go

@@ -132,7 +132,8 @@ func (sc *scaleCmd) load(cmd *cobra.Command) error {
 		return errors.Wrap(err, "failed to get client")
 	}
 
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	_, err = sc.client.EnsureResourceGroup(ctx, sc.resourceGroupName, sc.location, nil)
 	if err != nil {
 		return err
@@ -209,7 +210,8 @@ func (sc *scaleCmd) run(cmd *cobra.Command, args []string) error {
 		return errors.Wrap(err, "failed to load existing container service")
 	}
 
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	orchestratorInfo := sc.containerService.Properties.OrchestratorProfile
 	var currentNodeCount, highestUsedIndex, index, winPoolIndex int
 	winPoolIndex = -1

cmd/upgrade.go

@@ -122,7 +122,8 @@ func (uc *upgradeCmd) loadCluster(cmd *cobra.Command) error {
 		return errors.Wrap(err, "Failed to get client")
 	}
 
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	_, err = uc.client.EnsureResourceGroup(ctx, uc.resourceGroupName, uc.location, nil)
 	if err != nil {
 		return errors.Wrap(err, "Error ensuring resource group")

pkg/acsengine/tenantid.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"regexp"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2016-06-01/subscriptions"
 	"github.com/pkg/errors"
@@ -22,7 +23,9 @@ func GetTenantID(resourceManagerEndpoint string, subscriptionID string) (string,
 	// we expect this request to fail (err != nil), but we are only interested
 	// in headers, so surface the error if the Response is not present (i.e.
 	// network error etc)
-	subs, err := c.Get(context.Background(), subscriptionID)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
+	subs, err := c.Get(ctx, subscriptionID)
 	if subs.Response.Response == nil {
 		return "", errors.Wrap(err, "Request failed")
 	}

pkg/armhelpers/azureclient.go

@@ -106,7 +106,9 @@ func NewAzureClientWithDeviceAuth(env azure.Environment, subscriptionID string)
 		}
 	}
 
-	client := &autorest.Client{}
+	client := &autorest.Client{
+		PollingDuration: 1 * time.Hour,
+	}
 
 	deviceCode, err := adal.InitiateDeviceAuth(client, *oauthConfig, acsEngineClientID, env.ServiceManagementEndpoint)
 	if err != nil {
@@ -301,6 +303,21 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armSpt *a
 	c.deploymentsClient.PollingDelay = time.Second * 5
 	c.resourcesClient.PollingDelay = time.Second * 5
 
+	// Set permissive timeouts to accommodate long-running operations
+	c.deploymentsClient.PollingDuration = time.Hour * 1
+	c.deploymentOperationsClient.PollingDuration = time.Hour * 1
+	c.applicationsClient.PollingDuration = time.Hour * 1
+	c.authorizationClient.PollingDuration = time.Hour * 1
+	c.disksClient.PollingDuration = time.Hour * 1
+	c.groupsClient.PollingDuration = time.Hour * 1
+	c.interfacesClient.PollingDuration = time.Hour * 1
+	c.providersClient.PollingDuration = time.Hour * 1
+	c.resourcesClient.PollingDuration = time.Hour * 1
+	c.storageAccountsClient.PollingDuration = time.Hour * 1
+	c.virtualMachineScaleSetsClient.PollingDuration = time.Hour * 1
+	c.virtualMachineScaleSetVMsClient.PollingDuration = time.Hour * 1
+	c.virtualMachinesClient.PollingDuration = time.Hour * 1
+
 	graphAuthorizer := autorest.NewBearerAuthorizer(graphSpt)
 	c.applicationsClient.Authorizer = graphAuthorizer
 	c.servicePrincipalsClient.Authorizer = graphAuthorizer
@@ -310,7 +327,8 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armSpt *a
 
 // EnsureProvidersRegistered checks if the AzureClient is registered to required resource providers and, if not, register subscription to providers
 func (az *AzureClient) EnsureProvidersRegistered(subscriptionID string) error {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	registeredProviders, err := az.providersClient.List(ctx, to.Int32Ptr(100), "")
 	if err != nil {
 		return err

pkg/armhelpers/deploymentError.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/Azure/acs-engine/pkg/api"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
@@ -58,7 +59,8 @@ func (e *DeploymentValidationError) Error() string {
 
 // DeployTemplateSync deploys the template and returns ArmError
 func DeployTemplateSync(az ACSEngineClient, logger *logrus.Entry, resourceGroupName, deploymentName string, template map[string]interface{}, parameters map[string]interface{}) error {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	deploymentExtended, err := az.DeployTemplate(ctx, resourceGroupName, deploymentName, template, parameters)
 	if err == nil {
 		return nil

pkg/operations/deletevm.go

@@ -3,6 +3,7 @@ package operations
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/Azure/acs-engine/pkg/armhelpers"
 	"github.com/Azure/acs-engine/pkg/armhelpers/utils"
@@ -17,7 +18,8 @@ const (
 
 // CleanDeleteVirtualMachine deletes a VM and any associated OS disk
 func CleanDeleteVirtualMachine(az armhelpers.ACSEngineClient, logger *log.Entry, subscriptionID, resourceGroup, name string) error {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	logger.Infof("fetching VM: %s/%s", resourceGroup, name)
 	vm, err := az.GetVirtualMachine(ctx, resourceGroup, name)
 	if err != nil {

pkg/operations/kubernetesupgrade/upgradecluster.go

@@ -145,7 +145,8 @@ func (uc *UpgradeCluster) UpgradeCluster(subscriptionID uuid.UUID, kubeConfig, r
 func (uc *UpgradeCluster) getClusterNodeStatus(subscriptionID uuid.UUID, resourceGroup string) error {
 	targetOrchestratorTypeVersion := fmt.Sprintf("%s:%s", uc.DataModel.Properties.OrchestratorProfile.OrchestratorType, uc.DataModel.Properties.OrchestratorProfile.OrchestratorVersion)
 
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Hour)
+	defer cancel()
 	for vmScaleSetPage, err := uc.Client.ListVirtualMachineScaleSets(ctx, resourceGroup); vmScaleSetPage.NotDone(); err = vmScaleSetPage.Next() {
 		if err != nil {
 			return err

pkg/operations/kubernetesupgrade/upgrader.go

@@ -54,7 +54,8 @@ func (ku *Upgrader) Init(translator *i18n.Translator, logger *logrus.Entry, clus
 
 // RunUpgrade runs the upgrade pipeline
 func (ku *Upgrader) RunUpgrade() error {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Minute)
+	defer cancel()
 	if err := ku.upgradeMasterNodes(ctx); err != nil {
 		return err
 	}