Merge pull request #612 from kishen-v/deprecate-bluemix

Deprecate bluemix-go and clean-up code.

Author: Power Cloud Robot

cmd/dhcp-sync/dhcp-sync.go

@@ -87,7 +87,7 @@ func syncDHCPD() {
 		klog.Fatalf("failed to create a session with IBM cloud: %v", err)
 	}
 
-	pvmclient, err := client.NewPVMClientWithEnv(c, pkg.Options.WorkspaceID, "", "prod")
+	pvmclient, err := client.NewPVMClientWithEnv(c, pkg.Options.WorkspaceID, "", client.DefaultEnvProd)
 	if err != nil {
 		klog.Fatalf("failed to create a PVM client: %v", err)
 	}
@@ -162,9 +162,6 @@ var Cmd = &cobra.Command{
 		if pkg.Options.WorkspaceID == "" {
 			return fmt.Errorf("--workspace-id is required")
 		}
-		if pkg.Options.APIKey == "" {
-			return fmt.Errorf("api-key can't be empty, pass the token via --api-key or set IBMCLOUD_API_KEY environment variable")
-		}
 		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/dhcpserver/cmd.go

@@ -27,9 +27,6 @@ var Cmd = &cobra.Command{
 	Short:   "dhcpserver command",
 	GroupID: "dhcp",
 	PreRunE: func(cmd *cobra.Command, args []string) error {
-		if pkg.Options.APIKey == "" {
-			return fmt.Errorf("api-key can't be empty, pass the token via --api-key or set IBMCLOUD_API_KEY environment variable")
-		}
 		if pkg.Options.WorkspaceID == "" {
 			return fmt.Errorf("--workspace-id required")
 		}

cmd/get/cloudconnections/cloudconnections.go

@@ -15,7 +15,6 @@
 package cloudconnections
 
 import (
-	"os"
 	"strings"
 
 	"github.com/IBM-Cloud/power-go-client/ibmpisession"
@@ -46,13 +45,6 @@ var Cmd = &cobra.Command{
 	Use:   "cloud-connections",
 	Short: "List the existing cloud connections in the account",
 	Long:  "List the existing cloud connections enabled across all workspaces in the account",
-	PreRunE: func(md *cobra.Command, args []string) error {
-		opt := pkg.Options
-		// TODO: The GetAuthenticatorFromEnvironment function seems to refer to "IBMCLOUD_APIKEY" rather than IBMCLOUD_API_KEY as used in the project.
-		// In order to use the resourcecontrollerv2's functionality, the IBMCLOUD_API_KEY is re-exported as IBMCLOUD_APIKEY.
-		os.Setenv("IBMCLOUD_APIKEY", opt.APIKey)
-		return nil
-	},
 	RunE: func(cmd *cobra.Command, args []string) error {
 		opt := pkg.Options
 		c, err := client.NewClientWithEnv(opt.APIKey, opt.Environment, opt.Debug)
@@ -61,6 +53,10 @@ var Cmd = &cobra.Command{
 			return err
 		}
 
+		environment, err := client.GetEnvironment(opt.Environment)
+		if err != nil {
+			return err
+		}
 		// Retrieve all workspaces that are available in the account.
 		workspaceInstances, err := c.ListWorkspaceInstances()
 		if err != nil {
@@ -73,18 +69,24 @@ var Cmd = &cobra.Command{
 			zoneWorkspaces[*workspaceInstance.RegionID] = append(zoneWorkspaces[*workspaceInstance.RegionID], workspaceDetails{name: *workspaceInstance.Name, guid: *workspaceInstance.GUID})
 		}
 		cloudConnections := map[string]cloudConnectionDetails{}
-		authenticator := &core.IamAuthenticator{ApiKey: c.Config.BluemixAPIKey, URL: *c.Config.TokenProviderEndpoint}
+		authenticator := &core.IamAuthenticator{ApiKey: pkg.Options.APIKey, URL: environment[client.TPEndpoint]}
 		klog.Info("Listing cloud connections across all workspaces, please wait..")
 		// Create a IBM PI Session per zone and reuse them across the workspaces in the same zone.
 		for workspaceZone, workspaces := range zoneWorkspaces {
-			pvmclientOptions := ibmpisession.IBMPIOptions{Authenticator: authenticator, Debug: pkg.Options.Debug, UserAccount: c.User.Account, Zone: workspaceZone}
+			pvmclientOptions := ibmpisession.IBMPIOptions{
+				Authenticator: authenticator,
+				Debug:         pkg.Options.Debug,
+				URL:           environment[client.PIEndpoint],
+				UserAccount:   c.User.Account,
+				Zone:          workspaceZone,
+			}
 			piSession, err := ibmpisession.NewIBMPISession(&pvmclientOptions)
 			if err != nil {
 				return err
 			}
 			// Iterate over the workspaces available in the zone.
 			for _, workspace := range workspaces {
-				pvmClient, err := client.NewGenericPVMClientWithEnv(c, workspace.guid, workspace.name, opt.Environment, piSession)
+				pvmClient, err := client.NewGenericPVMClient(c, workspace.guid, piSession)
 				if err != nil {
 					return err
 				}
@@ -130,8 +132,4 @@ var Cmd = &cobra.Command{
 		klog.Info("There are no active cloud connections in this account.")
 		return nil
 	},
-	PostRunE: func(md *cobra.Command, args []string) error {
-		os.Unsetenv("IBMCLOUD_APIKEY")
-		return nil
-	},
 }

cmd/image/import/import.go

@@ -19,46 +19,69 @@ import (
 	"strings"
 	"time"
 
-	"github.com/IBM-Cloud/bluemix-go/api/resource/resourcev1/controller"
-	"github.com/IBM-Cloud/bluemix-go/api/resource/resourcev2/controllerv2"
-	"github.com/IBM-Cloud/bluemix-go/crn"
-	"github.com/IBM-Cloud/bluemix-go/models"
 	pmodels "github.com/IBM-Cloud/power-go-client/power/models"
+	"github.com/IBM/platform-services-go-sdk/resourcecontrollerv2"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 
 	"github.com/ppc64le-cloud/pvsadm/pkg"
 	"github.com/ppc64le-cloud/pvsadm/pkg/client"
 	"github.com/ppc64le-cloud/pvsadm/pkg/utils"
 )
 
 const (
+	accessKeyId          = "access_key_id"
+	cosHmacKeys          = "cos_hmac_keys"
+	crnServiceRoleWriter = "crn:v1:bluemix:public:iam::::serviceRole:Writer"
+	imageStateActive     = "active"
+	jobStateCompleted    = "completed"
+	jobStateFailed       = "failed"
+	secretAccessKey      = "secret_access_key"
+	// CosResourceID is IBM COS service id, can be retrieved using ibmcloud cli
+	// ibmcloud catalog service cloud-object-storage.
 	serviceCredPrefix = "pvsadm-service-cred"
-	imageStateActive  = "active"
-	jobStateCompleted = "completed"
-	jobStateFailed    = "failed"
 )
 
-// Find COSINSTANCE details of the Provided bucket
-func findCOSInstanceDetails(resources []models.ServiceInstanceV2, bxCli *client.Client) (string, string, crn.CRN) {
+// findCOSInstance retrieves the service instance in which the bucket is present.
+func findCOSInstanceDetails(resources []resourcecontrollerv2.ResourceInstance, pvsClient *client.Client) *resourcecontrollerv2.ResourceInstance {
 	for _, resource := range resources {
-		if resource.Crn.ServiceName == "cloud-object-storage" {
-			s3client, err := client.NewS3Client(bxCli, resource.Name, pkg.ImageCMDOptions.Region)
-			if err != nil {
-				continue
-			}
-			buckets, err := s3client.S3Session.ListBuckets(nil)
-			if err != nil {
-				continue
-			}
-			for _, bucket := range buckets.Buckets {
-				if *bucket.Name == pkg.ImageCMDOptions.BucketName {
-					return resource.Name, resource.Guid, resource.Crn
-				}
+		s3client, err := client.NewS3Client(pvsClient, *resource.Name, pkg.ImageCMDOptions.Region)
+		if err != nil {
+			klog.Warningf("cannot create a new s3 client. err: %v", err)
+			continue
+		}
+		buckets, err := s3client.S3Session.ListBuckets(nil)
+		if err != nil {
+			klog.Warningf("cannot list buckets in the resource instance. err: %v", err)
+			continue
+		}
+		for _, bucket := range buckets.Buckets {
+			if *bucket.Name == pkg.ImageCMDOptions.BucketName {
+				return &resource
 			}
 		}
 	}
-	return "", "", crn.CRN{}
+	return nil
+}
+
+// createNewCredentialsWithHMAC generates the service credentials in the given COS instance with HMAC keys.
+func createNewCredentialsWithHMAC(pvsClient *client.Client, cosCRN, serviceCredName string) (*resourcecontrollerv2.ResourceKey, error) {
+	klog.V(2).Infof("Auto generating COS service credentials to import image: %s", serviceCredName)
+	params := &resourcecontrollerv2.ResourceKeyPostParameters{}
+	params.SetProperty("HMAC", true)
+	key, _, err := pvsClient.ResourceControllerClient.CreateResourceKey(
+		&resourcecontrollerv2.CreateResourceKeyOptions{
+			Name:       ptr.To(serviceCredName),
+			Parameters: params,
+			Role:       ptr.To(crnServiceRoleWriter),
+			Source:     ptr.To(cosCRN),
+		},
+	)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create resource key for service instance: %v", err.Error())
+	}
+	return key, nil
 }
 
 // checkStorageTierAvailability confirms if the provided cloud instance ID supports the required storageType.
@@ -95,9 +118,9 @@ pvsadm image import --help for information
 # Set the API key or feed the --api-key commandline argument
 export IBMCLOUD_API_KEY=<IBM_CLOUD_API_KEY>
 
-# To Import the image across the two different IBM account use accesskey and secretkey options
+# To Import the image across the two different IBM account use "--accesskey" and "--secretkey" options
 
-# To Import the image from public bucket use public-bucket option
+# To Import the image from public bucket use the "--public-bucket" or "-p" option
 
 Examples:
 
@@ -132,12 +155,12 @@ pvsadm image import -n upstream-core-lon04 -b <BUCKETNAME> --object rhel-83-1003
 		opt := pkg.ImageCMDOptions
 		apikey := pkg.Options.APIKey
 
-		bxCli, err := client.NewClientWithEnv(apikey, pkg.Options.Environment, pkg.Options.Debug)
+		pvsClient, err := client.NewClientWithEnv(apikey, pkg.Options.Environment, pkg.Options.Debug)
 		if err != nil {
 			return err
 		}
 
-		pvmclient, err := client.NewPVMClientWithEnv(bxCli, opt.WorkspaceID, opt.WorkspaceName, pkg.Options.Environment)
+		pvmclient, err := client.NewPVMClientWithEnv(pvsClient, opt.WorkspaceID, opt.WorkspaceName, pkg.Options.Environment)
 		if err != nil {
 			return err
 		}
@@ -148,54 +171,87 @@ pvsadm image import -n upstream-core-lon04 -b <BUCKETNAME> --object rhel-83-1003
 
 		//Create AccessKey and SecretKey for the bucket provided if bucket access is private
 		if (opt.AccessKey == "" || opt.SecretKey == "") && (!opt.Public) {
-			//Find CosInstance of the bucket
-			var svcs []models.ServiceInstanceV2
-			svcs, err = bxCli.ResourceClientV2.ListInstances(controllerv2.ServiceInstanceQuery{
-				Type: "service_instance",
-			})
+			// Find COS instance of the bucket
+			listServiceInstanceOptions := &resourcecontrollerv2.ListResourceInstancesOptions{
+				ResourceID: ptr.To(utils.CosResourceID),
+			}
+
+			workspaces, _, err := pvsClient.ResourceControllerClient.ListResourceInstances(listServiceInstanceOptions)
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to list the resource instances: %v", err)
 			}
-			cosInstanceName, cosID, crn := findCOSInstanceDetails(svcs, bxCli)
-			if cosInstanceName == "" {
+			if len(workspaces.Resources) == 0 {
+				return fmt.Errorf("no service instances were found")
+			}
+
+			cosInstance := findCOSInstanceDetails(workspaces.Resources, pvsClient)
+			if cosInstance == nil {
 				return fmt.Errorf("failed to find the COS instance for the bucket mentioned: %s", opt.BucketName)
 			}
 
-			keys, err := bxCli.GetResourceKeys(cosID)
+			klog.Infof("Identified bucket %q in service instance: %s", opt.BucketName, *cosInstance.Name)
+			listResourceKeysInstanceOptions := &resourcecontrollerv2.ListResourceKeysForInstanceOptions{
+				ID: cosInstance.GUID,
+			}
+			keys, _, err := pvsClient.ResourceControllerClient.ListResourceKeysForInstance(listResourceKeysInstanceOptions)
 			if err != nil {
-				return fmt.Errorf("failed to list the service credentials: %v", err)
+				return fmt.Errorf("cannot list the resource keys for instance. err: %v", err)
 			}
 
-			var cred map[string]interface{}
-			var ok bool
-			if len(keys) == 0 {
-				if opt.ServiceCredName == "" {
-					opt.ServiceCredName = serviceCredPrefix + "-" + cosInstanceName
-				}
+			var ok, credentialsPresent bool
+			var hmacKeys map[string]interface{}
+			var key *resourcecontrollerv2.ResourceKey
 
-				// Create the service credential if does not exist
-				klog.V(2).Infof("Auto Generating the COS Service credential for importing the image with name: %s", opt.ServiceCredName)
-				CreateServiceKeyRequest := controller.CreateServiceKeyRequest{
-					Name:       opt.ServiceCredName,
-					SourceCRN:  crn,
-					Parameters: map[string]interface{}{"HMAC": true},
-				}
-				newKey, err := bxCli.ResourceServiceKey.CreateKey(CreateServiceKeyRequest)
-				if err != nil {
-					return err
+			if opt.ServiceCredName == "" {
+				opt.ServiceCredName = serviceCredPrefix + "-" + *cosInstance.Name
+			}
+
+			// Create the service credential if does not exist
+			if len(keys.Resources) == 0 {
+				if key, err = createNewCredentialsWithHMAC(pvsClient, *cosInstance.CRN, opt.ServiceCredName); err != nil {
+					return fmt.Errorf("error while creating HMAC credentials. err: %v", err)
 				}
-				cred, ok = newKey.Credentials["cos_hmac_keys"].(map[string]interface{})
 			} else {
-				// Use the service credential already created
 				klog.V(2).Info("Reading the existing service credential")
-				cred, ok = keys[0].Credentials["cos_hmac_keys"].(map[string]interface{})
+				// Use the service credential already created. There may be a possibility that multiple credentials exist, but the HMAC credentials may not be present.
+				// In such case, manually re-create the credentials.
+
+				for _, serviceCredential := range keys.Resources {
+					key, _, err = pvsClient.ResourceControllerClient.GetResourceKey(
+						&resourcecontrollerv2.GetResourceKeyOptions{
+							ID: serviceCredential.ID,
+						},
+					)
+					if err != nil {
+						return fmt.Errorf("an error occured while retriving the resource key. err: %v", err)
+					}
+					// if the current credential has COS HMAC keys, reuse the same for importing the image
+					if prop := key.Credentials.GetProperty(cosHmacKeys); prop != nil {
+						klog.Infof("HMAC keys are available from the credential %q, re-using the same for image upload", *key.Name)
+						credentialsPresent = true
+						break
+					}
+					klog.Infof("No credentials found in the key %q.", *key.Name)
+				}
+				// if all the available service credentials do not have HMAC, create one with HMAC.
+				if !credentialsPresent {
+					if key, err = createNewCredentialsWithHMAC(pvsClient, *cosInstance.CRN, opt.ServiceCredName); err != nil {
+						return fmt.Errorf("error while creating HMAC credentials. err: %v", err)
+					}
+				}
 			}
-			if !ok {
-				return fmt.Errorf("failed to get the accessKey and secretKey from service credential")
+
+			prop := key.Credentials.GetProperty(cosHmacKeys)
+			if prop == nil {
+				return fmt.Errorf("unable to retrieve COS HMAC keys")
+			}
+
+			if hmacKeys, ok = prop.(map[string]interface{}); !ok {
+				return fmt.Errorf("type assertion for HMAC keys failed")
 			}
-			//Assign the Access Key and Secret Key for further operation
-			opt.AccessKey = cred["access_key_id"].(string)
-			opt.SecretKey = cred["secret_access_key"].(string)
+			// Assign the Access Key and Secret Key for further operation
+			opt.AccessKey = hmacKeys[accessKeyId].(string)
+			opt.SecretKey = hmacKeys[secretAccessKey].(string)
 		}
 
 		//By default Bucket Access is private
@@ -231,7 +287,7 @@ pvsadm image import -n upstream-core-lon04 -b <BUCKETNAME> --object rhel-83-1003
 		}
 
 		var image *pmodels.ImageReference = &pmodels.ImageReference{}
-		klog.V(1).Info("Retrieving image details")
+		klog.Info("Retrieving image details")
 
 		if image.ImageID == nil {
 			image, err = pvmclient.ImgClient.GetImageByName(opt.ImageName)

cmd/image/sync/sync.go

@@ -43,8 +43,7 @@ type InstanceItem struct {
 
 // sync constants
 const (
-	serviceType = "cloud-object-storage"
-	maxWorkers  = 20
+	maxWorkers = 20
 )
 
 // Worker method to copy object from source bucket to target bucket
@@ -64,18 +63,18 @@ func copyWorker(copyJobs <-chan copyWorkload, results chan<- bool, workerId int)
 }
 
 // Method to create the list of required instances
-func createInstanceList(spec []pkg.Spec, bxCli *client.Client) ([]InstanceItem, error) {
+func createInstanceList(spec []pkg.Spec, client *client.Client) ([]InstanceItem, error) {
 	var instanceList []InstanceItem
 	for _, item := range spec {
 		instance := InstanceItem{}
-		s3Cli, err := NewS3Client(bxCli, item.Source.Cos, item.Source.Region)
+		s3Cli, err := NewS3Client(client, item.Source.Cos, item.Source.Region)
 		if err != nil {
 			return nil, err
 		}
 
 		instance.Source = s3Cli
 		for _, targetItem := range item.Target {
-			s3Cli, err := NewS3Client(bxCli, item.Source.Cos, targetItem.Region)
+			s3Cli, err := NewS3Client(client, item.Source.Cos, targetItem.Region)
 			if err != nil {
 				return nil, err
 			}
@@ -257,7 +256,7 @@ Sample spec.yaml file:
 		start := time.Now()
 
 		//Create bluemix client
-		bxCli, err := client.NewClientWithEnv(apikey, pkg.Options.Environment, pkg.Options.Debug)
+		pvsClient, err := client.NewClientWithEnv(apikey, pkg.Options.Environment, pkg.Options.Debug)
 		if err != nil {
 			return err
 		}
@@ -269,7 +268,7 @@ Sample spec.yaml file:
 		}
 
 		// Create necessary objects
-		instanceList, err := createInstanceList(spec, bxCli)
+		instanceList, err := createInstanceList(spec, pvsClient)
 		if err != nil {
 			return err
 		}