Check for existance of bastion instance and don't write init token during read operation

Author: Luke Addison

pkg/terraform/providers/tarmak/resource_vault_cluster.go

@@ -109,7 +109,6 @@ func resourceTarmakVaultClusterRead(d *schema.ResourceData, meta interface{}) (e
 
 	log.Print("[DEBUG] calling rpc vault cluster init status")
 	var reply tarmakRPC.VaultClusterStatusReply
-	// TODO: verify that all Ensure operations have succeeded, not just initialisation
 	err = client.Call(tarmakRPC.VaultClusterInitStatusCall, args, &reply)
 	if err != nil {
 		d.SetId("")

pkg/terraform/providers/tarmak/resource_vault_instance_role.go

@@ -70,10 +70,10 @@ func resourceTarmakVaultInstanceRoleCreate(d *schema.ResourceData, meta interfac
 
 	log.Printf("[DEBUG] calling rpc vault instance role for role %s", roleName)
 	var reply tarmakRPC.VaultInstanceRoleReply
-	err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply)
+	err = client.Call(tarmakRPC.VaultInstanceRoleCreate, args, &reply)
 	if err != nil {
 		d.SetId("")
-		return fmt.Errorf("call to %s failed: %s", tarmakRPC.VaultInstanceRole, err)
+		return fmt.Errorf("call to %s failed: %s", tarmakRPC.VaultInstanceRoleCreate, err)
 	}
 
 	if err = d.Set("init_token", reply.InitToken); err != nil {
@@ -106,7 +106,7 @@ func resourceTarmakVaultInstanceRoleRead(d *schema.ResourceData, meta interface{
 
 	log.Printf("[DEBUG] calling rpc vault instance role for role %s", roleName)
 	var reply tarmakRPC.VaultInstanceRoleReply
-	err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply)
+	err = client.Call(tarmakRPC.VaultInstanceRoleRead, args, &reply)
 	if err != nil {
 		d.SetId("")
 		return nil

pkg/terraform/providers/tarmak/rpc/bastion_instance_status.go

@@ -8,6 +8,10 @@ import (
 	cluster "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1"
 )
 
+const (
+	bastionVerifyTimeoutSeconds = 120
+)
+
 var (
 	BastionInstanceStatusCall = fmt.Sprintf("%s.BastionInstanceStatus", RPCName)
 )
@@ -29,17 +33,41 @@ func (r *tarmakRPC) BastionInstanceStatus(args *BastionInstanceStatusArgs, resul
 		return nil
 	}
 
-	var err error
-	for i := 1; i <= Retries; i++ {
-		if err = r.cluster.Environment().VerifyBastionAvailable(); err != nil {
-			r.tarmak.Log().Error(err)
-			time.Sleep(time.Second)
-		} else {
-			break
+	// check if instance exists
+	instances, err := r.cluster.Environment().Provider().ListHosts(r.cluster.Environment().Hub())
+	if err != nil {
+		return fmt.Errorf("failed to list instances in hub: %s", err)
+	}
+	bastionExists := false
+	for _, instance := range instances {
+		for _, role := range instance.Roles() {
+			if role == cluster.InstancePoolTypeBastion {
+				bastionExists = true
+			}
 		}
 	}
-	if err != nil {
-		return fmt.Errorf("bastion instance is not ready: %s", err)
+	if !bastionExists {
+		return fmt.Errorf("bastion instance does not exist")
+	}
+
+	// verify bastion responsiveness
+	verifyChannel := make(chan bool)
+	go func() {
+		for {
+			if err := r.cluster.Environment().VerifyBastionAvailable(); err != nil {
+				r.tarmak.Log().Error(err)
+				time.Sleep(time.Second)
+				continue
+			}
+			verifyChannel <- true
+			return
+		}
+	}()
+
+	select {
+	case <-verifyChannel:
+	case <-time.After(bastionVerifyTimeoutSeconds * time.Second):
+		return fmt.Errorf("failed to verify bastion instance")
 	}
 
 	result.Status = "ready"

pkg/terraform/providers/tarmak/rpc/rpc.go

@@ -15,7 +15,6 @@ import (
 const (
 	ConnectorSocket = "/tmp/tarmak-connector.sock"
 	RPCName         = "Tarmak"
-	Retries         = 60
 )
 
 type tarmakRPC struct {
@@ -38,7 +37,8 @@ type Tarmak interface {
 	BastionInstanceStatus(*BastionInstanceStatusArgs, *BastionInstanceStatusReply) error
 	VaultClusterStatus(*VaultClusterStatusArgs, *VaultClusterStatusReply) error
 	VaultClusterInitStatus(*VaultClusterStatusArgs, *VaultClusterStatusReply) error
-	VaultInstanceRole(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error
+	VaultInstanceRoleCreate(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error
+	VaultInstanceRoleRead(*VaultInstanceRoleArgs, *VaultInstanceRoleReply) error
 	Ping(*PingArgs, *PingReply) error
 	log() *logrus.Entry
 }

pkg/terraform/providers/tarmak/rpc/vault_cluster_status.go

@@ -3,7 +3,6 @@ package rpc
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/jetstack/vault-helper/pkg/kubernetes"
 
@@ -36,6 +35,7 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 
 	vault := r.cluster.Environment().Vault()
 
+	// initialise and unseal vault
 	err := vault.VerifyInitFromFQDNs(args.VaultInternalFQDNs, args.VaultCA, args.VaultKMSKeyID, args.VaultUnsealKeyName)
 	if err != nil {
 		err = fmt.Errorf("failed to initialise vault cluster: %s", err)
@@ -78,6 +78,7 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result *VaultClusterStatusReply) error {
 	r.tarmak.Log().Debug("received rpc vault cluster status")
 
+	// if destroying, return with unknown state
 	if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
 		result.Status = "unknown"
 		return nil
@@ -93,6 +94,7 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result
 	}
 	defer vaultTunnel.Stop()
 
+	// init vault client
 	vaultClient := vaultTunnel.VaultClient()
 
 	vaultRootToken, err := vault.RootToken()
@@ -104,27 +106,21 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result
 
 	vaultClient.SetToken(vaultRootToken)
 
-	up := false
-	err = nil
-	for i := 1; i <= Retries; i++ {
-		up, err = vaultClient.Sys().InitStatus()
-		if err != nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		break
-	}
+	// retrieve vault init status
+	up, err := vaultClient.Sys().InitStatus()
 	if err != nil {
 		err = fmt.Errorf("failed to retrieve init status: %s", err)
 		r.tarmak.Log().Error(err)
 		return err
 	}
 	if !up {
-		err = fmt.Errorf("failed to initialised vault cluster")
+		err = fmt.Errorf("vault cluster is not initialised")
 		r.tarmak.Log().Error(err)
 		return err
 	}
 
+	// TODO: verify that all Ensure operations have succeeded, not just initialisation
+
 	result.Status = "ready"
 	return nil
 }

pkg/terraform/providers/tarmak/rpc/vault_instance_role.go

@@ -10,7 +10,8 @@ import (
 )
 
 var (
-	VaultInstanceRole = fmt.Sprintf("%s.VaultInstanceRole", RPCName)
+	VaultInstanceRoleCreate = fmt.Sprintf("%s.VaultInstanceRoleCreate", RPCName)
+	VaultInstanceRoleRead   = fmt.Sprintf("%s.VaultInstanceRoleRead", RPCName)
 )
 
 type VaultInstanceRoleArgs struct {
@@ -25,8 +26,8 @@ type VaultInstanceRoleReply struct {
 	InitToken string
 }
 
-func (r *tarmakRPC) VaultInstanceRole(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error {
-	r.tarmak.Log().Debug("received rpc vault instance role")
+func (r *tarmakRPC) VaultInstanceRoleCreate(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error {
+	r.tarmak.Log().Debug("received rpc vault instance role create")
 
 	if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
 		result.InitToken = ""
@@ -58,18 +59,74 @@ func (r *tarmakRPC) VaultInstanceRole(args *VaultInstanceRoleArgs, result *Vault
 	k := kubernetes.New(vaultClient, r.tarmak.Log())
 	k.SetClusterID(r.tarmak.Cluster().ClusterName())
 
-	if err := k.Ensure(); err != nil {
-		err = fmt.Errorf("vault cluster is not ready: %s", err)
+	initToken, err := k.NewInitToken(roleName)
+	if err != nil {
+		return fmt.Errorf("could not get init token for role %s: %s", roleName, err)
+	}
+
+	err = initToken.Ensure()
+	if err != nil {
+		return fmt.Errorf("could not ensure init token for role %s: %s", roleName, err)
+	}
+
+	initTokenString, err := initToken.InitToken()
+	if err != nil {
+		return fmt.Errorf("could not retrieve init token for role %s: %s", roleName, err)
+	}
+
+	result.InitToken = initTokenString
+
+	r.tarmak.Log().Debug(roleName, " init token ", initTokenString)
+
+	return nil
+}
+
+func (r *tarmakRPC) VaultInstanceRoleRead(args *VaultInstanceRoleArgs, result *VaultInstanceRoleReply) error {
+	r.tarmak.Log().Debug("received rpc vault instance role read")
+
+	if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
+		result.InitToken = ""
+		return nil
+	}
+
+	roleName := args.RoleName
+
+	vault := r.cluster.Environment().Vault()
+	vaultTunnel, err := vault.TunnelFromFQDNs(args.VaultInternalFQDNs, args.VaultCA)
+	if err != nil {
+		err := fmt.Errorf("failed to create vault tunnel: %s", err)
+		r.tarmak.Log().Error(err)
+		return err
+	}
+	defer vaultTunnel.Stop()
+
+	vaultClient := vaultTunnel.VaultClient()
+
+	vaultRootToken, err := vault.RootToken()
+	if err != nil {
+		err := fmt.Errorf("failed to retrieve root token: %s", err)
 		r.tarmak.Log().Error(err)
 		return err
 	}
 
-	initTokens := k.InitTokens()
-	initToken, ok := initTokens[roleName]
-	if !ok {
+	vaultClient.SetToken(vaultRootToken)
+
+	k := kubernetes.New(vaultClient, r.tarmak.Log())
+	k.SetClusterID(r.tarmak.Cluster().ClusterName())
+
+	initToken, err := k.NewInitToken(roleName)
+	if err != nil {
 		return fmt.Errorf("could not get init token for role %s: %s", roleName, err)
 	}
 
-	result.InitToken = initToken
+	initTokenString, err := initToken.InitToken()
+	if err != nil {
+		return fmt.Errorf("could not retrieve init token for role %s: %s", roleName, err)
+	}
+
+	result.InitToken = initTokenString
+
+	r.tarmak.Log().Debug(roleName, " init token ", initTokenString)
+
 	return nil
 }