Fix cluster UUID race condition, add regression UT (#1977)

Author: david-kow

pkg/controller/elasticsearch/driver/bootstrap.go

@@ -15,6 +15,7 @@ import (
 const (
 	// ClusterUUIDAnnotationName used to store the cluster UUID as an annotation when cluster has been bootstrapped.
 	ClusterUUIDAnnotationName = "elasticsearch.k8s.elastic.co/cluster-uuid"
+	formingClusterUUID        = "_na_"
 )
 
 // AnnotatedForBootstrap returns true if the cluster has been annotated with the UUID already.
@@ -75,7 +76,9 @@ func clusterNeedsReBootstrap(client k8s.Client, es *v1beta1.Elasticsearch) (bool
 
 // clusterIsBootstrapped returns true if the cluster has formed and has a UUID.
 func clusterIsBootstrapped(observedState observer.State) bool {
-	return observedState.ClusterInfo != nil && len(observedState.ClusterInfo.ClusterUUID) > 0
+	return observedState.ClusterInfo != nil &&
+		len(observedState.ClusterInfo.ClusterUUID) > 0 &&
+		observedState.ClusterInfo.ClusterUUID != formingClusterUUID
 }
 
 // annotateWithUUID annotates the cluster with its UUID, to mark it as "bootstrapped".

pkg/controller/elasticsearch/driver/bootstrap_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1beta1"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client"
+	esclient "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/observer"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -81,6 +82,41 @@ func Test_annotateWithUUID(t *testing.T) {
 	require.True(t, AnnotatedForBootstrap(retrieved))
 }
 
+func Test_clusterIsBootstrapped(t *testing.T) {
+	tests := []struct {
+		name  string
+		state observer.State
+		want  bool
+	}{
+		{
+			name:  "empty state",
+			state: observer.State{},
+			want:  false,
+		},
+		{
+			name:  "cluster uuid empty",
+			state: observer.State{ClusterInfo: &esclient.Info{}},
+			want:  false,
+		},
+		{
+			name:  "cluster uuid _na_ (not available) yet, cluster is still forming",
+			state: observer.State{ClusterInfo: &esclient.Info{ClusterUUID: "_na_"}},
+			want:  false,
+		},
+		{
+			name:  "cluster uuid set, cluster bootstrapped",
+			state: observer.State{ClusterInfo: &esclient.Info{ClusterUUID: "6902c192-ec1d-11e9-81b4-2a2ae2dbcce4"}},
+			want:  true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.want, clusterIsBootstrapped(tt.state))
+		})
+	}
+}
+
 func TestReconcileClusterUUID(t *testing.T) {
 	require.NoError(t, v1beta1.AddToScheme(scheme.Scheme))
 	tests := []struct {