Prevent flakiness in instance groups controller test

Author: anyush

pkg/instancegroups/controller_test.go

@@ -155,33 +155,54 @@ func TestSync(t *testing.T) {
 	var expectedSyncedNodesCounter = 0
 	firstNode := testNode()
 	secondNode := testNode()
+	thirdNode := testNode()
 	secondNode.Name = "secondNode"
+	thirdNode.Name = "thirdNode"
 
-	// Add two nodes
+	// Freeze queue processing by running other long-running sync
 	fakeKubeClient.CoreV1().Nodes().Create(context.TODO(), firstNode, meta_v1.CreateOptions{})
-	// wait time > resync period
-	time.Sleep(2 * time.Second)
+	time.Sleep(time.Second) // ensure sync started
+
+	// Add two nodes
 	fakeKubeClient.CoreV1().Nodes().Create(context.TODO(), secondNode, meta_v1.CreateOptions{})
-	// The counter = 1 because it synced only once (for the first Create() call)
-	expectedSyncedNodesCounter += 1
+	fakeKubeClient.CoreV1().Nodes().Create(context.TODO(), thirdNode, meta_v1.CreateOptions{})
+	time.Sleep(7 * time.Second) // wait until both syncs complete
+	// Second and third nodes should trigger only a single sync (+1 for the first)
+	expectedSyncedNodesCounter += 2
 	verifyExpectedSyncerCount(t, fakeManager.syncedNodes, expectedSyncedNodesCounter)
 
-	// Update both nodes
-	firstNode.Annotations["key"] = "true"
-	firstNode.Spec.Unschedulable = false
-	secondNode.Annotations["key"] = "true"
+	// Freeze queue processing by running other long-running sync
+	firstNode.Spec.Unschedulable = true
 	fakeKubeClient.CoreV1().Nodes().Update(context.TODO(), firstNode, meta_v1.UpdateOptions{})
+	time.Sleep(time.Second) // ensure sync started
+
+	// Update two nodes
+	secondNode.Spec.Unschedulable = true
+	thirdNode.Spec.Unschedulable = true
 	fakeKubeClient.CoreV1().Nodes().Update(context.TODO(), secondNode, meta_v1.UpdateOptions{})
-	time.Sleep(2 * time.Second)
-	// nodes were updated
-	expectedSyncedNodesCounter += 1
+	fakeKubeClient.CoreV1().Nodes().Update(context.TODO(), thirdNode, meta_v1.UpdateOptions{})
+	time.Sleep(7 * time.Second) // wait until both syncs complete
+	// Second and third nodes should trigger only a single sync (+1 for the first)
+	expectedSyncedNodesCounter += 2
 	verifyExpectedSyncerCount(t, fakeManager.syncedNodes, expectedSyncedNodesCounter)
 
 	// no real update
 	fakeKubeClient.CoreV1().Nodes().Update(context.TODO(), firstNode, meta_v1.UpdateOptions{})
 	// Nothing should change
 	time.Sleep(2 * time.Second)
 	verifyExpectedSyncerCount(t, fakeManager.syncedNodes, expectedSyncedNodesCounter)
+
+	// Freeze queue processing by running other long-running sync
+	fakeKubeClient.CoreV1().Nodes().Delete(context.TODO(), firstNode.Name, meta_v1.DeleteOptions{})
+	time.Sleep(time.Second) // ensure sync started
+
+	// Delete two nodes
+	fakeKubeClient.CoreV1().Nodes().Delete(context.TODO(), secondNode.Name, meta_v1.DeleteOptions{})
+	fakeKubeClient.CoreV1().Nodes().Delete(context.TODO(), thirdNode.Name, meta_v1.DeleteOptions{})
+	time.Sleep(7 * time.Second) // wait until both syncs complete
+	// Second and third nodes should trigger only a single sync (+1 for the first)
+	expectedSyncedNodesCounter += 2
+	verifyExpectedSyncerCount(t, fakeManager.syncedNodes, expectedSyncedNodesCounter)
 }
 
 func verifyExpectedSyncerCount(t *testing.T, syncedNodes [][]string, expectedCount int) {
@@ -195,6 +216,7 @@ type IGManagerFake struct {
 }
 
 func (igmf *IGManagerFake) Sync(nodeNames []string, logger klog.Logger) error {
+	time.Sleep(2 * time.Second)
 	igmf.syncedNodes = append(igmf.syncedNodes, nodeNames)
 	return nil
 }