Improve health check logic in getClusterHealthStatus function to handle shutdown delay or missing readyz endpoint

Signed-off-by: huangyanfeng <[email protected]>

Author: yanfeng1992

pkg/controllers/status/cluster_status_controller.go

@@ -435,8 +435,11 @@ func (c *ClusterStatusController) initLeaseController(cluster *clusterv1alpha1.C
 
 func getClusterHealthStatus(clusterClient *util.ClusterClient) (online, healthy bool) {
 	healthStatus, err := healthEndpointCheck(clusterClient.KubeClient, "/readyz")
-	if err != nil && healthStatus == http.StatusNotFound {
-		// do health check with healthz endpoint if the readyz endpoint is not installed in member cluster
+	if err != nil && (healthStatus == http.StatusInternalServerError || healthStatus == http.StatusNotFound) {
+		// do health check with healthz endpoint in two cases:
+		// 1. StatusInternalServerError(500): When the server is configured with --shutdown-delay-duration,
+		//    /readyz returns failure but /healthz still serves success
+		// 2. StatusNotFound(404): When the readyz endpoint is not installed in member cluster
 		healthStatus, err = healthEndpointCheck(clusterClient.KubeClient, "/healthz")
 	}
 

pkg/controllers/status/cluster_status_controller_test.go

@@ -1105,6 +1105,28 @@ func mockServer(statusCode int, existError bool) *httptest.Server {
 	return server
 }
 
+func mockNotReadyServer() *httptest.Server {
+	respBody := "test"
+	resp := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(bytes.NewBufferString(respBody)),
+	}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		if request.URL.Path == "/readyz" {
+			statusCode := http.StatusInternalServerError
+			errorMessage := "An error occurred"
+			http.Error(w, errorMessage, statusCode)
+		} else {
+			w.WriteHeader(resp.StatusCode)
+			_, err := io.Copy(w, resp.Body)
+			if err != nil {
+				fmt.Printf("failed to copy, err: %v", err)
+			}
+		}
+	}))
+	return server
+}
+
 func TestHealthEndpointCheck(t *testing.T) {
 	server := mockServer(http.StatusOK, false)
 	defer server.Close()
@@ -1141,4 +1163,13 @@ func TestGetClusterHealthStatus(t *testing.T) {
 		assert.Equal(t, true, online)
 		assert.Equal(t, false, healthy)
 	})
+
+	t.Run("readyz return error and StatusInternalServerError healthz return http.StatusOK", func(t *testing.T) {
+		server := mockNotReadyServer()
+		defer server.Close()
+		clusterClient := generateClusterClient(server.URL)
+		online, healthy := getClusterHealthStatus(clusterClient)
+		assert.Equal(t, true, online)
+		assert.Equal(t, true, healthy)
+	})
 }