Improve health checking

Wait for a deployment to be created before running the liveness check

Author: stefanprodan

controllers/gitrepository_controller.go renamed to controllers/gitrepository_watcher.go

@@ -34,7 +34,8 @@ import (
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1"
 )
 
-// KustomizationReconciler watches a GitRepository object
+// GitRepositoryWatcher watches GitRepository objects for revision changes
+// and triggers a sync for all the Kustomizations that reference a changed source
 type GitRepositoryWatcher struct {
 	client.Client
 	Log    logr.Logger
@@ -54,7 +55,7 @@ func (r *GitRepositoryWatcher) Reconcile(req ctrl.Request) (ctrl.Result, error)
 	}
 
 	log := r.Log.WithValues(strings.ToLower(repo.Kind), req.NamespacedName)
-	log.Info("New artifact detected")
+	log.Info("new artifact detected")
 
 	// get the list of kustomizations that are using this Git repository
 	var list kustomizev1.KustomizationList

controllers/kustomization_controller.go

@@ -513,9 +513,17 @@ func (r *KustomizationReconciler) checkHealth(kustomization kustomizev1.Kustomiz
 	var alerts string
 
 	for _, check := range kustomization.Spec.HealthChecks {
-		cmd := fmt.Sprintf("kubectl -n %s rollout status %s %s --timeout=%s",
-			check.Namespace, check.Kind, check.Name, kustomization.GetTimeout())
+		cmd := fmt.Sprintf("until kubectl -n %s get %s %s ; do sleep 2; done",
+			check.Namespace, check.Kind, check.Name)
 		command := exec.CommandContext(ctx, "/bin/sh", "-c", cmd)
+		if _, err := command.CombinedOutput(); err != nil {
+			return fmt.Errorf("health check timeout for %s '%s/%s': %w",
+				check.Kind, check.Namespace, check.Name, err)
+		}
+
+		cmd = fmt.Sprintf("kubectl -n %s rollout status %s %s --timeout=%s",
+			check.Namespace, check.Kind, check.Name, kustomization.GetTimeout())
+		command = exec.CommandContext(ctx, "/bin/sh", "-c", cmd)
 		output, err := command.CombinedOutput()
 		if err != nil {
 			if errors.Is(err, context.DeadlineExceeded) {