Skip to content

Commit 4122003

Browse files
stefanprodanstefanprodan
authored
Merge pull request #299 from fluxcd/suspend-metric
Expose suspended status as Prometheus metric
2 parents c1ffccb + 446545c commit 4122003

File tree

5 files changed

+44
-18
lines changed

5 files changed

+44
-18
lines changed

‎api/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.15
55
require (
66
github.com/fluxcd/pkg/apis/kustomize v0.0.1
77
github.com/fluxcd/pkg/apis/meta v0.8.0
8-
github.com/fluxcd/pkg/runtime v0.8.4
8+
github.com/fluxcd/pkg/runtime v0.8.5
99
k8s.io/apiextensions-apiserver v0.20.2
1010
k8s.io/apimachinery v0.20.2
1111
sigs.k8s.io/controller-runtime v0.8.3

‎api/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7
9292
github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0=
9393
github.com/fluxcd/pkg/apis/meta v0.8.0 h1:wqWpUsxhKHB1ZztcvOz+vnyhdKW9cWmjFp8Vci/XOdk=
9494
github.com/fluxcd/pkg/apis/meta v0.8.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po=
95-
github.com/fluxcd/pkg/runtime v0.8.4 h1:amuhfoHGCUfFCPXg3Zrcyy7f9J+fho+/+FbQDDyewko=
96-
github.com/fluxcd/pkg/runtime v0.8.4/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
95+
github.com/fluxcd/pkg/runtime v0.8.5 h1:ynh8fszbLQ3QSisQBNOABEUTnvt+/QfCdaL6gOJQcoQ=
96+
github.com/fluxcd/pkg/runtime v0.8.5/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
9797
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
9898
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
9999
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=

‎controllers/kustomization_controller.go‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,9 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
136136
return ctrl.Result{}, client.IgnoreNotFound(err)
137137
}
138138

139+
// Record suspended status metric
140+
defer r.recordSuspension(ctx, kustomization)
141+
139142
// Add our finalizer if it does not exist
140143
if !controllerutil.ContainsFinalizer(&kustomization, kustomizev1.KustomizationFinalizer) {
141144
controllerutil.AddFinalizer(&kustomization, kustomizev1.KustomizationFinalizer)
@@ -221,7 +224,7 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
221224
// set the reconciliation status to progressing
222225
kustomization = kustomizev1.KustomizationProgressing(kustomization)
223226
if err := r.patchStatus(ctx, req, kustomization.Status); err != nil {
224-
(logr.FromContext(ctx)).Error(err, "unable to update status to progressing")
227+
log.Error(err, "unable to update status to progressing")
225228
return ctrl.Result{Requeue: true}, err
226229
}
227230
r.recordReadiness(ctx, kustomization)
@@ -572,6 +575,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
572575
return nil
573576
}
574577

578+
log := logr.FromContext(ctx)
575579
timeout := kustomization.GetTimeout() + (time.Second * 1)
576580
applyCtx, cancel := context.WithTimeout(ctx, timeout)
577581
defer cancel()
@@ -580,7 +584,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
580584
if validation == "server" && kustomization.Spec.Force {
581585
// Use client-side validation with force
582586
validation = "client"
583-
(logr.FromContext(ctx)).Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled"))
587+
log.Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled"))
584588
}
585589

586590
cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp --force=%t",
@@ -616,6 +620,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
616620
}
617621

618622
func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kustomizev1.Kustomization, imp *KustomizeImpersonation, dirPath string) (string, error) {
623+
log := logr.FromContext(ctx)
619624
start := time.Now()
620625
timeout := kustomization.GetTimeout() + (time.Second * 1)
621626
applyCtx, cancel := context.WithTimeout(ctx, timeout)
@@ -658,7 +663,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto
658663
}
659664

660665
resources := parseApplyOutput(output)
661-
(logr.FromContext(ctx)).Info(
666+
log.Info(
662667
fmt.Sprintf("Kustomization applied in %s",
663668
time.Now().Sub(start).String()),
664669
"output", resources,
@@ -674,12 +679,13 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto
674679
}
675680

676681
func (r *KustomizationReconciler) applyWithRetry(ctx context.Context, kustomization kustomizev1.Kustomization, imp *KustomizeImpersonation, revision, dirPath string, delay time.Duration) (string, error) {
682+
log := logr.FromContext(ctx)
677683
changeSet, err := r.apply(ctx, kustomization, imp, dirPath)
678684
if err != nil {
679685
// retry apply due to CRD/CR race
680686
if strings.Contains(err.Error(), "could not find the requested resource") ||
681687
strings.Contains(err.Error(), "no matches for kind") {
682-
(logr.FromContext(ctx)).Info("retrying apply", "error", err.Error())
688+
log.Info("retrying apply", "error", err.Error())
683689
time.Sleep(delay)
684690
if changeSet, err := r.apply(ctx, kustomization, imp, dirPath); err != nil {
685691
return "", err
@@ -707,6 +713,7 @@ func (r *KustomizationReconciler) prune(ctx context.Context, kubeClient client.C
707713
return nil
708714
}
709715

716+
log := logr.FromContext(ctx)
710717
gc := NewGarbageCollector(kubeClient, *kustomization.Status.Snapshot, newChecksum, logr.FromContext(ctx))
711718

712719
if output, ok := gc.Prune(kustomization.GetTimeout(),
@@ -716,7 +723,7 @@ func (r *KustomizationReconciler) prune(ctx context.Context, kubeClient client.C
716723
return fmt.Errorf("garbage collection failed: %s", output)
717724
} else {
718725
if output != "" {
719-
(logr.FromContext(ctx)).Info(fmt.Sprintf("garbage collection completed: %s", output))
726+
log.Info(fmt.Sprintf("garbage collection completed: %s", output))
720727
r.event(ctx, kustomization, newChecksum, events.EventSeverityInfo, output, nil)
721728
}
722729
}
@@ -744,13 +751,14 @@ func (r *KustomizationReconciler) checkHealth(ctx context.Context, statusPoller
744751
}
745752

746753
func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, kustomization kustomizev1.Kustomization) (ctrl.Result, error) {
754+
log := logr.FromContext(ctx)
747755
if kustomization.Spec.Prune && !kustomization.Spec.Suspend {
748756
// create any necessary kube-clients
749757
imp := NewKustomizeImpersonation(kustomization, r.Client, r.StatusPoller, "")
750758
client, _, err := imp.GetClient(ctx)
751759
if err != nil {
752760
err = fmt.Errorf("failed to build kube client for Kustomization: %w", err)
753-
(logr.FromContext(ctx)).Error(err, "Unable to prune for finalizer")
761+
log.Error(err, "Unable to prune for finalizer")
754762
return ctrl.Result{}, err
755763
}
756764
if err := r.prune(ctx, client, kustomization, ""); err != nil {
@@ -774,13 +782,11 @@ func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, kustomiza
774782
}
775783

776784
func (r *KustomizationReconciler) event(ctx context.Context, kustomization kustomizev1.Kustomization, revision, severity, msg string, metadata map[string]string) {
785+
log := logr.FromContext(ctx)
777786
r.EventRecorder.Event(&kustomization, "Normal", severity, msg)
778787
objRef, err := reference.GetReference(r.Scheme, &kustomization)
779788
if err != nil {
780-
(logr.FromContext(ctx)).WithValues(
781-
strings.ToLower(kustomization.Kind),
782-
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
783-
).Error(err, "unable to send event")
789+
log.Error(err, "unable to send event")
784790
return
785791
}
786792

@@ -798,7 +804,7 @@ func (r *KustomizationReconciler) event(ctx context.Context, kustomization kusto
798804
}
799805

800806
if err := r.ExternalEventRecorder.Eventf(*objRef, metadata, severity, reason, msg); err != nil {
801-
(logr.FromContext(ctx)).Error(err, "unable to send event")
807+
log.Error(err, "unable to send event")
802808
return
803809
}
804810
}
@@ -808,10 +814,11 @@ func (r *KustomizationReconciler) recordReadiness(ctx context.Context, kustomiza
808814
if r.MetricsRecorder == nil {
809815
return
810816
}
817+
log := logr.FromContext(ctx)
811818

812819
objRef, err := reference.GetReference(r.Scheme, &kustomization)
813820
if err != nil {
814-
(logr.FromContext(ctx)).Error(err, "unable to record readiness metric")
821+
log.Error(err, "unable to record readiness metric")
815822
return
816823
}
817824
if rc := apimeta.FindStatusCondition(kustomization.Status.Conditions, meta.ReadyCondition); rc != nil {
@@ -824,6 +831,25 @@ func (r *KustomizationReconciler) recordReadiness(ctx context.Context, kustomiza
824831
}
825832
}
826833

834+
func (r *KustomizationReconciler) recordSuspension(ctx context.Context, kustomization kustomizev1.Kustomization) {
835+
if r.MetricsRecorder == nil {
836+
return
837+
}
838+
log := logr.FromContext(ctx)
839+
840+
objRef, err := reference.GetReference(r.Scheme, &kustomization)
841+
if err != nil {
842+
log.Error(err, "unable to record suspended metric")
843+
return
844+
}
845+
846+
if !kustomization.DeletionTimestamp.IsZero() {
847+
r.MetricsRecorder.RecordSuspend(*objRef, false)
848+
} else {
849+
r.MetricsRecorder.RecordSuspend(*objRef, kustomization.Spec.Suspend)
850+
}
851+
}
852+
827853
func (r *KustomizationReconciler) patchStatus(ctx context.Context, req ctrl.Request, newStatus kustomizev1.KustomizationStatus) error {
828854
var kustomization kustomizev1.Kustomization
829855
if err := r.Get(ctx, req.NamespacedName, &kustomization); err != nil {

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ require (
1010
github.com/fluxcd/kustomize-controller/api v0.9.2
1111
github.com/fluxcd/pkg/apis/kustomize v0.0.1
1212
github.com/fluxcd/pkg/apis/meta v0.8.0
13-
github.com/fluxcd/pkg/runtime v0.8.4
13+
github.com/fluxcd/pkg/runtime v0.8.5
1414
github.com/fluxcd/pkg/testserver v0.0.2
1515
github.com/fluxcd/pkg/untar v0.0.5
1616
github.com/fluxcd/source-controller/api v0.9.1

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,8 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7
197197
github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0=
198198
github.com/fluxcd/pkg/apis/meta v0.8.0 h1:wqWpUsxhKHB1ZztcvOz+vnyhdKW9cWmjFp8Vci/XOdk=
199199
github.com/fluxcd/pkg/apis/meta v0.8.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po=
200-
github.com/fluxcd/pkg/runtime v0.8.4 h1:amuhfoHGCUfFCPXg3Zrcyy7f9J+fho+/+FbQDDyewko=
201-
github.com/fluxcd/pkg/runtime v0.8.4/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
200+
github.com/fluxcd/pkg/runtime v0.8.5 h1:ynh8fszbLQ3QSisQBNOABEUTnvt+/QfCdaL6gOJQcoQ=
201+
github.com/fluxcd/pkg/runtime v0.8.5/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
202202
github.com/fluxcd/pkg/testserver v0.0.2 h1:SoaMtO9cE5p/wl2zkGudzflnEHd9mk68CGjZOo7w0Uk=
203203
github.com/fluxcd/pkg/testserver v0.0.2/go.mod h1:pgUZTh9aQ44FSTQo+5NFlh7YMbUfdz1B80DalW7k96Y=
204204
github.com/fluxcd/pkg/untar v0.0.5 h1:UGI3Ch1UIEIaqQvMicmImL1s9npQa64DJ/ozqHKB7gk=

0 commit comments

Comments
 (0)