Merge pull request #1502 from fluxcd/status-history

Track reconciliation attempts over time in `.status.history`

Author: stefanprodan

api/v1/kustomization_types.go

@@ -297,6 +297,11 @@ type KustomizationStatus struct {
 	// have been successfully applied.
 	// +optional
 	Inventory *ResourceInventory `json:"inventory,omitempty"`
+
+	// History contains a set of snapshots of the last reconciliation attempts
+	// tracking the revision, the state and the duration of each attempt.
+	// +optional
+	History meta.History `json:"history,omitempty"`
 }
 
 // GetTimeout returns the timeout with default.

api/v1/zz_generated.deepcopy.go

@@ -591,6 +591,57 @@ spec:
                   - type
                   type: object
                 type: array
+              history:
+                description: |-
+                  History contains a set of snapshots of the last reconciliation attempts
+                  tracking the revision, the state and the duration of each attempt.
+                items:
+                  description: |-
+                    Snapshot represents a point-in-time record of a group of resources reconciliation,
+                    including timing information, status, and a unique digest identifier.
+                  properties:
+                    digest:
+                      description: Digest is the checksum in the format `<algo>:<hex>`
+                        of the resources in this snapshot.
+                      type: string
+                    firstReconciled:
+                      description: FirstReconciled is the time when this revision
+                        was first reconciled to the cluster.
+                      format: date-time
+                      type: string
+                    lastReconciled:
+                      description: LastReconciled is the time when this revision was
+                        last reconciled to the cluster.
+                      format: date-time
+                      type: string
+                    lastReconciledDuration:
+                      description: LastReconciledDuration is time it took to reconcile
+                        the resources in this revision.
+                      type: string
+                    lastReconciledStatus:
+                      description: LastReconciledStatus is the status of the last
+                        reconciliation.
+                      type: string
+                    metadata:
+                      additionalProperties:
+                        type: string
+                      description: Metadata contains additional information about
+                        the snapshot.
+                      type: object
+                    totalReconciliations:
+                      description: TotalReconciliations is the total number of reconciliations
+                        that have occurred for this snapshot.
+                      format: int64
+                      type: integer
+                  required:
+                  - digest
+                  - firstReconciled
+                  - lastReconciled
+                  - lastReconciledDuration
+                  - lastReconciledStatus
+                  - totalReconciliations
+                  type: object
+                type: array
               inventory:
                 description: |-
                   Inventory contains the list of Kubernetes resource object references that

config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml

@@ -1147,6 +1147,21 @@ ResourceInventory
 have been successfully applied.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>history</code><br>
+<em>
+<a href="https://godoc.org/github.com/fluxcd/pkg/apis/meta#History">
+github.com/fluxcd/pkg/apis/meta.History
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>History contains a set of snapshots of the last reconciliation attempts
+tracking the revision, the state and the duration of each attempt.</p>
+</td>
+</tr>
 </tbody>
 </table>
 </div>

docs/api/v1/kustomize.md

@@ -2066,6 +2066,37 @@ configuration issue in the Kustomization spec. When a reconciliation fails, the
 `Reconciling` Condition `reason` would be `ProgressingWithRetry`. When the
 reconciliation is performed again after the failure, the `reason` is updated to `Progressing`.
 
+### History
+
+The kustomize-controller maintains a history of the last 5 reconciliations
+in `.status.history`, including the digest of the applied manifests, the
+source and origin revision, the timestamps and the duration of the reconciliations,
+the status and the total number of times a specific digest was reconciled.
+
+```yaml
+status:
+  history:
+    - digest: sha256:43ad78c94b2655429d84f21488f29d7cca9cd45b7f54d2b27e16bbec8eff9228
+      firstReconciled: "2025-08-15T10:11:00Z"
+      lastReconciled: "2025-08-15T11:12:00Z"
+      lastReconciledDuration: 2.818583s
+      lastReconciledStatus: ReconciliationSucceeded
+      totalReconciliations: 2
+      metadata:
+        revision: "v1.0.1@sha1:450796ddb2ab6724ee1cc32a4be56da032d1cca0"
+    - digest: sha256:ec8dbfe61777b65001190260cf873ffe454451bd2e464bd6f9a154cffcdcd7e5
+      firstReconciled: "2025-07-14T13:10:00Z"
+      lastReconciled: "2025-08-15T10:00:00Z"
+      lastReconciledDuration: 49.813292s
+      lastReconciledStatus: HealthCheckFailed
+      totalReconciliations: 120
+      metadata:
+        revision: "v1.0.0@sha1:67e2c98a60dc92283531412a9e604dd4bae005a9"
+```
+
+The kustomize-controller deduplicates entries based on the digest and status, with the
+most recent reconciliation being the first entry in the list.
+
 ### Inventory
 
 In order to perform operations such as drift detection, garbage collection, etc.

docs/spec/v1/kustomizations.md

@@ -27,6 +27,7 @@ import (
 
 	securejoin "github.com/cyphar/filepath-securejoin"
 	celtypes "github.com/google/cel-go/common/types"
+	"github.com/opencontainers/go-digest"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -281,6 +282,7 @@ func (r *KustomizationReconciler) reconcile(
 	src sourcev1.Source,
 	patcher *patch.SerialPatcher,
 	statusReaders []func(apimeta.RESTMapper) engine.StatusReader) error {
+	reconcileStart := time.Now()
 	log := ctrl.LoggerFrom(ctx)
 
 	// Update status with the reconciliation progress.
@@ -314,13 +316,14 @@ func (r *KustomizationReconciler) reconcile(
 	}(tmpDir)
 
 	// Download artifact and extract files to the tmp dir.
-	if err = fetch.NewArchiveFetcherWithLogger(
-		r.artifactFetchRetries,
-		tar.UnlimitedUntarSize,
-		tar.UnlimitedUntarSize,
-		os.Getenv("SOURCE_CONTROLLER_LOCALHOST"),
-		ctrl.LoggerFrom(ctx),
-	).Fetch(src.GetArtifact().URL, src.GetArtifact().Digest, tmpDir); err != nil {
+	fetcher := fetch.New(
+		fetch.WithLogger(ctrl.LoggerFrom(ctx)),
+		fetch.WithRetries(r.artifactFetchRetries),
+		fetch.WithMaxDownloadSize(tar.UnlimitedUntarSize),
+		fetch.WithUntar(tar.WithMaxUntarSize(tar.UnlimitedUntarSize)),
+		fetch.WithHostnameOverwrite(os.Getenv("SOURCE_CONTROLLER_LOCALHOST")),
+	)
+	if err = fetcher.Fetch(src.GetArtifact().URL, src.GetArtifact().Digest, tmpDir); err != nil {
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.ArtifactFailedReason, "%s", err)
 		return err
 	}
@@ -398,6 +401,13 @@ func (r *KustomizationReconciler) reconcile(
 		return err
 	}
 
+	// Calculate the digest of the built resources for history tracking.
+	checksum := digest.FromBytes(resources).String()
+	historyMeta := map[string]string{"revision": revision}
+	if originRevision != "" {
+		historyMeta["originRevision"] = originRevision
+	}
+
 	// Convert the build result into Kubernetes unstructured objects.
 	objects, err := ssautil.ReadObjects(bytes.NewReader(resources))
 	if err != nil {
@@ -423,6 +433,7 @@ func (r *KustomizationReconciler) reconcile(
 	// Validate and apply resources in stages.
 	drifted, changeSet, err := r.apply(ctx, resourceManager, obj, revision, originRevision, objects)
 	if err != nil {
+		obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.ReconciliationFailedReason, historyMeta)
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err)
 		return err
 	}
@@ -431,6 +442,7 @@ func (r *KustomizationReconciler) reconcile(
 	newInventory := inventory.New()
 	err = inventory.AddChangeSet(newInventory, changeSet)
 	if err != nil {
+		obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.ReconciliationFailedReason, historyMeta)
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err)
 		return err
 	}
@@ -441,12 +453,14 @@ func (r *KustomizationReconciler) reconcile(
 	// Detect stale resources which are subject to garbage collection.
 	staleObjects, err := inventory.Diff(oldInventory, newInventory)
 	if err != nil {
+		obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.ReconciliationFailedReason, historyMeta)
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err)
 		return err
 	}
 
 	// Run garbage collection for stale resources that do not have pruning disabled.
 	if _, err := r.prune(ctx, resourceManager, obj, revision, originRevision, staleObjects); err != nil {
+		obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.PruneFailedReason, historyMeta)
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.PruneFailedReason, "%s", err)
 		return err
 	}
@@ -462,6 +476,7 @@ func (r *KustomizationReconciler) reconcile(
 		isNewRevision,
 		drifted,
 		changeSet.ToObjMetadataSet()); err != nil {
+		obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.HealthCheckFailedReason, historyMeta)
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.HealthCheckFailedReason, "%s", err)
 		return err
 	}
@@ -475,6 +490,11 @@ func (r *KustomizationReconciler) reconcile(
 		meta.ReadyCondition,
 		meta.ReconciliationSucceededReason,
 		"Applied revision: %s", revision)
+	obj.Status.History.Upsert(checksum,
+		time.Now(),
+		time.Since(reconcileStart),
+		meta.ReconciliationSucceededReason,
+		historyMeta)
 
 	return nil
 }

internal/controller/kustomization_controller.go

@@ -115,6 +115,15 @@ stringData:
 
 	g.Expect(resultK.Status.LastAppliedOriginRevision).To(Equal("orev"))
 
+	g.Expect(resultK.Status.History).To(HaveLen(1))
+	g.Expect(resultK.Status.History[0].Digest).ToNot(BeEmpty())
+	g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
+	g.Expect(resultK.Status.History[0].FirstReconciled.Time).ToNot(BeZero())
+	g.Expect(resultK.Status.History[0].LastReconciled.Time).ToNot(BeZero())
+	g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
+	g.Expect(resultK.Status.History[0].LastReconciledDuration.Duration).To(BeNumerically(">", 0))
+	g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision, "orev"))
+
 	events := getEvents(kustomizationKey.Name, nil)
 	g.Expect(events).To(Not(BeEmpty()))
 

internal/controller/kustomization_origin_revision_test.go

@@ -136,6 +136,11 @@ parameters:
 		g.Expect(resultK.Status.ObservedGeneration).To(BeIdenticalTo(resultK.Generation))
 
 		kstatusCheck.CheckErr(ctx, resultK)
+
+		g.Expect(resultK.Status.History).To(HaveLen(1))
+		g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
+		g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
+		g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
 	})
 
 	t.Run("reports progressing status", func(t *testing.T) {
@@ -192,6 +197,11 @@ parameters:
 		g.Expect(resultK.Status.ObservedGeneration).To(BeIdenticalTo(resultK.Generation - 1))
 
 		kstatusCheck.CheckErr(ctx, resultK)
+
+		g.Expect(resultK.Status.History).To(HaveLen(2))
+		g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
+		g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.HealthCheckFailedReason))
+		g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
 	})
 
 	t.Run("emits unhealthy event", func(t *testing.T) {
@@ -228,6 +238,11 @@ parameters:
 		g.Expect(resultK.Status.ObservedGeneration).To(BeIdenticalTo(resultK.Generation))
 
 		kstatusCheck.CheckErr(ctx, resultK)
+
+		g.Expect(resultK.Status.History).To(HaveLen(2))
+		g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(2))
+		g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
+		g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
 	})
 
 	t.Run("emits recovery event", func(t *testing.T) {
@@ -258,6 +273,11 @@ parameters:
 		g.Expect(resultK.Status.LastAttemptedRevision).To(BeIdenticalTo(resultK.Status.LastAppliedRevision))
 
 		kstatusCheck.CheckErr(ctx, resultK)
+
+		g.Expect(resultK.Status.History).To(HaveLen(3))
+		g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
+		g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
+		g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
 	})
 
 	t.Run("emits event for the new revision", func(t *testing.T) {