[RFC-0012] Add support for ExternalArtifact source type

.gitignore

@@ -19,6 +19,7 @@ config/release/
 config/crd/bases/ocirepositories.yaml
 config/crd/bases/gitrepositories.yaml
 config/crd/bases/buckets.yaml
+config/crd/bases/externalartifacts.yaml
 
 build/
 

Makefile

@@ -38,6 +38,7 @@ ENVTEST_ARCH ?= amd64
 GITREPO_CRD ?= config/crd/bases/gitrepositories.yaml
 BUCKET_CRD ?= config/crd/bases/buckets.yaml
 OCIREPO_CRD ?= config/crd/bases/ocirepositories.yaml
+EA_CRD ?= config/crd/bases/externalartifacts.yaml
 
 # Keep a record of the version of the downloaded source CRDs. It is used to
 # detect and download new CRDs when the SOURCE_VER changes.
@@ -90,12 +91,15 @@ $(BUCKET_CRD):
 $(OCIREPO_CRD):
 	curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml -o $(OCIREPO_CRD)
 
+$(EA_CRD):
+	curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_externalartifacts.yaml -o $(EA_CRD)
+
 # Download the CRDs the controller depends on
-download-crd-deps: $(SOURCE_CRD_VER) $(GITREPO_CRD) $(BUCKET_CRD) $(OCIREPO_CRD)
+download-crd-deps: $(SOURCE_CRD_VER) $(GITREPO_CRD) $(BUCKET_CRD) $(OCIREPO_CRD) $(EA_CRD)
 
 # Delete the downloaded CRD dependencies.
 cleanup-crd-deps:
-	rm -f $(GITREPO_CRD) $(BUCKET_CRD) $(OCIREPO_CRD)
+	rm -f $(GITREPO_CRD) $(BUCKET_CRD) $(OCIREPO_CRD) $(EA_CRD)
 
 # Install CRDs into a cluster
 install: manifests

api/v1/reference_types.go

@@ -28,7 +28,7 @@ type CrossNamespaceSourceReference struct {
 	APIVersion string `json:"apiVersion,omitempty"`
 
 	// Kind of the referent.
-	// +kubebuilder:validation:Enum=OCIRepository;GitRepository;Bucket
+	// +kubebuilder:validation:Enum=OCIRepository;GitRepository;Bucket;ExternalArtifact
 	// +required
 	Kind string `json:"kind"`
 

config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml

@@ -494,6 +494,7 @@ spec:
                     - OCIRepository
                     - GitRepository
                     - Bucket
+                    - ExternalArtifact
                     type: string
                   name:
                     description: Name of the referent.

config/default/kustomization.yaml

@@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: kustomize-system
 resources:
-- https://github.com/fluxcd/source-controller/releases/download/v1.6.0/source-controller.crds.yaml
-- https://github.com/fluxcd/source-controller/releases/download/v1.6.0/source-controller.deployment.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v1.7.0-rc.3/source-controller.crds.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v1.7.0-rc.3/source-controller.deployment.yaml
 - ../crd
 - ../rbac
 - ../manager

docs/spec/v1/kustomizations.md

@@ -117,6 +117,7 @@ Artifact containing the YAML manifests. It has two required fields:
   + [GitRepository](https://github.com/fluxcd/source-controller/blob/main/docs/spec/v1/gitrepositories.md)
   + [OCIRepository](https://github.com/fluxcd/source-controller/blob/main/docs/spec/v1/ocirepositories.md)
   + [Bucket](https://github.com/fluxcd/source-controller/blob/main/docs/spec/v1/buckets.md)
+  + [ExternalArtifact](https://github.com/fluxcd/source-controller/blob/main/docs/spec/v1/externalartifacts.md) (requires `--feature-gates=ExternalArtifact=true` flag)
 - `name`: The Name of the referred Source object.
 
 #### Cross-namespace references

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/fluxcd/pkg/ssa v0.53.0
 	github.com/fluxcd/pkg/tar v0.14.0
 	github.com/fluxcd/pkg/testserver v0.13.0
-	github.com/fluxcd/source-controller/api v1.6.0
+	github.com/fluxcd/source-controller/api v1.7.0-rc.3
 	github.com/getsops/sops/v3 v3.10.2
 	github.com/google/cel-go v0.26.1
 	github.com/hashicorp/vault/api v1.20.0

go.sum

@@ -217,8 +217,8 @@ github.com/fluxcd/pkg/tar v0.14.0 h1:9Gku8FIvPt2bixKldZnzXJ/t+7SloxePlzyVGOK8GVQ
 github.com/fluxcd/pkg/tar v0.14.0/go.mod h1:+rOWYk93qLEJ8WwmkvJOkB8i0dna1mrwJFybE8i9Udo=
 github.com/fluxcd/pkg/testserver v0.13.0 h1:xEpBcEYtD7bwvZ+i0ZmChxKkDo/wfQEV3xmnzVybSSg=
 github.com/fluxcd/pkg/testserver v0.13.0/go.mod h1:akRYv3FLQUsme15na9ihECRG6hBuqni4XEY9W8kzs8E=
-github.com/fluxcd/source-controller/api v1.6.0 h1:IxfjUczJ2pzbXIef6iQ0RHEH4AYA9anJfTGK8dzwODM=
-github.com/fluxcd/source-controller/api v1.6.0/go.mod h1:ZJcAi0nemsnBxjVgmJl0WQzNvB0rMETxQMTdoFosmMw=
+github.com/fluxcd/source-controller/api v1.7.0-rc.3 h1:+9cd//77LAgp0XRe97CXUaPnu78jsRNWTXq95GHGhgc=
+github.com/fluxcd/source-controller/api v1.7.0-rc.3/go.mod h1:sbJibK4Ik+2AuTRRLXPA+n2u6nLUIGaxC07ava+RqeM=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=

internal/controller/kustomization_acl_test.go

@@ -120,6 +120,7 @@ stringData:
 
 	t.Run("fails to reconcile from cross-namespace source", func(t *testing.T) {
 		reconciler.NoCrossNamespaceRefs = true
+		defer func() { reconciler.NoCrossNamespaceRefs = false }()
 
 		revision = "v2.0.0"
 		err = applyGitRepository(repositoryName, artifact, revision)
@@ -132,5 +133,6 @@ stringData:
 		}, timeout, time.Second).Should(BeTrue())
 
 		g.Expect(readyCondition.Reason).To(Equal(apiacl.AccessDeniedReason))
+		g.Expect(apimeta.IsStatusConditionTrue(resultK.Status.Conditions, meta.StalledCondition)).Should(BeTrue())
 	})
 }

internal/controller/kustomization_controller.go

@@ -70,6 +70,7 @@ import (
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
 	intcache "github.com/fluxcd/kustomize-controller/internal/cache"
 	"github.com/fluxcd/kustomize-controller/internal/decryptor"
+	"github.com/fluxcd/kustomize-controller/internal/features"
 	"github.com/fluxcd/kustomize-controller/internal/inventory"
 	intruntime "github.com/fluxcd/kustomize-controller/internal/runtime"
 )
@@ -89,26 +90,37 @@ type KustomizationReconciler struct {
 	kuberecorder.EventRecorder
 	runtimeCtrl.Metrics
 
-	artifactFetchRetries int
-	requeueDependency    time.Duration
+	// Kubernetes options
 
-	Mapper                     apimeta.RESTMapper
-	APIReader                  client.Reader
-	ClusterReader              engine.ClusterReaderFactory
-	ControllerName             string
-	statusManager              string
-	NoCrossNamespaceRefs       bool
-	NoRemoteBases              bool
+	APIReader      client.Reader
+	ClusterReader  engine.ClusterReaderFactory
+	ConcurrentSSA  int
+	ControllerName string
+	KubeConfigOpts runtimeClient.KubeConfigOptions
+	Mapper         apimeta.RESTMapper
+	StatusManager  string
+
+	// Multi-tenancy and security options
+
+	DefaultServiceAccount   string
+	DisallowedFieldManagers []string
+	NoCrossNamespaceRefs    bool
+	NoRemoteBases           bool
+	SOPSAgeSecret           string
+	TokenCache              *cache.TokenCache
+
+	// Retry and requeue options
+
+	ArtifactFetchRetries      int
+	DependencyRequeueInterval time.Duration
+
+	// Feature gates
+
+	AdditiveCELDependencyCheck bool
+	AllowExternalArtifact      bool
 	FailFast                   bool
-	DefaultServiceAccount      string
-	SOPSAgeSecret              string
-	KubeConfigOpts             runtimeClient.KubeConfigOptions
-	ConcurrentSSA              int
-	DisallowedFieldManagers    []string
-	StrictSubstitutions        bool
 	GroupChangeLog             bool
-	AdditiveCELDependencyCheck bool
-	TokenCache                 *cache.TokenCache
+	StrictSubstitutions        bool
 }
 
 func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, retErr error) {
@@ -207,9 +219,9 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 		if acl.IsAccessDenied(err) {
 			conditions.MarkFalse(obj, meta.ReadyCondition, apiacl.AccessDeniedReason, "%s", err)
-			log.Error(err, "Access denied to cross-namespace source")
+			conditions.MarkStalled(obj, apiacl.AccessDeniedReason, "%s", err)
 			r.event(obj, "", "", eventv1.EventSeverityError, err.Error(), nil)
-			return ctrl.Result{RequeueAfter: obj.GetRetryInterval()}, nil
+			return ctrl.Result{}, reconcile.TerminalError(err)
 		}
 
 		// Retry with backoff on transient errors.
@@ -218,10 +230,10 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	// Requeue the reconciliation if the source artifact is not found.
 	if artifactSource.GetArtifact() == nil {
-		msg := fmt.Sprintf("Source artifact not found, retrying in %s", r.requeueDependency.String())
+		msg := fmt.Sprintf("Source artifact not found, retrying in %s", r.DependencyRequeueInterval.String())
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.ArtifactFailedReason, "%s", msg)
 		log.Info(msg)
-		return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
+		return ctrl.Result{RequeueAfter: r.DependencyRequeueInterval}, nil
 	}
 	revision := artifactSource.GetArtifact().Revision
 	originRevision := getOriginRevision(artifactSource)
@@ -241,10 +253,10 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 			// Retry on transient errors.
 			conditions.MarkFalse(obj, meta.ReadyCondition, meta.DependencyNotReadyReason, "%s", err)
-			msg := fmt.Sprintf("Dependencies do not meet ready condition, retrying in %s", r.requeueDependency.String())
+			msg := fmt.Sprintf("Dependencies do not meet ready condition, retrying in %s", r.DependencyRequeueInterval.String())
 			log.Info(msg)
 			r.event(obj, revision, originRevision, eventv1.EventSeverityInfo, msg, nil)
-			return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
+			return ctrl.Result{RequeueAfter: r.DependencyRequeueInterval}, nil
 		}
 		log.Info("All dependencies are ready, proceeding with reconciliation")
 	}
@@ -254,10 +266,10 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	// Requeue at the specified retry interval if the artifact tarball is not found.
 	if errors.Is(reconcileErr, fetch.ErrFileNotFound) {
-		msg := fmt.Sprintf("Source is not ready, artifact not found, retrying in %s", r.requeueDependency.String())
+		msg := fmt.Sprintf("Source is not ready, artifact not found, retrying in %s", r.DependencyRequeueInterval.String())
 		conditions.MarkFalse(obj, meta.ReadyCondition, meta.ArtifactFailedReason, "%s", msg)
 		log.Info(msg)
-		return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
+		return ctrl.Result{RequeueAfter: r.DependencyRequeueInterval}, nil
 	}
 
 	// Broadcast the reconciliation failure and requeue at the specified retry interval.
@@ -318,7 +330,7 @@ func (r *KustomizationReconciler) reconcile(
 	// Download artifact and extract files to the tmp dir.
 	fetcher := fetch.New(
 		fetch.WithLogger(ctrl.LoggerFrom(ctx)),
-		fetch.WithRetries(r.artifactFetchRetries),
+		fetch.WithRetries(r.ArtifactFetchRetries),
 		fetch.WithMaxDownloadSize(tar.UnlimitedUntarSize),
 		fetch.WithUntar(tar.WithMaxUntarSize(tar.UnlimitedUntarSize)),
 		fetch.WithHostnameOverwrite(os.Getenv("SOURCE_CONTROLLER_LOCALHOST")),
@@ -613,6 +625,8 @@ func (r *KustomizationReconciler) evalReadyExpr(
 	return celExpr.EvaluateBoolean(ctx, vars)
 }
 
+// getSource resolves the source reference and returns the source object containing the artifact.
+// It returns an error if the source is not found or if access is denied.
 func (r *KustomizationReconciler) getSource(ctx context.Context,
 	obj *kustomizev1.Kustomization) (sourcev1.Source, error) {
 	var src sourcev1.Source
@@ -625,12 +639,20 @@ func (r *KustomizationReconciler) getSource(ctx context.Context,
 		Name:      obj.Spec.SourceRef.Name,
 	}
 
+	// Check if cross-namespace references are allowed.
 	if r.NoCrossNamespaceRefs && sourceNamespace != obj.GetNamespace() {
 		return src, acl.AccessDeniedError(
 			fmt.Sprintf("can't access '%s/%s', cross-namespace references have been blocked",
 				obj.Spec.SourceRef.Kind, namespacedName))
 	}
 
+	// Check if ExternalArtifact kind is allowed.
+	if obj.Spec.SourceRef.Kind == sourcev1.ExternalArtifactKind && !r.AllowExternalArtifact {
+		return src, acl.AccessDeniedError(
+			fmt.Sprintf("can't access '%s/%s', %s feature gate is disabled",
+				obj.Spec.SourceRef.Kind, namespacedName, features.ExternalArtifact))
+	}
+
 	switch obj.Spec.SourceRef.Kind {
 	case sourcev1.OCIRepositoryKind:
 		var repository sourcev1.OCIRepository
@@ -662,6 +684,16 @@ func (r *KustomizationReconciler) getSource(ctx context.Context,
 			return src, fmt.Errorf("unable to get source '%s': %w", namespacedName, err)
 		}
 		src = &bucket
+	case sourcev1.ExternalArtifactKind:
+		var ea sourcev1.ExternalArtifact
+		err := r.Client.Get(ctx, namespacedName, &ea)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return src, err
+			}
+			return src, fmt.Errorf("unable to get source '%s': %w", namespacedName, err)
+		}
+		src = &ea
 	default:
 		return src, fmt.Errorf("source `%s` kind '%s' not supported",
 			obj.Spec.SourceRef.Name, obj.Spec.SourceRef.Kind)
@@ -1194,7 +1226,7 @@ func (r *KustomizationReconciler) patch(ctx context.Context,
 	patchOpts = append(patchOpts,
 		patch.WithOwnedConditions{Conditions: ownedConditions},
 		patch.WithForceOverwriteConditions{},
-		patch.WithFieldOwner(r.statusManager),
+		patch.WithFieldOwner(r.StatusManager),
 	)
 
 	// Patch the object status, conditions and finalizers.