Skip to content

Commit e9e9719

Browse files
authored
Merge pull request #117 from fluxcd/s3-bucket-source
Add support for S3 bucket sources
2 parents c907900 + 36fd99a commit e9e9719

File tree

16 files changed

+290
-49
lines changed

16 files changed

+290
-49
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,4 @@
1616
bin/
1717
config/release/
1818
config/crd/bases/gitrepositories.yaml
19+
config/crd/bases/buckets.yaml

Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
IMG ?= fluxcd/kustomize-controller:latest
33
# Produce CRDs that work back to Kubernetes 1.16
44
CRD_OPTIONS ?= crd:crdVersions=v1
5-
SOURCE_VER ?= v0.0.16
5+
SOURCE_VER ?= v0.0.17
66

77
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
88
ifeq (,$(shell go env GOBIN))
@@ -29,6 +29,7 @@ run: generate fmt vet manifests
2929
# Download the CRDs the controller depends on
3030
download-crd-deps:
3131
curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml > config/crd/bases/gitrepositories.yaml
32+
curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml > config/crd/bases/buckets.yaml
3233

3334
# Install CRDs into a cluster
3435
install: manifests

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ Specifications:
3333

3434
The kustomize-controller is part of a composable [GitOps toolkit](https://toolkit.fluxcd.io)
3535
and depends on [source-controller](https://github.com/fluxcd/source-controller)
36-
to acquire the Kubernetes manifests from Git repositories.
36+
to acquire the Kubernetes manifests from Git repositories and S3 compatible storage buckets.
3737

3838
### Install the toolkit controllers
3939

api/v1alpha1/kustomization_types.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,11 @@ func (in Kustomization) GetDependsOn() (types.NamespacedName, []dependency.Cross
215215

216216
const (
217217
// SourceIndexKey is the key used for indexing kustomizations
218-
// based on their sources.
218+
// based on their Git sources.
219219
SourceIndexKey string = ".metadata.source"
220+
// BucketIndexKey is the key used for indexing kustomizations
221+
// based on their S3 sources.
222+
BucketIndexKey string = ".metadata.bucket"
220223
)
221224

222225
// +genclient

api/v1alpha1/reference_types.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ type CrossNamespaceSourceReference struct {
4444
APIVersion string `json:"apiVersion,omitempty"`
4545

4646
// Kind of the referent
47-
// +kubebuilder:validation:Enum=GitRepository
47+
// +kubebuilder:validation:Enum=GitRepository;Bucket
4848
// +required
4949
Kind string `json:"kind"`
5050

config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ spec:
146146
description: Kind of the referent
147147
enum:
148148
- GitRepository
149+
- Bucket
149150
type: string
150151
name:
151152
description: Name of the referent

config/default/kustomization.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@ resources:
55
- ../crd
66
- ../rbac
77
- ../manager
8-
- github.com/fluxcd/source-controller/config//crd?ref=v0.0.16
9-
- github.com/fluxcd/source-controller/config//manager?ref=v0.0.16
8+
- github.com/fluxcd/source-controller/config//crd?ref=v0.0.17
9+
- github.com/fluxcd/source-controller/config//manager?ref=v0.0.17
1010
- namespace.yaml

config/rbac/role.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,20 @@ rules:
3333
- get
3434
- patch
3535
- update
36+
- apiGroups:
37+
- source.toolkit.fluxcd.io
38+
resources:
39+
- buckets
40+
verbs:
41+
- get
42+
- list
43+
- watch
44+
- apiGroups:
45+
- source.toolkit.fluxcd.io
46+
resources:
47+
- buckets/status
48+
verbs:
49+
- get
3650
- apiGroups:
3751
- source.toolkit.fluxcd.io
3852
resources:

controllers/bucket_predicate.go

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
Copyright 2020 The Flux CD contributors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"sigs.k8s.io/controller-runtime/pkg/event"
21+
"sigs.k8s.io/controller-runtime/pkg/predicate"
22+
23+
sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
24+
)
25+
26+
type BucketRevisionChangePredicate struct {
27+
predicate.Funcs
28+
}
29+
30+
func (BucketRevisionChangePredicate) Update(e event.UpdateEvent) bool {
31+
if e.MetaOld == nil || e.MetaNew == nil {
32+
return false
33+
}
34+
35+
oldRepo, ok := e.ObjectOld.(*sourcev1.Bucket)
36+
if !ok {
37+
return false
38+
}
39+
40+
newRepo, ok := e.ObjectNew.(*sourcev1.Bucket)
41+
if !ok {
42+
return false
43+
}
44+
45+
if oldRepo.GetArtifact() == nil && newRepo.GetArtifact() != nil {
46+
return true
47+
}
48+
49+
if oldRepo.GetArtifact() != nil && newRepo.GetArtifact() != nil &&
50+
oldRepo.GetArtifact().Checksum != newRepo.GetArtifact().Checksum {
51+
return true
52+
}
53+
54+
return false
55+
}
56+
57+
func (BucketRevisionChangePredicate) Create(e event.CreateEvent) bool {
58+
return false
59+
}
60+
61+
func (BucketRevisionChangePredicate) Delete(e event.DeleteEvent) bool {
62+
return false
63+
}

controllers/bucket_watcher.go

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
Copyright 2020 The Flux CD contributors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"strings"
23+
"time"
24+
25+
"github.com/fluxcd/pkg/runtime/dependency"
26+
"github.com/go-logr/logr"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/apimachinery/pkg/runtime"
29+
"k8s.io/apimachinery/pkg/types"
30+
"k8s.io/client-go/util/retry"
31+
ctrl "sigs.k8s.io/controller-runtime"
32+
"sigs.k8s.io/controller-runtime/pkg/client"
33+
34+
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1"
35+
consts "github.com/fluxcd/pkg/runtime"
36+
sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
37+
)
38+
39+
// BucketWatcher watches Bucket objects for revision changes
40+
// and triggers a reconcile for all the Kustomizations that reference a changed source
41+
type BucketWatcher struct {
42+
client.Client
43+
Log logr.Logger
44+
Scheme *runtime.Scheme
45+
}
46+
47+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets,verbs=get;list;watch
48+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets/status,verbs=get
49+
50+
func (r *BucketWatcher) Reconcile(req ctrl.Request) (ctrl.Result, error) {
51+
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
52+
defer cancel()
53+
54+
var repo sourcev1.Bucket
55+
if err := r.Get(ctx, req.NamespacedName, &repo); err != nil {
56+
return ctrl.Result{}, client.IgnoreNotFound(err)
57+
}
58+
59+
log := r.Log.WithValues(strings.ToLower(repo.Kind), req.NamespacedName)
60+
log.Info("new artifact detected")
61+
62+
// get the list of kustomizations that are using this Bucket
63+
var list kustomizev1.KustomizationList
64+
if err := r.List(ctx, &list,
65+
client.MatchingFields{kustomizev1.BucketIndexKey: fmt.Sprintf("%s/%s", req.Namespace, req.Name)}); err != nil {
66+
log.Error(err, "unable to list kustomizations")
67+
return ctrl.Result{}, err
68+
}
69+
70+
var dd []dependency.Dependent
71+
for _, d := range list.Items {
72+
dd = append(dd, d)
73+
}
74+
sorted, err := dependency.Sort(dd)
75+
if err != nil {
76+
log.Error(err, "unable to dependency sort kustomizations")
77+
return ctrl.Result{}, err
78+
}
79+
80+
// trigger apply for each kustomization using this Bucket taking into account the dependency order
81+
for _, k := range sorted {
82+
name := types.NamespacedName(k)
83+
if err := r.requestReconciliation(name); err != nil {
84+
log.Error(err, "unable to annotate Kustomization", "kustomization", name)
85+
continue
86+
}
87+
log.Info("requested immediate reconciliation", "kustomization", name)
88+
}
89+
90+
return ctrl.Result{}, nil
91+
}
92+
93+
func (r *BucketWatcher) SetupWithManager(mgr ctrl.Manager) error {
94+
// create a kustomization index based on Bucket name
95+
err := mgr.GetFieldIndexer().IndexField(context.TODO(), &kustomizev1.Kustomization{}, kustomizev1.BucketIndexKey,
96+
func(rawObj runtime.Object) []string {
97+
k := rawObj.(*kustomizev1.Kustomization)
98+
if k.Spec.SourceRef.Kind == sourcev1.BucketKind {
99+
namespace := k.GetNamespace()
100+
if k.Spec.SourceRef.Namespace != "" {
101+
namespace = k.Spec.SourceRef.Namespace
102+
}
103+
return []string{fmt.Sprintf("%s/%s", namespace, k.Spec.SourceRef.Name)}
104+
}
105+
return nil
106+
},
107+
)
108+
if err != nil {
109+
return err
110+
}
111+
112+
return ctrl.NewControllerManagedBy(mgr).
113+
For(&sourcev1.Bucket{}).
114+
WithEventFilter(BucketRevisionChangePredicate{}).
115+
Complete(r)
116+
}
117+
118+
func (r *BucketWatcher) requestReconciliation(name types.NamespacedName) error {
119+
var kustomization kustomizev1.Kustomization
120+
return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
121+
if err := r.Get(context.TODO(), name, &kustomization); err != nil {
122+
return err
123+
}
124+
125+
if kustomization.Annotations == nil {
126+
kustomization.Annotations = make(map[string]string)
127+
}
128+
kustomization.Annotations[consts.ReconcileAtAnnotation] = metav1.Now().String()
129+
err = r.Update(context.TODO(), &kustomization)
130+
return
131+
})
132+
}

0 commit comments

Comments
 (0)