Merge pull request #32 from fluxcd/dep-sort

Sort dependencies with Tarjan's SCCS algorithm

Author: stefanprodan

api/v1alpha1/sort.go

@@ -0,0 +1,151 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"fmt"
+)
+
+// +kubebuilder:object:generate=false
+type Unsortable [][]string
+
+func (e Unsortable) Error() string {
+	return fmt.Sprintf("circular dependencies: %v", [][]string(e))
+}
+
+func DependencySort(ks []Kustomization) ([]Kustomization, error) {
+	n := make(graph)
+	lookup := map[string]*Kustomization{}
+	for i := 0; i < len(ks); i++ {
+		n[ks[i].Name] = after(ks[i].Spec.DependsOn)
+		lookup[ks[i].Name] = &ks[i]
+	}
+	sccs := tarjanSCC(n)
+	var sorted []Kustomization
+	var unsortable Unsortable
+	for i := 0; i < len(sccs); i++ {
+		s := sccs[i]
+		if len(s) != 1 {
+			unsortable = append(unsortable, s)
+			continue
+		}
+		if k, ok := lookup[s[0]]; ok {
+			sorted = append(sorted, *k.DeepCopy())
+		}
+	}
+	if unsortable != nil {
+		for i, j := 0, len(unsortable)-1; i < j; i, j = i+1, j-1 {
+			unsortable[i], unsortable[j] = unsortable[j], unsortable[i]
+		}
+		return nil, unsortable
+	}
+	return sorted, nil
+}
+
+// https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm#The_algorithm_in_pseudocode
+
+type graph map[string]edges
+
+type edges map[string]struct{}
+
+func after(i []string) edges {
+	if len(i) == 0 {
+		return nil
+	}
+	s := make(edges)
+	for _, v := range i {
+		s[v] = struct{}{}
+	}
+	return s
+}
+
+func tarjanSCC(g graph) [][]string {
+	t := tarjan{
+		g: g,
+
+		indexTable: make(map[string]int, len(g)),
+		lowLink:    make(map[string]int, len(g)),
+		onStack:    make(map[string]bool, len(g)),
+	}
+	for v := range t.g {
+		if t.indexTable[v] == 0 {
+			t.strongconnect(v)
+		}
+	}
+	return t.sccs
+}
+
+type tarjan struct {
+	g graph
+
+	index      int
+	indexTable map[string]int
+	lowLink    map[string]int
+	onStack    map[string]bool
+
+	stack []string
+
+	sccs [][]string
+}
+
+func (t *tarjan) strongconnect(v string) {
+	// Set the depth index for v to the smallest unused index.
+	t.index++
+	t.indexTable[v] = t.index
+	t.lowLink[v] = t.index
+	t.stack = append(t.stack, v)
+	t.onStack[v] = true
+
+	// Consider successors of v.
+	for w := range t.g[v] {
+		if t.indexTable[w] == 0 {
+			// Successor w has not yet been visited; recur on it.
+			t.strongconnect(w)
+			t.lowLink[v] = min(t.lowLink[v], t.lowLink[w])
+		} else if t.onStack[w] {
+			// Successor w is in stack s and hence in the current SCC.
+			t.lowLink[v] = min(t.lowLink[v], t.indexTable[w])
+		}
+	}
+
+	// If v is a root graph, pop the stack and generate an SCC.
+	if t.lowLink[v] == t.indexTable[v] {
+		// Start a new strongly connected component.
+		var (
+			scc []string
+			w   string
+		)
+		for {
+			w, t.stack = t.stack[len(t.stack)-1], t.stack[:len(t.stack)-1]
+			t.onStack[w] = false
+			// Add w to current strongly connected component.
+			scc = append(scc, w)
+			if w == v {
+				break
+			}
+		}
+		// Output the current strongly connected component.
+		t.sccs = append(t.sccs, scc)
+	}
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}

api/v1alpha1/sort_test.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"reflect"
+	"testing"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestDependencySort(t *testing.T) {
+	tests := []struct {
+		name    string
+		ks      []Kustomization
+		want    []Kustomization
+		wantErr bool
+	}{
+		{
+			"simple",
+			[]Kustomization{
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "frontend"},
+					Spec:       KustomizationSpec{DependsOn: []string{"backend"}},
+				},
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "common"},
+				},
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "backend"},
+					Spec:       KustomizationSpec{DependsOn: []string{"common"}},
+				},
+			},
+			[]Kustomization{
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "common"},
+				},
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "backend"},
+					Spec:       KustomizationSpec{DependsOn: []string{"common"}},
+				},
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "frontend"},
+					Spec:       KustomizationSpec{DependsOn: []string{"backend"}},
+				},
+			},
+			false,
+		},
+		{
+			"circle dependency",
+			[]Kustomization{
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "dependency"},
+					Spec:       KustomizationSpec{DependsOn: []string{"endless"}},
+				},
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "endless"},
+					Spec:       KustomizationSpec{DependsOn: []string{"circular"}},
+				},
+				{
+					ObjectMeta: v1.ObjectMeta{Name: "circular"},
+					Spec:       KustomizationSpec{DependsOn: []string{"dependency"}},
+				},
+			},
+			nil,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := DependencySort(tt.ks)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("DependencySort() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DependencySort() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestDependencySort_DeadEnd(t *testing.T) {
+	ks := []Kustomization{
+		{
+			ObjectMeta: v1.ObjectMeta{Name: "backend"},
+			Spec:       KustomizationSpec{DependsOn: []string{"common"}},
+		},
+		{
+			ObjectMeta: v1.ObjectMeta{Name: "frontend"},
+			Spec:       KustomizationSpec{DependsOn: []string{"infra"}},
+		},
+		{
+			ObjectMeta: v1.ObjectMeta{Name: "common"},
+		},
+	}
+	got, err := DependencySort(ks)
+	if err != nil {
+		t.Errorf("DependencySort() error = %v", err)
+		return
+	}
+	if len(got) != len(ks) {
+		t.Errorf("DependencySort() len = %v, want %v", len(got), len(ks))
+	}
+}

controllers/gitrepository_controller.go

@@ -29,8 +29,9 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
+
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1"
 )
 
 // KustomizationReconciler watches a GitRepository object
@@ -63,10 +64,16 @@ func (r *GitRepositoryWatcher) Reconcile(req ctrl.Request) (ctrl.Result, error)
 		return ctrl.Result{}, err
 	}
 
+	sorted, err := kustomizev1.DependencySort(list.Items)
+	if err != nil {
+		log.Error(err, "unable to dependency sort kustomizations")
+		return ctrl.Result{}, err
+	}
+
 	// trigger apply for each kustomization using this Git repository
-	for _, kustomization := range list.Items {
-		namespacedName := types.NamespacedName{Namespace: kustomization.Namespace, Name: kustomization.Name}
-		if err := r.requestKustomizationSync(kustomization); err != nil {
+	for _, k := range sorted {
+		namespacedName := types.NamespacedName{Namespace: k.Namespace, Name: k.Name}
+		if err := r.requestKustomizationSync(k); err != nil {
 			log.Error(err, "unable to annotate Kustomization", "kustomization", namespacedName)
 			continue
 		}