Skip to content

Commit d25ca99

Browse files
synthe102synthe102
committed
feat: add support for multiple Datastores
1 parent 17869a4 commit d25ca99

File tree

10 files changed

+270
-3
lines changed

10 files changed

+270
-3
lines changed

‎api/v1alpha1/tenantcontrolplane_types.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,14 @@ type AddonsSpec struct {
297297
KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
298298
}
299299

300+
// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
301+
type DataStoreOverride struct {
302+
// Resource specifies which kubernetes resource to target.
303+
Resource string `json:"resource,omitempty"`
304+
// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
305+
DataStore string `json:"dataStore,omitempty"`
306+
}
307+
300308
// TenantControlPlaneSpec defines the desired state of TenantControlPlane.
301309
// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
302310
// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
@@ -324,8 +332,10 @@ type TenantControlPlaneSpec struct {
324332
// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
325333
// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
326334
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
327-
DataStoreUsername string `json:"dataStoreUsername,omitempty"`
328-
ControlPlane ControlPlane `json:"controlPlane"`
335+
DataStoreUsername string `json:"dataStoreUsername,omitempty"`
336+
// DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
337+
DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"`
338+
ControlPlane ControlPlane `json:"controlPlane"`
329339
// Kubernetes specification for tenant control plane
330340
Kubernetes KubernetesSpec `json:"kubernetes"`
331341
// NetworkProfile specifies how the network is

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6760,6 +6760,19 @@ versions:
67606760
Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
67616761
Migration from one DataStore to another backed by a different Driver is not supported.
67626762
type: string
6763+
dataStoreOverrides:
6764+
description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
6765+
items:
6766+
description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
6767+
properties:
6768+
dataStore:
6769+
description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
6770+
type: string
6771+
resource:
6772+
description: Resource specifies which kubernetes resource to target.
6773+
type: string
6774+
type: object
6775+
type: array
67636776
dataStoreSchema:
67646777
description: |-
67656778
DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

‎charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6768,6 +6768,19 @@ spec:
67686768
Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
67696769
Migration from one DataStore to another backed by a different Driver is not supported.
67706770
type: string
6771+
dataStoreOverrides:
6772+
description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
6773+
items:
6774+
description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
6775+
properties:
6776+
dataStore:
6777+
description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
6778+
type: string
6779+
resource:
6780+
description: Resource specifies which kubernetes resource to target.
6781+
type: string
6782+
type: object
6783+
type: array
67716784
dataStoreSchema:
67726785
description: |-
67736786
DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

‎internal/builders/controlplane/controlplane_suite_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package controlplane_test
5+
6+
import (
7+
"testing"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
)
12+
13+
func TestControlpalne(t *testing.T) {
14+
RegisterFailHandler(Fail)
15+
RunSpecs(t, "ControlPlane Suite")
16+
}

‎internal/builders/controlplane/deployment.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,15 @@ const (
5252
kineInitContainerName = "chmod"
5353
)
5454

55+
type DataStoreOverrides struct {
56+
Resource string
57+
DataStore kamajiv1alpha1.DataStore
58+
}
59+
5560
type Deployment struct {
5661
KineContainerImage string
5762
DataStore kamajiv1alpha1.DataStore
63+
DataStoreOverrides []DataStoreOverrides
5864
Client client.Client
5965
}
6066

@@ -711,11 +717,28 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
711717
desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
712718
}
713719

720+
if len(d.DataStoreOverrides) != 0 {
721+
desiredArgs["--etcd-servers-overrides"] = d.etcdServersOverrides()
722+
}
723+
714724
// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
715725
// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
716726
return utilities.MergeMaps(current, desiredArgs, extraArgs)
717727
}
718728

729+
func (d Deployment) etcdServersOverrides() string {
730+
dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides))
731+
for _, dso := range d.DataStoreOverrides {
732+
httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints))
733+
734+
for _, ep := range dso.DataStore.Spec.Endpoints {
735+
httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
736+
}
737+
dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";")))
738+
}
739+
return strings.Join(dataStoreOverridesEndpoints, ",")
740+
}
741+
719742
func (d Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
720743
return &corev1.SecretProjection{
721744
LocalObjectReference: corev1.LocalObjectReference{

‎internal/builders/controlplane/deployment_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package controlplane
5+
6+
import (
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
10+
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
11+
)
12+
13+
var _ = Describe("Controlplane Deployment", func() {
14+
var (
15+
d Deployment
16+
)
17+
BeforeEach(func() {
18+
d = Deployment{
19+
DataStoreOverrides: []DataStoreOverrides{{
20+
Resource: "/events",
21+
DataStore: kamajiv1alpha1.DataStore{
22+
Spec: kamajiv1alpha1.DataStoreSpec{
23+
Endpoints: kamajiv1alpha1.Endpoints{"etcd-0", "etcd-1", "etcd-2"},
24+
},
25+
},
26+
}},
27+
}
28+
})
29+
30+
Describe("DataStoreOverrides flag generation", func() {
31+
It("should generate valid --etcd-servers-overrides value", func() {
32+
etcdSerVersOverrides := d.etcdServersOverrides()
33+
Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2"))
34+
})
35+
It("should generate valid --etcd-servers-overrides value with 2 DataStoreOverrides", func() {
36+
d.DataStoreOverrides = append(d.DataStoreOverrides, DataStoreOverrides{
37+
Resource: "/pods",
38+
DataStore: kamajiv1alpha1.DataStore{
39+
Spec: kamajiv1alpha1.DataStoreSpec{
40+
Endpoints: kamajiv1alpha1.Endpoints{"etcd-3", "etcd-4", "etcd-5"},
41+
},
42+
},
43+
})
44+
etcdSerVersOverrides := d.etcdServersOverrides()
45+
Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5"))
46+
})
47+
})
48+
})

‎internal/webhook/handlers/tcp_datastore.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func (t TenantControlPlaneDataStore) OnCreate(object runtime.Object) AdmissionRe
3030
return nil, t.check(ctx, tcp.Spec.DataStore)
3131
}
3232

33-
return nil, nil
33+
return nil, t.checkDataStoreOverrides(ctx, tcp)
3434
}
3535
}
3636

@@ -61,3 +61,19 @@ func (t TenantControlPlaneDataStore) check(ctx context.Context, dataStoreName st
6161

6262
return nil
6363
}
64+
65+
func (t TenantControlPlaneDataStore) checkDataStoreOverrides(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error {
66+
overrideCheck := make(map[string]struct{}, 0)
67+
for _, ds := range tcp.Spec.DataStoreOverrides {
68+
if _, exists := overrideCheck[ds.Resource]; !exists {
69+
overrideCheck[ds.Resource] = struct{}{}
70+
} else {
71+
return fmt.Errorf("duplicate resource override in Spec.DataStoreOverrides")
72+
}
73+
if err := t.check(ctx, ds.DataStore); err != nil {
74+
return err
75+
}
76+
}
77+
78+
return nil
79+
}

‎internal/webhook/handlers/tcp_datastore_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package handlers
5+
6+
import (
7+
"context"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/runtime"
13+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
14+
fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
15+
16+
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
17+
)
18+
19+
var _ = Describe("TCP Datastore webhook", func() {
20+
var (
21+
ctx context.Context
22+
t TenantControlPlaneDataStore
23+
tcp *kamajiv1alpha1.TenantControlPlane
24+
)
25+
BeforeEach(func() {
26+
scheme := runtime.NewScheme()
27+
utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
28+
29+
ctx = context.Background()
30+
t = TenantControlPlaneDataStore{
31+
Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(&kamajiv1alpha1.DataStore{
32+
ObjectMeta: metav1.ObjectMeta{
33+
Name: "foo",
34+
},
35+
}, &kamajiv1alpha1.DataStore{
36+
ObjectMeta: metav1.ObjectMeta{
37+
Name: "bar",
38+
},
39+
}).Build(),
40+
}
41+
tcp = &kamajiv1alpha1.TenantControlPlane{
42+
ObjectMeta: metav1.ObjectMeta{
43+
Name: "tcp",
44+
Namespace: "default",
45+
},
46+
Spec: kamajiv1alpha1.TenantControlPlaneSpec{},
47+
}
48+
})
49+
Describe("validation should succeed without DataStoreOverrides", func() {
50+
It("should validate TCP without DataStoreOverrides", func() {
51+
err := t.checkDataStoreOverrides(ctx, tcp)
52+
Expect(err).ToNot(HaveOccurred())
53+
})
54+
})
55+
56+
Describe("validation should fail with duplicate resources in DataStoreOverrides", func() {
57+
It("should fail to validate TCP with duplicate resources in DataStoreOverrides", func() {
58+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
59+
Resource: "/event",
60+
DataStore: "foo",
61+
}, {
62+
Resource: "/event",
63+
DataStore: "bar",
64+
}}
65+
err := t.checkDataStoreOverrides(ctx, tcp)
66+
Expect(err).To(HaveOccurred())
67+
})
68+
})
69+
70+
Describe("validation should succeed with valid DataStoreOverrides", func() {
71+
It("should validate TCP with valid DataStoreOverrides", func() {
72+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
73+
Resource: "/leases",
74+
DataStore: "foo",
75+
}, {
76+
Resource: "/event",
77+
DataStore: "bar",
78+
}}
79+
err := t.checkDataStoreOverrides(ctx, tcp)
80+
Expect(err).ToNot(HaveOccurred())
81+
})
82+
})
83+
84+
Describe("validation should fail with nonexistent DataStoreOverrides", func() {
85+
It("should fail to validate TCP with nonexistent DataStoreOverrides", func() {
86+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
87+
Resource: "/leases",
88+
DataStore: "baz",
89+
}}
90+
err := t.checkDataStoreOverrides(ctx, tcp)
91+
Expect(err).To(HaveOccurred())
92+
})
93+
})
94+
})

‎internal/webhook/handlers/tcp_deployment.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,20 @@ func (t TenantControlPlaneDeployment) OnUpdate(newObject runtime.Object, oldObje
6767
}
6868
t.DeploymentBuilder.DataStore = ds
6969

70+
dataStoreOverrides := make([]controlplane.DataStoreOverrides, 0, len(tcp.Spec.DataStoreOverrides))
71+
72+
for _, dso := range tcp.Spec.DataStoreOverrides {
73+
ds := kamajiv1alpha1.DataStore{}
74+
if err := t.Client.Get(ctx, types.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
75+
return nil, err
76+
}
77+
dataStoreOverrides = append(dataStoreOverrides, controlplane.DataStoreOverrides{
78+
Resource: dso.Resource,
79+
DataStore: ds,
80+
})
81+
}
82+
t.DeploymentBuilder.DataStoreOverrides = dataStoreOverrides
83+
7084
deployment := appsv1.Deployment{}
7185
deployment.Name = tcp.Name
7286
deployment.Namespace = tcp.Namespace

0 commit comments

Comments
 (0)