Skip to content

Commit 9819e8a

Browse files
synthe102synthe102
committed
feat: add support for multiple Datastores
1 parent ac7da57 commit 9819e8a

File tree

10 files changed

+270
-3
lines changed

10 files changed

+270
-3
lines changed

‎api/v1alpha1/tenantcontrolplane_types.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,14 @@ func (p *Permissions) HasAnyLimitation() bool {
341341
return false
342342
}
343343

344+
// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
345+
type DataStoreOverride struct {
346+
// Resource specifies which kubernetes resource to target.
347+
Resource string `json:"resource,omitempty"`
348+
// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
349+
DataStore string `json:"dataStore,omitempty"`
350+
}
351+
344352
// TenantControlPlaneSpec defines the desired state of TenantControlPlane.
345353
// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
346354
// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
@@ -375,8 +383,10 @@ type TenantControlPlaneSpec struct {
375383
// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
376384
// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
377385
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
378-
DataStoreUsername string `json:"dataStoreUsername,omitempty"`
379-
ControlPlane ControlPlane `json:"controlPlane"`
386+
DataStoreUsername string `json:"dataStoreUsername,omitempty"`
387+
// DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
388+
DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"`
389+
ControlPlane ControlPlane `json:"controlPlane"`
380390
// Kubernetes specification for tenant control plane
381391
Kubernetes KubernetesSpec `json:"kubernetes"`
382392
// NetworkProfile specifies how the network is

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6810,6 +6810,19 @@ versions:
68106810
Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
68116811
Migration from one DataStore to another backed by a different Driver is not supported.
68126812
type: string
6813+
dataStoreOverrides:
6814+
description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
6815+
items:
6816+
description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
6817+
properties:
6818+
dataStore:
6819+
description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
6820+
type: string
6821+
resource:
6822+
description: Resource specifies which kubernetes resource to target.
6823+
type: string
6824+
type: object
6825+
type: array
68136826
dataStoreSchema:
68146827
description: |-
68156828
DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

‎charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6818,6 +6818,19 @@ spec:
68186818
Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
68196819
Migration from one DataStore to another backed by a different Driver is not supported.
68206820
type: string
6821+
dataStoreOverrides:
6822+
description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
6823+
items:
6824+
description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
6825+
properties:
6826+
dataStore:
6827+
description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
6828+
type: string
6829+
resource:
6830+
description: Resource specifies which kubernetes resource to target.
6831+
type: string
6832+
type: object
6833+
type: array
68216834
dataStoreSchema:
68226835
description: |-
68236836
DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

‎internal/builders/controlplane/controlplane_suite_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package controlplane_test
5+
6+
import (
7+
"testing"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
)
12+
13+
func TestControlpalne(t *testing.T) {
14+
RegisterFailHandler(Fail)
15+
RunSpecs(t, "ControlPlane Suite")
16+
}

‎internal/builders/controlplane/deployment.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,15 @@ const (
5252
kineInitContainerName = "chmod"
5353
)
5454

55+
type DataStoreOverrides struct {
56+
Resource string
57+
DataStore kamajiv1alpha1.DataStore
58+
}
59+
5560
type Deployment struct {
5661
KineContainerImage string
5762
DataStore kamajiv1alpha1.DataStore
63+
DataStoreOverrides []DataStoreOverrides
5864
Client client.Client
5965
}
6066

@@ -711,11 +717,28 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
711717
desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
712718
}
713719

720+
if len(d.DataStoreOverrides) != 0 {
721+
desiredArgs["--etcd-servers-overrides"] = d.etcdServersOverrides()
722+
}
723+
714724
// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
715725
// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
716726
return utilities.MergeMaps(current, desiredArgs, extraArgs)
717727
}
718728

729+
func (d Deployment) etcdServersOverrides() string {
730+
dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides))
731+
for _, dso := range d.DataStoreOverrides {
732+
httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints))
733+
734+
for _, ep := range dso.DataStore.Spec.Endpoints {
735+
httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
736+
}
737+
dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";")))
738+
}
739+
return strings.Join(dataStoreOverridesEndpoints, ",")
740+
}
741+
719742
func (d Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
720743
return &corev1.SecretProjection{
721744
LocalObjectReference: corev1.LocalObjectReference{

‎internal/builders/controlplane/deployment_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package controlplane
5+
6+
import (
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
10+
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
11+
)
12+
13+
var _ = Describe("Controlplane Deployment", func() {
14+
var (
15+
d Deployment
16+
)
17+
BeforeEach(func() {
18+
d = Deployment{
19+
DataStoreOverrides: []DataStoreOverrides{{
20+
Resource: "/events",
21+
DataStore: kamajiv1alpha1.DataStore{
22+
Spec: kamajiv1alpha1.DataStoreSpec{
23+
Endpoints: kamajiv1alpha1.Endpoints{"etcd-0", "etcd-1", "etcd-2"},
24+
},
25+
},
26+
}},
27+
}
28+
})
29+
30+
Describe("DataStoreOverrides flag generation", func() {
31+
It("should generate valid --etcd-servers-overrides value", func() {
32+
etcdSerVersOverrides := d.etcdServersOverrides()
33+
Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2"))
34+
})
35+
It("should generate valid --etcd-servers-overrides value with 2 DataStoreOverrides", func() {
36+
d.DataStoreOverrides = append(d.DataStoreOverrides, DataStoreOverrides{
37+
Resource: "/pods",
38+
DataStore: kamajiv1alpha1.DataStore{
39+
Spec: kamajiv1alpha1.DataStoreSpec{
40+
Endpoints: kamajiv1alpha1.Endpoints{"etcd-3", "etcd-4", "etcd-5"},
41+
},
42+
},
43+
})
44+
etcdSerVersOverrides := d.etcdServersOverrides()
45+
Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5"))
46+
})
47+
})
48+
})

‎internal/webhook/handlers/tcp_datastore.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func (t TenantControlPlaneDataStore) OnCreate(object runtime.Object) AdmissionRe
3030
return nil, t.check(ctx, tcp.Spec.DataStore)
3131
}
3232

33-
return nil, nil
33+
return nil, t.checkDataStoreOverrides(ctx, tcp)
3434
}
3535
}
3636

@@ -61,3 +61,19 @@ func (t TenantControlPlaneDataStore) check(ctx context.Context, dataStoreName st
6161

6262
return nil
6363
}
64+
65+
func (t TenantControlPlaneDataStore) checkDataStoreOverrides(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error {
66+
overrideCheck := make(map[string]struct{}, 0)
67+
for _, ds := range tcp.Spec.DataStoreOverrides {
68+
if _, exists := overrideCheck[ds.Resource]; !exists {
69+
overrideCheck[ds.Resource] = struct{}{}
70+
} else {
71+
return fmt.Errorf("duplicate resource override in Spec.DataStoreOverrides")
72+
}
73+
if err := t.check(ctx, ds.DataStore); err != nil {
74+
return err
75+
}
76+
}
77+
78+
return nil
79+
}

‎internal/webhook/handlers/tcp_datastore_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package handlers
5+
6+
import (
7+
"context"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/runtime"
13+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
14+
fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
15+
16+
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
17+
)
18+
19+
var _ = Describe("TCP Datastore webhook", func() {
20+
var (
21+
ctx context.Context
22+
t TenantControlPlaneDataStore
23+
tcp *kamajiv1alpha1.TenantControlPlane
24+
)
25+
BeforeEach(func() {
26+
scheme := runtime.NewScheme()
27+
utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
28+
29+
ctx = context.Background()
30+
t = TenantControlPlaneDataStore{
31+
Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(&kamajiv1alpha1.DataStore{
32+
ObjectMeta: metav1.ObjectMeta{
33+
Name: "foo",
34+
},
35+
}, &kamajiv1alpha1.DataStore{
36+
ObjectMeta: metav1.ObjectMeta{
37+
Name: "bar",
38+
},
39+
}).Build(),
40+
}
41+
tcp = &kamajiv1alpha1.TenantControlPlane{
42+
ObjectMeta: metav1.ObjectMeta{
43+
Name: "tcp",
44+
Namespace: "default",
45+
},
46+
Spec: kamajiv1alpha1.TenantControlPlaneSpec{},
47+
}
48+
})
49+
Describe("validation should succeed without DataStoreOverrides", func() {
50+
It("should validate TCP without DataStoreOverrides", func() {
51+
err := t.checkDataStoreOverrides(ctx, tcp)
52+
Expect(err).ToNot(HaveOccurred())
53+
})
54+
})
55+
56+
Describe("validation should fail with duplicate resources in DataStoreOverrides", func() {
57+
It("should fail to validate TCP with duplicate resources in DataStoreOverrides", func() {
58+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
59+
Resource: "/event",
60+
DataStore: "foo",
61+
}, {
62+
Resource: "/event",
63+
DataStore: "bar",
64+
}}
65+
err := t.checkDataStoreOverrides(ctx, tcp)
66+
Expect(err).To(HaveOccurred())
67+
})
68+
})
69+
70+
Describe("validation should succeed with valid DataStoreOverrides", func() {
71+
It("should validate TCP with valid DataStoreOverrides", func() {
72+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
73+
Resource: "/leases",
74+
DataStore: "foo",
75+
}, {
76+
Resource: "/event",
77+
DataStore: "bar",
78+
}}
79+
err := t.checkDataStoreOverrides(ctx, tcp)
80+
Expect(err).ToNot(HaveOccurred())
81+
})
82+
})
83+
84+
Describe("validation should fail with nonexistent DataStoreOverrides", func() {
85+
It("should fail to validate TCP with nonexistent DataStoreOverrides", func() {
86+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
87+
Resource: "/leases",
88+
DataStore: "baz",
89+
}}
90+
err := t.checkDataStoreOverrides(ctx, tcp)
91+
Expect(err).To(HaveOccurred())
92+
})
93+
})
94+
})

‎internal/webhook/handlers/tcp_deployment.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,20 @@ func (t TenantControlPlaneDeployment) OnUpdate(newObject runtime.Object, oldObje
6767
}
6868
t.DeploymentBuilder.DataStore = ds
6969

70+
dataStoreOverrides := make([]controlplane.DataStoreOverrides, 0, len(tcp.Spec.DataStoreOverrides))
71+
72+
for _, dso := range tcp.Spec.DataStoreOverrides {
73+
ds := kamajiv1alpha1.DataStore{}
74+
if err := t.Client.Get(ctx, types.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
75+
return nil, err
76+
}
77+
dataStoreOverrides = append(dataStoreOverrides, controlplane.DataStoreOverrides{
78+
Resource: dso.Resource,
79+
DataStore: ds,
80+
})
81+
}
82+
t.DeploymentBuilder.DataStoreOverrides = dataStoreOverrides
83+
7084
deployment := appsv1.Deployment{}
7185
deployment.Name = tcp.Name
7286
deployment.Namespace = tcp.Namespace

0 commit comments

Comments
 (0)