Skip to content

Commit 78aaf25

Browse files
synthe102synthe102
committed
feat: add support for multiple Datastores
1 parent ac7da57 commit 78aaf25

File tree

11 files changed

+308
-3
lines changed

11 files changed

+308
-3
lines changed

‎api/v1alpha1/tenantcontrolplane_types.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,14 @@ func (p *Permissions) HasAnyLimitation() bool {
341341
return false
342342
}
343343

344+
// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
345+
type DataStoreOverride struct {
346+
// Resource specifies which kubernetes resource to target.
347+
Resource string `json:"resource,omitempty"`
348+
// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
349+
DataStore string `json:"dataStore,omitempty"`
350+
}
351+
344352
// TenantControlPlaneSpec defines the desired state of TenantControlPlane.
345353
// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
346354
// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
@@ -375,8 +383,10 @@ type TenantControlPlaneSpec struct {
375383
// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
376384
// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
377385
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
378-
DataStoreUsername string `json:"dataStoreUsername,omitempty"`
379-
ControlPlane ControlPlane `json:"controlPlane"`
386+
DataStoreUsername string `json:"dataStoreUsername,omitempty"`
387+
// DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
388+
DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"`
389+
ControlPlane ControlPlane `json:"controlPlane"`
380390
// Kubernetes specification for tenant control plane
381391
Kubernetes KubernetesSpec `json:"kubernetes"`
382392
// NetworkProfile specifies how the network is

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6810,6 +6810,19 @@ versions:
68106810
Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
68116811
Migration from one DataStore to another backed by a different Driver is not supported.
68126812
type: string
6813+
dataStoreOverrides:
6814+
description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
6815+
items:
6816+
description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
6817+
properties:
6818+
dataStore:
6819+
description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
6820+
type: string
6821+
resource:
6822+
description: Resource specifies which kubernetes resource to target.
6823+
type: string
6824+
type: object
6825+
type: array
68136826
dataStoreSchema:
68146827
description: |-
68156828
DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

‎charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6818,6 +6818,19 @@ spec:
68186818
Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
68196819
Migration from one DataStore to another backed by a different Driver is not supported.
68206820
type: string
6821+
dataStoreOverrides:
6822+
description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
6823+
items:
6824+
description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
6825+
properties:
6826+
dataStore:
6827+
description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
6828+
type: string
6829+
resource:
6830+
description: Resource specifies which kubernetes resource to target.
6831+
type: string
6832+
type: object
6833+
type: array
68216834
dataStoreSchema:
68226835
description: |-
68236836
DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

‎docs/content/reference/api.md‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28500,6 +28500,13 @@ Migration from one DataStore to another backed by the same Driver is possible. S
2850028500
Migration from one DataStore to another backed by a different Driver is not supported.<br/>
2850128501
</td>
2850228502
<td>false</td>
28503+
</tr><tr>
28504+
<td><b><a href="#tenantcontrolplanespecdatastoreoverridesindex">dataStoreOverrides</a></b></td>
28505+
<td>[]object</td>
28506+
<td>
28507+
DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.<br/>
28508+
</td>
28509+
<td>false</td>
2850328510
</tr><tr>
2850428511
<td><b>dataStoreSchema</b></td>
2850528512
<td>string</td>
@@ -41970,6 +41977,38 @@ In case this value is set, kubeadm does not change automatically the version of
4197041977
</table>
4197141978

4197241979

41980+
<span id="tenantcontrolplanespecdatastoreoverridesindex">`TenantControlPlane.spec.dataStoreOverrides[index]`</span>
41981+
41982+
41983+
DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
41984+
41985+
<table>
41986+
<thead>
41987+
<tr>
41988+
<th>Name</th>
41989+
<th>Type</th>
41990+
<th>Description</th>
41991+
<th>Required</th>
41992+
</tr>
41993+
</thead>
41994+
<tbody><tr>
41995+
<td><b>dataStore</b></td>
41996+
<td>string</td>
41997+
<td>
41998+
DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.<br/>
41999+
</td>
42000+
<td>false</td>
42001+
</tr><tr>
42002+
<td><b>resource</b></td>
42003+
<td>string</td>
42004+
<td>
42005+
Resource specifies which kubernetes resource to target.<br/>
42006+
</td>
42007+
<td>false</td>
42008+
</tr></tbody>
42009+
</table>
42010+
42011+
4197342012
<span id="tenantcontrolplanespecnetworkprofile">`TenantControlPlane.spec.networkProfile`</span>
4197442013

4197542014

‎internal/builders/controlplane/controlplane_suite_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package controlplane_test
5+
6+
import (
7+
"testing"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
)
12+
13+
func TestControlpalne(t *testing.T) {
14+
RegisterFailHandler(Fail)
15+
RunSpecs(t, "ControlPlane Suite")
16+
}

‎internal/builders/controlplane/deployment.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,15 @@ const (
5252
kineInitContainerName = "chmod"
5353
)
5454

55+
type DataStoreOverrides struct {
56+
Resource string
57+
DataStore kamajiv1alpha1.DataStore
58+
}
59+
5560
type Deployment struct {
5661
KineContainerImage string
5762
DataStore kamajiv1alpha1.DataStore
63+
DataStoreOverrides []DataStoreOverrides
5864
Client client.Client
5965
}
6066

@@ -711,11 +717,29 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
711717
desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
712718
}
713719

720+
if len(d.DataStoreOverrides) != 0 {
721+
desiredArgs["--etcd-servers-overrides"] = d.etcdServersOverrides()
722+
}
723+
714724
// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
715725
// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
716726
return utilities.MergeMaps(current, desiredArgs, extraArgs)
717727
}
718728

729+
func (d Deployment) etcdServersOverrides() string {
730+
dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides))
731+
for _, dso := range d.DataStoreOverrides {
732+
httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints))
733+
734+
for _, ep := range dso.DataStore.Spec.Endpoints {
735+
httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
736+
}
737+
dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";")))
738+
}
739+
740+
return strings.Join(dataStoreOverridesEndpoints, ",")
741+
}
742+
719743
func (d Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
720744
return &corev1.SecretProjection{
721745
LocalObjectReference: corev1.LocalObjectReference{

‎internal/builders/controlplane/deployment_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package controlplane
5+
6+
import (
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
10+
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
11+
)
12+
13+
var _ = Describe("Controlplane Deployment", func() {
14+
var d Deployment
15+
BeforeEach(func() {
16+
d = Deployment{
17+
DataStoreOverrides: []DataStoreOverrides{{
18+
Resource: "/events",
19+
DataStore: kamajiv1alpha1.DataStore{
20+
Spec: kamajiv1alpha1.DataStoreSpec{
21+
Endpoints: kamajiv1alpha1.Endpoints{"etcd-0", "etcd-1", "etcd-2"},
22+
},
23+
},
24+
}},
25+
}
26+
})
27+
28+
Describe("DataStoreOverrides flag generation", func() {
29+
It("should generate valid --etcd-servers-overrides value", func() {
30+
etcdSerVersOverrides := d.etcdServersOverrides()
31+
Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2"))
32+
})
33+
It("should generate valid --etcd-servers-overrides value with 2 DataStoreOverrides", func() {
34+
d.DataStoreOverrides = append(d.DataStoreOverrides, DataStoreOverrides{
35+
Resource: "/pods",
36+
DataStore: kamajiv1alpha1.DataStore{
37+
Spec: kamajiv1alpha1.DataStoreSpec{
38+
Endpoints: kamajiv1alpha1.Endpoints{"etcd-3", "etcd-4", "etcd-5"},
39+
},
40+
},
41+
})
42+
etcdSerVersOverrides := d.etcdServersOverrides()
43+
Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5"))
44+
})
45+
})
46+
})

‎internal/webhook/handlers/tcp_datastore.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func (t TenantControlPlaneDataStore) OnCreate(object runtime.Object) AdmissionRe
3030
return nil, t.check(ctx, tcp.Spec.DataStore)
3131
}
3232

33-
return nil, nil
33+
return nil, t.checkDataStoreOverrides(ctx, tcp)
3434
}
3535
}
3636

@@ -61,3 +61,19 @@ func (t TenantControlPlaneDataStore) check(ctx context.Context, dataStoreName st
6161

6262
return nil
6363
}
64+
65+
func (t TenantControlPlaneDataStore) checkDataStoreOverrides(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error {
66+
overrideCheck := make(map[string]struct{}, 0)
67+
for _, ds := range tcp.Spec.DataStoreOverrides {
68+
if _, exists := overrideCheck[ds.Resource]; !exists {
69+
overrideCheck[ds.Resource] = struct{}{}
70+
} else {
71+
return fmt.Errorf("duplicate resource override in Spec.DataStoreOverrides")
72+
}
73+
if err := t.check(ctx, ds.DataStore); err != nil {
74+
return err
75+
}
76+
}
77+
78+
return nil
79+
}

‎internal/webhook/handlers/tcp_datastore_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package handlers
5+
6+
import (
7+
"context"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/runtime"
13+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
14+
fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
15+
16+
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
17+
)
18+
19+
var _ = Describe("TCP Datastore webhook", func() {
20+
var (
21+
ctx context.Context
22+
t TenantControlPlaneDataStore
23+
tcp *kamajiv1alpha1.TenantControlPlane
24+
)
25+
BeforeEach(func() {
26+
scheme := runtime.NewScheme()
27+
utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
28+
29+
ctx = context.Background()
30+
t = TenantControlPlaneDataStore{
31+
Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(&kamajiv1alpha1.DataStore{
32+
ObjectMeta: metav1.ObjectMeta{
33+
Name: "foo",
34+
},
35+
}, &kamajiv1alpha1.DataStore{
36+
ObjectMeta: metav1.ObjectMeta{
37+
Name: "bar",
38+
},
39+
}).Build(),
40+
}
41+
tcp = &kamajiv1alpha1.TenantControlPlane{
42+
ObjectMeta: metav1.ObjectMeta{
43+
Name: "tcp",
44+
Namespace: "default",
45+
},
46+
Spec: kamajiv1alpha1.TenantControlPlaneSpec{},
47+
}
48+
})
49+
Describe("validation should succeed without DataStoreOverrides", func() {
50+
It("should validate TCP without DataStoreOverrides", func() {
51+
err := t.checkDataStoreOverrides(ctx, tcp)
52+
Expect(err).ToNot(HaveOccurred())
53+
})
54+
})
55+
56+
Describe("validation should fail with duplicate resources in DataStoreOverrides", func() {
57+
It("should fail to validate TCP with duplicate resources in DataStoreOverrides", func() {
58+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
59+
Resource: "/event",
60+
DataStore: "foo",
61+
}, {
62+
Resource: "/event",
63+
DataStore: "bar",
64+
}}
65+
err := t.checkDataStoreOverrides(ctx, tcp)
66+
Expect(err).To(HaveOccurred())
67+
})
68+
})
69+
70+
Describe("validation should succeed with valid DataStoreOverrides", func() {
71+
It("should validate TCP with valid DataStoreOverrides", func() {
72+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
73+
Resource: "/leases",
74+
DataStore: "foo",
75+
}, {
76+
Resource: "/event",
77+
DataStore: "bar",
78+
}}
79+
err := t.checkDataStoreOverrides(ctx, tcp)
80+
Expect(err).ToNot(HaveOccurred())
81+
})
82+
})
83+
84+
Describe("validation should fail with nonexistent DataStoreOverrides", func() {
85+
It("should fail to validate TCP with nonexistent DataStoreOverrides", func() {
86+
tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
87+
Resource: "/leases",
88+
DataStore: "baz",
89+
}}
90+
err := t.checkDataStoreOverrides(ctx, tcp)
91+
Expect(err).To(HaveOccurred())
92+
})
93+
})
94+
})

0 commit comments

Comments
 (0)