Skip to content

Commit a828e55

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #47 from agradouski/issue/20
Allow to specify external cluster for hollow pods
2 parents ec638a9 + 57bc2ad commit a828e55

10 files changed

+312
-15
lines changed

‎api/v1alpha4/kubemarkmachine_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ type KubemarkMachineSpec struct {
4747
// will be passed to the `kubemark` binary.
4848
// +optional
4949
KubemarkOptions KubemarkProcessOptions `json:"kubemarkOptions,omitempty"`
50+
51+
// KubemarkHollowPodClusterSecretRef is a reference to a secret with a kubeconfig for an external cluster used for kubemark pods.
52+
KubemarkHollowPodClusterSecretRef *corev1.ObjectReference `json:"kubemarkHollowPodClusterSecretRef,omitempty"`
5053
}
5154

5255
// Mount specifies a host volume to mount into a container.

‎api/v1alpha4/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/infrastructure.cluster.x-k8s.io_kubemarkmachines.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,44 @@ spec:
7575
- name
7676
type: object
7777
type: array
78+
kubemarkHollowPodClusterSecretRef:
79+
description: KubemarkHollowPodClusterSecretRef is a reference to a
80+
secret with a kubeconfig for an external cluster used for kubemark
81+
pods.
82+
properties:
83+
apiVersion:
84+
description: API version of the referent.
85+
type: string
86+
fieldPath:
87+
description: 'If referring to a piece of an object instead of
88+
an entire object, this string should contain a valid JSON/Go
89+
field access statement, such as desiredState.manifest.containers[2].
90+
For example, if the object reference is to a container within
91+
a pod, this would take on a value like: "spec.containers{name}"
92+
(where "name" refers to the name of the container that triggered
93+
the event) or if no container name is specified "spec.containers[2]"
94+
(container with index 2 in this pod). This syntax is chosen
95+
only to have some well-defined way of referencing a part of
96+
an object. TODO: this design is not final and this field is
97+
subject to change in the future.'
98+
type: string
99+
kind:
100+
description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
101+
type: string
102+
name:
103+
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
104+
type: string
105+
namespace:
106+
description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
107+
type: string
108+
resourceVersion:
109+
description: 'Specific resourceVersion to which this reference
110+
is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
111+
type: string
112+
uid:
113+
description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
114+
type: string
115+
type: object
78116
kubemarkOptions:
79117
description: KubemarkOptions are API representations of command line
80118
flags that will be passed to the `kubemark` binary.

‎config/crd/bases/infrastructure.cluster.x-k8s.io_kubemarkmachinetemplates.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,45 @@ spec:
8787
- name
8888
type: object
8989
type: array
90+
kubemarkHollowPodClusterSecretRef:
91+
description: KubemarkHollowPodClusterSecretRef is a reference
92+
to a secret with a kubeconfig for an external cluster used
93+
for kubemark pods.
94+
properties:
95+
apiVersion:
96+
description: API version of the referent.
97+
type: string
98+
fieldPath:
99+
description: 'If referring to a piece of an object instead
100+
of an entire object, this string should contain a valid
101+
JSON/Go field access statement, such as desiredState.manifest.containers[2].
102+
For example, if the object reference is to a container
103+
within a pod, this would take on a value like: "spec.containers{name}"
104+
(where "name" refers to the name of the container that
105+
triggered the event) or if no container name is specified
106+
"spec.containers[2]" (container with index 2 in this
107+
pod). This syntax is chosen only to have some well-defined
108+
way of referencing a part of an object. TODO: this design
109+
is not final and this field is subject to change in
110+
the future.'
111+
type: string
112+
kind:
113+
description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
114+
type: string
115+
name:
116+
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
117+
type: string
118+
namespace:
119+
description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
120+
type: string
121+
resourceVersion:
122+
description: 'Specific resourceVersion to which this reference
123+
is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
124+
type: string
125+
uid:
126+
description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
127+
type: string
128+
type: object
90129
kubemarkOptions:
91130
description: KubemarkOptions are API representations of command
92131
line flags that will be passed to the `kubemark` binary.

‎controllers/kubemarkcluster.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package controllers
2+
3+
import (
4+
gocontext "context"
5+
"strings"
6+
7+
"github.com/pkg/errors"
8+
corev1 "k8s.io/api/core/v1"
9+
"k8s.io/client-go/tools/clientcmd"
10+
"sigs.k8s.io/controller-runtime/pkg/client"
11+
)
12+
13+
type KubemarkCluster interface {
14+
GenerateKubemarkClusterClient(kubemarkClusterSecretRef *corev1.ObjectReference, ownerNamespace string, context gocontext.Context) (client.Client, string, error)
15+
}
16+
17+
// NewKubemarkCluster creates new KubemarkCluster instance
18+
func NewKubemarkCluster(client client.Client) KubemarkCluster {
19+
return &kubemarkCluster{
20+
Client: client,
21+
}
22+
}
23+
24+
type kubemarkCluster struct {
25+
client.Client
26+
}
27+
28+
// GenerateKubemarkClusterClient creates a client for kubemark cluster.
29+
func (w *kubemarkCluster) GenerateKubemarkClusterClient(kubemarkClusterSecretRef *corev1.ObjectReference, ownerNamespace string, context gocontext.Context) (client.Client, string, error) {
30+
if kubemarkClusterSecretRef == nil {
31+
return w.Client, ownerNamespace, nil
32+
}
33+
34+
kubemarkClusterKubeconfigSecret := &corev1.Secret{}
35+
kubemarkClusterKubeconfigSecretKey := client.ObjectKey{Namespace: kubemarkClusterSecretRef.Namespace, Name: kubemarkClusterSecretRef.Name}
36+
if err := w.Client.Get(context, kubemarkClusterKubeconfigSecretKey, kubemarkClusterKubeconfigSecret); err != nil {
37+
return nil, "", errors.Wrapf(err, "failed to fetch kubemark cluster kubeconfig secret %s/%s", kubemarkClusterSecretRef.Namespace, kubemarkClusterSecretRef.Name)
38+
}
39+
40+
kubeConfig, ok := kubemarkClusterKubeconfigSecret.Data["kubeconfig"]
41+
if !ok {
42+
return nil, "", errors.New("Failed to retrieve kubemark cluster kubeconfig from secret: 'kubeconfig' key is missing.")
43+
}
44+
45+
namespace := "default"
46+
namespaceBytes, ok := kubemarkClusterKubeconfigSecret.Data["namespace"]
47+
if ok {
48+
namespace = string(namespaceBytes)
49+
namespace = strings.TrimSpace(namespace)
50+
}
51+
52+
// generate REST config
53+
restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig)
54+
if err != nil {
55+
return nil, "", errors.Wrap(err, "failed to create REST config")
56+
}
57+
58+
// create the client
59+
kubemarkClusterClient, err := client.New(restConfig, client.Options{Scheme: w.Client.Scheme()})
60+
if err != nil {
61+
return nil, "", errors.Wrap(err, "failed to create kubemark cluster client")
62+
}
63+
64+
return kubemarkClusterClient, namespace, nil
65+
}

‎controllers/kubemarkmachine_controller.go‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,10 @@ const (
6969
// KubemarkMachineReconciler reconciles a KubemarkMachine object
7070
type KubemarkMachineReconciler struct {
7171
client.Client
72-
Log logr.Logger
73-
Scheme *runtime.Scheme
74-
KubemarkImage string
72+
KubemarkCluster KubemarkCluster
73+
Log logr.Logger
74+
Scheme *runtime.Scheme
75+
KubemarkImage string
7576
}
7677

7778
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=kubemarkmachines,verbs=get;list;watch;create;update;patch;delete
@@ -114,24 +115,34 @@ func (r *KubemarkMachineReconciler) Reconcile(ctx context.Context, req ctrl.Requ
114115
}
115116
}()
116117

118+
kubemarkClusterClient, kubemarkClusterNamespace, err := r.KubemarkCluster.GenerateKubemarkClusterClient(kubemarkMachine.Spec.KubemarkHollowPodClusterSecretRef, kubemarkMachine.Namespace, ctx)
119+
if err != nil {
120+
return ctrl.Result{RequeueAfter: 10 * time.Second}, err
121+
}
122+
123+
if kubemarkClusterClient == nil {
124+
logger.Info("Waiting for kubemark cluster client...")
125+
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
126+
}
127+
117128
if !kubemarkMachine.ObjectMeta.DeletionTimestamp.IsZero() {
118129
logger.Info("deleting machine")
119130

120-
if err := r.Delete(ctx, &v1.Pod{
131+
if err := kubemarkClusterClient.Delete(ctx, &v1.Pod{
121132
ObjectMeta: metav1.ObjectMeta{
122133
Name: kubemarkMachine.Name,
123-
Namespace: kubemarkMachine.Namespace,
134+
Namespace: kubemarkClusterNamespace,
124135
},
125136
}); err != nil {
126137
if !apierrors.IsNotFound(err) {
127138
logger.Error(err, "error deleting kubemark pod")
128139
return ctrl.Result{}, err
129140
}
130141
}
131-
if err := r.Delete(ctx, &v1.Secret{
142+
if err := kubemarkClusterClient.Delete(ctx, &v1.Secret{
132143
ObjectMeta: metav1.ObjectMeta{
133144
Name: kubemarkMachine.Name,
134-
Namespace: kubemarkMachine.Namespace,
145+
Namespace: kubemarkClusterNamespace,
135146
},
136147
}); err != nil {
137148
if !apierrors.IsNotFound(err) {
@@ -266,14 +277,14 @@ func (r *KubemarkMachineReconciler) Reconcile(ctx context.Context, req ctrl.Requ
266277
secret := &v1.Secret{
267278
ObjectMeta: metav1.ObjectMeta{
268279
Name: kubemarkMachine.Name,
269-
Namespace: kubemarkMachine.Namespace,
280+
Namespace: kubemarkClusterNamespace,
270281
},
271282
Data: map[string][]byte{
272283
"kubeconfig": kubeconfig,
273284
"cert.pem": stackedCert.Bytes(),
274285
},
275286
}
276-
if err := r.Create(ctx, secret); err != nil {
287+
if err := kubemarkClusterClient.Create(ctx, secret); err != nil {
277288
if !apierrors.IsAlreadyExists(err) {
278289
logger.Error(err, "failed to create secret")
279290
return ctrl.Result{}, err
@@ -323,7 +334,7 @@ func (r *KubemarkMachineReconciler) Reconcile(ctx context.Context, req ctrl.Requ
323334
ObjectMeta: metav1.ObjectMeta{
324335
Name: kubemarkMachine.Name,
325336
Labels: map[string]string{"app": kubemarkName},
326-
Namespace: kubemarkMachine.Namespace,
337+
Namespace: kubemarkClusterNamespace,
327338
},
328339
Spec: v1.PodSpec{
329340
Containers: []v1.Container{
@@ -391,7 +402,7 @@ func (r *KubemarkMachineReconciler) Reconcile(ctx context.Context, req ctrl.Requ
391402
})
392403
}
393404

394-
if err = r.Create(ctx, pod); err != nil {
405+
if err = kubemarkClusterClient.Create(ctx, pod); err != nil {
395406
if !apierrors.IsAlreadyExists(err) {
396407
logger.Error(err, "failed to create pod")
397408
return ctrl.Result{}, err

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.16
55
require (
66
github.com/Masterminds/semver v1.5.0 // indirect
77
github.com/go-logr/logr v0.4.0
8+
github.com/pkg/errors v0.9.1 // indirect
89
k8s.io/api v0.21.3
910
k8s.io/apimachinery v0.22.0-alpha.0.0.20210417144234-8daf28983e6e
1011
k8s.io/client-go v0.21.3

‎hack/tests/resources/kubemark-machine-deployment-external-cluster.yaml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: MachineDeployment
3+
metadata:
4+
annotations:
5+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5"
6+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1"
7+
name: km-wl-kubemark-external-md-0
8+
namespace: default
9+
spec:
10+
clusterName: km-cp
11+
replicas: 4
12+
selector:
13+
matchLabels:
14+
cluster.x-k8s.io/cluster-name: km-cp
15+
cluster.x-k8s.io/deployment-name: km-wl-kubemark-external-md-0
16+
template:
17+
metadata:
18+
labels:
19+
cluster.x-k8s.io/cluster-name: km-cp
20+
cluster.x-k8s.io/deployment-name: km-wl-kubemark-external-md-0
21+
spec:
22+
bootstrap:
23+
configRef:
24+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
25+
kind: KubeadmConfigTemplate
26+
name: km-wl-kubemark-external-md-0
27+
clusterName: km-cp
28+
infrastructureRef:
29+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
30+
kind: KubemarkMachineTemplate
31+
name: km-wl-kubemark-external-md-0
32+
version: 1.21.1
33+
---
34+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
35+
kind: KubemarkMachineTemplate
36+
metadata:
37+
labels:
38+
cluster.x-k8s.io/cluster-name: km-cp
39+
name: km-wl-kubemark-external-md-0
40+
namespace: default
41+
spec:
42+
template:
43+
spec:
44+
extraMounts:
45+
- name: containerd-sock
46+
containerPath: /run/containerd/containerd.sock
47+
hostPath: /run/containerd/containerd.sock
48+
type: Socket
49+
kubemarkHollowPodClusterSecretRef:
50+
apiVersion: v1
51+
kind: Secret
52+
name: kubemark-cluster-kubeconfig
53+
namespace: capk-system
54+
---
55+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
56+
kind: KubeadmConfigTemplate
57+
metadata:
58+
name: km-wl-kubemark-external-md-0
59+
namespace: default
60+
spec:
61+
template:
62+
spec:
63+
joinConfiguration:
64+
nodeRegistration:
65+
name: ''
66+

‎main.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,11 @@ func main() {
7575
}
7676
ctx := ctrl.SetupSignalHandler()
7777
if err = (&controllers.KubemarkMachineReconciler{
78-
Client: mgr.GetClient(),
79-
Log: ctrl.Log.WithName("controllers").WithName("KubemarkMachine"),
80-
Scheme: mgr.GetScheme(),
81-
KubemarkImage: kubemarkImage,
78+
Client: mgr.GetClient(),
79+
KubemarkCluster: controllers.NewKubemarkCluster(mgr.GetClient()),
80+
Log: ctrl.Log.WithName("controllers").WithName("KubemarkMachine"),
81+
Scheme: mgr.GetScheme(),
82+
KubemarkImage: kubemarkImage,
8283
}).SetupWithManager(ctx, mgr); err != nil {
8384
setupLog.Error(err, "unable to create controller", "controller", "KubemarkMachine")
8485
os.Exit(1)

0 commit comments

Comments
 (0)