Skip to content

Commit a3bd209

Browse files
committed
feat(core): detect if kvm enabled
Signed-off-by: Pavel Tishkov <[email protected]>
1 parent 965f446 commit a3bd209

File tree

8 files changed

+400
-2
lines changed

8 files changed

+400
-2
lines changed
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
Copyright 2025 Flant JSC
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// The purpose of this hook is to prevent already launched virt-handler pods from flapping, since the node group configuration virtualization-detect-kvm.sh will be responsible for installing the label virtualization.deckhouse.io/kvm-enabled.
18+
19+
package main
20+
21+
import (
22+
"context"
23+
"fmt"
24+
25+
"github.com/deckhouse/module-sdk/pkg"
26+
"github.com/deckhouse/module-sdk/pkg/app"
27+
"github.com/deckhouse/module-sdk/pkg/registry"
28+
v1 "k8s.io/api/core/v1"
29+
"k8s.io/utils/ptr"
30+
31+
"hooks/pkg/common"
32+
33+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34+
)
35+
36+
const (
37+
nodesMetadataSnapshot = "virthandler-nodes"
38+
virtHandlerLabel = "kubevirt.internal.virtualization.deckhouse.io/schedulable"
39+
virtHandlerNodeLabelValue = "true"
40+
kvmEnabledLabel = "virtualization.deckhouse.io/kvm-enabled"
41+
kvmEnabledLabelValue = "true"
42+
nodeJQFilter = ".metadata"
43+
)
44+
45+
var _ = registry.RegisterFunc(configDiscoveryService, handleVirtHandlerNodes)
46+
47+
var configDiscoveryService = &pkg.HookConfig{
48+
OnBeforeHelm: &pkg.OrderedConfig{Order: 1},
49+
Kubernetes: []pkg.KubernetesConfig{
50+
{
51+
Name: nodesMetadataSnapshot,
52+
APIVersion: "v1",
53+
Kind: "Node",
54+
JqFilter: nodeJQFilter,
55+
LabelSelector: &metav1.LabelSelector{
56+
MatchLabels: map[string]string{
57+
virtHandlerLabel: virtHandlerNodeLabelValue,
58+
},
59+
},
60+
ExecuteHookOnSynchronization: ptr.To(false),
61+
ExecuteHookOnEvents: ptr.To(false),
62+
},
63+
},
64+
65+
Queue: fmt.Sprintf("modules/%s", common.MODULE_NAME),
66+
}
67+
68+
func handleVirtHandlerNodes(_ context.Context, input *pkg.HookInput) error {
69+
nodeMetadatas := input.Snapshots.Get(nodesMetadataSnapshot)
70+
if len(nodeMetadatas) == 0 {
71+
return nil
72+
}
73+
74+
for _, nodeMetadata := range nodeMetadatas {
75+
metadata := &metav1.ObjectMeta{}
76+
if err := nodeMetadata.UnmarshalTo(metadata); err != nil {
77+
input.Logger.Error(fmt.Sprintf("Failed to unmarshal node metadata %v", err))
78+
}
79+
80+
_, ok := metadata.Labels[kvmEnabledLabel]
81+
if ok {
82+
continue
83+
} else {
84+
metadata.Labels[kvmEnabledLabel] = kvmEnabledLabelValue
85+
nodePatch := v1.Node{}
86+
nodePatch.ObjectMeta = *metadata
87+
input.PatchCollector.PatchWithMerge(nodePatch, "v1", "Node", "", metadata.Name)
88+
}
89+
}
90+
91+
return nil
92+
}
93+
94+
func main() {
95+
app.Run()
96+
}
Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
/*
2+
Copyright 2025 Flant JSC
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"bytes"
21+
"context"
22+
"encoding/json"
23+
"os"
24+
"testing"
25+
"time"
26+
27+
"github.com/deckhouse/deckhouse/pkg/log"
28+
"github.com/deckhouse/module-sdk/pkg"
29+
"github.com/deckhouse/module-sdk/pkg/jq"
30+
"github.com/deckhouse/module-sdk/testing/mock"
31+
. "github.com/onsi/ginkgo"
32+
. "github.com/onsi/gomega"
33+
v1 "k8s.io/api/core/v1"
34+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35+
"sigs.k8s.io/yaml"
36+
)
37+
38+
func TestMigrateVirthandlerKVMLabels(t *testing.T) {
39+
RegisterFailHandler(Fail)
40+
RunSpecs(t, "Migrate virthandler KVM labels Suite")
41+
}
42+
43+
var _ = Describe("Migrate virthandler KVM labels", func() {
44+
err := os.Setenv("D8_IS_TESTS_ENVIRONMENT", "true")
45+
Expect(err).ShouldNot(HaveOccurred())
46+
47+
const (
48+
node1YAML = `
49+
---
50+
apiVersion: v1
51+
kind: Node
52+
metadata:
53+
labels:
54+
kubevirt.internal.virtualization.deckhouse.io/schedulable: "true"
55+
name: node1
56+
`
57+
58+
node2YAML = `
59+
---
60+
apiVersion: v1
61+
kind: Node
62+
metadata:
63+
labels:
64+
kubevirt.internal.virtualization.deckhouse.io/schedulable: "true"
65+
virtualization.deckhouse.io/kvm-enabled: "true"
66+
name: node2
67+
`
68+
69+
node3YAML = `
70+
---
71+
apiVersion: v1
72+
kind: Node
73+
metadata:
74+
labels:
75+
kubevirt.internal.virtualization.deckhouse.io/schedulable: "true"
76+
virtualization.deckhouse.io/kvm-enabled: "false"
77+
name: node3
78+
`
79+
80+
node4YAML = `
81+
---
82+
apiVersion: v1
83+
kind: Node
84+
metadata:
85+
labels:
86+
kubevirt.internal.virtualization.deckhouse.io/schedulable: "true"
87+
name: node6
88+
`
89+
)
90+
91+
var (
92+
snapshots *mock.SnapshotsMock
93+
values *mock.PatchableValuesCollectorMock
94+
patchCollector *mock.PatchCollectorMock
95+
input *pkg.HookInput
96+
buf *bytes.Buffer
97+
)
98+
99+
filterResultNode1, err := nodeYamlToSnapshot(node1YAML)
100+
if err != nil {
101+
Expect(err).ShouldNot(HaveOccurred())
102+
}
103+
104+
filterResultNode2, err := nodeYamlToSnapshot(node2YAML)
105+
if err != nil {
106+
Expect(err).ShouldNot(HaveOccurred())
107+
}
108+
109+
filterResultNode3, err := nodeYamlToSnapshot(node3YAML)
110+
if err != nil {
111+
Expect(err).ShouldNot(HaveOccurred())
112+
}
113+
114+
filterResultNode4, err := nodeYamlToSnapshot(node4YAML)
115+
if err != nil {
116+
Expect(err).ShouldNot(HaveOccurred())
117+
}
118+
119+
BeforeEach(func() {
120+
snapshots = mock.NewSnapshotsMock(GinkgoT())
121+
values = mock.NewPatchableValuesCollectorMock(GinkgoT())
122+
patchCollector = mock.NewPatchCollectorMock(GinkgoT())
123+
124+
buf = bytes.NewBuffer([]byte{})
125+
126+
input = &pkg.HookInput{
127+
Values: values,
128+
Snapshots: snapshots,
129+
Logger: log.NewLogger(log.Options{
130+
Level: log.LevelDebug.Level(),
131+
Output: buf,
132+
TimeFunc: func(_ time.Time) time.Time {
133+
parsedTime, err := time.Parse(time.DateTime, "2006-01-02 15:04:05")
134+
Expect(err).ShouldNot(HaveOccurred())
135+
return parsedTime
136+
},
137+
}),
138+
PatchCollector: patchCollector,
139+
}
140+
})
141+
142+
Context("Empty cluster", func() {
143+
It("Hook must execute successfully", func() {
144+
snapshots.GetMock.When(nodesMetadataSnapshot).Then(
145+
[]pkg.Snapshot{},
146+
)
147+
err := handleVirtHandlerNodes(context.Background(), input)
148+
Expect(err).ShouldNot(HaveOccurred())
149+
})
150+
})
151+
152+
Context("Four nodes but only two should be patched.", func() {
153+
It("Hook must execute successfully", func() {
154+
155+
expectedVMLabels := map[string]map[string]string{
156+
"node1": {
157+
virtHandlerLabel: virtHandlerNodeLabelValue,
158+
},
159+
"node6": {
160+
virtHandlerLabel: virtHandlerNodeLabelValue,
161+
},
162+
}
163+
164+
snapshots.GetMock.When(nodesMetadataSnapshot).Then(
165+
[]pkg.Snapshot{
166+
mock.NewSnapshotMock(GinkgoT()).UnmarshalToMock.Set(getNodeSnapshot(filterResultNode1)),
167+
mock.NewSnapshotMock(GinkgoT()).UnmarshalToMock.Set(getNodeSnapshot(filterResultNode2)),
168+
mock.NewSnapshotMock(GinkgoT()).UnmarshalToMock.Set(getNodeSnapshot(filterResultNode3)),
169+
mock.NewSnapshotMock(GinkgoT()).UnmarshalToMock.Set(getNodeSnapshot(filterResultNode4)),
170+
},
171+
)
172+
173+
patchCollector.PatchWithMergeMock.Set(func(patch any, apiVersion, kind, namespace, name string, opts ...pkg.PatchCollectorOption) {
174+
node, ok := patch.(v1.Node)
175+
Expect(ok).To(BeTrue())
176+
Expect(expectedVMLabels).To(HaveKey(name))
177+
for k, v := range expectedVMLabels[name] {
178+
Expect(node.Labels).Should(HaveKeyWithValue(k, v))
179+
}
180+
delete(expectedVMLabels, name)
181+
})
182+
err := handleVirtHandlerNodes(context.Background(), input)
183+
Expect(err).ShouldNot(HaveOccurred())
184+
Expect(expectedVMLabels).To(HaveLen(0))
185+
})
186+
})
187+
188+
})
189+
190+
func nodeYamlToSnapshot(manifest string) (string, error) {
191+
node := new(v1.Node)
192+
err := yaml.Unmarshal([]byte(manifest), node)
193+
if err != nil {
194+
return "", err
195+
}
196+
197+
query, err := jq.NewQuery(nodeJQFilter)
198+
if err != nil {
199+
return "", err
200+
}
201+
202+
filterResult, err := query.FilterObject(context.Background(), node)
203+
if err != nil {
204+
return "", err
205+
}
206+
207+
return filterResult.String(), nil
208+
}
209+
210+
func getNodeSnapshot(nodeManifest string) func(v any) (err error) {
211+
return func(v any) (err error) {
212+
rt := v.(*metav1.ObjectMeta)
213+
if err := json.Unmarshal([]byte(nodeManifest), rt); err != nil {
214+
return err
215+
}
216+
217+
return nil
218+
}
219+
}

images/hooks/go.mod

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@ require (
88
github.com/deckhouse/deckhouse/pkg/log v0.0.0-20250424095005-9ab587d01d7a
99
github.com/deckhouse/module-sdk v0.2.2
1010
github.com/deckhouse/virtualization/api v0.15.0
11+
github.com/onsi/ginkgo v1.16.4
1112
github.com/onsi/ginkgo/v2 v2.17.1
1213
github.com/onsi/gomega v1.32.0
1314
github.com/tidwall/gjson v1.14.4
1415
golang.org/x/crypto v0.38.0
1516
k8s.io/api v0.30.11
1617
k8s.io/apimachinery v0.30.11
1718
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
19+
sigs.k8s.io/yaml v1.4.0
1820
)
1921

2022
require (
@@ -50,6 +52,8 @@ require (
5052
github.com/google/uuid v1.6.0 // indirect
5153
github.com/imdario/mergo v0.3.16 // indirect
5254
github.com/inconshreveable/mousetrap v1.1.0 // indirect
55+
github.com/itchyny/gojq v0.12.17 // indirect
56+
github.com/itchyny/timefmt-go v0.1.6 // indirect
5357
github.com/jmoiron/sqlx v1.3.5 // indirect
5458
github.com/jonboulle/clockwork v0.4.0 // indirect
5559
github.com/josharian/intern v1.0.0 // indirect
@@ -60,6 +64,7 @@ require (
6064
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
6165
github.com/modern-go/reflect2 v1.0.2 // indirect
6266
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
67+
github.com/nxadm/tail v1.4.8 // indirect
6368
github.com/opencontainers/go-digest v1.0.0 // indirect
6469
github.com/opencontainers/image-spec v1.1.0 // indirect
6570
github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183 // indirect
@@ -94,6 +99,7 @@ require (
9499
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
95100
google.golang.org/protobuf v1.35.1 // indirect
96101
gopkg.in/inf.v0 v0.9.1 // indirect
102+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
97103
gopkg.in/yaml.v2 v2.4.0 // indirect
98104
gopkg.in/yaml.v3 v3.0.1 // indirect
99105
k8s.io/apiextensions-apiserver v0.30.11 // indirect
@@ -106,7 +112,6 @@ require (
106112
sigs.k8s.io/controller-runtime v0.18.7 // indirect
107113
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
108114
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
109-
sigs.k8s.io/yaml v1.4.0 // indirect
110115
)
111116

112117
replace golang.org/x/net => golang.org/x/net v0.40.0 // CVE-2025-22870, CVE-2025-22872

images/hooks/go.sum

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,11 +204,13 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
204204
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
205205
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
206206
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
207+
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
207208
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
208209
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
209210
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
210211
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
211212
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
213+
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
212214
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
213215
github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
214216
github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8=
@@ -468,6 +470,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
468470
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
469471
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
470472
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
473+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
471474
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
472475
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
473476
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

images/hooks/werf.inc.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,4 @@ shell:
3535
- go build -ldflags="-s -w" -o /hooks/generate-secret-for-dvcr ./cmd/generate-secret-for-dvcr
3636
- go build -ldflags="-s -w" -o /hooks/discovery-clusterip-service-for-dvcr ./cmd/discovery-clusterip-service-for-dvcr
3737
- go build -ldflags="-s -w" -o /hooks/discovery-workload-nodes ./cmd/discovery-workload-nodes
38+
- go build -ldflags="-s -w" -o /hooks/migrate-virthandler-kvm-node-labels ./cmd/migrate-virthandler-kvm-node-labels

0 commit comments

Comments
 (0)