Skip to content

Commit 0b72b10

Browse files
tobiasgiesetobiasgiese
committed
Get rid of kubectl depepdency in container image
We should not use the kubectl binary inside our containers because of potential CVEs. To get rid of the binary we can use the crdutil from the k8s-operator-libs. Signed-off-by: Tobias Giese <[email protected]>
1 parent fe9595a commit 0b72b10

File tree

11 files changed

+434
-25
lines changed

11 files changed

+434
-25
lines changed

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,11 @@ gpu-operator:
8686
CGO_ENABLED=0 GOOS=$(GOOS) \
8787
go build -ldflags "-s -w -X $(VERSION_PKG).gitCommit=$(GIT_COMMIT) -X $(VERSION_PKG).version=$(VERSION)" -o gpu-operator ./cmd/gpu-operator/...
8888

89+
# Build apply-crds binary
90+
apply-crds:
91+
CGO_ENABLED=0 GOOS=$(GOOS) \
92+
go build -ldflags "-s -w -X $(VERSION_PKG).gitCommit=$(GIT_COMMIT) -X $(VERSION_PKG).version=$(VERSION)" -o apply-crds ./cmd/apply-crds/...
93+
8994
# Run against the configured Kubernetes cluster in ~/.kube/config
9095
run: generate check manifests
9196
go run ./cmd/gpu-operator/...

‎cmd/apply-crds/main.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
2024 NVIDIA CORPORATION & AFFILIATES
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package main uses the crdutil package to ensure CRDs are created
18+
// or updated in the cluster during Helm chart installation.
19+
package main
20+
21+
import (
22+
"github.com/NVIDIA/k8s-operator-libs/pkg/crdutil"
23+
)
24+
25+
func main() {
26+
crdutil.EnsureCRDsCmd()
27+
}

‎deployments/gpu-operator/templates/upgrade_crd.yaml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,12 @@ spec:
8383
image: {{ include "gpu-operator.fullimage" . }}
8484
imagePullPolicy: {{ .Values.operator.imagePullPolicy }}
8585
command:
86-
- /bin/sh
87-
- -c
88-
- >
89-
kubectl apply -f /opt/gpu-operator/nvidia.com_clusterpolicies.yaml;
90-
kubectl apply -f /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml;
86+
- /usr/bin/apply-crds
87+
args:
88+
- --crds-file=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml
89+
- --crds-file=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml
9190
{{- if .Values.nfd.enabled }}
92-
kubectl apply -f /opt/gpu-operator/nfd-api-crds.yaml;
91+
- --crds-file=/opt/gpu-operator/nfd-api-crds.yaml
9392
{{- end }}
9493
restartPolicy: OnFailure
9594
{{- end }}

‎docker/Dockerfile‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ COPY *.mk .
5353
# Build
5454
ARG VERSION="unknown"
5555
ARG GIT_COMMIT="unknown"
56-
RUN make gpu-operator
56+
RUN make gpu-operator apply-crds
5757

5858
FROM nvcr.io/nvidia/cuda:12.6.2-base-ubi9
5959

@@ -76,19 +76,14 @@ LABEL vsc-ref=${GIT_COMMIT}
7676

7777
WORKDIR /
7878
COPY --from=builder /workspace/gpu-operator /usr/bin/
79+
COPY --from=builder /workspace/apply-crds /usr/bin/
7980

8081
RUN mkdir -p /opt/gpu-operator/manifests
8182
COPY assets /opt/gpu-operator/
8283
COPY manifests /opt/gpu-operator/manifests
8384
RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
8485
COPY hack/must-gather.sh /usr/bin/gather
8586

86-
# Install must-gather dependency: `kubectl`
87-
ARG TARGETARCH
88-
RUN OS_ARCH=${TARGETARCH/x86_64/amd64} && OS_ARCH=${OS_ARCH/aarch64/arm64} && curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${OS_ARCH}/kubectl
89-
RUN chmod +x ./kubectl
90-
RUN mv ./kubectl /usr/local/bin
91-
9287
# Add CRD resource into the image for helm upgrades
9388
COPY deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml /opt/gpu-operator/nvidia.com_clusterpolicies.yaml
9489
COPY deployments/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml

‎go.mod‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ require (
88
github.com/Masterminds/sprig/v3 v3.3.0
99
github.com/NVIDIA/go-nvlib v0.7.0
1010
github.com/NVIDIA/k8s-kata-manager v0.2.2
11-
github.com/NVIDIA/k8s-operator-libs v0.0.0-20240826221728-249ba446fa35
11+
github.com/NVIDIA/k8s-operator-libs v0.0.0-20241120073822-1ad8938d7274
1212
github.com/NVIDIA/nvidia-container-toolkit v1.17.2
1313
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
1414
github.com/go-logr/logr v1.4.2
@@ -169,14 +169,19 @@ require (
169169
gopkg.in/yaml.v3 v3.0.1 // indirect
170170
helm.sh/helm/v3 v3.16.1 // indirect
171171
k8s.io/apiserver v0.31.2 // indirect
172-
k8s.io/cli-runtime v0.31.1 // indirect
172+
k8s.io/cli-runtime v0.31.2 // indirect
173173
k8s.io/component-base v0.31.2 // indirect
174174
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
175-
k8s.io/kubectl v0.31.0 // indirect
175+
k8s.io/kubectl v0.31.2 // indirect
176176
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 // indirect
177177
oras.land/oras-go v1.2.5 // indirect
178178
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
179179
sigs.k8s.io/kustomize/api v0.17.2 // indirect
180180
sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
181181
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
182182
)
183+
184+
// DROP BEFORE MERGE!
185+
// Implements https://github.com/NVIDIA/k8s-operator-libs/pull/58
186+
// This is only for testing.
187+
replace github.com/NVIDIA/k8s-operator-libs => github.com/tobiasgiese/k8s-operator-libs v0.0.0-20241125092837-e8a080621717

‎go.sum‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,6 @@ github.com/NVIDIA/go-nvlib v0.7.0 h1:Z/J7skMdLbTiHvomKVsGYsttfQMZj5FwNYIFXhZ4i/c
2828
github.com/NVIDIA/go-nvlib v0.7.0/go.mod h1:9UrsLGx/q1OrENygXjOuM5Ey5KCtiZhbvBlbUIxtGWY=
2929
github.com/NVIDIA/k8s-kata-manager v0.2.2 h1:+xVIp4yLfCjZ31Dfrm9LOKo4T47b4g+DV6XkwAqalns=
3030
github.com/NVIDIA/k8s-kata-manager v0.2.2/go.mod h1:UGjGQUcpXTegwyOc5IwcyLTzPKwO9lOIkqw/qUzk8Q0=
31-
github.com/NVIDIA/k8s-operator-libs v0.0.0-20240826221728-249ba446fa35 h1:w9DXPTJCq9k2PVpdBQJrWE4vAmZcFaSHKLpM/xos9WI=
32-
github.com/NVIDIA/k8s-operator-libs v0.0.0-20240826221728-249ba446fa35/go.mod h1:sw6XRI5wq0Q+nSgaWa1Pyo/ZKxQebc70x6VIznDAxtM=
3331
github.com/NVIDIA/nvidia-container-toolkit v1.17.2 h1:iE6PK9SQH3HyDrOolu27xn3CJgURR3bDtnbfFrxdML8=
3432
github.com/NVIDIA/nvidia-container-toolkit v1.17.2/go.mod h1:R6bNf6ca0IjjACa0ncKGvsrx6zSjsgz8QkFyBDk5szU=
3533
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
@@ -351,6 +349,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
351349
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
352350
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
353351
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
352+
github.com/tobiasgiese/k8s-operator-libs v0.0.0-20241125092837-e8a080621717 h1:456lFgNispD2ff9fpni9sYB3838p14O30zN0cyoeFmI=
353+
github.com/tobiasgiese/k8s-operator-libs v0.0.0-20241125092837-e8a080621717/go.mod h1:g8DW2t4Vit91uLdqCxsjKbKYrwCdb/oB9q/YOXdUjmQ=
354354
github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
355355
github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
356356
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
@@ -488,8 +488,8 @@ k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
488488
k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
489489
k8s.io/apiserver v0.31.2 h1:VUzOEUGRCDi6kX1OyQ801m4A7AUPglpsmGvdsekmcI4=
490490
k8s.io/apiserver v0.31.2/go.mod h1:o3nKZR7lPlJqkU5I3Ove+Zx3JuoFjQobGX1Gctw6XuE=
491-
k8s.io/cli-runtime v0.31.1 h1:/ZmKhmZ6hNqDM+yf9s3Y4KEYakNXUn5sod2LWGGwCuk=
492-
k8s.io/cli-runtime v0.31.1/go.mod h1:pKv1cDIaq7ehWGuXQ+A//1OIF+7DI+xudXtExMCbe9U=
491+
k8s.io/cli-runtime v0.31.2 h1:7FQt4C4Xnqx8V1GJqymInK0FFsoC+fAZtbLqgXYVOLQ=
492+
k8s.io/cli-runtime v0.31.2/go.mod h1:XROyicf+G7rQ6FQJMbeDV9jqxzkWXTYD6Uxd15noe0Q=
493493
k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
494494
k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
495495
k8s.io/component-base v0.31.2 h1:Z1J1LIaC0AV+nzcPRFqfK09af6bZ4D1nAOpWsy9owlA=
@@ -498,8 +498,8 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
498498
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
499499
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
500500
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
501-
k8s.io/kubectl v0.31.0 h1:kANwAAPVY02r4U4jARP/C+Q1sssCcN/1p9Nk+7BQKVg=
502-
k8s.io/kubectl v0.31.0/go.mod h1:pB47hhFypGsaHAPjlwrNbvhXgmuAr01ZBvAIIUaI8d4=
501+
k8s.io/kubectl v0.31.2 h1:gTxbvRkMBwvTSAlobiTVqsH6S8Aa1aGyBcu5xYLsn8M=
502+
k8s.io/kubectl v0.31.2/go.mod h1:EyASYVU6PY+032RrTh5ahtSOMgoDRIux9V1JLKtG5xM=
503503
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI=
504504
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
505505
oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=

‎vendor/github.com/NVIDIA/k8s-operator-libs/pkg/crdutil/README.md‎

Lines changed: 61 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)