Skip to content

Commit 5235bed

Browse files
jgehrckejgehrcke
authored
Merge pull request #658 from jgehrcke/jp/log-full-component-config-on-startup
Log full startup config in all CLIs in `Before` hook
2 parents 7b5e2cd + aa15924 commit 5235bed

File tree

6 files changed

+90
-54
lines changed

6 files changed

+90
-54
lines changed

‎cmd/compute-domain-controller/main.go‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ import (
4040
_ "k8s.io/component-base/metrics/prometheus/workqueue" // register work queues in the default legacy registry
4141

4242
"github.com/NVIDIA/k8s-dra-driver-gpu/internal/info"
43-
"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
43+
pkgflags "github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
4444
)
4545

4646
const (
@@ -54,9 +54,7 @@ const (
5454
)
5555

5656
type Flags struct {
57-
kubeClientConfig flags.KubeClientConfig
58-
loggingConfig *flags.LoggingConfig
59-
featureGateConfig *flags.FeatureGateConfig
57+
kubeClientConfig pkgflags.KubeClientConfig
6058

6159
podName string
6260
namespace string
@@ -74,7 +72,7 @@ type Flags struct {
7472
type Config struct {
7573
driverName string
7674
flags *Flags
77-
clientsets flags.ClientSets
75+
clientsets pkgflags.ClientSets
7876
mux *http.ServeMux
7977
}
8078

@@ -86,10 +84,10 @@ func main() {
8684
}
8785

8886
func newApp() *cli.App {
89-
flags := &Flags{
90-
loggingConfig: flags.NewLoggingConfig(),
91-
featureGateConfig: flags.NewFeatureGateConfig(),
92-
}
87+
loggingConfig := pkgflags.NewLoggingConfig()
88+
featureGateConfig := pkgflags.NewFeatureGateConfig()
89+
flags := &Flags{}
90+
9391
cliFlags := []cli.Flag{
9492
&cli.StringFlag{
9593
Name: "pod-name",
@@ -157,8 +155,8 @@ func newApp() *cli.App {
157155
}
158156

159157
cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...)
160-
cliFlags = append(cliFlags, flags.featureGateConfig.Flags()...)
161-
cliFlags = append(cliFlags, flags.loggingConfig.Flags()...)
158+
cliFlags = append(cliFlags, featureGateConfig.Flags()...)
159+
cliFlags = append(cliFlags, loggingConfig.Flags()...)
162160

163161
app := &cli.App{
164162
Name: "compute-domain-controller",
@@ -170,7 +168,10 @@ func newApp() *cli.App {
170168
if c.Args().Len() > 0 {
171169
return fmt.Errorf("arguments not supported: %v", c.Args().Slice())
172170
}
173-
return flags.loggingConfig.Apply()
171+
// `loggingConfig` must be applied before doing any logging
172+
err := loggingConfig.Apply()
173+
pkgflags.LogStartupConfig(flags, loggingConfig)
174+
return err
174175
},
175176
Action: func(c *cli.Context) error {
176177
mux := http.NewServeMux()

‎cmd/compute-domain-daemon/main.go‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import (
3434

3535
nvapi "github.com/NVIDIA/k8s-dra-driver-gpu/api/nvidia.com/resource/v1beta1"
3636
"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/featuregates"
37-
"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
37+
pkgflags "github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
3838
)
3939

4040
const (
@@ -55,8 +55,6 @@ type Flags struct {
5555
podName string
5656
podNamespace string
5757
maxNodesPerIMEXDomain int
58-
loggingConfig *flags.LoggingConfig
59-
featureGateConfig *flags.FeatureGateConfig
6058
}
6159

6260
type IMEXConfigTemplateData struct {
@@ -71,10 +69,9 @@ func main() {
7169
}
7270

7371
func newApp() *cli.App {
74-
flags := Flags{
75-
loggingConfig: flags.NewLoggingConfig(),
76-
featureGateConfig: flags.NewFeatureGateConfig(),
77-
}
72+
loggingConfig := pkgflags.NewLoggingConfig()
73+
featureGateConfig := pkgflags.NewFeatureGateConfig()
74+
flags := &Flags{}
7875

7976
// Create a wrapper that will be used to gracefully shut down all subcommands
8077
wrapper := func(ctx context.Context, f func(ctx context.Context, cancel context.CancelFunc, flags *Flags) error) error {
@@ -91,7 +88,7 @@ func newApp() *cli.App {
9188
}()
9289

9390
// Call the wrapped function
94-
return f(ctx, cancel, &flags)
91+
return f(ctx, cancel, flags)
9592
}
9693

9794
cliFlags := []cli.Flag{
@@ -151,16 +148,19 @@ func newApp() *cli.App {
151148
Destination: &flags.maxNodesPerIMEXDomain,
152149
},
153150
}
154-
cliFlags = append(cliFlags, flags.featureGateConfig.Flags()...)
155-
cliFlags = append(cliFlags, flags.loggingConfig.Flags()...)
151+
cliFlags = append(cliFlags, featureGateConfig.Flags()...)
152+
cliFlags = append(cliFlags, loggingConfig.Flags()...)
156153

157154
// Create the app
158155
app := &cli.App{
159156
Name: "compute-domain-daemon",
160157
Usage: "compute-domain-daemon manages the IMEX daemon for NVIDIA compute domains.",
161158
Flags: cliFlags,
162159
Before: func(c *cli.Context) error {
163-
return flags.loggingConfig.Apply()
160+
// `loggingConfig` must be applied before doing any logging
161+
err := loggingConfig.Apply()
162+
pkgflags.LogStartupConfig(flags, loggingConfig)
163+
return err
164164
},
165165
Commands: []*cli.Command{
166166
{
@@ -197,7 +197,6 @@ func run(ctx context.Context, cancel context.CancelFunc, flags *Flags) error {
197197
podNamespace: flags.podNamespace,
198198
maxNodesPerIMEXDomain: flags.maxNodesPerIMEXDomain,
199199
}
200-
klog.Infof("config: %v", config)
201200

202201
// Support heterogeneous ComputeDomains. That means that a CD may contain
203202
// nodes that do not take part in Multi-Node NVLink communication. On such

‎cmd/compute-domain-kubelet-plugin/main.go‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333
"k8s.io/component-base/logs"
3434

3535
"github.com/NVIDIA/k8s-dra-driver-gpu/internal/info"
36-
"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
36+
pkgflags "github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
3737
)
3838

3939
const (
@@ -43,9 +43,7 @@ const (
4343
)
4444

4545
type Flags struct {
46-
kubeClientConfig flags.KubeClientConfig
47-
loggingConfig *flags.LoggingConfig
48-
featureGateConfig *flags.FeatureGateConfig
46+
kubeClientConfig pkgflags.KubeClientConfig
4947

5048
nodeName string
5149
namespace string
@@ -60,7 +58,7 @@ type Flags struct {
6058

6159
type Config struct {
6260
flags *Flags
63-
clientsets flags.ClientSets
61+
clientsets pkgflags.ClientSets
6462
}
6563

6664
func (c Config) DriverPluginPath() string {
@@ -75,10 +73,10 @@ func main() {
7573
}
7674

7775
func newApp() *cli.App {
78-
flags := &Flags{
79-
loggingConfig: flags.NewLoggingConfig(),
80-
featureGateConfig: flags.NewFeatureGateConfig(),
81-
}
76+
loggingConfig := pkgflags.NewLoggingConfig()
77+
featureGateConfig := pkgflags.NewFeatureGateConfig()
78+
flags := &Flags{}
79+
8280
cliFlags := []cli.Flag{
8381
&cli.StringFlag{
8482
Name: "node-name",
@@ -145,8 +143,8 @@ func newApp() *cli.App {
145143
},
146144
}
147145
cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...)
148-
cliFlags = append(cliFlags, flags.featureGateConfig.Flags()...)
149-
cliFlags = append(cliFlags, flags.loggingConfig.Flags()...)
146+
cliFlags = append(cliFlags, featureGateConfig.Flags()...)
147+
cliFlags = append(cliFlags, loggingConfig.Flags()...)
150148

151149
app := &cli.App{
152150
Name: "compute-domain-kubelet-plugin",
@@ -158,11 +156,12 @@ func newApp() *cli.App {
158156
if c.Args().Len() > 0 {
159157
return fmt.Errorf("arguments not supported: %v", c.Args().Slice())
160158
}
161-
return flags.loggingConfig.Apply()
159+
// `loggingConfig` must be applied before doing any logging
160+
err := loggingConfig.Apply()
161+
pkgflags.LogStartupConfig(flags, loggingConfig)
162+
return err
162163
},
163164
Action: func(c *cli.Context) error {
164-
ctx := c.Context
165-
166165
clientSets, err := flags.kubeClientConfig.NewClientSets()
167166
if err != nil {
168167
return fmt.Errorf("create client: %w", err)
@@ -173,7 +172,7 @@ func newApp() *cli.App {
173172
clientsets: clientSets,
174173
}
175174

176-
return RunPlugin(ctx, config)
175+
return RunPlugin(c.Context, config)
177176
},
178177
After: func(c *cli.Context) error {
179178
// Runs after `Action` (regardless of success/error). In urfave cli

‎cmd/gpu-kubelet-plugin/main.go‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import (
3232
"k8s.io/klog/v2"
3333

3434
"github.com/NVIDIA/k8s-dra-driver-gpu/internal/info"
35-
"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
35+
pkgflags "github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flags"
3636
)
3737

3838
const (
@@ -41,9 +41,7 @@ const (
4141
)
4242

4343
type Flags struct {
44-
kubeClientConfig flags.KubeClientConfig
45-
loggingConfig *flags.LoggingConfig
46-
featureGateConfig *flags.FeatureGateConfig
44+
kubeClientConfig pkgflags.KubeClientConfig
4745

4846
nodeName string
4947
namespace string
@@ -59,7 +57,7 @@ type Flags struct {
5957

6058
type Config struct {
6159
flags *Flags
62-
clientsets flags.ClientSets
60+
clientsets pkgflags.ClientSets
6361
}
6462

6563
func (c Config) DriverPluginPath() string {
@@ -74,10 +72,10 @@ func main() {
7472
}
7573

7674
func newApp() *cli.App {
77-
flags := &Flags{
78-
loggingConfig: flags.NewLoggingConfig(),
79-
featureGateConfig: flags.NewFeatureGateConfig(),
80-
}
75+
loggingConfig := pkgflags.NewLoggingConfig()
76+
featureGateConfig := pkgflags.NewFeatureGateConfig()
77+
flags := &Flags{}
78+
8179
cliFlags := []cli.Flag{
8280
&cli.StringFlag{
8381
Name: "node-name",
@@ -151,8 +149,8 @@ func newApp() *cli.App {
151149
},
152150
}
153151
cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...)
154-
cliFlags = append(cliFlags, flags.featureGateConfig.Flags()...)
155-
cliFlags = append(cliFlags, flags.loggingConfig.Flags()...)
152+
cliFlags = append(cliFlags, featureGateConfig.Flags()...)
153+
cliFlags = append(cliFlags, loggingConfig.Flags()...)
156154

157155
app := &cli.App{
158156
Name: "gpu-kubelet-plugin",
@@ -164,11 +162,12 @@ func newApp() *cli.App {
164162
if c.Args().Len() > 0 {
165163
return fmt.Errorf("arguments not supported: %v", c.Args().Slice())
166164
}
167-
return flags.loggingConfig.Apply()
165+
// `loggingConfig` must be applied before doing any logging
166+
err := loggingConfig.Apply()
167+
pkgflags.LogStartupConfig(flags, loggingConfig)
168+
return err
168169
},
169170
Action: func(c *cli.Context) error {
170-
ctx := c.Context
171-
172171
clientSets, err := flags.kubeClientConfig.NewClientSets()
173172
if err != nil {
174173
return fmt.Errorf("create client: %w", err)
@@ -179,7 +178,7 @@ func newApp() *cli.App {
179178
clientsets: clientSets,
180179
}
181180

182-
return RunPlugin(ctx, config)
181+
return RunPlugin(c.Context, config)
183182
},
184183
After: func(c *cli.Context) error {
185184
// Runs after `Action` (regardless of success/error). In urfave cli

‎pkg/flags/utils.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ import (
2121

2222
"github.com/spf13/pflag"
2323
"github.com/urfave/cli/v2"
24+
"k8s.io/apimachinery/pkg/util/dump"
25+
"k8s.io/klog/v2"
26+
27+
"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/featuregates"
2428
)
2529

2630
func pflagToCLI(flag *pflag.Flag, category string) cli.Flag {
@@ -33,3 +37,21 @@ func pflagToCLI(flag *pflag.Flag, category string) cli.Flag {
3337
EnvVars: []string{strings.ToUpper(strings.ReplaceAll(flag.Name, "-", "_"))},
3438
}
3539
}
40+
41+
func LogStartupConfig(parsedFlags any, loggingConfig *LoggingConfig) {
42+
// Always log component startup config (level 0).
43+
klog.Infof("\nFeature gates: %#v\nVerbosity: %d\nFlags: %s",
44+
// Flat boolean map -- no pretty-printing needed.
45+
featuregates.ToMap(),
46+
loggingConfig.config.Verbosity,
47+
// Based on go-spew's Sdump(), with indentation. Type information is
48+
// always displayed (cannot be disabled).
49+
dump.Pretty(parsedFlags),
50+
)
51+
52+
// This is a complex object, comprised of largely static default klog
53+
// component configuration. Various parts can be overridden via environment
54+
// variables or CLI flags: it makes sense to log the interpolated config,
55+
// but only on a high verbosity level.
56+
klog.V(6).Infof("Logging config: %s", dump.Pretty(loggingConfig))
57+
}

‎tests/bats/tests.bats‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,22 @@ log_objects() {
266266
echo "${output}" | grep -E '^.*SUM multinode_device_to_device_memcpy_read_ce [0-9]+\.[0-9]+.*$'
267267
}
268268

269+
@test "Confirm startup config / detail in logs on level 0" {
270+
local _iargs=("--set" "logVerbosity=0")
271+
iupgrade_wait "${TEST_CHART_REPO}" "${TEST_CHART_VERSION}" _iargs
272+
273+
run kubectl logs -l nvidia-dra-driver-gpu-component=controller -n nvidia-dra-driver-gpu --tail=-1
274+
assert_output --partial "Verbosity:"
275+
assert_output --partial '"MPSSupport":false'
276+
assert_output --partial 'additionalNamespaces:'
277+
278+
run kubectl logs -l nvidia-dra-driver-gpu-component=kubelet-plugin -n nvidia-dra-driver-gpu --tail=-1
279+
assert_output --partial "Verbosity"
280+
assert_output --partial "nodeName"
281+
assert_output --partial "identified fabric clique"
282+
assert_output --partial "driver version validation"
283+
}
284+
269285
@test "CD controller: test log verbosity levels" {
270286
iupgrade_wait "${TEST_CHART_REPO}" "${TEST_CHART_VERSION}" NOARGS
271287

0 commit comments

Comments
 (0)