From 9d469f5ce8bcdede46ac616d8cb07e0cda6cec59 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Mon, 3 Nov 2025 14:36:45 -0800 Subject: [PATCH 1/2] e2e-k8s.sh drop CLUSTER_LOG_FORMAT and add versioned kubeadm config patches --- hack/ci/e2e-k8s.sh | 96 ++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/hack/ci/e2e-k8s.sh b/hack/ci/e2e-k8s.sh index 6c7e523e5c..f168a12655 100755 --- a/hack/ci/e2e-k8s.sh +++ b/hack/ci/e2e-k8s.sh @@ -82,47 +82,11 @@ build() { export PATH="${PWD}/_output/bin:$PATH" } -check_structured_log_support() { - case "${KUBE_VERSION}" in - v1.1[0-8].*) - echo "$1 is only supported on versions >= v1.19, got ${KUBE_VERSION}" - exit 1 - ;; - esac -} - # up a cluster with kind create_cluster() { - # Grab the version of the cluster we're about to start - KUBE_VERSION="$(docker run --rm --entrypoint=cat "kindest/node:latest" /kind/version)" - # Default Log level for all components in test clusters KIND_CLUSTER_LOG_LEVEL=${KIND_CLUSTER_LOG_LEVEL:-4} - # potentially enable --logging-format - CLUSTER_LOG_FORMAT=${CLUSTER_LOG_FORMAT:-} - scheduler_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"" - controllerManager_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"" - apiServer_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"" - if [ -n "$CLUSTER_LOG_FORMAT" ]; then - check_structured_log_support "CLUSTER_LOG_FORMAT" - scheduler_extra_args="${scheduler_extra_args} - \"logging-format\": \"${CLUSTER_LOG_FORMAT}\"" - controllerManager_extra_args="${controllerManager_extra_args} - \"logging-format\": \"${CLUSTER_LOG_FORMAT}\"" - apiServer_extra_args="${apiServer_extra_args} - \"logging-format\": \"${CLUSTER_LOG_FORMAT}\"" - fi - kubelet_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\" - \"container-log-max-files\": \"10\" - \"container-log-max-size\": \"100Mi\"" - KUBELET_LOG_FORMAT=${KUBELET_LOG_FORMAT:-$CLUSTER_LOG_FORMAT} - if [ -n "$KUBELET_LOG_FORMAT" ]; then - check_structured_log_support "KUBECTL_LOG_FORMAT" - kubelet_extra_args="${kubelet_extra_args} - \"logging-format\": \"${KUBELET_LOG_FORMAT}\"" - fi - # JSON or YAML map injected into featureGates config feature_gates="${FEATURE_GATES:-{\}}" # --runtime-config argument value passed to the API server, again as a map @@ -146,29 +110,79 @@ nodes: featureGates: ${feature_gates} runtimeConfig: ${runtime_config} kubeadmConfigPatches: +# v1beta4 for the future (v1.35.0+ ?) +# https://github.com/kubernetes-sigs/kind/issues/3847 +# TODO: drop v1beta3 when we no longer need versions that use it +- | + kind: ClusterConfiguration + apiVersion: kubeadm.k8s.io/v1beta4 + metadata: + name: config + apiServer: + extraArgs: + - name: "v" + value: "${KIND_CLUSTER_LOG_LEVEL}" + controllerManager: + extraArgs: + - name: "v" + value: "${KIND_CLUSTER_LOG_LEVEL}" + scheduler: + extraArgs: + - name: "v" + value: "${KIND_CLUSTER_LOG_LEVEL}" + --- + kind: InitConfiguration + apiVersion: kubeadm.k8s.io/v1beta4 + nodeRegistration: + kubeletExtraArgs: + - name: "v" + value: "${KIND_CLUSTER_LOG_LEVEL}" + - name: "container-log-max-files" + value: "10" + - name: "container-log-max-size" + value: "100Mi" + --- + kind: JoinConfiguration + apiVersion: kubeadm.k8s.io/v1beta4 + nodeRegistration: + kubeletExtraArgs: + - name: "v" + value: "${KIND_CLUSTER_LOG_LEVEL}" + - name: "container-log-max-files" + value: "10" + - name: "container-log-max-size" + value: "100Mi" +# v1beta3 for v1.23.0 ... ? - | kind: ClusterConfiguration + apiVersion: kubeadm.k8s.io/v1beta3 metadata: name: config apiServer: extraArgs: -${apiServer_extra_args} + "v": "${KIND_CLUSTER_LOG_LEVEL}" controllerManager: extraArgs: -${controllerManager_extra_args} + "v": "${KIND_CLUSTER_LOG_LEVEL}" scheduler: extraArgs: -${scheduler_extra_args} + "v": "${KIND_CLUSTER_LOG_LEVEL}" --- kind: InitConfiguration + apiVersion: kubeadm.k8s.io/v1beta3 nodeRegistration: kubeletExtraArgs: -${kubelet_extra_args} + "v": "${KIND_CLUSTER_LOG_LEVEL}" + "container-log-max-files": "10" + "container-log-max-size": "100Mi" --- kind: JoinConfiguration + apiVersion: kubeadm.k8s.io/v1beta3 nodeRegistration: kubeletExtraArgs: -${kubelet_extra_args} + "v": "${KIND_CLUSTER_LOG_LEVEL}" + "container-log-max-files": "10" + "container-log-max-size": "100Mi" EOF # NOTE: must match the number of workers above NUM_NODES=2 From 56265ef8b7e9ac24db80ba40d8fce609694a072b Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Thu, 6 Nov 2025 14:56:37 -0800 Subject: [PATCH 2/2] add warning to container-log* flags --- hack/ci/e2e-k8s.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hack/ci/e2e-k8s.sh b/hack/ci/e2e-k8s.sh index f168a12655..134aa8fa51 100755 --- a/hack/ci/e2e-k8s.sh +++ b/hack/ci/e2e-k8s.sh @@ -116,8 +116,6 @@ kubeadmConfigPatches: - | kind: ClusterConfiguration apiVersion: kubeadm.k8s.io/v1beta4 - metadata: - name: config apiServer: extraArgs: - name: "v" @@ -148,6 +146,10 @@ kubeadmConfigPatches: kubeletExtraArgs: - name: "v" value: "${KIND_CLUSTER_LOG_LEVEL}" + # Warning: these flags appear to be load bearing / impact performance + # See: https://github.com/kubernetes-sigs/kind/pull/4046 + # Be careful when updating these. + # Most CI jobs should not need them, but some CI jobs might. - name: "container-log-max-files" value: "10" - name: "container-log-max-size" @@ -156,8 +158,6 @@ kubeadmConfigPatches: - | kind: ClusterConfiguration apiVersion: kubeadm.k8s.io/v1beta3 - metadata: - name: config apiServer: extraArgs: "v": "${KIND_CLUSTER_LOG_LEVEL}" @@ -181,6 +181,10 @@ kubeadmConfigPatches: nodeRegistration: kubeletExtraArgs: "v": "${KIND_CLUSTER_LOG_LEVEL}" + # Warning: these flags appear to be load bearing / impact performance + # See: https://github.com/kubernetes-sigs/kind/pull/4046 + # Be careful when updating these. + # Most CI jobs should not need them, but some CI jobs might. "container-log-max-files": "10" "container-log-max-size": "100Mi" EOF