FIXME: break this up

Author: thockin

cmd/kube-controller-manager/app/storageversionmigrator.go

@@ -26,8 +26,9 @@ import (
 	"k8s.io/controller-manager/controller"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/names"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/utils/feature"
 
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	oldutilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientgofeaturegate "k8s.io/client-go/features"
 	svm "k8s.io/kubernetes/pkg/controller/storageversionmigrator"
 )
@@ -45,8 +46,8 @@ func startSVMController(
 	controllerContext ControllerContext,
 	controllerName string,
 ) (controller.Interface, bool, error) {
-	if !utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionMigrator) ||
-		!clientgofeaturegate.FeatureGates().Enabled(clientgofeaturegate.InformerResourceVersion) {
+	if !oldutilfeature.DefaultFeatureGate.Enabled(features.StorageVersionMigrator) &&
+		!feature.Enabled(clientgofeaturegate.InformerResourceVersionGate) {
 		return nil, false, nil
 	}
 

cmd/kube-controller-manager/controller-manager.go

@@ -24,6 +24,8 @@ import (
 	"os"
 	_ "time/tzdata" // for CronJob Time Zone support
 
+	clientfeatures "k8s.io/client-go/features"
+
 	"k8s.io/component-base/cli"
 	_ "k8s.io/component-base/logs/json/register"          // for JSON log format registration
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin
@@ -32,6 +34,9 @@ import (
 )
 
 func main() {
+	// FIXME: Example, do not merge
+	clientfeatures.WatchListClientGate.Set(true)
+
 	command := app.NewControllerManagerCommand()
 	code := cli.Run(command)
 	os.Exit(code)

pkg/api/pod/util.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/core/helper"
 	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/utils/feature"
 )
 
 // ContainerType signifies container type
@@ -685,7 +686,8 @@ func dropDisabledFields(
 		// For other types of containers, validateContainers will handle them.
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) && !rroInUse(oldPodSpec) {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+		!feature.Enabled(features.RecursiveReadOnlyMountsGate) && !rroInUse(oldPodSpec) {
 		for i := range podSpec.Containers {
 			for j := range podSpec.Containers[i].VolumeMounts {
 				podSpec.Containers[i].VolumeMounts[j].RecursiveReadOnly = nil
@@ -809,7 +811,8 @@ func dropDisabledPodStatusFields(podStatus, oldPodStatus *api.PodStatus, podSpec
 		podStatus.HostIPs = nil
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) && !rroInUse(oldPodSpec) {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+		!feature.Enabled(features.RecursiveReadOnlyMountsGate) && !rroInUse(oldPodSpec) {
 		for i := range podStatus.ContainerStatuses {
 			podStatus.ContainerStatuses[i].VolumeMounts = nil
 		}

pkg/kubelet/kubelet_pods.go

@@ -69,6 +69,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 	volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
 	"k8s.io/kubernetes/third_party/forked/golang/expansion"
+	"k8s.io/utils/feature"
 	utilnet "k8s.io/utils/net"
 )
 
@@ -348,7 +349,8 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 		if err != nil {
 			return nil, cleanupAction, fmt.Errorf("failed to resolve recursive read-only mode: %w", err)
 		}
-		if rro && !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+		if rro && !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+			!feature.Enabled(features.RecursiveReadOnlyMountsGate) {
 			return nil, cleanupAction, fmt.Errorf("recursive read-only mount needs feature gate %q to be enabled", features.RecursiveReadOnlyMounts)
 		}
 
@@ -2136,7 +2138,8 @@ func (kl *Kubelet) convertToAPIContainerStatuses(pod *v1.Pod, podStatus *kubecon
 		}
 		// status.VolumeMounts cannot be propagated from kubecontainer.Status
 		// because the CRI API is unaware of the volume names.
-		if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+		if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) ||
+			feature.Enabled(features.RecursiveReadOnlyMountsGate) {
 			for _, vol := range container.VolumeMounts {
 				volStatus := v1.VolumeMountStatus{
 					Name:      vol.Name,
@@ -2148,7 +2151,8 @@ func (kl *Kubelet) convertToAPIContainerStatuses(pod *v1.Pod, podStatus *kubecon
 					if b, err := resolveRecursiveReadOnly(vol, supportsRRO); err != nil {
 						klog.ErrorS(err, "failed to resolve recursive read-only mode", "mode", *vol.RecursiveReadOnly)
 					} else if b {
-						if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+						if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) ||
+							feature.Enabled(features.RecursiveReadOnlyMountsGate) {
 							rroMode = v1.RecursiveReadOnlyEnabled
 						} else {
 							klog.ErrorS(nil, "recursive read-only mount needs feature gate to be enabled",

pkg/kubelet/nodestatus/setters.go

@@ -43,6 +43,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/events"
+	"k8s.io/utils/feature"
 	netutils "k8s.io/utils/net"
 
 	"k8s.io/klog/v2"
@@ -485,7 +486,8 @@ func GoRuntime() Setter {
 // RuntimeHandlers returns a Setter that sets RuntimeHandlers on the node.
 func RuntimeHandlers(fn func() []kubecontainer.RuntimeHandler) Setter {
 	return func(ctx context.Context, node *v1.Node) error {
-		if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+			!feature.Enabled(features.RecursiveReadOnlyMountsGate) {
 			return nil
 		}
 		handlers := fn()

pkg/registry/core/node/strategy.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/client"
+	"k8s.io/utils/feature"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
@@ -103,7 +104,8 @@ func dropDisabledFields(node *api.Node, oldNode *api.Node) {
 		node.Spec.ConfigSource = nil
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+		!feature.Enabled(features.RecursiveReadOnlyMountsGate) {
 		node.Status.RuntimeHandlers = nil
 	}
 }

staging/src/k8s.io/client-go/tools/cache/shared_informer.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/tools/cache/synctrack"
 	"k8s.io/utils/buffer"
 	"k8s.io/utils/clock"
+	"k8s.io/utils/feature"
 
 	"k8s.io/klog/v2"
 
@@ -411,7 +412,8 @@ func (v *dummyController) HasSynced() bool {
 }
 
 func (v *dummyController) LastSyncResourceVersion() string {
-	if clientgofeaturegate.FeatureGates().Enabled(clientgofeaturegate.InformerResourceVersion) {
+	if clientgofeaturegate.FeatureGates().Enabled(clientgofeaturegate.InformerResourceVersion) ||
+		feature.Enabled(clientgofeaturegate.InformerResourceVersionGate) {
 		return v.informer.LastSyncResourceVersion()
 	}
 

test/integration/storageversionmigrator/storageversionmigrator_test.go

@@ -30,9 +30,9 @@ import (
 	encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientgofeaturegate "k8s.io/client-go/features"
-	"k8s.io/component-base/featuregate"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/utils/feature"
 )
 
 // TestStorageVersionMigration is an integration test that verifies storage version migration works.
@@ -47,7 +47,7 @@ import (
 // 8. Verify that the resource version of the secret is not updated. i.e. it was a no-op update
 func TestStorageVersionMigration(t *testing.T) {
 	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionMigrator, true)
-	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, featuregate.Feature(clientgofeaturegate.InformerResourceVersion), true)
+	feature.SetForTesting(clientgofeaturegate.InformerResourceVersionGate, t, true)
 
 	// this makes the test super responsive. It's set to a default of 1 minute.
 	encryptionconfigcontroller.EncryptionConfigFileChangePollDuration = time.Millisecond
@@ -149,7 +149,7 @@ func TestStorageVersionMigration(t *testing.T) {
 // 11. Verify the list of CRs at v2 works
 func TestStorageVersionMigrationWithCRD(t *testing.T) {
 	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionMigrator, true)
-	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, featuregate.Feature(clientgofeaturegate.InformerResourceVersion), true)
+	feature.SetForTesting(clientgofeaturegate.InformerResourceVersionGate, t, true)
 	// decode errors are expected when using conversation webhooks
 	etcd3watcher.TestOnlySetFatalOnDecodeError(false)
 	defer etcd3watcher.TestOnlySetFatalOnDecodeError(true)