From 5e9977270f40ad073e3e6fd5b9e51b7d49977310 Mon Sep 17 00:00:00 2001 From: Cici Huang Date: Thu, 5 Jun 2025 01:40:21 -0700 Subject: [PATCH] Add KEP for Implicit Tolerations. --- keps/prod-readiness/sig-scheduling/5282.yaml | 3 + .../5282-implicit-tolerations/README.md | 727 ++++++++++++++++++ .../5282-implicit-tolerations/kep.yaml | 43 ++ 3 files changed, 773 insertions(+) create mode 100644 keps/prod-readiness/sig-scheduling/5282.yaml create mode 100644 keps/sig-scheduling/5282-implicit-tolerations/README.md create mode 100644 keps/sig-scheduling/5282-implicit-tolerations/kep.yaml diff --git a/keps/prod-readiness/sig-scheduling/5282.yaml b/keps/prod-readiness/sig-scheduling/5282.yaml new file mode 100644 index 00000000000..51c7d7ad17e --- /dev/null +++ b/keps/prod-readiness/sig-scheduling/5282.yaml @@ -0,0 +1,3 @@ +kep-number: 5282 +alpha: + approver: "@johnbelamaric" diff --git a/keps/sig-scheduling/5282-implicit-tolerations/README.md b/keps/sig-scheduling/5282-implicit-tolerations/README.md new file mode 100644 index 00000000000..17ecd5b0d19 --- /dev/null +++ b/keps/sig-scheduling/5282-implicit-tolerations/README.md @@ -0,0 +1,727 @@ +# KEP-5282: Implicit tolerations + + +- [Release Signoff Checklist](#release-signoff-checklist) +- [Summary](#summary) +- [Motivation](#motivation) + - [Goals](#goals) + - [Non-Goals](#non-goals) +- [Proposal](#proposal) + - [User Stories (Optional)](#user-stories-optional) + - [Story 1 - GPU-Based ML Training](#story-1---gpu-based-ml-training) + - [Story 2 - Multi-Optional Resource Requests](#story-2---multi-optional-resource-requests) + - [Notes/Constraints/Caveats (Optional)](#notesconstraintscaveats-optional) + - [Risks and Mitigations](#risks-and-mitigations) + - [Increased Scheduler Complexity](#increased-scheduler-complexity) + - [Inconsistent Scheduling Behavior Across Clusters](#inconsistent-scheduling-behavior-across-clusters) + - [API Server Load from Pod Patching](#api-server-load-from-pod-patching) +- [Design Details](#design-details) + - [Plugin Lifecycle](#plugin-lifecycle) + - [Scheduler configuration API](#scheduler-configuration-api) + - [Pod mutation](#pod-mutation) + - [Failure Handling](#failure-handling) + - [Test Plan](#test-plan) + - [Prerequisite testing updates](#prerequisite-testing-updates) + - [Unit tests](#unit-tests) + - [Integration tests](#integration-tests) + - [e2e tests](#e2e-tests) + - [Graduation Criteria](#graduation-criteria) + - [Alpha](#alpha) + - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy) + - [Version Skew Strategy](#version-skew-strategy) +- [Production Readiness Review Questionnaire](#production-readiness-review-questionnaire) + - [Feature Enablement and Rollback](#feature-enablement-and-rollback) + - [Rollout, Upgrade and Rollback Planning](#rollout-upgrade-and-rollback-planning) + - [Monitoring Requirements](#monitoring-requirements) + - [Dependencies](#dependencies) + - [Scalability](#scalability) + - [Troubleshooting](#troubleshooting) +- [Implementation History](#implementation-history) +- [Drawbacks](#drawbacks) +- [Alternatives](#alternatives) +- [Infrastructure Needed (Optional)](#infrastructure-needed-optional) + + +## Release Signoff Checklist + + + +Items marked with (R) are required *prior to targeting to a milestone / release*. + +- [ ] (R) Enhancement issue in release milestone, which links to KEP dir in [kubernetes/enhancements] (not the initial KEP PR) +- [ ] (R) KEP approvers have approved the KEP status as `implementable` +- [ ] (R) Design details are appropriately documented +- [ ] (R) Test plan is in place, giving consideration to SIG Architecture and SIG Testing input (including test refactors) + - [ ] e2e Tests for all Beta API Operations (endpoints) + - [ ] (R) Ensure GA e2e tests meet requirements for [Conformance Tests](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md) + - [ ] (R) Minimum Two Week Window for GA e2e tests to prove flake free +- [ ] (R) Graduation criteria is in place + - [ ] (R) [all GA Endpoints](https://github.com/kubernetes/community/pull/1806) must be hit by [Conformance Tests](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md) +- [ ] (R) Production readiness review completed +- [ ] (R) Production readiness review approved +- [ ] "Implementation History" section is up-to-date for milestone +- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io] +- [ ] Supporting documentation—e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes + + + +[kubernetes.io]: https://kubernetes.io/ +[kubernetes/enhancements]: https://git.k8s.io/enhancements +[kubernetes/kubernetes]: https://git.k8s.io/kubernetes +[kubernetes/website]: https://git.k8s.io/website + +## Summary + +This proposal introduces a scheduler-based mechanism to automatically apply tolerations to pods based on their Dynamic Resource Allocation (DRA) requirements. +The tolerations are derived from the resource classes and claims bound to a pod and applied within the scheduler during the PreEnqueue extension point. + +This avoids reliance on user-specified tolerations or external admission webhooks, reducing friction for users running workloads that consume extended resources like GPUs. + +## Motivation + +Cluster administrators often taint nodes with special resources (e.g., GPUs, FPGAs, DPUs) to prevent generic workloads from landing on them. +For pods to schedule on those nodes, users must add appropriate tolerations, which is often forgotten or error-prone. + +DRA introduced a more dynamic and decoupled way to specify resource requirements using ResourceClaims and DeviceClasses. +However, these claims are not fully resolved at API admission time, making it impossible for admission-time webhooks or admission plugins to inject correct tolerations. + +This proposal moves toleration logic into the scheduler to make it aware of the actual resolved resource requests, +enabling dynamic and reliable toleration injection. + +### Goals + +- Automatically add tolerations to pods based on their resolved DRA ResourceClaims and DeviceClasses. + +- Reduce user errors due to forgotten tolerations. + +- Avoid scheduling failures for pods that legitimately need tainted resources. + +### Non-Goals + +- Supporting non-DRA resource inference. + +- Inferring tolerations for arbitrary annotations or labels. + +- Changing the DRA APIs or Pod API. + +- Change the tolerations mechanism in any way. + +## Proposal + +Introduce a new PreEnqueue scheduler plugin (or extend the existing TaintToleration plugin) that: + +- Depends on the DRA PreEnqueue plugin to ensure all ResourceClaims and DeviceClasses are resolved. + +- Inspects the resolved resources and applies appropriate tolerations in-memory. + +- Optionally patches the Pod via the API server to persist these tolerations (configurable). + +This ensures that pods are only admitted to the scheduling queue when all resource info is ready, and allows toleration logic to be based on full context. + + +### User Stories (Optional) + + + +#### Story 1 - GPU-Based ML Training + +When a pod that uses a DRA claim for a GPU-backed DeviceClass is submitted. +The node with GPUs is tainted. +The user may forget to specify the toleration. +Cluster admin could config through scheduler configuration. +The scheduler sees the resolved DeviceClass and injects the toleration. +The pod schedules successfully. + + +#### Story 2 - Multi-Optional Resource Requests +A pod declares multiple optional claims. Depending on binding, +the scheduler injects tolerations for the correct resource class without user intervention. + +### Notes/Constraints/Caveats (Optional) + + + +### Risks and Mitigations + +#### Increased Scheduler Complexity + +Introducing dynamic toleration injection logic into the scheduler increases the cognitive and maintenance complexity of the scheduling logic. + +Mitigation: Isolate the logic within a dedicated PreEnqueue plugin to limit impact on other scheduling components. +Maintain strict test coverage and fuzzing for rule evaluation. + +#### Inconsistent Scheduling Behavior Across Clusters + +Clusters with different plugin configurations may behave differently for identical pods, potentially surprising users. + +Mitigation: Clearly document expected behavior and make the plugin opt-in. +Encourage platform providers to standardize configurations where applicable. + +#### API Server Load from Pod Patching + +The scheduler might issue one PATCH request per scheduled pod, increasing API server traffic and write load. + +Mitigation: Make PATCHing optional and off by default. Coordinate with SIG Scalability to assess and benchmark impact before enabling by default. + +## Design Details + +### Plugin Lifecycle + +Introduce a new PreEnqueue scheduler plugin (or extend the existing TaintToleration plugin) in scheduler plugin lifecycle: + +- Implement a PreEnqueue plugin. + +- Ensure it runs after DRA readiness is guaranteed. + +- Inspect bound ResourceClaims and DeviceClasses. + +- Inject tolerations based on scheduler config rules. + +### Scheduler configuration API + +The plugin will be configurable through the scheduler policy config, using pluginConfig entries. +The API will support defining toleration rules based on device class selectors. + +```go +type ImplicitTolerationRule struct { + Selector RuleSelector + Toleration corev1.Toleration +} + +type RuleSelector struct { + Type string + + // for Type == 'ExtendedResource' + ResourceNames []string + + // for Type == 'Device' + DevicePrototypes []resourcev1.Device + + // for Type == 'CEL' + Expression *string +} + +type ImplicitTolerationArgs struct { +TolerationRules []ImplicitTolerationRule +PatchPods bool +} +``` + +### Pod mutation + +To make user and other components stay informed, the tolerations could be injected into pod. +A asynchronous call to api-server would be needed to make the patch. +The asynchronous API call discussion happens in: https://github.com/kubernetes/enhancements/issues/5229 + +### Failure Handling + +If ResourceClaims are not resolved, DRA plugin returns Unschedulable. +This plugin never runs until DRA PreEnqueue permits. + +### Test Plan + + + +[X] I/we understand the owners of the involved components may require updates to +existing tests to make this code solid enough prior to committing the changes necessary +to implement this enhancement. + +##### Prerequisite testing updates + + + +##### Unit tests + +- ``: `` - `` + +##### Integration tests + +- [test name](https://github.com/kubernetes/kubernetes/blob/2334b8469e1983c525c0c6382125710093a25883/test/integration/...): [integration master](https://testgrid.k8s.io/sig-release-master-blocking#integration-master?include-filter-by-regex=MyCoolFeature), [triage search](https://storage.googleapis.com/k8s-triage/index.html?test=MyCoolFeature) + +##### e2e tests + +- [test name](https://github.com/kubernetes/kubernetes/blob/2334b8469e1983c525c0c6382125710093a25883/test/e2e/...): [SIG ...](https://testgrid.k8s.io/sig-...?include-filter-by-regex=MyCoolFeature), [triage search](https://storage.googleapis.com/k8s-triage/index.html?test=MyCoolFeature) + +### Graduation Criteria +#### Alpha + +- Feature implemented behind a feature flag +- Initial e2e tests completed and enabled + + + +### Upgrade / Downgrade Strategy + +Implicit Toleration gets disabled when downgrading to a release without support for it or +when disabling the feature. The effect is as if the toleration weren't set. + +### Version Skew Strategy + +During version skew where the apiserver supports the feature and the scheduler +doesn't, an older scheduler will simply not be configured with the plugin or the plugin will be disabled by default. +No impact to API server behavior. + +During version skew where the apiserver does not support the feature and the scheduler +does, when the patching of Pods is enabled, the API server may reject unknown toleration keys or object structure. + +Mitigation: The scheduler should treat failed PATCH operations as non-fatal and proceed with in-memory toleration injection. + +## Production Readiness Review Questionnaire + +### Feature Enablement and Rollback + + + +###### How can this feature be enabled / disabled in a live cluster? + + + +- [ ] Feature gate (also fill in values in `kep.yaml`) + - Feature gate name: ImplicitToleration + - Components depending on the feature gate: scheduler, apiserver + +###### Does enabling the feature change any default behavior? + + +No. + +###### Can the feature be disabled once it has been enabled (i.e. can we roll back the enablement)? + + +Yes. It can be disabled through the feature gate added. + +###### What happens if we reenable the feature if it was previously rolled back? + +The implicit toleration will takes effect again for scheduling after feature reenabled. + +###### Are there any tests for feature enablement/disablement? + +This will be covered through unit tests and e2e test for the apiserver and scheduler. + + +### Rollout, Upgrade and Rollback Planning + + + +###### How can a rollout or rollback fail? Can it impact already running workloads? + + + +###### What specific metrics should inform a rollback? + + + +###### Were upgrade and rollback tested? Was the upgrade->downgrade->upgrade path tested? + + + +###### Is the rollout accompanied by any deprecations and/or removals of features, APIs, fields of API types, flags, etc.? + + + +### Monitoring Requirements + + + +###### How can an operator determine if the feature is in use by workloads? + + + +###### How can someone using this feature know that it is working for their instance? + + + +- [ ] Events + - Event Reason: +- [ ] API .status + - Condition name: + - Other field: +- [ ] Other (treat as last resort) + - Details: + +###### What are the reasonable SLOs (Service Level Objectives) for the enhancement? + + + +###### What are the SLIs (Service Level Indicators) an operator can use to determine the health of the service? + + + +- [ ] Metrics + - Metric name: + - [Optional] Aggregation method: + - Components exposing the metric: +- [ ] Other (treat as last resort) + - Details: + +###### Are there any missing metrics that would be useful to have to improve observability of this feature? + + + +### Dependencies + + + +###### Does this feature depend on any specific services running in the cluster? + + + +### Scalability + + + +###### Will enabling / using this feature result in any new API calls? + + + +###### Will enabling / using this feature result in introducing new API types? + + + +###### Will enabling / using this feature result in any new calls to the cloud provider? + + + +###### Will enabling / using this feature result in increasing size or count of the existing API objects? + + + +###### Will enabling / using this feature result in increasing time taken by any operations covered by existing SLIs/SLOs? + + + +###### Will enabling / using this feature result in non-negligible increase of resource usage (CPU, RAM, disk, IO, ...) in any components? + + + +###### Can enabling / using this feature result in resource exhaustion of some node resources (PIDs, sockets, inodes, etc.)? + + + +### Troubleshooting + + + +###### How does this feature react if the API server and/or etcd is unavailable? + +###### What are other known failure modes? + + + +###### What steps should be taken if SLOs are not being met to determine the problem? + +## Implementation History + + + +## Drawbacks + + + +## Alternatives + +1. Admission Webhook + +Not viable due to incomplete resource resolution at admission time. + +2. Controller-Based Solution + +Separate controller updates pod with tolerations post-claim resolution. +Pros: + - Clean separation + - avoids scheduler API writes +Cons: + - Adds latency + - introduces new component + + +## Infrastructure Needed (Optional) + + diff --git a/keps/sig-scheduling/5282-implicit-tolerations/kep.yaml b/keps/sig-scheduling/5282-implicit-tolerations/kep.yaml new file mode 100644 index 00000000000..4a22d1248ac --- /dev/null +++ b/keps/sig-scheduling/5282-implicit-tolerations/kep.yaml @@ -0,0 +1,43 @@ +title: Implicit tolerations +kep-number: 5282 +authors: + - "@cici37" +owning-sig: sig-scheduling +participating-sigs: + - sig-scheduling +status: implementable +creation-date: 2025-06-04 +reviewers: + - "@sanposhiho" + - "@dom4ha" + - "@johnbelamaric" +approvers: + - "@sanposhiho" + - "@dom4ha" + - "@johnbelamaric" + +# The target maturity stage in the current dev cycle for this KEP. +# If the purpose of this KEP is to deprecate a user-visible feature +# and a Deprecated feature gates are added, they should be deprecated|disabled|removed. +stage: alpha + +# The most recent milestone for which work toward delivery of this KEP has been +# done. This can be the current (upcoming) milestone, if it is being actively +# worked on. +latest-milestone: "v1.34" + +# The milestone at which this feature was, or is targeted to be, at each stage. +milestone: + alpha: "v1.34" + +# The following PRR answers are required at alpha release +# List the feature gate name and the components for which it must be enabled +feature-gates: + - name: ImplicitTolerations + components: + - kube-apiserver + - kube-scheduler +disable-supported: true + +# The following PRR answers are required at beta release +metrics: