diff --git a/pkg/scheduler/api/job_info.go b/pkg/scheduler/api/job_info.go index 03ce5f70f64..9ecb2f5db3f 100644 --- a/pkg/scheduler/api/job_info.go +++ b/pkg/scheduler/api/job_info.go @@ -29,7 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/klog" - volumescheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding" + volumescheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" batch "volcano.sh/apis/pkg/apis/batch/v1alpha1" "volcano.sh/apis/pkg/apis/scheduling" diff --git a/pkg/scheduler/cache/cache.go b/pkg/scheduler/cache/cache.go index 0a84341d5a6..594f7d05754 100644 --- a/pkg/scheduler/cache/cache.go +++ b/pkg/scheduler/cache/cache.go @@ -50,7 +50,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/v1/pod" - volumescheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding" + volumescheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" batch "volcano.sh/apis/pkg/apis/batch/v1alpha1" "volcano.sh/apis/pkg/apis/scheduling" diff --git a/pkg/scheduler/cache/interface.go b/pkg/scheduler/cache/interface.go index aa984ffa843..99bcd0a23c5 100644 --- a/pkg/scheduler/cache/interface.go +++ b/pkg/scheduler/cache/interface.go @@ -22,7 +22,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" - scheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding" + scheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" "volcano.sh/volcano/pkg/scheduler/api" ) diff --git a/pkg/scheduler/plugins/volumebinding/assume_cache.go b/pkg/scheduler/capabilities/volumebinding/assume_cache.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/assume_cache.go rename to pkg/scheduler/capabilities/volumebinding/assume_cache.go diff --git a/pkg/scheduler/plugins/volumebinding/assume_cache_test.go b/pkg/scheduler/capabilities/volumebinding/assume_cache_test.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/assume_cache_test.go rename to pkg/scheduler/capabilities/volumebinding/assume_cache_test.go diff --git a/pkg/scheduler/plugins/volumebinding/binder.go b/pkg/scheduler/capabilities/volumebinding/binder.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/binder.go rename to pkg/scheduler/capabilities/volumebinding/binder.go diff --git a/pkg/scheduler/plugins/volumebinding/binder_test.go b/pkg/scheduler/capabilities/volumebinding/binder_test.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/binder_test.go rename to pkg/scheduler/capabilities/volumebinding/binder_test.go diff --git a/pkg/scheduler/plugins/volumebinding/fake_binder.go b/pkg/scheduler/capabilities/volumebinding/fake_binder.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/fake_binder.go rename to pkg/scheduler/capabilities/volumebinding/fake_binder.go diff --git a/pkg/scheduler/capabilities/volumebinding/readme.md b/pkg/scheduler/capabilities/volumebinding/readme.md new file mode 100644 index 00000000000..7ddf0346483 --- /dev/null +++ b/pkg/scheduler/capabilities/volumebinding/readme.md @@ -0,0 +1,20 @@ +## Introduction +When adapting to K8S V1.25, Volcano introduces the volumebinding package from K8S for self-maintenance. It is expected that the current package will be deleted from Volcano when adapting to K8S V1.29. + +## Background +Volcano encounters forward compatibility issues when adapting to K8S V1.25. Volcano can run properly in K8S V1.25, but cannot run in earlier K8S versions (for example, v1.23, v1.21). The error information is `reflector.go:424] k8s.io/client-go/informers/factory.go:134: failed to list *v1.CSIStorageCapacity: the server could not find the requested resource`. + +## Reason +The `volumeBinder.csiStorageCapacityLister` in the `k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding` package uses the `k8s.io/client-go/listers/storage/v1beta1` interface in V1.23, and is updated to the `k8s.io/client-go/listers/storage/v1` interface in V1.25. (csiStorageCapacity is officially released v1 in v1.24) In addition, the EnableCSIStorageCapacity feature switch is removed. + +The volumeBinder structure is statically defined. If the external plug-in imports the volumebinding package and uses the volumeBinder structure, forward compatibility issues may occur, for example, `v1.CSIStorageCapacity` is not defined. + +Volcano references the volumebinding package of the K8S during scheduling for PVC binding. Therefore, after the K8S V1.25 is upgraded, the V1.23 and V1.21 versions cannot run properly. + +## Current solution +The `k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding` package is introduced to Volcano for self-maintenance. The `volumeBinder.csiStorageCapacityLister` version is changed back to `k8s.io/client-go/listers/storage/v1beta1` to implement compatibility processing of K8S v1.25, v1.23, and v1.21. + +## Future evolution +Solution 1: After the Volcano adapts to K8S V1.27, the `volumeBinder.csiStorageCapacityLister` interface version is upgraded from V1beta1 to V1, the self-maintained volumebinding package is deleted, and the volumebinding package in K8S is referenced again. After the upgrade, however, Only K8S V1.27 and V1.25 are compatible. Earlier versions, such as V1.23, are not compatible. + +Solution 2: Add an abstract layer to volumeBinder in the volumebinding package and change the static version dependency to dynamic definition to ensure compatibility between v1beta1 and v1 interfaces. diff --git a/pkg/scheduler/plugins/volumebinding/scorer.go b/pkg/scheduler/capabilities/volumebinding/scorer.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/scorer.go rename to pkg/scheduler/capabilities/volumebinding/scorer.go diff --git a/pkg/scheduler/plugins/volumebinding/scorer_test.go b/pkg/scheduler/capabilities/volumebinding/scorer_test.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/scorer_test.go rename to pkg/scheduler/capabilities/volumebinding/scorer_test.go diff --git a/pkg/scheduler/plugins/volumebinding/test_utils.go b/pkg/scheduler/capabilities/volumebinding/test_utils.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/test_utils.go rename to pkg/scheduler/capabilities/volumebinding/test_utils.go diff --git a/pkg/scheduler/plugins/volumebinding/volume_binding.go b/pkg/scheduler/capabilities/volumebinding/volume_binding.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/volume_binding.go rename to pkg/scheduler/capabilities/volumebinding/volume_binding.go diff --git a/pkg/scheduler/plugins/volumebinding/volume_binding_test.go b/pkg/scheduler/capabilities/volumebinding/volume_binding_test.go similarity index 100% rename from pkg/scheduler/plugins/volumebinding/volume_binding_test.go rename to pkg/scheduler/capabilities/volumebinding/volume_binding_test.go diff --git a/pkg/scheduler/plugins/nodeorder/nodeorder.go b/pkg/scheduler/plugins/nodeorder/nodeorder.go index efeeee84042..17260b0b366 100644 --- a/pkg/scheduler/plugins/nodeorder/nodeorder.go +++ b/pkg/scheduler/plugins/nodeorder/nodeorder.go @@ -36,9 +36,9 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/selectorspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" - "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding" "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" "volcano.sh/volcano/pkg/scheduler/framework" "volcano.sh/volcano/pkg/scheduler/plugins/util" "volcano.sh/volcano/pkg/scheduler/plugins/util/k8s" diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index 5b95f2fcd0e..f344faa5fea 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -24,7 +24,9 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilFeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" k8sframework "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" @@ -36,9 +38,9 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumezone" - "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding" "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" "volcano.sh/volcano/pkg/scheduler/framework" "volcano.sh/volcano/pkg/scheduler/plugins/util" "volcano.sh/volcano/pkg/scheduler/plugins/util/k8s" @@ -291,6 +293,13 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { }, }) + features := feature.Features{ + EnableReadWriteOncePod: utilFeature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod), + EnableVolumeCapacityPriority: utilFeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority), + EnableMinDomainsInPodTopologySpread: utilFeature.DefaultFeatureGate.Enabled(features.MinDomainsInPodTopologySpread), + EnableNodeInclusionPolicyInPodTopologySpread: utilFeature.DefaultFeatureGate.Enabled(features.NodeInclusionPolicyInPodTopologySpread), + EnableMatchLabelKeysInPodTopologySpread: utilFeature.DefaultFeatureGate.Enabled(features.MatchLabelKeysInPodTopologySpread), + } // Initialize k8s plugins // TODO: Add more predicates, k8s.io/kubernetes/pkg/scheduler/framework/plugins/legacy_registry.go handle := k8s.NewFrameworkHandle(nodeMap, ssn.KubeClient(), ssn.InformerFactory()) @@ -314,7 +323,6 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { plugin, _ = interpodaffinity.New(plArgs, handle) podAffinityFilter := plugin.(*interpodaffinity.InterPodAffinity) // 6. NodeVolumeLimits - features := feature.Features{} plugin, _ = nodevolumelimits.NewCSI(nil, handle, features) nodeVolumeLimitsCSIFilter := plugin.(*nodevolumelimits.CSILimits) // 7. VolumeBinding @@ -398,7 +406,15 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { } // InterPodAffinity Predicate - // TODO use framework.PreFilterResult + // TODO: Update the node information to be processed by the filer based on the node list returned by the prefilter. + // In K8S V1.25, the return value result is added to the Prefile interface, + // indicating the list of nodes that meet filtering conditions. + // If the value of result is nil, all nodes meet the conditions. + // If the specified node information exists, only the node information in result meets the conditions. + // The value of Prefile in the current InterPodAffinity package always returns nil. + // The outer layer does not need to be processed temporarily. + // If the filtering logic is added to the Prefile node in the Volumebinding package in the future, + // the processing logic needs to be added to the return value result. _, status = podAffinityFilter.PreFilter(context.TODO(), state, task.Pod) if !status.IsSuccess() { return fmt.Errorf("plugin %s pre-predicates failed %s", interpodaffinity.Name, status.Message()) @@ -416,7 +432,15 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { } // Check VolumeBinding: handle immediate claims unbounded case - // TODO use framework.PreFilterResult + // TODO: Update the node information to be processed by the filer based on the node list returned by the prefilter. + // In K8S V1.25, the return value result is added to the Prefile interface, + // indicating the list of nodes that meet filtering conditions. + // If the value of result is nil, all nodes meet the conditions. + // If the specified node information exists, only the node information in result meets the conditions. + // The value of Prefile in the current Volumebinding package always returns nil. + // The outer layer does not need to be processed temporarily. + // If the filtering logic is added to the Prefile node in the Volumebinding package in the future, + // the processing logic needs to be added to the return value result. _, status = volumebindingFilter.PreFilter(context.TODO(), state, task.Pod) if !status.IsSuccess() { return fmt.Errorf("plugin %s pre-predicates failed %s", volumebindingFilter.Name(), status.Message()) @@ -435,7 +459,15 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { } // Check PodTopologySpread - // TODO use framework.PreFilterResult + // TODO: Update the node information to be processed by the filer based on the node list returned by the prefilter. + // In K8S V1.25, the return value result is added to the Prefile interface, + // indicating the list of nodes that meet filtering conditions. + // If the value of result is nil, all nodes meet the conditions. + // If the specified node information exists, only the node information in result meets the conditions. + // The value of Prefile in the current PodTopologySpread package always returns nil. + // The outer layer does not need to be processed temporarily. + // If the filtering logic is added to the Prefile node in the Volumebinding package in the future, + // the processing logic needs to be added to the return value result. _, status = podTopologySpreadFilter.PreFilter(context.TODO(), state, task.Pod) if !status.IsSuccess() { return fmt.Errorf("plugin %s pre-predicates failed %s", podTopologySpreadFilter.Name(), status.Message()) diff --git a/pkg/scheduler/plugins/util/k8s/framework.go b/pkg/scheduler/plugins/util/k8s/framework.go index f585aae9916..89877742654 100644 --- a/pkg/scheduler/plugins/util/k8s/framework.go +++ b/pkg/scheduler/plugins/util/k8s/framework.go @@ -28,7 +28,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" - scheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding" + scheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" ) // Framework is a K8S framework who mainly provides some methods diff --git a/pkg/scheduler/util/test_utils.go b/pkg/scheduler/util/test_utils.go index 1297f2834a3..1921e85c962 100644 --- a/pkg/scheduler/util/test_utils.go +++ b/pkg/scheduler/util/test_utils.go @@ -32,7 +32,7 @@ import ( "k8s.io/client-go/tools/cache" schedulingv2 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" "volcano.sh/volcano/pkg/scheduler/api" - volumescheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding" + volumescheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" ) // BuildResourceList builts resource list object