Migrate volumebinding from scheduler/plugin/ to scheduler/capabilitie…

…s/ to avoid dependency of cache on the plugin directory.

Signed-off-by: wangyang <wangyang289@huawei.com>

pkg/scheduler/api/job_info.go

@@ -29,7 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog"
-	volumescheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding"
+	volumescheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 
 	batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
 	"volcano.sh/apis/pkg/apis/scheduling"

pkg/scheduler/cache/cache.go

@@ -50,7 +50,7 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
-	volumescheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding"
+	volumescheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 
 	batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
 	"volcano.sh/apis/pkg/apis/scheduling"

pkg/scheduler/cache/interface.go

@@ -22,7 +22,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
-	scheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding"
+	scheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 
 	"volcano.sh/volcano/pkg/scheduler/api"
 )

pkg/scheduler/capabilities/volumebinding/readme.md

@@ -0,0 +1,20 @@
+## Introduction
+When adapting to K8S V1.25, Volcano introduces the volumebinding package from K8S for self-maintenance. It is expected that the current package will be deleted from Volcano when adapting to K8S V1.29.
+
+## Background
+Volcano encounters forward compatibility issues when adapting to K8S V1.25. Volcano can run properly in K8S V1.25, but cannot run in earlier K8S versions (for example, v1.23, v1.21). The error information is `reflector.go:424] k8s.io/client-go/informers/factory.go:134: failed to list *v1.CSIStorageCapacity: the server could not find the requested resource`.
+
+## Reason
+The `volumeBinder.csiStorageCapacityLister` in the `k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding` package uses the `k8s.io/client-go/listers/storage/v1beta1` interface in V1.23, and is updated to the `k8s.io/client-go/listers/storage/v1` interface in V1.25. (csiStorageCapacity is officially released v1 in v1.24) In addition, the EnableCSIStorageCapacity feature switch is removed.
+
+The volumeBinder structure is statically defined. If the external plug-in imports the volumebinding package and uses the volumeBinder structure, forward compatibility issues may occur, for example, `v1.CSIStorageCapacity` is not defined.
+
+Volcano references the volumebinding package of the K8S during scheduling for PVC binding. Therefore, after the K8S V1.25 is upgraded, the V1.23 and V1.21 versions cannot run properly.
+
+## Current solution
+The `k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding` package is introduced to Volcano for self-maintenance. The `volumeBinder.csiStorageCapacityLister` version is changed back to `k8s.io/client-go/listers/storage/v1beta1` to implement compatibility processing of K8S v1.25, v1.23, and v1.21.
+
+## Future evolution
+Solution 1: After the Volcano adapts to K8S V1.27, the `volumeBinder.csiStorageCapacityLister` interface version is upgraded from V1beta1 to V1, the self-maintained volumebinding package is deleted, and the volumebinding package in K8S is referenced again. After the upgrade, however, Only K8S V1.27 and V1.25 are compatible. Earlier versions, such as V1.23, are not compatible.
+
+Solution 2: Add an abstract layer to volumeBinder in the volumebinding package and change the static version dependency to dynamic definition to ensure compatibility between v1beta1 and v1 interfaces.

pkg/scheduler/plugins/nodeorder/nodeorder.go

@@ -36,9 +36,9 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/selectorspread"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
-	"volcano.sh/volcano/pkg/scheduler/plugins/volumebinding"
 
 	"volcano.sh/volcano/pkg/scheduler/api"
+	"volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 	"volcano.sh/volcano/pkg/scheduler/framework"
 	"volcano.sh/volcano/pkg/scheduler/plugins/util"
 	"volcano.sh/volcano/pkg/scheduler/plugins/util/k8s"

pkg/scheduler/plugins/predicates/predicates.go

@@ -24,7 +24,9 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	utilFeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/scheduler/apis/config"
 	k8sframework "k8s.io/kubernetes/pkg/scheduler/framework"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
@@ -36,9 +38,9 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumezone"
-	"volcano.sh/volcano/pkg/scheduler/plugins/volumebinding"
 
 	"volcano.sh/volcano/pkg/scheduler/api"
+	"volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 	"volcano.sh/volcano/pkg/scheduler/framework"
 	"volcano.sh/volcano/pkg/scheduler/plugins/util"
 	"volcano.sh/volcano/pkg/scheduler/plugins/util/k8s"
@@ -291,6 +293,13 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
 		},
 	})
 
+	features := feature.Features{
+		EnableReadWriteOncePod:                       utilFeature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod),
+		EnableVolumeCapacityPriority:                 utilFeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority),
+		EnableMinDomainsInPodTopologySpread:          utilFeature.DefaultFeatureGate.Enabled(features.MinDomainsInPodTopologySpread),
+		EnableNodeInclusionPolicyInPodTopologySpread: utilFeature.DefaultFeatureGate.Enabled(features.NodeInclusionPolicyInPodTopologySpread),
+		EnableMatchLabelKeysInPodTopologySpread:      utilFeature.DefaultFeatureGate.Enabled(features.MatchLabelKeysInPodTopologySpread),
+	}
 	// Initialize k8s plugins
 	// TODO: Add more predicates, k8s.io/kubernetes/pkg/scheduler/framework/plugins/legacy_registry.go
 	handle := k8s.NewFrameworkHandle(nodeMap, ssn.KubeClient(), ssn.InformerFactory())
@@ -314,7 +323,6 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
 	plugin, _ = interpodaffinity.New(plArgs, handle)
 	podAffinityFilter := plugin.(*interpodaffinity.InterPodAffinity)
 	// 6. NodeVolumeLimits
-	features := feature.Features{}
 	plugin, _ = nodevolumelimits.NewCSI(nil, handle, features)
 	nodeVolumeLimitsCSIFilter := plugin.(*nodevolumelimits.CSILimits)
 	// 7. VolumeBinding
@@ -398,7 +406,15 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
 		}
 
 		// InterPodAffinity Predicate
-		// TODO use framework.PreFilterResult
+		// TODO: Update the node information to be processed by the filer based on the node list returned by the prefilter.
+		// In K8S V1.25, the return value result is added to the Prefile interface,
+		// indicating the list of nodes that meet filtering conditions.
+		// If the value of result is nil, all nodes meet the conditions.
+		// If the specified node information exists, only the node information in result meets the conditions.
+		// The value of Prefile in the current InterPodAffinity package always returns nil.
+		// The outer layer does not need to be processed temporarily.
+		// If the filtering logic is added to the Prefile node in the Volumebinding package in the future,
+		// the processing logic needs to be added to the return value result.
 		_, status = podAffinityFilter.PreFilter(context.TODO(), state, task.Pod)
 		if !status.IsSuccess() {
 			return fmt.Errorf("plugin %s pre-predicates failed %s", interpodaffinity.Name, status.Message())
@@ -416,7 +432,15 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
 		}
 
 		// Check VolumeBinding: handle immediate claims unbounded case
-		// TODO use framework.PreFilterResult
+		// TODO: Update the node information to be processed by the filer based on the node list returned by the prefilter.
+		// In K8S V1.25, the return value result is added to the Prefile interface,
+		// indicating the list of nodes that meet filtering conditions.
+		// If the value of result is nil, all nodes meet the conditions.
+		// If the specified node information exists, only the node information in result meets the conditions.
+		// The value of Prefile in the current Volumebinding package always returns nil.
+		// The outer layer does not need to be processed temporarily.
+		// If the filtering logic is added to the Prefile node in the Volumebinding package in the future,
+		// the processing logic needs to be added to the return value result.
 		_, status = volumebindingFilter.PreFilter(context.TODO(), state, task.Pod)
 		if !status.IsSuccess() {
 			return fmt.Errorf("plugin %s pre-predicates failed %s", volumebindingFilter.Name(), status.Message())
@@ -435,7 +459,15 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
 		}
 
 		// Check PodTopologySpread
-		// TODO use framework.PreFilterResult
+		// TODO: Update the node information to be processed by the filer based on the node list returned by the prefilter.
+		// In K8S V1.25, the return value result is added to the Prefile interface,
+		// indicating the list of nodes that meet filtering conditions.
+		// If the value of result is nil, all nodes meet the conditions.
+		// If the specified node information exists, only the node information in result meets the conditions.
+		// The value of Prefile in the current PodTopologySpread package always returns nil.
+		// The outer layer does not need to be processed temporarily.
+		// If the filtering logic is added to the Prefile node in the Volumebinding package in the future,
+		// the processing logic needs to be added to the return value result.
 		_, status = podTopologySpreadFilter.PreFilter(context.TODO(), state, task.Pod)
 		if !status.IsSuccess() {
 			return fmt.Errorf("plugin %s pre-predicates failed %s", podTopologySpreadFilter.Name(), status.Message())

pkg/scheduler/plugins/util/k8s/framework.go

@@ -28,7 +28,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
 	"k8s.io/kubernetes/pkg/scheduler/framework/parallelize"
-	scheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding"
+	scheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 )
 
 // Framework is a K8S framework who mainly provides some methods

pkg/scheduler/util/test_utils.go

@@ -32,7 +32,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	schedulingv2 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
 	"volcano.sh/volcano/pkg/scheduler/api"
-	volumescheduling "volcano.sh/volcano/pkg/scheduler/plugins/volumebinding"
+	volumescheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 )
 
 // BuildResourceList builts resource list object